mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-07-14 16:01:13 +03:00
Implement per intel-gpu stats collection
This commit is contained in:
parent
c8cfb9400a
commit
740c03e627
76
frigate/stats/intel_gpu_info.py
Normal file
76
frigate/stats/intel_gpu_info.py
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
"""Resolve human-readable names for Intel GPUs via OpenVINO."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class IntelGpuNameResolver:
|
||||||
|
"""Build a pdev -> normalized device name map by enumerating OpenVINO GPUs.
|
||||||
|
|
||||||
|
The lookup is performed once on first access and cached for the process
|
||||||
|
lifetime. OpenVINO exposes DEVICE_PCI_INFO (domain/bus/device/function) and
|
||||||
|
FULL_DEVICE_NAME for each GPU it can see, which is enough to associate the
|
||||||
|
name with the pdev string used by DRM fdinfo.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_names: Optional[dict[str, str]] = None
|
||||||
|
|
||||||
|
def get_names(self) -> dict[str, str]:
|
||||||
|
if self._names is not None:
|
||||||
|
return self._names
|
||||||
|
|
||||||
|
names: dict[str, str] = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
from openvino import Core
|
||||||
|
except ImportError:
|
||||||
|
logger.debug("OpenVINO unavailable; cannot resolve Intel GPU names")
|
||||||
|
self._names = names
|
||||||
|
return names
|
||||||
|
|
||||||
|
try:
|
||||||
|
core = Core()
|
||||||
|
devices = core.available_devices
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug(f"OpenVINO Core initialization failed: {exc}")
|
||||||
|
self._names = names
|
||||||
|
return names
|
||||||
|
|
||||||
|
for device in devices:
|
||||||
|
if not device.startswith("GPU"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
pci = core.get_property(device, "DEVICE_PCI_INFO")
|
||||||
|
raw_name = core.get_property(device, "FULL_DEVICE_NAME")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug(f"Failed to read properties for {device}: {exc}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
pdev = self._format_pdev(pci)
|
||||||
|
if not pdev:
|
||||||
|
continue
|
||||||
|
|
||||||
|
names[pdev] = self._normalize_name(raw_name)
|
||||||
|
|
||||||
|
self._names = names
|
||||||
|
return names
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _format_pdev(pci) -> Optional[str]:
|
||||||
|
try:
|
||||||
|
return f"{pci.domain:04x}:{pci.bus:02x}:{pci.device:02x}.{pci.function:x}"
|
||||||
|
except AttributeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _normalize_name(name: str) -> str:
|
||||||
|
cleaned = re.sub(r"\(R\)|\(TM\)", "", name)
|
||||||
|
cleaned = re.sub(r"\s*\((?:i|d)GPU\)\s*$", "", cleaned, flags=re.IGNORECASE)
|
||||||
|
return " ".join(cleaned.split())
|
||||||
|
|
||||||
|
|
||||||
|
intel_gpu_name_resolver = IntelGpuNameResolver()
|
||||||
@ -230,6 +230,7 @@ async def set_gpu_stats(
|
|||||||
hwaccel_args.append(args)
|
hwaccel_args.append(args)
|
||||||
|
|
||||||
stats: dict[str, dict] = {}
|
stats: dict[str, dict] = {}
|
||||||
|
intel_gpu_collected = False
|
||||||
|
|
||||||
for args in hwaccel_args:
|
for args in hwaccel_args:
|
||||||
if args in hwaccel_errors:
|
if args in hwaccel_errors:
|
||||||
@ -265,14 +266,17 @@ async def set_gpu_stats(
|
|||||||
if not config.telemetry.stats.intel_gpu_stats:
|
if not config.telemetry.stats.intel_gpu_stats:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if "intel-gpu" not in stats:
|
if not intel_gpu_collected:
|
||||||
# intel GPU (QSV or VAAPI both use the same physical GPU)
|
# intel GPU (QSV or VAAPI both use the same physical GPU)
|
||||||
|
intel_gpu_collected = True
|
||||||
intel_usage = get_intel_gpu_stats(
|
intel_usage = get_intel_gpu_stats(
|
||||||
config.telemetry.stats.intel_gpu_device
|
config.telemetry.stats.intel_gpu_device
|
||||||
)
|
)
|
||||||
|
|
||||||
if intel_usage is not None:
|
if intel_usage:
|
||||||
stats["intel-gpu"] = intel_usage or {"gpu": "", "mem": ""}
|
for entry in intel_usage.values():
|
||||||
|
name = entry.pop("name")
|
||||||
|
stats[name] = entry
|
||||||
else:
|
else:
|
||||||
stats["intel-gpu"] = {"gpu": "", "mem": ""}
|
stats["intel-gpu"] = {"gpu": "", "mem": ""}
|
||||||
hwaccel_errors.append(args)
|
hwaccel_errors.append(args)
|
||||||
|
|||||||
@ -17,12 +17,14 @@ class TestGpuStats(unittest.TestCase):
|
|||||||
amd_stats = get_amd_gpu_stats()
|
amd_stats = get_amd_gpu_stats()
|
||||||
assert amd_stats == {"gpu": "4.17%", "mem": "60.37%"}
|
assert amd_stats == {"gpu": "4.17%", "mem": "60.37%"}
|
||||||
|
|
||||||
|
@patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names")
|
||||||
@patch("frigate.util.services.time.sleep")
|
@patch("frigate.util.services.time.sleep")
|
||||||
@patch("frigate.util.services.time.monotonic")
|
@patch("frigate.util.services.time.monotonic")
|
||||||
@patch("frigate.util.services._read_intel_drm_fdinfo")
|
@patch("frigate.util.services._read_intel_drm_fdinfo")
|
||||||
def test_intel_gpu_stats_fdinfo(self, read_fdinfo, monotonic, sleep):
|
def test_intel_gpu_stats_fdinfo(self, read_fdinfo, monotonic, sleep, get_names):
|
||||||
# 1 second of wall clock between snapshots
|
# 1 second of wall clock between snapshots
|
||||||
monotonic.side_effect = [0.0, 1.0]
|
monotonic.side_effect = [0.0, 1.0]
|
||||||
|
get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
|
||||||
|
|
||||||
# Two i915 clients on the same iGPU. Engine values are cumulative ns.
|
# Two i915 clients on the same iGPU. Engine values are cumulative ns.
|
||||||
# Deltas over the 1s window:
|
# Deltas over the 1s window:
|
||||||
@ -79,11 +81,14 @@ class TestGpuStats(unittest.TestCase):
|
|||||||
|
|
||||||
sleep.assert_called_once()
|
sleep.assert_called_once()
|
||||||
assert intel_stats == {
|
assert intel_stats == {
|
||||||
"gpu": "90.0%",
|
"0000:00:02.0": {
|
||||||
"mem": "-%",
|
"name": "Intel Graphics",
|
||||||
"compute": "30.0%",
|
"gpu": "90.0%",
|
||||||
"dec": "60.0%",
|
"mem": "-%",
|
||||||
"clients": {"100": "80.0%", "200": "10.0%"},
|
"compute": "30.0%",
|
||||||
|
"dec": "60.0%",
|
||||||
|
"clients": {"100": "80.0%", "200": "10.0%"},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@patch("frigate.util.services._read_intel_drm_fdinfo")
|
@patch("frigate.util.services._read_intel_drm_fdinfo")
|
||||||
|
|||||||
@ -393,8 +393,10 @@ def _read_intel_drm_fdinfo(target_pdev: Optional[str]) -> dict:
|
|||||||
return snapshot
|
return snapshot
|
||||||
|
|
||||||
|
|
||||||
def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, Any]]:
|
def get_intel_gpu_stats(
|
||||||
"""Get stats by reading DRM fdinfo files.
|
intel_gpu_device: Optional[str],
|
||||||
|
) -> Optional[dict[str, dict[str, Any]]]:
|
||||||
|
"""Get stats by reading DRM fdinfo files, bucketed per-pdev.
|
||||||
|
|
||||||
Each DRM client FD exposes monotonic per-engine busy counters via
|
Each DRM client FD exposes monotonic per-engine busy counters via
|
||||||
/proc/<pid>/fdinfo/<fd> (i915 since kernel 5.19, Xe since first release).
|
/proc/<pid>/fdinfo/<fd> (i915 since kernel 5.19, Xe since first release).
|
||||||
@ -402,7 +404,14 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, A
|
|||||||
utilization. Render/3D and Compute are pooled into "compute"; Video and
|
utilization. Render/3D and Compute are pooled into "compute"; Video and
|
||||||
VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped
|
VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped
|
||||||
to 100%).
|
to 100%).
|
||||||
|
|
||||||
|
The return value is keyed by the GPU's drm-pdev string so multiple Intel
|
||||||
|
GPUs in the same system are reported separately. Each entry carries a
|
||||||
|
"name" populated from OpenVINO (falling back to the pdev) so callers can
|
||||||
|
surface a real device name in the UI.
|
||||||
"""
|
"""
|
||||||
|
from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
|
||||||
|
|
||||||
target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device)
|
target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device)
|
||||||
|
|
||||||
snapshot_a = _read_intel_drm_fdinfo(target_pdev)
|
snapshot_a = _read_intel_drm_fdinfo(target_pdev)
|
||||||
@ -417,19 +426,21 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, A
|
|||||||
if not snapshot_b or elapsed_ns <= 0:
|
if not snapshot_b or elapsed_ns <= 0:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
engine_pct: dict[str, float] = {
|
def _new_engine_pct() -> dict[str, float]:
|
||||||
"render": 0.0,
|
return {"render": 0.0, "video": 0.0, "video-enhance": 0.0, "compute": 0.0}
|
||||||
"video": 0.0,
|
|
||||||
"video-enhance": 0.0,
|
per_pdev_engine_pct: dict[str, dict[str, float]] = {}
|
||||||
"compute": 0.0,
|
per_pdev_pid_pct: dict[str, dict[str, float]] = {}
|
||||||
}
|
|
||||||
pid_pct: dict[str, float] = {}
|
|
||||||
|
|
||||||
for key, data_b in snapshot_b.items():
|
for key, data_b in snapshot_b.items():
|
||||||
data_a = snapshot_a.get(key)
|
data_a = snapshot_a.get(key)
|
||||||
if not data_a or data_a["driver"] != data_b["driver"]:
|
if not data_a or data_a["driver"] != data_b["driver"]:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
pdev = key[0]
|
||||||
|
engine_pct = per_pdev_engine_pct.setdefault(pdev, _new_engine_pct())
|
||||||
|
pid_pct = per_pdev_pid_pct.setdefault(pdev, {})
|
||||||
|
|
||||||
client_total = 0.0
|
client_total = 0.0
|
||||||
for engine, (busy_b, total_b) in data_b["engines"].items():
|
for engine, (busy_b, total_b) in data_b["engines"].items():
|
||||||
if engine not in engine_pct:
|
if engine not in engine_pct:
|
||||||
@ -452,25 +463,36 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, A
|
|||||||
|
|
||||||
pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total
|
pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total
|
||||||
|
|
||||||
for engine in engine_pct:
|
if not per_pdev_engine_pct:
|
||||||
engine_pct[engine] = min(100.0, engine_pct[engine])
|
return None
|
||||||
|
|
||||||
compute_pct = min(100.0, engine_pct["render"] + engine_pct["compute"])
|
names = intel_gpu_name_resolver.get_names()
|
||||||
dec_pct = min(100.0, engine_pct["video"] + engine_pct["video-enhance"])
|
results: dict[str, dict[str, Any]] = {}
|
||||||
overall_pct = min(100.0, compute_pct + dec_pct)
|
|
||||||
|
|
||||||
results: dict[str, Any] = {
|
for pdev, engine_pct in per_pdev_engine_pct.items():
|
||||||
"gpu": f"{round(overall_pct, 2)}%",
|
for engine in engine_pct:
|
||||||
"mem": "-%",
|
engine_pct[engine] = min(100.0, engine_pct[engine])
|
||||||
"compute": f"{round(compute_pct, 2)}%",
|
|
||||||
"dec": f"{round(dec_pct, 2)}%",
|
|
||||||
}
|
|
||||||
|
|
||||||
if pid_pct:
|
compute_pct = min(100.0, engine_pct["render"] + engine_pct["compute"])
|
||||||
results["clients"] = {
|
dec_pct = min(100.0, engine_pct["video"] + engine_pct["video-enhance"])
|
||||||
pid: f"{round(min(100.0, pct), 2)}%" for pid, pct in pid_pct.items()
|
overall_pct = min(100.0, compute_pct + dec_pct)
|
||||||
|
|
||||||
|
entry: dict[str, Any] = {
|
||||||
|
"name": names.get(pdev) or f"Intel GPU {pdev}",
|
||||||
|
"gpu": f"{round(overall_pct, 2)}%",
|
||||||
|
"mem": "-%",
|
||||||
|
"compute": f"{round(compute_pct, 2)}%",
|
||||||
|
"dec": f"{round(dec_pct, 2)}%",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pid_pct = per_pdev_pid_pct.get(pdev)
|
||||||
|
if pid_pct:
|
||||||
|
entry["clients"] = {
|
||||||
|
pid: f"{round(min(100.0, pct), 2)}%" for pid, pct in pid_pct.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
results[pdev] = entry
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user