Implement per intel-gpu stats collection

This commit is contained in:
Nicolas Mowen 2026-05-13 14:22:33 -06:00
parent c8cfb9400a
commit 740c03e627
4 changed files with 139 additions and 32 deletions

View File

@ -0,0 +1,76 @@
"""Resolve human-readable names for Intel GPUs via OpenVINO."""
import logging
import re
from typing import Optional
logger = logging.getLogger(__name__)
class IntelGpuNameResolver:
"""Build a pdev -> normalized device name map by enumerating OpenVINO GPUs.
The lookup is performed once on first access and cached for the process
lifetime. OpenVINO exposes DEVICE_PCI_INFO (domain/bus/device/function) and
FULL_DEVICE_NAME for each GPU it can see, which is enough to associate the
name with the pdev string used by DRM fdinfo.
"""
_names: Optional[dict[str, str]] = None
def get_names(self) -> dict[str, str]:
if self._names is not None:
return self._names
names: dict[str, str] = {}
try:
from openvino import Core
except ImportError:
logger.debug("OpenVINO unavailable; cannot resolve Intel GPU names")
self._names = names
return names
try:
core = Core()
devices = core.available_devices
except Exception as exc:
logger.debug(f"OpenVINO Core initialization failed: {exc}")
self._names = names
return names
for device in devices:
if not device.startswith("GPU"):
continue
try:
pci = core.get_property(device, "DEVICE_PCI_INFO")
raw_name = core.get_property(device, "FULL_DEVICE_NAME")
except Exception as exc:
logger.debug(f"Failed to read properties for {device}: {exc}")
continue
pdev = self._format_pdev(pci)
if not pdev:
continue
names[pdev] = self._normalize_name(raw_name)
self._names = names
return names
@staticmethod
def _format_pdev(pci) -> Optional[str]:
try:
return f"{pci.domain:04x}:{pci.bus:02x}:{pci.device:02x}.{pci.function:x}"
except AttributeError:
return None
@staticmethod
def _normalize_name(name: str) -> str:
cleaned = re.sub(r"\(R\)|\(TM\)", "", name)
cleaned = re.sub(r"\s*\((?:i|d)GPU\)\s*$", "", cleaned, flags=re.IGNORECASE)
return " ".join(cleaned.split())
intel_gpu_name_resolver = IntelGpuNameResolver()

View File

@ -230,6 +230,7 @@ async def set_gpu_stats(
hwaccel_args.append(args) hwaccel_args.append(args)
stats: dict[str, dict] = {} stats: dict[str, dict] = {}
intel_gpu_collected = False
for args in hwaccel_args: for args in hwaccel_args:
if args in hwaccel_errors: if args in hwaccel_errors:
@ -265,14 +266,17 @@ async def set_gpu_stats(
if not config.telemetry.stats.intel_gpu_stats: if not config.telemetry.stats.intel_gpu_stats:
continue continue
if "intel-gpu" not in stats: if not intel_gpu_collected:
# intel GPU (QSV or VAAPI both use the same physical GPU) # intel GPU (QSV or VAAPI both use the same physical GPU)
intel_gpu_collected = True
intel_usage = get_intel_gpu_stats( intel_usage = get_intel_gpu_stats(
config.telemetry.stats.intel_gpu_device config.telemetry.stats.intel_gpu_device
) )
if intel_usage is not None: if intel_usage:
stats["intel-gpu"] = intel_usage or {"gpu": "", "mem": ""} for entry in intel_usage.values():
name = entry.pop("name")
stats[name] = entry
else: else:
stats["intel-gpu"] = {"gpu": "", "mem": ""} stats["intel-gpu"] = {"gpu": "", "mem": ""}
hwaccel_errors.append(args) hwaccel_errors.append(args)

View File

@ -17,12 +17,14 @@ class TestGpuStats(unittest.TestCase):
amd_stats = get_amd_gpu_stats() amd_stats = get_amd_gpu_stats()
assert amd_stats == {"gpu": "4.17%", "mem": "60.37%"} assert amd_stats == {"gpu": "4.17%", "mem": "60.37%"}
@patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names")
@patch("frigate.util.services.time.sleep") @patch("frigate.util.services.time.sleep")
@patch("frigate.util.services.time.monotonic") @patch("frigate.util.services.time.monotonic")
@patch("frigate.util.services._read_intel_drm_fdinfo") @patch("frigate.util.services._read_intel_drm_fdinfo")
def test_intel_gpu_stats_fdinfo(self, read_fdinfo, monotonic, sleep): def test_intel_gpu_stats_fdinfo(self, read_fdinfo, monotonic, sleep, get_names):
# 1 second of wall clock between snapshots # 1 second of wall clock between snapshots
monotonic.side_effect = [0.0, 1.0] monotonic.side_effect = [0.0, 1.0]
get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
# Two i915 clients on the same iGPU. Engine values are cumulative ns. # Two i915 clients on the same iGPU. Engine values are cumulative ns.
# Deltas over the 1s window: # Deltas over the 1s window:
@ -79,11 +81,14 @@ class TestGpuStats(unittest.TestCase):
sleep.assert_called_once() sleep.assert_called_once()
assert intel_stats == { assert intel_stats == {
"gpu": "90.0%", "0000:00:02.0": {
"mem": "-%", "name": "Intel Graphics",
"compute": "30.0%", "gpu": "90.0%",
"dec": "60.0%", "mem": "-%",
"clients": {"100": "80.0%", "200": "10.0%"}, "compute": "30.0%",
"dec": "60.0%",
"clients": {"100": "80.0%", "200": "10.0%"},
},
} }
@patch("frigate.util.services._read_intel_drm_fdinfo") @patch("frigate.util.services._read_intel_drm_fdinfo")

View File

@ -393,8 +393,10 @@ def _read_intel_drm_fdinfo(target_pdev: Optional[str]) -> dict:
return snapshot return snapshot
def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, Any]]: def get_intel_gpu_stats(
"""Get stats by reading DRM fdinfo files. intel_gpu_device: Optional[str],
) -> Optional[dict[str, dict[str, Any]]]:
"""Get stats by reading DRM fdinfo files, bucketed per-pdev.
Each DRM client FD exposes monotonic per-engine busy counters via Each DRM client FD exposes monotonic per-engine busy counters via
/proc/<pid>/fdinfo/<fd> (i915 since kernel 5.19, Xe since first release). /proc/<pid>/fdinfo/<fd> (i915 since kernel 5.19, Xe since first release).
@ -402,7 +404,14 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, A
utilization. Render/3D and Compute are pooled into "compute"; Video and utilization. Render/3D and Compute are pooled into "compute"; Video and
VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped
to 100%). to 100%).
The return value is keyed by the GPU's drm-pdev string so multiple Intel
GPUs in the same system are reported separately. Each entry carries a
"name" populated from OpenVINO (falling back to the pdev) so callers can
surface a real device name in the UI.
""" """
from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device) target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device)
snapshot_a = _read_intel_drm_fdinfo(target_pdev) snapshot_a = _read_intel_drm_fdinfo(target_pdev)
@ -417,19 +426,21 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, A
if not snapshot_b or elapsed_ns <= 0: if not snapshot_b or elapsed_ns <= 0:
return None return None
engine_pct: dict[str, float] = { def _new_engine_pct() -> dict[str, float]:
"render": 0.0, return {"render": 0.0, "video": 0.0, "video-enhance": 0.0, "compute": 0.0}
"video": 0.0,
"video-enhance": 0.0, per_pdev_engine_pct: dict[str, dict[str, float]] = {}
"compute": 0.0, per_pdev_pid_pct: dict[str, dict[str, float]] = {}
}
pid_pct: dict[str, float] = {}
for key, data_b in snapshot_b.items(): for key, data_b in snapshot_b.items():
data_a = snapshot_a.get(key) data_a = snapshot_a.get(key)
if not data_a or data_a["driver"] != data_b["driver"]: if not data_a or data_a["driver"] != data_b["driver"]:
continue continue
pdev = key[0]
engine_pct = per_pdev_engine_pct.setdefault(pdev, _new_engine_pct())
pid_pct = per_pdev_pid_pct.setdefault(pdev, {})
client_total = 0.0 client_total = 0.0
for engine, (busy_b, total_b) in data_b["engines"].items(): for engine, (busy_b, total_b) in data_b["engines"].items():
if engine not in engine_pct: if engine not in engine_pct:
@ -452,25 +463,36 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, A
pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total
for engine in engine_pct: if not per_pdev_engine_pct:
engine_pct[engine] = min(100.0, engine_pct[engine]) return None
compute_pct = min(100.0, engine_pct["render"] + engine_pct["compute"]) names = intel_gpu_name_resolver.get_names()
dec_pct = min(100.0, engine_pct["video"] + engine_pct["video-enhance"]) results: dict[str, dict[str, Any]] = {}
overall_pct = min(100.0, compute_pct + dec_pct)
results: dict[str, Any] = { for pdev, engine_pct in per_pdev_engine_pct.items():
"gpu": f"{round(overall_pct, 2)}%", for engine in engine_pct:
"mem": "-%", engine_pct[engine] = min(100.0, engine_pct[engine])
"compute": f"{round(compute_pct, 2)}%",
"dec": f"{round(dec_pct, 2)}%",
}
if pid_pct: compute_pct = min(100.0, engine_pct["render"] + engine_pct["compute"])
results["clients"] = { dec_pct = min(100.0, engine_pct["video"] + engine_pct["video-enhance"])
pid: f"{round(min(100.0, pct), 2)}%" for pid, pct in pid_pct.items() overall_pct = min(100.0, compute_pct + dec_pct)
entry: dict[str, Any] = {
"name": names.get(pdev) or f"Intel GPU {pdev}",
"gpu": f"{round(overall_pct, 2)}%",
"mem": "-%",
"compute": f"{round(compute_pct, 2)}%",
"dec": f"{round(dec_pct, 2)}%",
} }
pid_pct = per_pdev_pid_pct.get(pdev)
if pid_pct:
entry["clients"] = {
pid: f"{round(min(100.0, pct), 2)}%" for pid, pct in pid_pct.items()
}
results[pdev] = entry
return results return results