diff --git a/frigate/test/test_gpu_stats.py b/frigate/test/test_gpu_stats.py index e94625f645..c99414a419 100644 --- a/frigate/test/test_gpu_stats.py +++ b/frigate/test/test_gpu_stats.py @@ -21,8 +21,12 @@ class TestGpuStats(unittest.TestCase): @patch("frigate.util.services.time.sleep") @patch("frigate.util.services.time.monotonic") @patch("frigate.util.services._read_intel_drm_fdinfo") - def test_intel_gpu_stats_fdinfo(self, read_fdinfo, monotonic, sleep, get_names): + @patch("frigate.util.services._enumerate_drm_devices") + def test_intel_gpu_stats_fdinfo( + self, drm_devices, read_fdinfo, monotonic, sleep, get_names + ): # 1 second of wall clock between snapshots + drm_devices.return_value = {"0000:00:02.0": "i915"} monotonic.side_effect = [0.0, 1.0] get_names.return_value = {"0000:00:02.0": "Intel Graphics"} @@ -96,13 +100,15 @@ class TestGpuStats(unittest.TestCase): @patch("frigate.util.services.time.sleep") @patch("frigate.util.services.time.monotonic") @patch("frigate.util.services._read_intel_drm_fdinfo") + @patch("frigate.util.services._enumerate_drm_devices") def test_intel_gpu_stats_xe_capacity( - self, read_fdinfo, monotonic, sleep, get_names + self, drm_devices, read_fdinfo, monotonic, sleep, get_names ): # Xe engines report cumulative cycles paired with total cycles, plus a # per-class capacity. drm-cycles-* is summed across every instance of a # class, so on Battlemage (capacity 2 for vcs/vecs) busy/total must be # divided by capacity to land in 0-100%. + drm_devices.return_value = {"0000:03:00.0": "xe"} monotonic.side_effect = [0.0, 1.0] get_names.return_value = {"0000:03:00.0": "Intel Arc"} @@ -150,7 +156,145 @@ class TestGpuStats(unittest.TestCase): }, } + @patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names") + @patch("frigate.util.services.time.sleep") @patch("frigate.util.services._read_intel_drm_fdinfo") - def test_intel_gpu_stats_no_clients(self, read_fdinfo): + @patch("frigate.util.services._enumerate_drm_devices") + def test_intel_gpu_stats_no_clients_reports_idle( + self, drm_devices, read_fdinfo, sleep, get_names + ): + # The device exists but nothing holds it open, e.g. while camera + # processes are restarting. This is an idle state, not an error: + # returning None here would latch the hwaccel error cooldown and + # blank GPU stats for an hour over a momentary gap. + drm_devices.return_value = {"0000:00:02.0": "i915"} read_fdinfo.return_value = {} + get_names.return_value = {"0000:00:02.0": "Intel Graphics"} + + assert get_intel_gpu_stats(None) == { + "0000:00:02.0": { + "name": "Intel Graphics", + "vendor": "intel", + "gpu": "0.0%", + "mem": "-%", + "compute": "0.0%", + "dec": "0.0%", + }, + } + # Idle short-circuits before spending the sample window + sleep.assert_not_called() + read_fdinfo.assert_called_once() + + @patch("frigate.util.services.time.sleep") + @patch("frigate.util.services._read_intel_drm_fdinfo") + @patch("frigate.util.services._enumerate_drm_devices") + def test_intel_gpu_stats_clients_without_engine_counters( + self, drm_devices, read_fdinfo, sleep + ): + # i915 publishes drm-driver/drm-pdev/drm-client-id but no drm-engine-* + # lines while GuC submission is active on kernels older than 6.5, so + # clients are found with nothing to sample. Reporting idle here would + # be a lie, and sampling a second time cannot help. + drm_devices.return_value = {"0000:00:02.0": "i915"} + read_fdinfo.return_value = { + ("0000:00:02.0", "48", "1109"): { + "driver": "i915", + "pid": "1109", + "engines": {}, + }, + ("0000:00:02.0", "51", "1258"): { + "driver": "i915", + "pid": "1258", + "engines": {}, + }, + } + assert get_intel_gpu_stats(None) is None + sleep.assert_not_called() + read_fdinfo.assert_called_once() + + @patch("frigate.util.services._read_intel_drm_fdinfo") + @patch("frigate.util.services._enumerate_drm_devices") + def test_intel_gpu_stats_no_intel_device(self, drm_devices, read_fdinfo): + # Only a non-Intel GPU is visible in sysfs; /proc is never scanned + drm_devices.return_value = {"0000:01:00.0": "nvidia"} + + assert get_intel_gpu_stats(None) is None + read_fdinfo.assert_not_called() + + @patch("frigate.util.services._read_intel_drm_fdinfo") + @patch("frigate.util.services._enumerate_drm_devices") + @patch("frigate.util.services._resolve_intel_gpu_pdev") + def test_intel_gpu_stats_unresolvable_device_hint( + self, resolve_pdev, drm_devices, read_fdinfo + ): + # A configured intel_gpu_device that cannot be resolved is a config + # error, not a reason to silently fall back to reporting all GPUs + resolve_pdev.return_value = None + + assert get_intel_gpu_stats("/dev/dri/renderD999") is None + drm_devices.assert_not_called() + read_fdinfo.assert_not_called() + + @patch("frigate.util.services._read_intel_drm_fdinfo") + @patch("frigate.util.services._enumerate_drm_devices") + @patch("frigate.util.services._resolve_intel_gpu_pdev") + def test_intel_gpu_stats_hint_resolves_to_non_intel_gpu( + self, resolve_pdev, drm_devices, read_fdinfo + ): + # card numbering can reorder across reboots on multi-GPU hosts, so a + # configured hint may point at another vendor's card; call it out + # instead of reporting nothing + resolve_pdev.return_value = "0000:01:00.0" + drm_devices.return_value = { + "0000:00:02.0": "i915", + "0000:01:00.0": "nvidia", + } + + assert get_intel_gpu_stats("/dev/dri/card0") is None + read_fdinfo.assert_not_called() + + @patch("frigate.util.services._read_intel_drm_fdinfo") + @patch("frigate.util.services._enumerate_drm_devices") + def test_intel_gpu_stats_unreadable_proc(self, drm_devices, read_fdinfo): + # A scan failure (None) is a different condition than a scan that + # finds no clients ({}) and must not report idle + drm_devices.return_value = {"0000:00:02.0": "i915"} + read_fdinfo.return_value = None + + assert get_intel_gpu_stats(None) is None + + @patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names") + @patch("frigate.util.services.time.sleep") + @patch("frigate.util.services.time.monotonic") + @patch("frigate.util.services._read_intel_drm_fdinfo") + @patch("frigate.util.services._enumerate_drm_devices") + def test_intel_gpu_stats_clients_lost_between_samples( + self, drm_devices, read_fdinfo, monotonic, sleep, get_names + ): + # Clients disappearing during the sample window is transient process + # churn, so report idle rather than latching an error + drm_devices.return_value = {"0000:00:02.0": "i915"} + monotonic.side_effect = [0.0, 1.0] + get_names.return_value = {"0000:00:02.0": "Intel Graphics"} + read_fdinfo.side_effect = [ + { + ("0000:00:02.0", "1", "100"): { + "driver": "i915", + "pid": "100", + "engines": {"video": (5_000_000_000, 0, 1)}, + }, + }, + {}, + ] + + assert get_intel_gpu_stats(None) == { + "0000:00:02.0": { + "name": "Intel Graphics", + "vendor": "intel", + "gpu": "0.0%", + "mem": "-%", + "compute": "0.0%", + "dec": "0.0%", + }, + } diff --git a/frigate/util/services.py b/frigate/util/services.py index cc25de1fb7..38737907ef 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -285,6 +285,10 @@ _XE_ENGINE_KEYS = { "vecs": "video-enhance", "ccs": "compute", } +_INTEL_DRM_DRIVERS = ("i915", "xe") +_PCI_ADDRESS_RE = re.compile( + r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$" +) def _resolve_intel_gpu_pdev(device: str | None) -> str | None: @@ -294,29 +298,70 @@ def _resolve_intel_gpu_pdev(device: str | None) -> str | None: if not device: return None - if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$", device): + if _PCI_ADDRESS_RE.match(device): return device name = os.path.basename(device.rstrip("/")) try: - return os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device")) + pdev = os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device")) except OSError: return None + # realpath does not raise on a nonexistent node; it returns the input + # path unchanged, so validate the result actually looks like a PCI + # address before trusting it. + return pdev if _PCI_ADDRESS_RE.match(pdev) else None -def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict: + +def _enumerate_drm_devices() -> dict[str, str]: + """Map each PCI-attached DRM device to its bound kernel driver. + + Reads /sys/class/drm, which reflects every GPU on the host even when only + some render nodes are mapped into the container, so device presence can be + verified without /dev access. Returns {pdev: driver}, e.g. + {"0000:00:02.0": "i915"}. + """ + devices: dict[str, str] = {} + + try: + entries = os.listdir("/sys/class/drm") + except OSError: + return devices + + for entry in entries: + device_dir = f"/sys/class/drm/{entry}/device" + pdev = os.path.basename(os.path.realpath(device_dir)) + + if not _PCI_ADDRESS_RE.match(pdev): + continue + + try: + driver = os.path.basename(os.readlink(f"{device_dir}/driver")) + except OSError: + continue + + devices[pdev] = driver + + return devices + + +def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict | None: """Snapshot DRM fdinfo for every Intel client visible in /proc. Returns a dict keyed by (pdev, drm-client-id, pid) so the same context seen via multiple file descriptors on a single process collapses to one - entry. + entry. Clients whose fdinfo carries no engine counters are still included + with an empty "engines" dict so the caller can distinguish "clients exist + but the kernel publishes no busyness" from "no clients at all". Returns + None when /proc itself cannot be scanned, which is a different failure + than a scan that finds nothing. """ snapshot: dict = {} try: proc_entries = os.listdir("/proc") except OSError: - return snapshot + return None for entry in proc_entries: if not entry.isdigit(): @@ -400,42 +445,124 @@ def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict: except (ValueError, IndexError): continue - if not engines: - continue - snapshot[key] = {"driver": driver, "pid": entry, "engines": engines} return snapshot +def _idle_intel_gpu_stats( + target_pdev: str | None, intel_pdevs: dict[str, str] +) -> dict[str, dict[str, Any]]: + """Build a 0% reading for the configured (or every) Intel GPU. + + Used when the device is confirmed present but no DRM client is currently + attached, e.g. while camera processes are restarting. That is an idle + state, not a collection failure, so it must produce a valid reading: + returning None would latch the hwaccel error cooldown and blank GPU stats + for an hour over a momentary gap. + """ + from frigate.stats.intel_gpu_info import intel_gpu_name_resolver + + names = intel_gpu_name_resolver.get_names() + pdevs = [target_pdev] if target_pdev else sorted(intel_pdevs) + + return { + pdev: { + "name": names.get(pdev) or "Intel iGPU", + "vendor": "intel", + "gpu": "0.0%", + "mem": "-%", + "compute": "0.0%", + "dec": "0.0%", + } + for pdev in pdevs + } + + def get_intel_gpu_stats( intel_gpu_device: str | None, ) -> dict[str, dict[str, Any]] | None: """Get stats by reading DRM fdinfo files, bucketed per-pdev. Each DRM client FD exposes monotonic per-engine busy counters via - /proc//fdinfo/ (i915 since kernel 5.19, Xe since first release). - We sample twice and divide busy-time deltas by wall-clock to derive - utilization. Render/3D and Compute are pooled into "compute"; Video and - VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped - to 100%). + /proc//fdinfo/. For i915 this requires kernel 6.5 or newer: + earlier kernels omit the per-engine counters whenever GuC submission is + active, which is the default on 12th gen and newer. Xe has exposed them + since its first release. We sample twice and divide busy-time deltas by + wall-clock to derive utilization. Render/3D and Compute are pooled into + "compute"; Video and VideoEnhance into "dec". Overall "gpu" is the sum of + those pools (clamped to 100%). The return value is keyed by the GPU's drm-pdev string so multiple Intel GPUs in the same system are reported separately. Each entry carries a "name" populated from OpenVINO (falling back to the pdev) so callers can surface a real device name in the UI. + + A device that exists but has no attached DRM clients reports an idle 0% + reading. None is returned only for durable failures (no Intel GPU, a bad + intel_gpu_device config, unreadable /proc, or a kernel that publishes no + counters), each of which logs a distinct warning, and the caller latches + it against retries for an hour. """ from frigate.stats.intel_gpu_info import intel_gpu_name_resolver target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device) + if intel_gpu_device and not target_pdev: + logger.warning( + "Unable to collect Intel GPU stats: configured intel_gpu_device %s " + "does not exist or could not be resolved to a PCI device", + intel_gpu_device, + ) + return None + + drm_devices = _enumerate_drm_devices() + intel_pdevs = { + pdev: driver + for pdev, driver in drm_devices.items() + if driver in _INTEL_DRM_DRIVERS + } + + if not intel_pdevs: + logger.warning( + "Unable to collect Intel GPU stats: no Intel GPU (i915/xe) found in " + "/sys/class/drm. Check that the driver is loaded on the host" + ) + return None + + if target_pdev and target_pdev not in intel_pdevs: + logger.warning( + "Unable to collect Intel GPU stats: configured intel_gpu_device %s " + "resolved to %s (driver: %s), which is not an Intel GPU", + intel_gpu_device, + target_pdev, + drm_devices.get(target_pdev, "unknown"), + ) + return None snapshot_a = _read_intel_drm_fdinfo(target_pdev) + if snapshot_a is None: + logger.warning("Unable to collect Intel GPU stats: /proc could not be read") + return None + if not snapshot_a: + # No process currently holds the GPU open, e.g. while camera processes + # are restarting. The device is confirmed present, so report idle + # rather than an error; the next stats cycle re-samples normally. + logger.debug("No active DRM clients for Intel GPU, reporting idle") + return _idle_intel_gpu_stats(target_pdev, intel_pdevs) + + if not any(client["engines"] for client in snapshot_a.values()): + # Clients exist but the kernel published no busyness for them, so + # there is nothing to sample and a second snapshot would not help. + # i915 suppresses per-client engine counters while GuC submission is + # active on kernels older than 6.5 (kernel commit 1324680a80eb lifted + # this), which covers stock Debian 12 and Ubuntu 22.04 on 12th gen + # and newer. logger.warning( - "Unable to collect Intel GPU stats: no DRM fdinfo entries found" - "%s. Check kernel version (required >= 6.5) and that /proc is readable" - " and the i915/xe driver is loaded", - f" for pdev {target_pdev}" if target_pdev else "", + "Unable to collect Intel GPU stats: found %d DRM client(s) for %s but " + "no per-engine counters. Kernel 6.5 or newer is required.", + len(snapshot_a), + "/".join(sorted({client["driver"] for client in snapshot_a.values()})), ) return None @@ -444,12 +571,18 @@ def get_intel_gpu_stats( elapsed_ns = (time.monotonic() - start) * 1e9 snapshot_b = _read_intel_drm_fdinfo(target_pdev) - if not snapshot_b or elapsed_ns <= 0: - logger.warning( - "Unable to collect Intel GPU stats: second DRM fdinfo sample was empty" - ) + if snapshot_b is None: + logger.warning("Unable to collect Intel GPU stats: /proc could not be read") return None + if not snapshot_b or elapsed_ns <= 0: + # Every client disappeared during the sample window; transient by + # definition, so report idle instead of latching an error. + logger.debug( + "No DRM clients persisted across Intel GPU samples, reporting idle" + ) + return _idle_intel_gpu_stats(target_pdev, intel_pdevs) + def _new_engine_pct() -> dict[str, float]: return {"render": 0.0, "video": 0.0, "video-enhance": 0.0, "compute": 0.0} @@ -461,6 +594,11 @@ def get_intel_gpu_stats( if not data_a or data_a["driver"] != data_b["driver"]: continue + # Skip before the setdefault below so a counter-less client cannot + # register its pdev on its own. + if not data_b["engines"]: + continue + pdev = key[0] engine_pct = per_pdev_engine_pct.setdefault(pdev, _new_engine_pct()) pid_pct = per_pdev_pid_pct.setdefault(pdev, {}) @@ -492,11 +630,12 @@ def get_intel_gpu_stats( pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total if not per_pdev_engine_pct: - logger.warning( - "Unable to collect Intel GPU stats: no per-engine counters available " - "(i915 requires kernel >= 5.19)" + # Clients were seen in both snapshots but none persisted as the same + # (pdev, client-id, pid); process churn, so report idle. + logger.debug( + "No DRM clients persisted across Intel GPU samples, reporting idle" ) - return None + return _idle_intel_gpu_stats(target_pdev, intel_pdevs) names = intel_gpu_name_resolver.get_names() results: dict[str, dict[str, Any]] = {} diff --git a/web/public/locales/en/views/system.json b/web/public/locales/en/views/system.json index 9f387a7f30..84014aaffe 100644 --- a/web/public/locales/en/views/system.json +++ b/web/public/locales/en/views/system.json @@ -107,12 +107,7 @@ }, "npuUsage": "NPU Usage", "npuMemory": "NPU Memory", - "npuTemperature": "NPU Temperature", - "intelGpuWarning": { - "title": "Intel GPU Stats Warning", - "message": "GPU stats unavailable", - "description": "This is a known bug in Intel's GPU stats reporting tools (intel_gpu_top) where it will break and repeatedly return a GPU usage of 0% even in cases where hardware acceleration and object detection are correctly running on the (i)GPU. This is not a Frigate bug. You can restart the host to temporarily fix the issue and confirm that the GPU is working correctly. This does not affect performance." - } + "npuTemperature": "NPU Temperature" }, "otherProcesses": { "title": "Other Processes", diff --git a/web/src/views/system/GeneralMetrics.tsx b/web/src/views/system/GeneralMetrics.tsx index 6d032de840..4cba965298 100644 --- a/web/src/views/system/GeneralMetrics.tsx +++ b/web/src/views/system/GeneralMetrics.tsx @@ -470,50 +470,6 @@ export default function GeneralMetrics({ return Object.keys(series).length > 0 ? Object.values(series) : undefined; }, [statsHistory]); - // Check if Intel GPU has all 0% usage values (known bug) - const showIntelGpuWarning = useMemo(() => { - if (!statsHistory || statsHistory.length < 3) { - return false; - } - - const hasIntelGpu = Object.values(statsHistory[0]?.gpu_usages ?? {}).some( - (stats) => stats.vendor === "intel", - ); - - if (!hasIntelGpu) { - return false; - } - - // Check if all GPU usage values are 0% across all stats - let allZero = true; - let hasDataPoints = false; - - for (const stats of statsHistory) { - if (!stats) { - continue; - } - - Object.values(stats.gpu_usages || {}).forEach((gpuStats) => { - if (gpuStats.vendor !== "intel") { - return; - } - if (gpuStats.gpu) { - hasDataPoints = true; - const gpuValue = parseFloat(gpuStats.gpu.slice(0, -1)); - if (!isNaN(gpuValue) && gpuValue > 0) { - allZero = false; - } - } - }); - - if (!allZero) { - break; - } - } - - return hasDataPoints && allZero; - }, [statsHistory]); - // npu stats const npuSeries = useMemo(() => { @@ -819,46 +775,8 @@ export default function GeneralMetrics({ <> {statsHistory.length != 0 ? (
-
+
{t("general.hardwareInfo.gpuUsage")} - {showIntelGpuWarning && ( - - - - - -
-
- {t( - "general.hardwareInfo.intelGpuWarning.title", - )} -
-
- {t( - "general.hardwareInfo.intelGpuWarning.description", - )} -
-
- - - )}
{gpuSeries.map((series) => (