improve intel gpu stats error handling and remove stale zero-usage warning

Verify an i915/xe device exists in /sys/class/drm before scanning /proc, and log a distinct warning for each durable failure: an intel_gpu_device that does not resolve, one that resolves to a non-Intel card, an unreadable /proc, and a kernel that publishes no per-client engine counters. A present GPU with no attached DRM clients now reports an idle 0% reading instead of latching the hour-long hwaccel error cooldown, which blanked stats after momentary gaps such as camera restarts. Remove the System page warning that blamed intel_gpu_top for all-zero readings, since 0.18 no longer uses intel_gpu_top and 0% is now a legitimate idle reading.

frigate/test/test_gpu_stats.py

@@ -21,8 +21,12 @@ class TestGpuStats(unittest.TestCase):
     @patch("frigate.util.services.time.sleep")
     @patch("frigate.util.services.time.monotonic")
     @patch("frigate.util.services._read_intel_drm_fdinfo")
-    def test_intel_gpu_stats_fdinfo(self, read_fdinfo, monotonic, sleep, get_names):
+    @patch("frigate.util.services._enumerate_drm_devices")
+    def test_intel_gpu_stats_fdinfo(
+        self, drm_devices, read_fdinfo, monotonic, sleep, get_names
+    ):
         # 1 second of wall clock between snapshots
+        drm_devices.return_value = {"0000:00:02.0": "i915"}
         monotonic.side_effect = [0.0, 1.0]
         get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
 
@@ -96,13 +100,15 @@ class TestGpuStats(unittest.TestCase):
     @patch("frigate.util.services.time.sleep")
     @patch("frigate.util.services.time.monotonic")
     @patch("frigate.util.services._read_intel_drm_fdinfo")
+    @patch("frigate.util.services._enumerate_drm_devices")
     def test_intel_gpu_stats_xe_capacity(
-        self, read_fdinfo, monotonic, sleep, get_names
+        self, drm_devices, read_fdinfo, monotonic, sleep, get_names
     ):
         # Xe engines report cumulative cycles paired with total cycles, plus a
         # per-class capacity. drm-cycles-* is summed across every instance of a
         # class, so on Battlemage (capacity 2 for vcs/vecs) busy/total must be
         # divided by capacity to land in 0-100%.
+        drm_devices.return_value = {"0000:03:00.0": "xe"}
         monotonic.side_effect = [0.0, 1.0]
         get_names.return_value = {"0000:03:00.0": "Intel Arc"}
 
@@ -150,7 +156,145 @@ class TestGpuStats(unittest.TestCase):
             },
         }
 
+    @patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names")
+    @patch("frigate.util.services.time.sleep")
     @patch("frigate.util.services._read_intel_drm_fdinfo")
-    def test_intel_gpu_stats_no_clients(self, read_fdinfo):
+    @patch("frigate.util.services._enumerate_drm_devices")
+    def test_intel_gpu_stats_no_clients_reports_idle(
+        self, drm_devices, read_fdinfo, sleep, get_names
+    ):
+        # The device exists but nothing holds it open, e.g. while camera
+        # processes are restarting. This is an idle state, not an error:
+        # returning None here would latch the hwaccel error cooldown and
+        # blank GPU stats for an hour over a momentary gap.
+        drm_devices.return_value = {"0000:00:02.0": "i915"}
         read_fdinfo.return_value = {}
+        get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
+
+        assert get_intel_gpu_stats(None) == {
+            "0000:00:02.0": {
+                "name": "Intel Graphics",
+                "vendor": "intel",
+                "gpu": "0.0%",
+                "mem": "-%",
+                "compute": "0.0%",
+                "dec": "0.0%",
+            },
+        }
+        # Idle short-circuits before spending the sample window
+        sleep.assert_not_called()
+        read_fdinfo.assert_called_once()
+
+    @patch("frigate.util.services.time.sleep")
+    @patch("frigate.util.services._read_intel_drm_fdinfo")
+    @patch("frigate.util.services._enumerate_drm_devices")
+    def test_intel_gpu_stats_clients_without_engine_counters(
+        self, drm_devices, read_fdinfo, sleep
+    ):
+        # i915 publishes drm-driver/drm-pdev/drm-client-id but no drm-engine-*
+        # lines while GuC submission is active on kernels older than 6.5, so
+        # clients are found with nothing to sample. Reporting idle here would
+        # be a lie, and sampling a second time cannot help.
+        drm_devices.return_value = {"0000:00:02.0": "i915"}
+        read_fdinfo.return_value = {
+            ("0000:00:02.0", "48", "1109"): {
+                "driver": "i915",
+                "pid": "1109",
+                "engines": {},
+            },
+            ("0000:00:02.0", "51", "1258"): {
+                "driver": "i915",
+                "pid": "1258",
+                "engines": {},
+            },
+        }
+
         assert get_intel_gpu_stats(None) is None
+        sleep.assert_not_called()
+        read_fdinfo.assert_called_once()
+
+    @patch("frigate.util.services._read_intel_drm_fdinfo")
+    @patch("frigate.util.services._enumerate_drm_devices")
+    def test_intel_gpu_stats_no_intel_device(self, drm_devices, read_fdinfo):
+        # Only a non-Intel GPU is visible in sysfs; /proc is never scanned
+        drm_devices.return_value = {"0000:01:00.0": "nvidia"}
+
+        assert get_intel_gpu_stats(None) is None
+        read_fdinfo.assert_not_called()
+
+    @patch("frigate.util.services._read_intel_drm_fdinfo")
+    @patch("frigate.util.services._enumerate_drm_devices")
+    @patch("frigate.util.services._resolve_intel_gpu_pdev")
+    def test_intel_gpu_stats_unresolvable_device_hint(
+        self, resolve_pdev, drm_devices, read_fdinfo
+    ):
+        # A configured intel_gpu_device that cannot be resolved is a config
+        # error, not a reason to silently fall back to reporting all GPUs
+        resolve_pdev.return_value = None
+
+        assert get_intel_gpu_stats("/dev/dri/renderD999") is None
+        drm_devices.assert_not_called()
+        read_fdinfo.assert_not_called()
+
+    @patch("frigate.util.services._read_intel_drm_fdinfo")
+    @patch("frigate.util.services._enumerate_drm_devices")
+    @patch("frigate.util.services._resolve_intel_gpu_pdev")
+    def test_intel_gpu_stats_hint_resolves_to_non_intel_gpu(
+        self, resolve_pdev, drm_devices, read_fdinfo
+    ):
+        # card numbering can reorder across reboots on multi-GPU hosts, so a
+        # configured hint may point at another vendor's card; call it out
+        # instead of reporting nothing
+        resolve_pdev.return_value = "0000:01:00.0"
+        drm_devices.return_value = {
+            "0000:00:02.0": "i915",
+            "0000:01:00.0": "nvidia",
+        }
+
+        assert get_intel_gpu_stats("/dev/dri/card0") is None
+        read_fdinfo.assert_not_called()
+
+    @patch("frigate.util.services._read_intel_drm_fdinfo")
+    @patch("frigate.util.services._enumerate_drm_devices")
+    def test_intel_gpu_stats_unreadable_proc(self, drm_devices, read_fdinfo):
+        # A scan failure (None) is a different condition than a scan that
+        # finds no clients ({}) and must not report idle
+        drm_devices.return_value = {"0000:00:02.0": "i915"}
+        read_fdinfo.return_value = None
+
+        assert get_intel_gpu_stats(None) is None
+
+    @patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names")
+    @patch("frigate.util.services.time.sleep")
+    @patch("frigate.util.services.time.monotonic")
+    @patch("frigate.util.services._read_intel_drm_fdinfo")
+    @patch("frigate.util.services._enumerate_drm_devices")
+    def test_intel_gpu_stats_clients_lost_between_samples(
+        self, drm_devices, read_fdinfo, monotonic, sleep, get_names
+    ):
+        # Clients disappearing during the sample window is transient process
+        # churn, so report idle rather than latching an error
+        drm_devices.return_value = {"0000:00:02.0": "i915"}
+        monotonic.side_effect = [0.0, 1.0]
+        get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
+        read_fdinfo.side_effect = [
+            {
+                ("0000:00:02.0", "1", "100"): {
+                    "driver": "i915",
+                    "pid": "100",
+                    "engines": {"video": (5_000_000_000, 0, 1)},
+                },
+            },
+            {},
+        ]
+
+        assert get_intel_gpu_stats(None) == {
+            "0000:00:02.0": {
+                "name": "Intel Graphics",
+                "vendor": "intel",
+                "gpu": "0.0%",
+                "mem": "-%",
+                "compute": "0.0%",
+                "dec": "0.0%",
+            },
+        }

frigate/util/services.py

@@ -285,6 +285,10 @@ _XE_ENGINE_KEYS = {
     "vecs": "video-enhance",
     "ccs": "compute",
 }
+_INTEL_DRM_DRIVERS = ("i915", "xe")
+_PCI_ADDRESS_RE = re.compile(
+    r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$"
+)
 
 
 def _resolve_intel_gpu_pdev(device: str | None) -> str | None:
@@ -294,29 +298,70 @@ def _resolve_intel_gpu_pdev(device: str | None) -> str | None:
     if not device:
         return None
 
-    if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$", device):
+    if _PCI_ADDRESS_RE.match(device):
         return device
 
     name = os.path.basename(device.rstrip("/"))
     try:
-        return os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device"))
+        pdev = os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device"))
     except OSError:
         return None
 
+    # realpath does not raise on a nonexistent node; it returns the input
+    # path unchanged, so validate the result actually looks like a PCI
+    # address before trusting it.
+    return pdev if _PCI_ADDRESS_RE.match(pdev) else None
 
-def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict:
+
+def _enumerate_drm_devices() -> dict[str, str]:
+    """Map each PCI-attached DRM device to its bound kernel driver.
+
+    Reads /sys/class/drm, which reflects every GPU on the host even when only
+    some render nodes are mapped into the container, so device presence can be
+    verified without /dev access. Returns {pdev: driver}, e.g.
+    {"0000:00:02.0": "i915"}.
+    """
+    devices: dict[str, str] = {}
+
+    try:
+        entries = os.listdir("/sys/class/drm")
+    except OSError:
+        return devices
+
+    for entry in entries:
+        device_dir = f"/sys/class/drm/{entry}/device"
+        pdev = os.path.basename(os.path.realpath(device_dir))
+
+        if not _PCI_ADDRESS_RE.match(pdev):
+            continue
+
+        try:
+            driver = os.path.basename(os.readlink(f"{device_dir}/driver"))
+        except OSError:
+            continue
+
+        devices[pdev] = driver
+
+    return devices
+
+
+def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict | None:
     """Snapshot DRM fdinfo for every Intel client visible in /proc.
 
     Returns a dict keyed by (pdev, drm-client-id, pid) so the same context
     seen via multiple file descriptors on a single process collapses to one
-    entry.
+    entry. Clients whose fdinfo carries no engine counters are still included
+    with an empty "engines" dict so the caller can distinguish "clients exist
+    but the kernel publishes no busyness" from "no clients at all". Returns
+    None when /proc itself cannot be scanned, which is a different failure
+    than a scan that finds nothing.
     """
     snapshot: dict = {}
 
     try:
         proc_entries = os.listdir("/proc")
     except OSError:
-        return snapshot
+        return None
 
     for entry in proc_entries:
         if not entry.isdigit():
@@ -400,42 +445,124 @@ def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict:
                     except (ValueError, IndexError):
                         continue
 
-            if not engines:
-                continue
-
             snapshot[key] = {"driver": driver, "pid": entry, "engines": engines}
 
     return snapshot
 
 
+def _idle_intel_gpu_stats(
+    target_pdev: str | None, intel_pdevs: dict[str, str]
+) -> dict[str, dict[str, Any]]:
+    """Build a 0% reading for the configured (or every) Intel GPU.
+
+    Used when the device is confirmed present but no DRM client is currently
+    attached, e.g. while camera processes are restarting. That is an idle
+    state, not a collection failure, so it must produce a valid reading:
+    returning None would latch the hwaccel error cooldown and blank GPU stats
+    for an hour over a momentary gap.
+    """
+    from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
+
+    names = intel_gpu_name_resolver.get_names()
+    pdevs = [target_pdev] if target_pdev else sorted(intel_pdevs)
+
+    return {
+        pdev: {
+            "name": names.get(pdev) or "Intel iGPU",
+            "vendor": "intel",
+            "gpu": "0.0%",
+            "mem": "-%",
+            "compute": "0.0%",
+            "dec": "0.0%",
+        }
+        for pdev in pdevs
+    }
+
+
 def get_intel_gpu_stats(
     intel_gpu_device: str | None,
 ) -> dict[str, dict[str, Any]] | None:
     """Get stats by reading DRM fdinfo files, bucketed per-pdev.
 
     Each DRM client FD exposes monotonic per-engine busy counters via
-    /proc/<pid>/fdinfo/<fd> (i915 since kernel 5.19, Xe since first release).
+    /proc/<pid>/fdinfo/<fd>. For i915 this requires kernel 6.5 or newer:
-    We sample twice and divide busy-time deltas by wall-clock to derive
+    earlier kernels omit the per-engine counters whenever GuC submission is
-    utilization. Render/3D and Compute are pooled into "compute"; Video and
+    active, which is the default on 12th gen and newer. Xe has exposed them
-    VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped
+    since its first release. We sample twice and divide busy-time deltas by
-    to 100%).
+    wall-clock to derive utilization. Render/3D and Compute are pooled into
+    "compute"; Video and VideoEnhance into "dec". Overall "gpu" is the sum of
+    those pools (clamped to 100%).
 
     The return value is keyed by the GPU's drm-pdev string so multiple Intel
     GPUs in the same system are reported separately. Each entry carries a
     "name" populated from OpenVINO (falling back to the pdev) so callers can
     surface a real device name in the UI.
+
+    A device that exists but has no attached DRM clients reports an idle 0%
+    reading. None is returned only for durable failures (no Intel GPU, a bad
+    intel_gpu_device config, unreadable /proc, or a kernel that publishes no
+    counters), each of which logs a distinct warning, and the caller latches
+    it against retries for an hour.
     """
     from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
 
     target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device)
+    if intel_gpu_device and not target_pdev:
+        logger.warning(
+            "Unable to collect Intel GPU stats: configured intel_gpu_device %s "
+            "does not exist or could not be resolved to a PCI device",
+            intel_gpu_device,
+        )
+        return None
+
+    drm_devices = _enumerate_drm_devices()
+    intel_pdevs = {
+        pdev: driver
+        for pdev, driver in drm_devices.items()
+        if driver in _INTEL_DRM_DRIVERS
+    }
+
+    if not intel_pdevs:
+        logger.warning(
+            "Unable to collect Intel GPU stats: no Intel GPU (i915/xe) found in "
+            "/sys/class/drm. Check that the driver is loaded on the host"
+        )
+        return None
+
+    if target_pdev and target_pdev not in intel_pdevs:
+        logger.warning(
+            "Unable to collect Intel GPU stats: configured intel_gpu_device %s "
+            "resolved to %s (driver: %s), which is not an Intel GPU",
+            intel_gpu_device,
+            target_pdev,
+            drm_devices.get(target_pdev, "unknown"),
+        )
+        return None
 
     snapshot_a = _read_intel_drm_fdinfo(target_pdev)
+    if snapshot_a is None:
+        logger.warning("Unable to collect Intel GPU stats: /proc could not be read")
+        return None
+
     if not snapshot_a:
+        # No process currently holds the GPU open, e.g. while camera processes
+        # are restarting. The device is confirmed present, so report idle
+        # rather than an error; the next stats cycle re-samples normally.
+        logger.debug("No active DRM clients for Intel GPU, reporting idle")
+        return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
+
+    if not any(client["engines"] for client in snapshot_a.values()):
+        # Clients exist but the kernel published no busyness for them, so
+        # there is nothing to sample and a second snapshot would not help.
+        # i915 suppresses per-client engine counters while GuC submission is
+        # active on kernels older than 6.5 (kernel commit 1324680a80eb lifted
+        # this), which covers stock Debian 12 and Ubuntu 22.04 on 12th gen
+        # and newer.
         logger.warning(
-            "Unable to collect Intel GPU stats: no DRM fdinfo entries found"
+            "Unable to collect Intel GPU stats: found %d DRM client(s) for %s but "
-            "%s. Check kernel version (required >= 6.5) and that /proc is readable"
+            "no per-engine counters. Kernel 6.5 or newer is required.",
-            " and the i915/xe driver is loaded",
+            len(snapshot_a),
-            f" for pdev {target_pdev}" if target_pdev else "",
+            "/".join(sorted({client["driver"] for client in snapshot_a.values()})),
         )
         return None
 
@@ -444,12 +571,18 @@ def get_intel_gpu_stats(
     elapsed_ns = (time.monotonic() - start) * 1e9
 
     snapshot_b = _read_intel_drm_fdinfo(target_pdev)
-    if not snapshot_b or elapsed_ns <= 0:
+    if snapshot_b is None:
-        logger.warning(
+        logger.warning("Unable to collect Intel GPU stats: /proc could not be read")
-            "Unable to collect Intel GPU stats: second DRM fdinfo sample was empty"
-        )
         return None
 
+    if not snapshot_b or elapsed_ns <= 0:
+        # Every client disappeared during the sample window; transient by
+        # definition, so report idle instead of latching an error.
+        logger.debug(
+            "No DRM clients persisted across Intel GPU samples, reporting idle"
+        )
+        return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
+
     def _new_engine_pct() -> dict[str, float]:
         return {"render": 0.0, "video": 0.0, "video-enhance": 0.0, "compute": 0.0}
 
@@ -461,6 +594,11 @@ def get_intel_gpu_stats(
         if not data_a or data_a["driver"] != data_b["driver"]:
             continue
 
+        # Skip before the setdefault below so a counter-less client cannot
+        # register its pdev on its own.
+        if not data_b["engines"]:
+            continue
+
         pdev = key[0]
         engine_pct = per_pdev_engine_pct.setdefault(pdev, _new_engine_pct())
         pid_pct = per_pdev_pid_pct.setdefault(pdev, {})
@@ -492,11 +630,12 @@ def get_intel_gpu_stats(
         pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total
 
     if not per_pdev_engine_pct:
-        logger.warning(
+        # Clients were seen in both snapshots but none persisted as the same
-            "Unable to collect Intel GPU stats: no per-engine counters available "
+        # (pdev, client-id, pid); process churn, so report idle.
-            "(i915 requires kernel >= 5.19)"
+        logger.debug(
+            "No DRM clients persisted across Intel GPU samples, reporting idle"
         )
-        return None
+        return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
 
     names = intel_gpu_name_resolver.get_names()
     results: dict[str, dict[str, Any]] = {}

web/public/locales/en/views/system.json

@@ -107,12 +107,7 @@
       },
       "npuUsage": "NPU Usage",
       "npuMemory": "NPU Memory",
-      "npuTemperature": "NPU Temperature",
+      "npuTemperature": "NPU Temperature"
-      "intelGpuWarning": {
-        "title": "Intel GPU Stats Warning",
-        "message": "GPU stats unavailable",
-        "description": "This is a known bug in Intel's GPU stats reporting tools (intel_gpu_top) where it will break and repeatedly return a GPU usage of 0% even in cases where hardware acceleration and object detection are correctly running on the (i)GPU. This is not a Frigate bug. You can restart the host to temporarily fix the issue and confirm that the GPU is working correctly. This does not affect performance."
-      }
     },
     "otherProcesses": {
       "title": "Other Processes",

web/src/views/system/GeneralMetrics.tsx

@@ -470,50 +470,6 @@ export default function GeneralMetrics({
     return Object.keys(series).length > 0 ? Object.values(series) : undefined;
   }, [statsHistory]);
 
-  // Check if Intel GPU has all 0% usage values (known bug)
-  const showIntelGpuWarning = useMemo(() => {
-    if (!statsHistory || statsHistory.length < 3) {
-      return false;
-    }
-
-    const hasIntelGpu = Object.values(statsHistory[0]?.gpu_usages ?? {}).some(
-      (stats) => stats.vendor === "intel",
-    );
-
-    if (!hasIntelGpu) {
-      return false;
-    }
-
-    // Check if all GPU usage values are 0% across all stats
-    let allZero = true;
-    let hasDataPoints = false;
-
-    for (const stats of statsHistory) {
-      if (!stats) {
-        continue;
-      }
-
-      Object.values(stats.gpu_usages || {}).forEach((gpuStats) => {
-        if (gpuStats.vendor !== "intel") {
-          return;
-        }
-        if (gpuStats.gpu) {
-          hasDataPoints = true;
-          const gpuValue = parseFloat(gpuStats.gpu.slice(0, -1));
-          if (!isNaN(gpuValue) && gpuValue > 0) {
-            allZero = false;
-          }
-        }
-      });
-
-      if (!allZero) {
-        break;
-      }
-    }
-
-    return hasDataPoints && allZero;
-  }, [statsHistory]);
-
   // npu stats
 
   const npuSeries = useMemo(() => {
@@ -819,46 +775,8 @@ export default function GeneralMetrics({
                 <>
                   {statsHistory.length != 0 ? (
                     <div className="rounded-lg bg-background_alt p-2.5 md:rounded-2xl">
-                      <div className="mb-5 flex flex-row items-center justify-between">
+                      <div className="mb-5">
                         {t("general.hardwareInfo.gpuUsage")}
-                        {showIntelGpuWarning && (
-                          <Popover>
-                            <PopoverTrigger asChild>
-                              <button
-                                className="flex flex-row items-center gap-1.5 text-yellow-600 focus:outline-none dark:text-yellow-500"
-                                aria-label={t(
-                                  "general.hardwareInfo.intelGpuWarning.title",
-                                )}
-                              >
-                                <CiCircleAlert
-                                  className="size-5"
-                                  aria-label={t(
-                                    "general.hardwareInfo.intelGpuWarning.title",
-                                  )}
-                                />
-                                <span className="text-sm">
-                                  {t(
-                                    "general.hardwareInfo.intelGpuWarning.message",
-                                  )}
-                                </span>
-                              </button>
-                            </PopoverTrigger>
-                            <PopoverContent className="w-80">
-                              <div className="space-y-2">
-                                <div className="font-semibold">
-                                  {t(
-                                    "general.hardwareInfo.intelGpuWarning.title",
-                                  )}
-                                </div>
-                                <div>
-                                  {t(
-                                    "general.hardwareInfo.intelGpuWarning.description",
-                                  )}
-                                </div>
-                              </div>
-                            </PopoverContent>
-                          </Popover>
-                        )}
                       </div>
                       {gpuSeries.map((series) => (
                         <ThresholdBarGraph