Miscellaneous fixes (0.18 beta) (#23725)

frigate/util/services.py

@@ -285,6 +285,10 @@ _XE_ENGINE_KEYS = {
     "vecs": "video-enhance",
     "ccs": "compute",
 }
+_INTEL_DRM_DRIVERS = ("i915", "xe")
+_PCI_ADDRESS_RE = re.compile(
+    r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$"
+)
 
 
 def _resolve_intel_gpu_pdev(device: str | None) -> str | None:
@@ -294,29 +298,70 @@ def _resolve_intel_gpu_pdev(device: str | None) -> str | None:
     if not device:
         return None
 
-    if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$", device):
+    if _PCI_ADDRESS_RE.match(device):
         return device
 
     name = os.path.basename(device.rstrip("/"))
     try:
-        return os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device"))
+        pdev = os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device"))
     except OSError:
         return None
 
+    # realpath does not raise on a nonexistent node; it returns the input
+    # path unchanged, so validate the result actually looks like a PCI
+    # address before trusting it.
+    return pdev if _PCI_ADDRESS_RE.match(pdev) else None
 
-def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict:
+
+def _enumerate_drm_devices() -> dict[str, str]:
+    """Map each PCI-attached DRM device to its bound kernel driver.
+
+    Reads /sys/class/drm, which reflects every GPU on the host even when only
+    some render nodes are mapped into the container, so device presence can be
+    verified without /dev access. Returns {pdev: driver}, e.g.
+    {"0000:00:02.0": "i915"}.
+    """
+    devices: dict[str, str] = {}
+
+    try:
+        entries = os.listdir("/sys/class/drm")
+    except OSError:
+        return devices
+
+    for entry in entries:
+        device_dir = f"/sys/class/drm/{entry}/device"
+        pdev = os.path.basename(os.path.realpath(device_dir))
+
+        if not _PCI_ADDRESS_RE.match(pdev):
+            continue
+
+        try:
+            driver = os.path.basename(os.readlink(f"{device_dir}/driver"))
+        except OSError:
+            continue
+
+        devices[pdev] = driver
+
+    return devices
+
+
+def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict | None:
     """Snapshot DRM fdinfo for every Intel client visible in /proc.
 
     Returns a dict keyed by (pdev, drm-client-id, pid) so the same context
     seen via multiple file descriptors on a single process collapses to one
-    entry.
+    entry. Clients whose fdinfo carries no engine counters are still included
+    with an empty "engines" dict so the caller can distinguish "clients exist
+    but the kernel publishes no busyness" from "no clients at all". Returns
+    None when /proc itself cannot be scanned, which is a different failure
+    than a scan that finds nothing.
     """
     snapshot: dict = {}
 
     try:
         proc_entries = os.listdir("/proc")
     except OSError:
-        return snapshot
+        return None
 
     for entry in proc_entries:
         if not entry.isdigit():
@@ -400,41 +445,124 @@ def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict:
                     except (ValueError, IndexError):
                         continue
 
-            if not engines:
-                continue
-
             snapshot[key] = {"driver": driver, "pid": entry, "engines": engines}
 
     return snapshot
 
 
+def _idle_intel_gpu_stats(
+    target_pdev: str | None, intel_pdevs: dict[str, str]
+) -> dict[str, dict[str, Any]]:
+    """Build a 0% reading for the configured (or every) Intel GPU.
+
+    Used when the device is confirmed present but no DRM client is currently
+    attached, e.g. while camera processes are restarting. That is an idle
+    state, not a collection failure, so it must produce a valid reading:
+    returning None would latch the hwaccel error cooldown and blank GPU stats
+    for an hour over a momentary gap.
+    """
+    from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
+
+    names = intel_gpu_name_resolver.get_names()
+    pdevs = [target_pdev] if target_pdev else sorted(intel_pdevs)
+
+    return {
+        pdev: {
+            "name": names.get(pdev) or "Intel iGPU",
+            "vendor": "intel",
+            "gpu": "0.0%",
+            "mem": "-%",
+            "compute": "0.0%",
+            "dec": "0.0%",
+        }
+        for pdev in pdevs
+    }
+
+
 def get_intel_gpu_stats(
     intel_gpu_device: str | None,
 ) -> dict[str, dict[str, Any]] | None:
     """Get stats by reading DRM fdinfo files, bucketed per-pdev.
 
     Each DRM client FD exposes monotonic per-engine busy counters via
-    /proc/<pid>/fdinfo/<fd> (i915 since kernel 5.19, Xe since first release).
-    We sample twice and divide busy-time deltas by wall-clock to derive
-    utilization. Render/3D and Compute are pooled into "compute"; Video and
-    VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped
-    to 100%).
+    /proc/<pid>/fdinfo/<fd>. For i915 this requires kernel 6.5 or newer:
+    earlier kernels omit the per-engine counters whenever GuC submission is
+    active, which is the default on 12th gen and newer. Xe has exposed them
+    since its first release. We sample twice and divide busy-time deltas by
+    wall-clock to derive utilization. Render/3D and Compute are pooled into
+    "compute"; Video and VideoEnhance into "dec". Overall "gpu" is the sum of
+    those pools (clamped to 100%).
 
     The return value is keyed by the GPU's drm-pdev string so multiple Intel
     GPUs in the same system are reported separately. Each entry carries a
     "name" populated from OpenVINO (falling back to the pdev) so callers can
     surface a real device name in the UI.
+
+    A device that exists but has no attached DRM clients reports an idle 0%
+    reading. None is returned only for durable failures (no Intel GPU, a bad
+    intel_gpu_device config, unreadable /proc, or a kernel that publishes no
+    counters), each of which logs a distinct warning, and the caller latches
+    it against retries for an hour.
     """
     from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
 
     target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device)
+    if intel_gpu_device and not target_pdev:
+        logger.warning(
+            "Unable to collect Intel GPU stats: configured intel_gpu_device %s "
+            "does not exist or could not be resolved to a PCI device",
+            intel_gpu_device,
+        )
+        return None
+
+    drm_devices = _enumerate_drm_devices()
+    intel_pdevs = {
+        pdev: driver
+        for pdev, driver in drm_devices.items()
+        if driver in _INTEL_DRM_DRIVERS
+    }
+
+    if not intel_pdevs:
+        logger.warning(
+            "Unable to collect Intel GPU stats: no Intel GPU (i915/xe) found in "
+            "/sys/class/drm. Check that the driver is loaded on the host"
+        )
+        return None
+
+    if target_pdev and target_pdev not in intel_pdevs:
+        logger.warning(
+            "Unable to collect Intel GPU stats: configured intel_gpu_device %s "
+            "resolved to %s (driver: %s), which is not an Intel GPU",
+            intel_gpu_device,
+            target_pdev,
+            drm_devices.get(target_pdev, "unknown"),
+        )
+        return None
 
     snapshot_a = _read_intel_drm_fdinfo(target_pdev)
+    if snapshot_a is None:
+        logger.warning("Unable to collect Intel GPU stats: /proc could not be read")
+        return None
+
     if not snapshot_a:
+        # No process currently holds the GPU open, e.g. while camera processes
+        # are restarting. The device is confirmed present, so report idle
+        # rather than an error; the next stats cycle re-samples normally.
+        logger.debug("No active DRM clients for Intel GPU, reporting idle")
+        return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
+
+    if not any(client["engines"] for client in snapshot_a.values()):
+        # Clients exist but the kernel published no busyness for them, so
+        # there is nothing to sample and a second snapshot would not help.
+        # i915 suppresses per-client engine counters while GuC submission is
+        # active on kernels older than 6.5 (kernel commit 1324680a80eb lifted
+        # this), which covers stock Debian 12 and Ubuntu 22.04 on 12th gen
+        # and newer.
         logger.warning(
-            "Unable to collect Intel GPU stats: no DRM fdinfo entries found"
-            "%s. Check that /proc is readable and the i915/xe driver is loaded",
-            f" for pdev {target_pdev}" if target_pdev else "",
+            "Unable to collect Intel GPU stats: found %d DRM client(s) for %s but "
+            "no per-engine counters. Kernel 6.5 or newer is required.",
+            len(snapshot_a),
+            "/".join(sorted({client["driver"] for client in snapshot_a.values()})),
         )
         return None
 
@@ -443,12 +571,18 @@ def get_intel_gpu_stats(
     elapsed_ns = (time.monotonic() - start) * 1e9
 
     snapshot_b = _read_intel_drm_fdinfo(target_pdev)
-    if not snapshot_b or elapsed_ns <= 0:
-        logger.warning(
-            "Unable to collect Intel GPU stats: second DRM fdinfo sample was empty"
-        )
+    if snapshot_b is None:
+        logger.warning("Unable to collect Intel GPU stats: /proc could not be read")
         return None
 
+    if not snapshot_b or elapsed_ns <= 0:
+        # Every client disappeared during the sample window; transient by
+        # definition, so report idle instead of latching an error.
+        logger.debug(
+            "No DRM clients persisted across Intel GPU samples, reporting idle"
+        )
+        return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
+
     def _new_engine_pct() -> dict[str, float]:
         return {"render": 0.0, "video": 0.0, "video-enhance": 0.0, "compute": 0.0}
 
@@ -460,6 +594,11 @@ def get_intel_gpu_stats(
         if not data_a or data_a["driver"] != data_b["driver"]:
             continue
 
+        # Skip before the setdefault below so a counter-less client cannot
+        # register its pdev on its own.
+        if not data_b["engines"]:
+            continue
+
         pdev = key[0]
         engine_pct = per_pdev_engine_pct.setdefault(pdev, _new_engine_pct())
         pid_pct = per_pdev_pid_pct.setdefault(pdev, {})
@@ -491,11 +630,12 @@ def get_intel_gpu_stats(
         pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total
 
     if not per_pdev_engine_pct:
-        logger.warning(
-            "Unable to collect Intel GPU stats: no per-engine counters available "
-            "(i915 requires kernel >= 5.19)"
+        # Clients were seen in both snapshots but none persisted as the same
+        # (pdev, client-id, pid); process churn, so report idle.
+        logger.debug(
+            "No DRM clients persisted across Intel GPU samples, reporting idle"
         )
-        return None
+        return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
 
     names = intel_gpu_name_resolver.get_names()
     results: dict[str, dict[str, Any]] = {}