Miscellaneous fixes (#23317)

* resolve global record.export.hwaccel_args to fix phantom camera override

* auto-stop debug replay sessions after 12 hours

* docs tweaks

* add more tips to object classification docs

* tweak language

* Store hwaccel errors with timeout so it can retry

* Add error logs for Intel GPU stats

* add area

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

frigate/api/fastapi_app.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import re
 from typing import Optional
@@ -36,7 +37,7 @@ from frigate.comms.event_metadata_updater import (
 from frigate.config import FrigateConfig
 from frigate.config.camera.updater import CameraConfigUpdatePublisher
 from frigate.config.profile_manager import ProfileManager
-from frigate.debug_replay import DebugReplayManager
+from frigate.debug_replay import DebugReplayManager, debug_replay_auto_stop_watchdog
 from frigate.embeddings import EmbeddingsContext
 from frigate.genai import GenAIClientManager
 from frigate.ptz.onvif import OnvifController
@@ -116,6 +117,11 @@ def create_fastapi_app(
     @app.on_event("startup")
     async def startup():
         logger.info("FastAPI started")
+        asyncio.create_task(
+            debug_replay_auto_stop_watchdog(
+                replay_manager, frigate_config, config_publisher
+            )
+        )
 
     # Rate limiter (used for login endpoint)
     if frigate_config.auth.failed_login_rate_limit is None:

frigate/config/config.py

@@ -680,6 +680,13 @@ class FrigateConfig(FrigateBaseModel):
         if self.ffmpeg.hwaccel_args == "auto":
             self.ffmpeg.hwaccel_args = auto_detect_hwaccel()
 
+        # Resolve global export hwaccel_args so it matches the per-camera
+        # resolution below. Without this, every camera reads as overriding
+        # record.export.hwaccel_args because the global stays "auto" while
+        # the camera value gets resolved to the actual args list.
+        if self.record.export.hwaccel_args == "auto":
+            self.record.export.hwaccel_args = self.ffmpeg.hwaccel_args
+
         # Populate global audio filters from listen. Existing user-defined
         # entries for labels not in listen are preserved but unused at runtime.
         if self.audio.filters is None:

frigate/debug_replay.py

@@ -5,6 +5,7 @@ frigate.jobs.debug_replay. This module owns only session presence
 (active), session metadata, and post-session cleanup.
 """
 
+import asyncio
 import logging
 import os
 import shutil
@@ -40,6 +41,9 @@ from frigate.util.config import find_config_file
 
 logger = logging.getLogger(__name__)
 
+MAX_SESSION_DURATION_SECONDS = 12 * 60 * 60
+AUTO_STOP_CHECK_INTERVAL_SECONDS = 60
+
 
 class DebugReplayManager:
     """Owns the lifecycle pointers for a single debug replay session.
@@ -58,6 +62,7 @@ class DebugReplayManager:
         self.clip_path: str | None = None
         self.start_ts: float | None = None
         self.end_ts: float | None = None
+        self.session_started_at: float | None = None
         self._job_state_publisher = JobStatePublisher()
 
     @property
@@ -83,6 +88,7 @@ class DebugReplayManager:
             self.start_ts = start_ts
             self.end_ts = end_ts
             self.clip_path = None
+            self.session_started_at = time.time()
 
     def mark_session_ready(self, clip_path: str) -> None:
         """Record the on-disk clip path after the camera has been published."""
@@ -104,6 +110,7 @@ class DebugReplayManager:
         self.clip_path = None
         self.start_ts = None
         self.end_ts = None
+        self.session_started_at = None
 
     def publish_camera(
         self,
@@ -351,3 +358,41 @@ def cleanup_replay_cameras() -> None:
             shutil.rmtree(REPLAY_DIR)
         except Exception as e:
             logger.error("Failed to remove replay cache directory: %s", e)
+
+
+async def debug_replay_auto_stop_watchdog(
+    manager: DebugReplayManager,
+    frigate_config: FrigateConfig,
+    config_publisher: CameraConfigUpdatePublisher,
+) -> None:
+    """Auto-stop debug replay sessions that exceed MAX_SESSION_DURATION_SECONDS.
+
+    Backstop against a session left running for days. The cap is intentionally
+    generous so realistic tuning and overnight soak workflows aren't disrupted.
+    """
+    while True:
+        try:
+            await asyncio.sleep(AUTO_STOP_CHECK_INTERVAL_SECONDS)
+
+            started_at = manager.session_started_at
+            if not manager.active or started_at is None:
+                continue
+
+            if time.time() - started_at < MAX_SESSION_DURATION_SECONDS:
+                continue
+
+            replay_name = manager.replay_camera_name
+            await asyncio.to_thread(
+                manager.stop,
+                frigate_config=frigate_config,
+                config_publisher=config_publisher,
+            )
+            logger.info(
+                "Debug replay auto-stopped after exceeding max session duration of %d hours: %s",
+                MAX_SESSION_DURATION_SECONDS // 3600,
+                replay_name,
+            )
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            logger.exception("Error in debug replay auto-stop watchdog")

frigate/stats/emitter.py

@@ -32,7 +32,7 @@ class StatsEmitter(threading.Thread):
         self.config = config
         self.stats_tracking = stats_tracking
         self.stop_event = stop_event
-        self.hwaccel_errors: list[str] = []
+        self.hwaccel_errors: dict[str, float] = {}
         self.stats_history: list[dict[str, Any]] = []
 
         # create communication for stats

frigate/stats/util.py

@@ -1,6 +1,7 @@
 """Utilities for stats."""
 
 import asyncio
+import logging
 import os
 import shutil
 import time
@@ -34,6 +35,10 @@ from frigate.util.services import (
 )
 from frigate.version import VERSION
 
+logger = logging.getLogger(__name__)
+
+HWACCEL_ERROR_COOLDOWN_SECONDS = 3600
+
 
 def get_latest_version(config: FrigateConfig) -> str:
     if not config.telemetry.version_check:
@@ -167,7 +172,9 @@ def get_detector_stats(
 
 
 def get_processing_stats(
-    config: FrigateConfig, stats: dict[str, str], hwaccel_errors: list[str]
+    config: FrigateConfig,
+    stats: dict[str, str],
+    hwaccel_errors: dict[str, float],
 ) -> None:
     """Get stats for cpu / gpu."""
 
@@ -206,7 +213,9 @@ async def set_bandwidth_stats(config: FrigateConfig, all_stats: dict[str, Any])
 
 
 async def set_gpu_stats(
-    config: FrigateConfig, all_stats: dict[str, Any], hwaccel_errors: list[str]
+    config: FrigateConfig,
+    all_stats: dict[str, Any],
+    hwaccel_errors: dict[str, float],
 ) -> None:
     """Parse GPUs from hwaccel args and use for stats."""
     hwaccel_args = []
@@ -231,12 +240,16 @@ async def set_gpu_stats(
 
     stats: dict[str, dict] = {}
     intel_gpu_collected = False
+    now = time.monotonic()
 
     for args in hwaccel_args:
-        if args in hwaccel_errors:
-            # known erroring args should automatically return as error
-            stats["error-gpu"] = {"gpu": "", "mem": ""}
-        elif "cuvid" in args or "nvidia" in args:
+        last_error = hwaccel_errors.get(args)
+        if last_error is not None:
+            if now - last_error < HWACCEL_ERROR_COOLDOWN_SECONDS:
+                continue
+            hwaccel_errors.pop(args, None)
+
+        if "cuvid" in args or "nvidia" in args:
             # nvidia GPU
             nvidia_usage = get_nvidia_gpu_stats()
 
@@ -253,7 +266,7 @@ async def set_gpu_stats(
 
             else:
                 stats["nvidia-gpu"] = {"vendor": "nvidia", "gpu": "", "mem": ""}
-                hwaccel_errors.append(args)
+                hwaccel_errors[args] = time.monotonic()
         elif "nvmpi" in args or "jetson" in args:
             # nvidia Jetson
             jetson_usage = get_jetson_stats()
@@ -262,7 +275,7 @@ async def set_gpu_stats(
                 stats["jetson-gpu"] = {"vendor": "nvidia", **jetson_usage}
             else:
                 stats["jetson-gpu"] = {"vendor": "nvidia", "gpu": "", "mem": ""}
-                hwaccel_errors.append(args)
+                hwaccel_errors[args] = time.monotonic()
         elif "qsv" in args or ("vaapi" in args and not is_vaapi_amd_driver()):
             if not config.telemetry.stats.intel_gpu_stats:
                 continue
@@ -280,7 +293,7 @@ async def set_gpu_stats(
                         stats[name] = entry
                 else:
                     stats["intel-gpu"] = {"vendor": "intel", "gpu": "", "mem": ""}
-                    hwaccel_errors.append(args)
+                    hwaccel_errors[args] = time.monotonic()
         elif "vaapi" in args:
             if not config.telemetry.stats.amd_gpu_stats:
                 continue
@@ -292,7 +305,7 @@ async def set_gpu_stats(
                 stats["amd-vaapi"] = {"vendor": "amd", **amd_usage}
             else:
                 stats["amd-vaapi"] = {"vendor": "amd", "gpu": "", "mem": ""}
-                hwaccel_errors.append(args)
+                hwaccel_errors[args] = time.monotonic()
         elif "preset-rk" in args:
             rga_usage = get_rockchip_gpu_stats()
 
@@ -328,7 +341,9 @@ async def set_npu_usages(config: FrigateConfig, all_stats: dict[str, Any]) -> No
 
 
 def stats_snapshot(
-    config: FrigateConfig, stats_tracking: StatsTrackingTypes, hwaccel_errors: list[str]
+    config: FrigateConfig,
+    stats_tracking: StatsTrackingTypes,
+    hwaccel_errors: dict[str, float],
 ) -> dict[str, Any]:
     """Get a snapshot of the current stats that are being tracked."""
     camera_metrics = stats_tracking["camera_metrics"]

frigate/util/services.py

@@ -416,6 +416,11 @@ def get_intel_gpu_stats(
 
     snapshot_a = _read_intel_drm_fdinfo(target_pdev)
     if not snapshot_a:
+        logger.warning(
+            "Unable to collect Intel GPU stats: no DRM fdinfo entries found"
+            "%s. Check that /proc is readable and the i915/xe driver is loaded",
+            f" for pdev {target_pdev}" if target_pdev else "",
+        )
         return None
 
     start = time.monotonic()
@@ -424,6 +429,9 @@ def get_intel_gpu_stats(
 
     snapshot_b = _read_intel_drm_fdinfo(target_pdev)
     if not snapshot_b or elapsed_ns <= 0:
+        logger.warning(
+            "Unable to collect Intel GPU stats: second DRM fdinfo sample was empty"
+        )
         return None
 
     def _new_engine_pct() -> dict[str, float]:
@@ -464,6 +472,10 @@ def get_intel_gpu_stats(
         pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total
 
     if not per_pdev_engine_pct:
+        logger.warning(
+            "Unable to collect Intel GPU stats: no per-engine counters available "
+            "(i915 requires kernel >= 5.19)"
+        )
         return None
 
     names = intel_gpu_name_resolver.get_names()