reuse pooled upstream connections for nginx auth subrequests

The /auth subrequest proxied to a variable address, which nginx can't pool, and its location-level headers dropped the inherited Connection header so nginx sent Connection: close. Every authenticated request opened and closed its own loopback connection to the API. It now proxies to the frigate_api upstream with an empty Connection header so the keepalive pool is reused.

docker/main/rootfs/usr/local/nginx/conf/auth_location.conf

@@ -1,10 +1,8 @@
-set $upstream_auth http://127.0.0.1:5001/auth;
-
 ## Virtual endpoint created by nginx to forward auth requests.
 location /auth {
     ## Essential Proxy Configuration
     internal;
-    proxy_pass $upstream_auth;
+    proxy_pass http://frigate_api/auth;
 
     ## Headers
 
@@ -39,6 +37,7 @@ location /auth {
     proxy_next_upstream error timeout invalid_header http_500 http_502 http_503; # Timeout if the real server is dead
     proxy_redirect http:// $scheme://;
     proxy_http_version 1.1;
+    proxy_set_header Connection "";
     proxy_cache_bypass $cookie_session;
     proxy_no_cache $cookie_session;
     proxy_buffers 4 32k;

docs/docs/configuration/metrics.md

@@ -69,7 +69,7 @@ These gauges report the operating system's figures for the whole filesystem (the
 
 ### Event Metrics
 
-- `frigate_camera_events{camera="", label=""}` - Count of camera events since exporter started
+- `frigate_camera_events{camera="", label=""}` - Current number of events stored in the database for each camera and label
 
 ## Configuring Prometheus
 
@@ -101,8 +101,8 @@ rate(frigate_detection_fps{camera_name="front_door"}[5m])
 # Storage usage percentage
 (frigate_storage_used_bytes / frigate_storage_total_bytes) * 100
 
-# Event count by camera in last hour
-increase(frigate_camera_events[1h])
+# Number of events currently stored in the database by camera
+sum by (camera) (frigate_camera_events)
 ```
 
 ## Grafana Dashboard
@@ -121,8 +121,7 @@ A sample Grafana dashboard JSON will be provided in a future update.
 
 The metrics exposed by Frigate use the following Prometheus metric types:
 
-- **Counter**: Cumulative values that only increase (e.g., `frigate_camera_events`)
-- **Gauge**: Values that can go up and down (e.g., `frigate_cpu_usage_percent`)
+- **Gauge**: Values that can go up and down (e.g., `frigate_cpu_usage_percent`, `frigate_camera_events`)
 - **Info**: Key-value pairs for metadata (e.g., `frigate_storage_mount_type`)
 
 For more information about Prometheus metric types, see the [Prometheus documentation](https://prometheus.io/docs/concepts/metric_types/).

frigate/api/app.py

@@ -177,7 +177,9 @@ def metrics(request: Request):
     # Retrieve the latest statistics and update the Prometheus metrics
     stats = request.app.stats_emitter.get_latest_stats()
     # query DB for count of events by camera, label
-    event_counts: list[dict[str, Any]] = (
+    # Materialize the rows; the collector is shared across request threads and a
+    # lazy query would step one sqlite cursor from two threads.
+    event_counts: list[dict[str, Any]] = list(
         Event.select(Event.camera, Event.label, fn.Count())
         .group_by(Event.camera, Event.label)
         .dicts()

frigate/api/camera.py

@@ -84,9 +84,13 @@ def _is_valid_host(host: str) -> bool:
 
 @router.get("/go2rtc/streams", dependencies=[Depends(allow_any_authenticated())])
 async def go2rtc_streams(request: Request):
-    r = await asyncio.to_thread(requests.get, "http://127.0.0.1:1984/api/streams")
-    if not r.ok:
-        logger.error("Failed to fetch streams from go2rtc")
+    try:
+        r = await asyncio.to_thread(
+            requests.get, "http://127.0.0.1:1984/api/streams", timeout=10
+        )
+        r.raise_for_status()
+    except requests.RequestException as e:
+        logger.error("Failed to fetch streams from go2rtc: %s", e)
         return JSONResponse(
             content=({"success": False, "message": "Error fetching stream data"}),
             status_code=500,
@@ -121,16 +125,21 @@ async def go2rtc_streams(request: Request):
     dependencies=[Depends(require_go2rtc_stream_access)],
 )
 def go2rtc_camera_stream(request: Request, stream_name: str):
-    r = requests.get(
-        "http://127.0.0.1:1984/api/streams",
-        params={
-            "src": stream_name,
-            "video": "all",
-            "audio": "all",
-            "microphone": "",
-        },
-    )
-    if not r.ok:
+    try:
+        r = requests.get(
+            "http://127.0.0.1:1984/api/streams",
+            params={
+                "src": stream_name,
+                "video": "all",
+                "audio": "all",
+                "microphone": "",
+            },
+            # go2rtc probes each source in turn, so one unresponsive source
+            # can take 10s or more
+            timeout=(2, 35),
+        )
+        r.raise_for_status()
+    except requests.RequestException as e:
         camera_config = request.app.frigate_config.cameras.get(stream_name)
 
         if camera_config is None:
@@ -140,7 +149,7 @@ def go2rtc_camera_stream(request: Request, stream_name: str):
                     break
 
         if camera_config and camera_config.enabled:
-            logger.error("Failed to fetch streams from go2rtc")
+            logger.error("Failed to fetch streams from go2rtc: %s", e)
 
         return JSONResponse(
             content=({"success": False, "message": "Error fetching stream data"}),

frigate/app.py

@@ -731,6 +731,8 @@ class FrigateApp:
                 host="127.0.0.1",
                 port=5001,
                 log_level="error",
+                # outlive nginx's 60s upstream keepalive so nginx closes idle connections first
+                timeout_keep_alive=75,
             )
         finally:
             self.stop()

frigate/output/output.py

@@ -285,11 +285,16 @@ class OutputProcess(FrigateProcess):
             )
             preview_write_times[camera] = frame_time
 
+            manager = websocket_server.manager
+            with manager.lock:
+                websockets = list(manager.websockets.values())
+
             # send camera frame to ffmpeg process if websockets are connected
             if any(
-                ws.environ["PATH_INFO"].endswith(camera)
+                not ws.terminated
+                and ws.environ["PATH_INFO"].endswith(camera)
                 and ws_has_camera_access(ws, camera, self.config)
-                for ws in websocket_server.manager
+                for ws in websockets
             ):
                 # write to the converter for the camera if clients are listening to the specific camera
                 jsmpeg_cameras[camera].write_frame(frame.tobytes())
@@ -301,9 +306,10 @@ class OutputProcess(FrigateProcess):
                 and (
                     self.config.birdseye.restream
                     or any(
-                        ws.environ["PATH_INFO"].endswith("birdseye")
+                        not ws.terminated
+                        and ws.environ["PATH_INFO"].endswith("birdseye")
                         and ws_has_camera_access(ws, "birdseye", self.config)
-                        for ws in websocket_server.manager
+                        for ws in websockets
                     )
                 )
             ):

frigate/output/preview.py

@@ -392,18 +392,20 @@ class PreviewRecorder:
     ) -> None:
         self.offline = False
 
-        # always write the first frame
         if self.start_time == 0:
             self.start_time = frame_time
-            self.output_frames.append(frame_time)
-            self.write_frame_to_cache(frame_time, frame)
+
+            if self.config.record.enabled:
+                self.output_frames.append(frame_time)
+                self.write_frame_to_cache(frame_time, frame)
+
             return
 
         # check if PREVIEW clip should be generated and cached frames reset
         if frame_time >= self.segment_end:
             if len(self.output_frames) > 0:
                 # save last frame to ensure consistent duration
-                if self.config.record:
+                if self.config.record.enabled:
                     self.output_frames.append(frame_time)
                     self.write_frame_to_cache(frame_time, frame)
 

frigate/record/cleanup.py

@@ -516,10 +516,13 @@ class RecordingCleanup(threading.Thread):
                 logger.info("Exiting recording cleanup...")
                 break
 
-            self.clean_tmp_previews()
+            try:
+                self.clean_tmp_previews()
 
-            if counter == 0:
-                self.clean_tmp_clips()
-                maybe_empty_dirs = self.expire_recordings()
-                remove_empty_directories(Path(RECORD_DIR), maybe_empty_dirs)
-                self.truncate_wal()
+                if counter == 0:
+                    self.clean_tmp_clips()
+                    maybe_empty_dirs = self.expire_recordings()
+                    remove_empty_directories(Path(RECORD_DIR), maybe_empty_dirs)
+                    self.truncate_wal()
+            except Exception:
+                logger.exception("Recording cleanup pass failed")

frigate/stats/prometheus.py

@@ -5,7 +5,6 @@ from typing import Any
 from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
 from prometheus_client.core import (
     REGISTRY,
-    CounterMetricFamily,
     GaugeMetricFamily,
     InfoMetricFamily,
 )
@@ -472,9 +471,9 @@ class CustomCollector:
         yield storage_total
         yield storage_used
 
-        camera_events = CounterMetricFamily(
+        camera_events = GaugeMetricFamily(
             "frigate_camera_events",
-            "Count of camera events since exporter started",
+            "Current number of events stored in the database per camera and label",
             labels=["camera", "label"],
         )
 

frigate/test/http_api/test_http_event.py

@@ -563,23 +563,20 @@ class TestHttpApp(BaseTestHttp):
         assert "# TYPE frigate_detection_total_fps gauge" in event.text
         assert "frigate_detection_total_fps 13.7" in event.text
         assert (
-            "# HELP frigate_camera_events_total Count of camera events since exporter started"
+            "# HELP frigate_camera_events Current number of events stored in the database per camera and label"
             in event.text
         )
-        assert "# TYPE frigate_camera_events_total counter" in event.text
+        assert "# TYPE frigate_camera_events gauge" in event.text
         assert (
-            'frigate_camera_events_total{camera="front_door",label="Mock"} 3.0'
+            'frigate_camera_events{camera="front_door",label="Mock"} 3.0' in event.text
+        )
+        assert (
+            'frigate_camera_events{camera="front_door",label="inside"} 2.0'
             in event.text
         )
         assert (
-            'frigate_camera_events_total{camera="front_door",label="inside"} 2.0'
+            'frigate_camera_events{camera="front_door",label="outside"} 2.0'
             in event.text
         )
-        assert (
-            'frigate_camera_events_total{camera="front_door",label="outside"} 2.0'
-            in event.text
-        )
-        assert (
-            'frigate_camera_events_total{camera="porch",label="Mock"} 2.0' in event.text
-        )
-        assert 'frigate_camera_events_total{camera="porch",label="inside"} 2.0'
+        assert 'frigate_camera_events{camera="porch",label="Mock"} 2.0' in event.text
+        assert 'frigate_camera_events{camera="porch",label="inside"} 2.0' in event.text

frigate/util/services.py

@@ -261,7 +261,7 @@ def is_vaapi_amd_driver() -> bool:
         output = p.stdout.decode("unicode_escape").split("\n")
 
         # VA Info will print out the friendly name of the driver
-        return any("AMD Radeon Graphics" in line for line in output)
+        return any(" for AMD " in line for line in output)
 
 
 def get_amd_gpu_stats() -> dict[str, str] | None:

frigate/video/ffmpeg.py

@@ -483,16 +483,18 @@ class CameraWatchdog(threading.Thread):
             elif self.camera_fps.value >= (self.config.detect.fps + 10):
                 self.fps_overflow_count += 1
 
-                if self.fps_overflow_count == 3:
+                # Require 3 retry intervals of sustained overflow. camera_fps
+                # averages the last 10s, so one burst of buffered frames stays
+                # over the limit for several 1s ticks.
+                if self.fps_overflow_count >= 3 * self.sleeptime and can_restart:
                     self.detect_status.send("offline", now)
                     self.fps_overflow_count = 0
                     self.camera_fps.value = 0
                     self.logger.info(
                         f"{self.config.name} exceeded fps limit. Exiting ffmpeg..."
                     )
-                    if can_restart:
-                        self.reset_capture_thread()
-                        last_restart_time = now
+                    self.reset_capture_thread()
+                    last_restart_time = now
             elif now - self.capture_thread.current_frame.value > 20:
                 self.detect_status.send("offline", now)
                 self.camera_fps.value = 0

web/package-lock.json

@@ -5498,9 +5498,9 @@
       "license": "ISC"
     },
     "node_modules/follow-redirects": {
-      "version": "1.16.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
-      "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
+      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
       "funding": [
         {
           "type": "individual",

web/src/components/camera/AutoUpdatingCameraImage.tsx

@@ -12,6 +12,7 @@ type AutoUpdatingCameraImageProps = {
 };
 
 const MIN_LOAD_TIMEOUT_MS = 200;
+const ERROR_RETRY_MS = 1000;
 
 export default function AutoUpdatingCameraImage({
   camera,
@@ -68,6 +69,24 @@ export default function AutoUpdatingCameraImage({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [key, setFps]);
 
+  const handleError = useCallback(() => {
+    if (reloadInterval == -1) {
+      return;
+    }
+
+    // the periodic cache url doesn't change between retries, so drop it
+    setIsCached(true);
+
+    if (timeoutRef.current) {
+      clearTimeout(timeoutRef.current);
+    }
+
+    timeoutRef.current = setTimeout(
+      () => setKey(Date.now()),
+      Math.max(reloadInterval, ERROR_RETRY_MS),
+    );
+  }, [reloadInterval]);
+
   // periodic cache to reduce loading indicator
 
   const [isCached, setIsCached] = useState(false);
@@ -92,6 +111,7 @@ export default function AutoUpdatingCameraImage({
       <CameraImage
         camera={camera}
         onload={handleLoad}
+        onerror={handleError}
         searchParams={cacheKey}
         className={cameraClasses}
       />

web/src/components/camera/CameraImage.tsx

@@ -11,6 +11,7 @@ type CameraImageProps = {
   className?: string;
   camera: string;
   onload?: () => void;
+  onerror?: () => void;
   searchParams?: string;
 };
 
@@ -18,6 +19,7 @@ export default function CameraImage({
   className,
   camera,
   onload,
+  onerror,
   searchParams = "",
 }: CameraImageProps) {
   const { data: config } = useSWR("config");
@@ -96,6 +98,7 @@ export default function CameraImage({
             "rounded-lg md:rounded-2xl",
           )}
           onLoad={handleImageLoad}
+          onError={onerror}
           loading="lazy"
         />
       ) : (

web/src/components/overlay/CameraInfoDialog.tsx

@@ -149,7 +149,7 @@ export default function CameraInfoDialog({
                                       <span>
                                         {t("cameras.info.resolution")}{" "}
                                         <span className="text-primary">
-                                          t("cameras.info.unknown")
+                                          {t("cameras.info.unknown")}
                                         </span>
                                       </span>
                                     )}