diff --git a/docker/main/rootfs/usr/local/nginx/conf/auth_location.conf b/docker/main/rootfs/usr/local/nginx/conf/auth_location.conf
index 2ff7bf9d9f..6f27ea7e2d 100644
--- a/docker/main/rootfs/usr/local/nginx/conf/auth_location.conf
+++ b/docker/main/rootfs/usr/local/nginx/conf/auth_location.conf
@@ -1,10 +1,8 @@
-set $upstream_auth http://127.0.0.1:5001/auth;
-
## Virtual endpoint created by nginx to forward auth requests.
location /auth {
## Essential Proxy Configuration
internal;
- proxy_pass $upstream_auth;
+ proxy_pass http://frigate_api/auth;
## Headers
@@ -39,6 +37,7 @@ location /auth {
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503; # Timeout if the real server is dead
proxy_redirect http:// $scheme://;
proxy_http_version 1.1;
+ proxy_set_header Connection "";
proxy_cache_bypass $cookie_session;
proxy_no_cache $cookie_session;
proxy_buffers 4 32k;
diff --git a/docs/docs/configuration/metrics.md b/docs/docs/configuration/metrics.md
index 69db159cc5..480fada5f4 100644
--- a/docs/docs/configuration/metrics.md
+++ b/docs/docs/configuration/metrics.md
@@ -69,7 +69,7 @@ These gauges report the operating system's figures for the whole filesystem (the
### Event Metrics
-- `frigate_camera_events{camera="", label=""}` - Count of camera events since exporter started
+- `frigate_camera_events{camera="", label=""}` - Current number of events stored in the database for each camera and label
## Configuring Prometheus
@@ -101,8 +101,8 @@ rate(frigate_detection_fps{camera_name="front_door"}[5m])
# Storage usage percentage
(frigate_storage_used_bytes / frigate_storage_total_bytes) * 100
-# Event count by camera in last hour
-increase(frigate_camera_events[1h])
+# Number of events currently stored in the database by camera
+sum by (camera) (frigate_camera_events)
```
## Grafana Dashboard
@@ -121,8 +121,7 @@ A sample Grafana dashboard JSON will be provided in a future update.
The metrics exposed by Frigate use the following Prometheus metric types:
-- **Counter**: Cumulative values that only increase (e.g., `frigate_camera_events`)
-- **Gauge**: Values that can go up and down (e.g., `frigate_cpu_usage_percent`)
+- **Gauge**: Values that can go up and down (e.g., `frigate_cpu_usage_percent`, `frigate_camera_events`)
- **Info**: Key-value pairs for metadata (e.g., `frigate_storage_mount_type`)
For more information about Prometheus metric types, see the [Prometheus documentation](https://prometheus.io/docs/concepts/metric_types/).
diff --git a/frigate/api/app.py b/frigate/api/app.py
index 8f0fcb5f06..c500f2b141 100644
--- a/frigate/api/app.py
+++ b/frigate/api/app.py
@@ -177,7 +177,9 @@ def metrics(request: Request):
# Retrieve the latest statistics and update the Prometheus metrics
stats = request.app.stats_emitter.get_latest_stats()
# query DB for count of events by camera, label
- event_counts: list[dict[str, Any]] = (
+ # Materialize the rows; the collector is shared across request threads and a
+ # lazy query would step one sqlite cursor from two threads.
+ event_counts: list[dict[str, Any]] = list(
Event.select(Event.camera, Event.label, fn.Count())
.group_by(Event.camera, Event.label)
.dicts()
diff --git a/frigate/api/camera.py b/frigate/api/camera.py
index 5dd3f4a700..faab4dc98e 100644
--- a/frigate/api/camera.py
+++ b/frigate/api/camera.py
@@ -84,9 +84,13 @@ def _is_valid_host(host: str) -> bool:
@router.get("/go2rtc/streams", dependencies=[Depends(allow_any_authenticated())])
async def go2rtc_streams(request: Request):
- r = await asyncio.to_thread(requests.get, "http://127.0.0.1:1984/api/streams")
- if not r.ok:
- logger.error("Failed to fetch streams from go2rtc")
+ try:
+ r = await asyncio.to_thread(
+ requests.get, "http://127.0.0.1:1984/api/streams", timeout=10
+ )
+ r.raise_for_status()
+ except requests.RequestException as e:
+ logger.error("Failed to fetch streams from go2rtc: %s", e)
return JSONResponse(
content=({"success": False, "message": "Error fetching stream data"}),
status_code=500,
@@ -121,16 +125,21 @@ async def go2rtc_streams(request: Request):
dependencies=[Depends(require_go2rtc_stream_access)],
)
def go2rtc_camera_stream(request: Request, stream_name: str):
- r = requests.get(
- "http://127.0.0.1:1984/api/streams",
- params={
- "src": stream_name,
- "video": "all",
- "audio": "all",
- "microphone": "",
- },
- )
- if not r.ok:
+ try:
+ r = requests.get(
+ "http://127.0.0.1:1984/api/streams",
+ params={
+ "src": stream_name,
+ "video": "all",
+ "audio": "all",
+ "microphone": "",
+ },
+ # go2rtc probes each source in turn, so one unresponsive source
+ # can take 10s or more
+ timeout=(2, 35),
+ )
+ r.raise_for_status()
+ except requests.RequestException as e:
camera_config = request.app.frigate_config.cameras.get(stream_name)
if camera_config is None:
@@ -140,7 +149,7 @@ def go2rtc_camera_stream(request: Request, stream_name: str):
break
if camera_config and camera_config.enabled:
- logger.error("Failed to fetch streams from go2rtc")
+ logger.error("Failed to fetch streams from go2rtc: %s", e)
return JSONResponse(
content=({"success": False, "message": "Error fetching stream data"}),
diff --git a/frigate/app.py b/frigate/app.py
index fe7f1e237c..7c7b152696 100644
--- a/frigate/app.py
+++ b/frigate/app.py
@@ -731,6 +731,8 @@ class FrigateApp:
host="127.0.0.1",
port=5001,
log_level="error",
+ # outlive nginx's 60s upstream keepalive so nginx closes idle connections first
+ timeout_keep_alive=75,
)
finally:
self.stop()
diff --git a/frigate/output/output.py b/frigate/output/output.py
index b40a0f4cd0..0872ca7b91 100644
--- a/frigate/output/output.py
+++ b/frigate/output/output.py
@@ -285,11 +285,16 @@ class OutputProcess(FrigateProcess):
)
preview_write_times[camera] = frame_time
+ manager = websocket_server.manager
+ with manager.lock:
+ websockets = list(manager.websockets.values())
+
# send camera frame to ffmpeg process if websockets are connected
if any(
- ws.environ["PATH_INFO"].endswith(camera)
+ not ws.terminated
+ and ws.environ["PATH_INFO"].endswith(camera)
and ws_has_camera_access(ws, camera, self.config)
- for ws in websocket_server.manager
+ for ws in websockets
):
# write to the converter for the camera if clients are listening to the specific camera
jsmpeg_cameras[camera].write_frame(frame.tobytes())
@@ -301,9 +306,10 @@ class OutputProcess(FrigateProcess):
and (
self.config.birdseye.restream
or any(
- ws.environ["PATH_INFO"].endswith("birdseye")
+ not ws.terminated
+ and ws.environ["PATH_INFO"].endswith("birdseye")
and ws_has_camera_access(ws, "birdseye", self.config)
- for ws in websocket_server.manager
+ for ws in websockets
)
)
):
diff --git a/frigate/output/preview.py b/frigate/output/preview.py
index 09ee4f3407..4a1b99feb0 100644
--- a/frigate/output/preview.py
+++ b/frigate/output/preview.py
@@ -392,18 +392,20 @@ class PreviewRecorder:
) -> None:
self.offline = False
- # always write the first frame
if self.start_time == 0:
self.start_time = frame_time
- self.output_frames.append(frame_time)
- self.write_frame_to_cache(frame_time, frame)
+
+ if self.config.record.enabled:
+ self.output_frames.append(frame_time)
+ self.write_frame_to_cache(frame_time, frame)
+
return
# check if PREVIEW clip should be generated and cached frames reset
if frame_time >= self.segment_end:
if len(self.output_frames) > 0:
# save last frame to ensure consistent duration
- if self.config.record:
+ if self.config.record.enabled:
self.output_frames.append(frame_time)
self.write_frame_to_cache(frame_time, frame)
diff --git a/frigate/record/cleanup.py b/frigate/record/cleanup.py
index 509c43ae14..429193f822 100644
--- a/frigate/record/cleanup.py
+++ b/frigate/record/cleanup.py
@@ -516,10 +516,13 @@ class RecordingCleanup(threading.Thread):
logger.info("Exiting recording cleanup...")
break
- self.clean_tmp_previews()
+ try:
+ self.clean_tmp_previews()
- if counter == 0:
- self.clean_tmp_clips()
- maybe_empty_dirs = self.expire_recordings()
- remove_empty_directories(Path(RECORD_DIR), maybe_empty_dirs)
- self.truncate_wal()
+ if counter == 0:
+ self.clean_tmp_clips()
+ maybe_empty_dirs = self.expire_recordings()
+ remove_empty_directories(Path(RECORD_DIR), maybe_empty_dirs)
+ self.truncate_wal()
+ except Exception:
+ logger.exception("Recording cleanup pass failed")
diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py
index 9797d569e1..db879bd38a 100644
--- a/frigate/stats/prometheus.py
+++ b/frigate/stats/prometheus.py
@@ -5,7 +5,6 @@ from typing import Any
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
from prometheus_client.core import (
REGISTRY,
- CounterMetricFamily,
GaugeMetricFamily,
InfoMetricFamily,
)
@@ -472,9 +471,9 @@ class CustomCollector:
yield storage_total
yield storage_used
- camera_events = CounterMetricFamily(
+ camera_events = GaugeMetricFamily(
"frigate_camera_events",
- "Count of camera events since exporter started",
+ "Current number of events stored in the database per camera and label",
labels=["camera", "label"],
)
diff --git a/frigate/test/http_api/test_http_event.py b/frigate/test/http_api/test_http_event.py
index 959fe6dbc2..8de5eb9dbc 100644
--- a/frigate/test/http_api/test_http_event.py
+++ b/frigate/test/http_api/test_http_event.py
@@ -563,23 +563,20 @@ class TestHttpApp(BaseTestHttp):
assert "# TYPE frigate_detection_total_fps gauge" in event.text
assert "frigate_detection_total_fps 13.7" in event.text
assert (
- "# HELP frigate_camera_events_total Count of camera events since exporter started"
+ "# HELP frigate_camera_events Current number of events stored in the database per camera and label"
in event.text
)
- assert "# TYPE frigate_camera_events_total counter" in event.text
+ assert "# TYPE frigate_camera_events gauge" in event.text
assert (
- 'frigate_camera_events_total{camera="front_door",label="Mock"} 3.0'
+ 'frigate_camera_events{camera="front_door",label="Mock"} 3.0' in event.text
+ )
+ assert (
+ 'frigate_camera_events{camera="front_door",label="inside"} 2.0'
in event.text
)
assert (
- 'frigate_camera_events_total{camera="front_door",label="inside"} 2.0'
+ 'frigate_camera_events{camera="front_door",label="outside"} 2.0'
in event.text
)
- assert (
- 'frigate_camera_events_total{camera="front_door",label="outside"} 2.0'
- in event.text
- )
- assert (
- 'frigate_camera_events_total{camera="porch",label="Mock"} 2.0' in event.text
- )
- assert 'frigate_camera_events_total{camera="porch",label="inside"} 2.0'
+ assert 'frigate_camera_events{camera="porch",label="Mock"} 2.0' in event.text
+ assert 'frigate_camera_events{camera="porch",label="inside"} 2.0' in event.text
diff --git a/frigate/util/services.py b/frigate/util/services.py
index a88e44b5c7..f0bb7e5abb 100644
--- a/frigate/util/services.py
+++ b/frigate/util/services.py
@@ -261,7 +261,7 @@ def is_vaapi_amd_driver() -> bool:
output = p.stdout.decode("unicode_escape").split("\n")
# VA Info will print out the friendly name of the driver
- return any("AMD Radeon Graphics" in line for line in output)
+ return any(" for AMD " in line for line in output)
def get_amd_gpu_stats() -> dict[str, str] | None:
diff --git a/frigate/video/ffmpeg.py b/frigate/video/ffmpeg.py
index 27cf325f2f..35b22458c7 100644
--- a/frigate/video/ffmpeg.py
+++ b/frigate/video/ffmpeg.py
@@ -483,16 +483,18 @@ class CameraWatchdog(threading.Thread):
elif self.camera_fps.value >= (self.config.detect.fps + 10):
self.fps_overflow_count += 1
- if self.fps_overflow_count == 3:
+ # Require 3 retry intervals of sustained overflow. camera_fps
+ # averages the last 10s, so one burst of buffered frames stays
+ # over the limit for several 1s ticks.
+ if self.fps_overflow_count >= 3 * self.sleeptime and can_restart:
self.detect_status.send("offline", now)
self.fps_overflow_count = 0
self.camera_fps.value = 0
self.logger.info(
f"{self.config.name} exceeded fps limit. Exiting ffmpeg..."
)
- if can_restart:
- self.reset_capture_thread()
- last_restart_time = now
+ self.reset_capture_thread()
+ last_restart_time = now
elif now - self.capture_thread.current_frame.value > 20:
self.detect_status.send("offline", now)
self.camera_fps.value = 0
diff --git a/web/src/components/camera/AutoUpdatingCameraImage.tsx b/web/src/components/camera/AutoUpdatingCameraImage.tsx
index 43a59daa93..8e968c37d7 100644
--- a/web/src/components/camera/AutoUpdatingCameraImage.tsx
+++ b/web/src/components/camera/AutoUpdatingCameraImage.tsx
@@ -12,6 +12,7 @@ type AutoUpdatingCameraImageProps = {
};
const MIN_LOAD_TIMEOUT_MS = 200;
+const ERROR_RETRY_MS = 1000;
export default function AutoUpdatingCameraImage({
camera,
@@ -68,6 +69,24 @@ export default function AutoUpdatingCameraImage({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [key, setFps]);
+ const handleError = useCallback(() => {
+ if (reloadInterval == -1) {
+ return;
+ }
+
+ // the periodic cache url doesn't change between retries, so drop it
+ setIsCached(true);
+
+ if (timeoutRef.current) {
+ clearTimeout(timeoutRef.current);
+ }
+
+ timeoutRef.current = setTimeout(
+ () => setKey(Date.now()),
+ Math.max(reloadInterval, ERROR_RETRY_MS),
+ );
+ }, [reloadInterval]);
+
// periodic cache to reduce loading indicator
const [isCached, setIsCached] = useState(false);
@@ -92,6 +111,7 @@ export default function AutoUpdatingCameraImage({
diff --git a/web/src/components/camera/CameraImage.tsx b/web/src/components/camera/CameraImage.tsx
index 0ba7ea19e7..f328d77387 100644
--- a/web/src/components/camera/CameraImage.tsx
+++ b/web/src/components/camera/CameraImage.tsx
@@ -11,6 +11,7 @@ type CameraImageProps = {
className?: string;
camera: string;
onload?: () => void;
+ onerror?: () => void;
searchParams?: string;
};
@@ -18,6 +19,7 @@ export default function CameraImage({
className,
camera,
onload,
+ onerror,
searchParams = "",
}: CameraImageProps) {
const { data: config } = useSWR("config");
@@ -96,6 +98,7 @@ export default function CameraImage({
"rounded-lg md:rounded-2xl",
)}
onLoad={handleImageLoad}
+ onError={onerror}
loading="lazy"
/>
) : (
diff --git a/web/src/components/overlay/CameraInfoDialog.tsx b/web/src/components/overlay/CameraInfoDialog.tsx
index 3696f3175c..2731202077 100644
--- a/web/src/components/overlay/CameraInfoDialog.tsx
+++ b/web/src/components/overlay/CameraInfoDialog.tsx
@@ -149,7 +149,7 @@ export default function CameraInfoDialog({
{t("cameras.info.resolution")}{" "}
- t("cameras.info.unknown")
+ {t("cameras.info.unknown")}
)}