add camera connection quality metrics and indicator

This commit is contained in:
Josh Hawkins 2025-12-15 10:13:54 -06:00
parent b962c95725
commit 6358cec686
7 changed files with 207 additions and 2 deletions

View File

@ -19,6 +19,8 @@ class CameraMetrics:
process_pid: Synchronized
capture_process_pid: Synchronized
ffmpeg_pid: Synchronized
reconnects_last_hour: Synchronized
stalls_last_hour: Synchronized
def __init__(self, manager: SyncManager):
self.camera_fps = manager.Value("d", 0)
@ -35,6 +37,8 @@ class CameraMetrics:
self.process_pid = manager.Value("i", 0)
self.capture_process_pid = manager.Value("i", 0)
self.ffmpeg_pid = manager.Value("i", 0)
self.reconnects_last_hour = manager.Value("i", 0)
self.stalls_last_hour = manager.Value("i", 0)
class PTZMetrics:

View File

@ -279,6 +279,32 @@ def stats_snapshot(
if camera_stats.capture_process_pid.value
else None
)
# Calculate connection quality based on current state
# This is computed at stats-collection time so offline cameras
# correctly show as unusable rather than excellent
expected_fps = config.cameras[name].detect.fps
current_fps = camera_stats.camera_fps.value
reconnects = camera_stats.reconnects_last_hour.value
stalls = camera_stats.stalls_last_hour.value
if current_fps < 0.1:
quality_str = "unusable"
elif reconnects == 0 and current_fps >= 0.9 * expected_fps and stalls < 5:
quality_str = "excellent"
elif reconnects <= 2 and current_fps >= 0.6 * expected_fps:
quality_str = "fair"
elif reconnects > 10 or current_fps < 1.0 or stalls > 100:
quality_str = "unusable"
else:
quality_str = "poor"
connection_quality = {
"connection_quality": quality_str,
"expected_fps": expected_fps,
"reconnects_last_hour": reconnects,
"stalls_last_hour": stalls,
}
stats["cameras"][name] = {
"camera_fps": round(camera_stats.camera_fps.value, 2),
"process_fps": round(camera_stats.process_fps.value, 2),
@ -290,6 +316,7 @@ def stats_snapshot(
"ffmpeg_pid": ffmpeg_pid,
"audio_rms": round(camera_stats.audio_rms.value, 4),
"audio_dBFS": round(camera_stats.audio_dBFS.value, 4),
**connection_quality,
}
stats["detectors"] = {}

View File

@ -3,6 +3,7 @@ import queue
import subprocess as sp
import threading
import time
from collections import deque
from datetime import datetime, timedelta, timezone
from multiprocessing import Queue, Value
from multiprocessing.synchronize import Event as MpEvent
@ -108,6 +109,7 @@ def capture_frames(
fps: Value,
skipped_fps: Value,
current_frame: Value,
stalls: Value,
stop_event: MpEvent,
) -> None:
frame_size = frame_shape[0] * frame_shape[1]
@ -115,6 +117,12 @@ def capture_frames(
frame_rate.start()
skipped_eps = EventsPerSecond()
skipped_eps.start()
# Stall detection
stall_timestamps = deque()
last_frame_time = 0.0
stall_active = False
config_subscriber = CameraConfigUpdateSubscriber(
None, {config.name: config}, [CameraConfigUpdateEnum.enabled]
)
@ -156,6 +164,32 @@ def capture_frames(
frame_rate.update()
# Update stall metrics
now = datetime.now().timestamp()
if last_frame_time > 0:
delta = now - last_frame_time
# Use observed fps when available; fall back to expected
observed_fps = fps.value if fps.value > 0 else config.detect.fps
# Compute a robust threshold: 2x frame interval, but at least 1s to ignore jitter
interval = 1.0 / max(observed_fps, 0.1)
stall_threshold = max(2.0 * interval, 2.0)
# Count a single stall per continuous gap exceeding threshold
if delta > stall_threshold:
if not stall_active:
stall_timestamps.append(now)
stall_active = True
else:
stall_active = False
last_frame_time = now
while stall_timestamps and stall_timestamps[0] < now - 3600:
stall_timestamps.popleft()
if stalls:
stalls.value = len(stall_timestamps)
# don't lock the queue to check, just try since it should rarely be full
try:
# add to the queue
@ -179,6 +213,8 @@ class CameraWatchdog(threading.Thread):
camera_fps,
skipped_fps,
ffmpeg_pid,
stalls,
reconnects,
stop_event,
):
threading.Thread.__init__(self)
@ -199,6 +235,9 @@ class CameraWatchdog(threading.Thread):
self.frame_index = 0
self.stop_event = stop_event
self.sleeptime = self.config.ffmpeg.retry_interval
self.reconnect_timestamps = deque()
self.stalls = stalls
self.reconnects = reconnects
self.config_subscriber = CameraConfigUpdateSubscriber(
None,
@ -239,6 +278,14 @@ class CameraWatchdog(threading.Thread):
else:
self.ffmpeg_detect_process.wait()
# Update reconnects
now = datetime.now().timestamp()
self.reconnect_timestamps.append(now)
while self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600:
self.reconnect_timestamps.popleft()
if self.reconnects:
self.reconnects.value = len(self.reconnect_timestamps)
# Wait for old capture thread to fully exit before starting a new one
if self.capture_thread is not None and self.capture_thread.is_alive():
self.logger.info("Waiting for capture thread to exit...")
@ -461,6 +508,7 @@ class CameraWatchdog(threading.Thread):
self.frame_queue,
self.camera_fps,
self.skipped_fps,
self.stalls,
self.stop_event,
)
self.capture_thread.start()
@ -514,6 +562,7 @@ class CameraCaptureRunner(threading.Thread):
frame_queue: Queue,
fps: Value,
skipped_fps: Value,
stalls: Value,
stop_event: MpEvent,
):
threading.Thread.__init__(self)
@ -530,6 +579,7 @@ class CameraCaptureRunner(threading.Thread):
self.ffmpeg_process = ffmpeg_process
self.current_frame = Value("d", 0.0)
self.last_frame = 0
self.stalls = stalls
def run(self):
capture_frames(
@ -543,6 +593,7 @@ class CameraCaptureRunner(threading.Thread):
self.fps,
self.skipped_fps,
self.current_frame,
self.stalls,
self.stop_event,
)
@ -576,6 +627,8 @@ class CameraCapture(FrigateProcess):
self.camera_metrics.camera_fps,
self.camera_metrics.skipped_fps,
self.camera_metrics.ffmpeg_pid,
self.camera_metrics.stalls_last_hour,
self.camera_metrics.reconnects_last_hour,
self.stop_event,
)
camera_watchdog.start()

View File

@ -151,6 +151,17 @@
"cameraDetectionsPerSecond": "{{camName}} detections per second",
"cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second"
},
"connectionQuality": {
"title": "Connection Quality",
"excellent": "Excellent",
"fair": "Fair",
"poor": "Poor",
"unusable": "Unusable",
"fps": "FPS",
"expectedFps": "Expected FPS",
"reconnectsLastHour": "Reconnects (last hour)",
"stallsLastHour": "Stalls (last hour)"
},
"toast": {
"success": {
"copyToClipboard": "Copied probe data to clipboard."

View File

@ -0,0 +1,76 @@
import { useTranslation } from "react-i18next";
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { cn } from "@/lib/utils";
type ConnectionQualityIndicatorProps = {
quality: "excellent" | "fair" | "poor" | "unusable";
expectedFps: number;
reconnects: number;
stalls: number;
};
export function ConnectionQualityIndicator({
quality,
expectedFps,
reconnects,
stalls,
}: ConnectionQualityIndicatorProps) {
const { t } = useTranslation(["views/system"]);
const getColorClass = (quality: string): string => {
switch (quality) {
case "excellent":
return "bg-success";
case "fair":
return "bg-yellow-500";
case "poor":
return "bg-orange-500";
case "unusable":
return "bg-destructive";
default:
return "bg-gray-500";
}
};
const qualityLabel = t(`cameras.connectionQuality.${quality}`);
return (
<Tooltip>
<TooltipTrigger asChild>
<div
className={cn(
"inline-block size-3 cursor-pointer rounded-full",
getColorClass(quality),
)}
/>
</TooltipTrigger>
<TooltipContent className="max-w-xs">
<div className="space-y-2">
<div className="font-semibold">
{t("cameras.connectionQuality.title")}
</div>
<div className="text-sm">
<div className="capitalize">{qualityLabel}</div>
<div className="mt-2 space-y-1 text-xs">
<div>
{t("cameras.connectionQuality.expectedFps")}:{" "}
{expectedFps.toFixed(1)} {t("cameras.connectionQuality.fps")}
</div>
<div>
{t("cameras.connectionQuality.reconnectsLastHour")}:{" "}
{reconnects}
</div>
<div>
{t("cameras.connectionQuality.stallsLastHour")}: {stalls}
</div>
</div>
</div>
</div>
</TooltipContent>
</Tooltip>
);
}

View File

@ -24,6 +24,10 @@ export type CameraStats = {
pid: number;
process_fps: number;
skipped_fps: number;
connection_quality: "excellent" | "fair" | "poor" | "unusable";
expected_fps: number;
reconnects_last_hour: number;
stalls_last_hour: number;
};
export type CpuStats = {

View File

@ -1,6 +1,7 @@
import { useFrigateStats } from "@/api/ws";
import { CameraLineGraph } from "@/components/graph/LineGraph";
import CameraInfoDialog from "@/components/overlay/CameraInfoDialog";
import { ConnectionQualityIndicator } from "@/components/camera/ConnectionQualityIndicator";
import { Skeleton } from "@/components/ui/skeleton";
import { FrigateConfig } from "@/types/frigateConfig";
import { FrigateStats } from "@/types/stats";
@ -282,8 +283,37 @@ export default function CameraMetrics({
)}
<div className="flex w-full flex-col gap-3">
<div className="flex flex-row items-center justify-between">
<div className="text-sm font-medium text-muted-foreground smart-capitalize">
<CameraNameLabel camera={camera} />
<div className="flex items-center gap-2">
<div className="text-sm font-medium text-muted-foreground smart-capitalize">
<CameraNameLabel camera={camera} />
</div>
{statsHistory.length > 0 &&
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
] && (
<ConnectionQualityIndicator
quality={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.connection_quality
}
expectedFps={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.expected_fps || 0
}
reconnects={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.reconnects_last_hour || 0
}
stalls={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.stalls_last_hour || 0
}
/>
)}
</div>
<Tooltip>
<TooltipTrigger>