add camera connection quality metrics and indicator

This commit is contained in:
Josh Hawkins 2025-12-15 10:13:54 -06:00
parent b962c95725
commit 6358cec686
7 changed files with 207 additions and 2 deletions

View File

@ -19,6 +19,8 @@ class CameraMetrics:
process_pid: Synchronized process_pid: Synchronized
capture_process_pid: Synchronized capture_process_pid: Synchronized
ffmpeg_pid: Synchronized ffmpeg_pid: Synchronized
reconnects_last_hour: Synchronized
stalls_last_hour: Synchronized
def __init__(self, manager: SyncManager): def __init__(self, manager: SyncManager):
self.camera_fps = manager.Value("d", 0) self.camera_fps = manager.Value("d", 0)
@ -35,6 +37,8 @@ class CameraMetrics:
self.process_pid = manager.Value("i", 0) self.process_pid = manager.Value("i", 0)
self.capture_process_pid = manager.Value("i", 0) self.capture_process_pid = manager.Value("i", 0)
self.ffmpeg_pid = manager.Value("i", 0) self.ffmpeg_pid = manager.Value("i", 0)
self.reconnects_last_hour = manager.Value("i", 0)
self.stalls_last_hour = manager.Value("i", 0)
class PTZMetrics: class PTZMetrics:

View File

@ -279,6 +279,32 @@ def stats_snapshot(
if camera_stats.capture_process_pid.value if camera_stats.capture_process_pid.value
else None else None
) )
# Calculate connection quality based on current state
# This is computed at stats-collection time so offline cameras
# correctly show as unusable rather than excellent
expected_fps = config.cameras[name].detect.fps
current_fps = camera_stats.camera_fps.value
reconnects = camera_stats.reconnects_last_hour.value
stalls = camera_stats.stalls_last_hour.value
if current_fps < 0.1:
quality_str = "unusable"
elif reconnects == 0 and current_fps >= 0.9 * expected_fps and stalls < 5:
quality_str = "excellent"
elif reconnects <= 2 and current_fps >= 0.6 * expected_fps:
quality_str = "fair"
elif reconnects > 10 or current_fps < 1.0 or stalls > 100:
quality_str = "unusable"
else:
quality_str = "poor"
connection_quality = {
"connection_quality": quality_str,
"expected_fps": expected_fps,
"reconnects_last_hour": reconnects,
"stalls_last_hour": stalls,
}
stats["cameras"][name] = { stats["cameras"][name] = {
"camera_fps": round(camera_stats.camera_fps.value, 2), "camera_fps": round(camera_stats.camera_fps.value, 2),
"process_fps": round(camera_stats.process_fps.value, 2), "process_fps": round(camera_stats.process_fps.value, 2),
@ -290,6 +316,7 @@ def stats_snapshot(
"ffmpeg_pid": ffmpeg_pid, "ffmpeg_pid": ffmpeg_pid,
"audio_rms": round(camera_stats.audio_rms.value, 4), "audio_rms": round(camera_stats.audio_rms.value, 4),
"audio_dBFS": round(camera_stats.audio_dBFS.value, 4), "audio_dBFS": round(camera_stats.audio_dBFS.value, 4),
**connection_quality,
} }
stats["detectors"] = {} stats["detectors"] = {}

View File

@ -3,6 +3,7 @@ import queue
import subprocess as sp import subprocess as sp
import threading import threading
import time import time
from collections import deque
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from multiprocessing import Queue, Value from multiprocessing import Queue, Value
from multiprocessing.synchronize import Event as MpEvent from multiprocessing.synchronize import Event as MpEvent
@ -108,6 +109,7 @@ def capture_frames(
fps: Value, fps: Value,
skipped_fps: Value, skipped_fps: Value,
current_frame: Value, current_frame: Value,
stalls: Value,
stop_event: MpEvent, stop_event: MpEvent,
) -> None: ) -> None:
frame_size = frame_shape[0] * frame_shape[1] frame_size = frame_shape[0] * frame_shape[1]
@ -115,6 +117,12 @@ def capture_frames(
frame_rate.start() frame_rate.start()
skipped_eps = EventsPerSecond() skipped_eps = EventsPerSecond()
skipped_eps.start() skipped_eps.start()
# Stall detection
stall_timestamps = deque()
last_frame_time = 0.0
stall_active = False
config_subscriber = CameraConfigUpdateSubscriber( config_subscriber = CameraConfigUpdateSubscriber(
None, {config.name: config}, [CameraConfigUpdateEnum.enabled] None, {config.name: config}, [CameraConfigUpdateEnum.enabled]
) )
@ -156,6 +164,32 @@ def capture_frames(
frame_rate.update() frame_rate.update()
# Update stall metrics
now = datetime.now().timestamp()
if last_frame_time > 0:
delta = now - last_frame_time
# Use observed fps when available; fall back to expected
observed_fps = fps.value if fps.value > 0 else config.detect.fps
# Compute a robust threshold: 2x frame interval, but at least 1s to ignore jitter
interval = 1.0 / max(observed_fps, 0.1)
stall_threshold = max(2.0 * interval, 2.0)
# Count a single stall per continuous gap exceeding threshold
if delta > stall_threshold:
if not stall_active:
stall_timestamps.append(now)
stall_active = True
else:
stall_active = False
last_frame_time = now
while stall_timestamps and stall_timestamps[0] < now - 3600:
stall_timestamps.popleft()
if stalls:
stalls.value = len(stall_timestamps)
# don't lock the queue to check, just try since it should rarely be full # don't lock the queue to check, just try since it should rarely be full
try: try:
# add to the queue # add to the queue
@ -179,6 +213,8 @@ class CameraWatchdog(threading.Thread):
camera_fps, camera_fps,
skipped_fps, skipped_fps,
ffmpeg_pid, ffmpeg_pid,
stalls,
reconnects,
stop_event, stop_event,
): ):
threading.Thread.__init__(self) threading.Thread.__init__(self)
@ -199,6 +235,9 @@ class CameraWatchdog(threading.Thread):
self.frame_index = 0 self.frame_index = 0
self.stop_event = stop_event self.stop_event = stop_event
self.sleeptime = self.config.ffmpeg.retry_interval self.sleeptime = self.config.ffmpeg.retry_interval
self.reconnect_timestamps = deque()
self.stalls = stalls
self.reconnects = reconnects
self.config_subscriber = CameraConfigUpdateSubscriber( self.config_subscriber = CameraConfigUpdateSubscriber(
None, None,
@ -239,6 +278,14 @@ class CameraWatchdog(threading.Thread):
else: else:
self.ffmpeg_detect_process.wait() self.ffmpeg_detect_process.wait()
# Update reconnects
now = datetime.now().timestamp()
self.reconnect_timestamps.append(now)
while self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600:
self.reconnect_timestamps.popleft()
if self.reconnects:
self.reconnects.value = len(self.reconnect_timestamps)
# Wait for old capture thread to fully exit before starting a new one # Wait for old capture thread to fully exit before starting a new one
if self.capture_thread is not None and self.capture_thread.is_alive(): if self.capture_thread is not None and self.capture_thread.is_alive():
self.logger.info("Waiting for capture thread to exit...") self.logger.info("Waiting for capture thread to exit...")
@ -461,6 +508,7 @@ class CameraWatchdog(threading.Thread):
self.frame_queue, self.frame_queue,
self.camera_fps, self.camera_fps,
self.skipped_fps, self.skipped_fps,
self.stalls,
self.stop_event, self.stop_event,
) )
self.capture_thread.start() self.capture_thread.start()
@ -514,6 +562,7 @@ class CameraCaptureRunner(threading.Thread):
frame_queue: Queue, frame_queue: Queue,
fps: Value, fps: Value,
skipped_fps: Value, skipped_fps: Value,
stalls: Value,
stop_event: MpEvent, stop_event: MpEvent,
): ):
threading.Thread.__init__(self) threading.Thread.__init__(self)
@ -530,6 +579,7 @@ class CameraCaptureRunner(threading.Thread):
self.ffmpeg_process = ffmpeg_process self.ffmpeg_process = ffmpeg_process
self.current_frame = Value("d", 0.0) self.current_frame = Value("d", 0.0)
self.last_frame = 0 self.last_frame = 0
self.stalls = stalls
def run(self): def run(self):
capture_frames( capture_frames(
@ -543,6 +593,7 @@ class CameraCaptureRunner(threading.Thread):
self.fps, self.fps,
self.skipped_fps, self.skipped_fps,
self.current_frame, self.current_frame,
self.stalls,
self.stop_event, self.stop_event,
) )
@ -576,6 +627,8 @@ class CameraCapture(FrigateProcess):
self.camera_metrics.camera_fps, self.camera_metrics.camera_fps,
self.camera_metrics.skipped_fps, self.camera_metrics.skipped_fps,
self.camera_metrics.ffmpeg_pid, self.camera_metrics.ffmpeg_pid,
self.camera_metrics.stalls_last_hour,
self.camera_metrics.reconnects_last_hour,
self.stop_event, self.stop_event,
) )
camera_watchdog.start() camera_watchdog.start()

View File

@ -151,6 +151,17 @@
"cameraDetectionsPerSecond": "{{camName}} detections per second", "cameraDetectionsPerSecond": "{{camName}} detections per second",
"cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second" "cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second"
}, },
"connectionQuality": {
"title": "Connection Quality",
"excellent": "Excellent",
"fair": "Fair",
"poor": "Poor",
"unusable": "Unusable",
"fps": "FPS",
"expectedFps": "Expected FPS",
"reconnectsLastHour": "Reconnects (last hour)",
"stallsLastHour": "Stalls (last hour)"
},
"toast": { "toast": {
"success": { "success": {
"copyToClipboard": "Copied probe data to clipboard." "copyToClipboard": "Copied probe data to clipboard."

View File

@ -0,0 +1,76 @@
import { useTranslation } from "react-i18next";
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { cn } from "@/lib/utils";
type ConnectionQualityIndicatorProps = {
quality: "excellent" | "fair" | "poor" | "unusable";
expectedFps: number;
reconnects: number;
stalls: number;
};
export function ConnectionQualityIndicator({
quality,
expectedFps,
reconnects,
stalls,
}: ConnectionQualityIndicatorProps) {
const { t } = useTranslation(["views/system"]);
const getColorClass = (quality: string): string => {
switch (quality) {
case "excellent":
return "bg-success";
case "fair":
return "bg-yellow-500";
case "poor":
return "bg-orange-500";
case "unusable":
return "bg-destructive";
default:
return "bg-gray-500";
}
};
const qualityLabel = t(`cameras.connectionQuality.${quality}`);
return (
<Tooltip>
<TooltipTrigger asChild>
<div
className={cn(
"inline-block size-3 cursor-pointer rounded-full",
getColorClass(quality),
)}
/>
</TooltipTrigger>
<TooltipContent className="max-w-xs">
<div className="space-y-2">
<div className="font-semibold">
{t("cameras.connectionQuality.title")}
</div>
<div className="text-sm">
<div className="capitalize">{qualityLabel}</div>
<div className="mt-2 space-y-1 text-xs">
<div>
{t("cameras.connectionQuality.expectedFps")}:{" "}
{expectedFps.toFixed(1)} {t("cameras.connectionQuality.fps")}
</div>
<div>
{t("cameras.connectionQuality.reconnectsLastHour")}:{" "}
{reconnects}
</div>
<div>
{t("cameras.connectionQuality.stallsLastHour")}: {stalls}
</div>
</div>
</div>
</div>
</TooltipContent>
</Tooltip>
);
}

View File

@ -24,6 +24,10 @@ export type CameraStats = {
pid: number; pid: number;
process_fps: number; process_fps: number;
skipped_fps: number; skipped_fps: number;
connection_quality: "excellent" | "fair" | "poor" | "unusable";
expected_fps: number;
reconnects_last_hour: number;
stalls_last_hour: number;
}; };
export type CpuStats = { export type CpuStats = {

View File

@ -1,6 +1,7 @@
import { useFrigateStats } from "@/api/ws"; import { useFrigateStats } from "@/api/ws";
import { CameraLineGraph } from "@/components/graph/LineGraph"; import { CameraLineGraph } from "@/components/graph/LineGraph";
import CameraInfoDialog from "@/components/overlay/CameraInfoDialog"; import CameraInfoDialog from "@/components/overlay/CameraInfoDialog";
import { ConnectionQualityIndicator } from "@/components/camera/ConnectionQualityIndicator";
import { Skeleton } from "@/components/ui/skeleton"; import { Skeleton } from "@/components/ui/skeleton";
import { FrigateConfig } from "@/types/frigateConfig"; import { FrigateConfig } from "@/types/frigateConfig";
import { FrigateStats } from "@/types/stats"; import { FrigateStats } from "@/types/stats";
@ -282,8 +283,37 @@ export default function CameraMetrics({
)} )}
<div className="flex w-full flex-col gap-3"> <div className="flex w-full flex-col gap-3">
<div className="flex flex-row items-center justify-between"> <div className="flex flex-row items-center justify-between">
<div className="text-sm font-medium text-muted-foreground smart-capitalize"> <div className="flex items-center gap-2">
<CameraNameLabel camera={camera} /> <div className="text-sm font-medium text-muted-foreground smart-capitalize">
<CameraNameLabel camera={camera} />
</div>
{statsHistory.length > 0 &&
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
] && (
<ConnectionQualityIndicator
quality={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.connection_quality
}
expectedFps={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.expected_fps || 0
}
reconnects={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.reconnects_last_hour || 0
}
stalls={
statsHistory[statsHistory.length - 1]?.cameras[
camera.name
]?.stalls_last_hour || 0
}
/>
)}
</div> </div>
<Tooltip> <Tooltip>
<TooltipTrigger> <TooltipTrigger>