mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-01-23 20:48:31 +03:00
add camera connection quality metrics and indicator
This commit is contained in:
parent
b962c95725
commit
6358cec686
@ -19,6 +19,8 @@ class CameraMetrics:
|
|||||||
process_pid: Synchronized
|
process_pid: Synchronized
|
||||||
capture_process_pid: Synchronized
|
capture_process_pid: Synchronized
|
||||||
ffmpeg_pid: Synchronized
|
ffmpeg_pid: Synchronized
|
||||||
|
reconnects_last_hour: Synchronized
|
||||||
|
stalls_last_hour: Synchronized
|
||||||
|
|
||||||
def __init__(self, manager: SyncManager):
|
def __init__(self, manager: SyncManager):
|
||||||
self.camera_fps = manager.Value("d", 0)
|
self.camera_fps = manager.Value("d", 0)
|
||||||
@ -35,6 +37,8 @@ class CameraMetrics:
|
|||||||
self.process_pid = manager.Value("i", 0)
|
self.process_pid = manager.Value("i", 0)
|
||||||
self.capture_process_pid = manager.Value("i", 0)
|
self.capture_process_pid = manager.Value("i", 0)
|
||||||
self.ffmpeg_pid = manager.Value("i", 0)
|
self.ffmpeg_pid = manager.Value("i", 0)
|
||||||
|
self.reconnects_last_hour = manager.Value("i", 0)
|
||||||
|
self.stalls_last_hour = manager.Value("i", 0)
|
||||||
|
|
||||||
|
|
||||||
class PTZMetrics:
|
class PTZMetrics:
|
||||||
|
|||||||
@ -279,6 +279,32 @@ def stats_snapshot(
|
|||||||
if camera_stats.capture_process_pid.value
|
if camera_stats.capture_process_pid.value
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
|
# Calculate connection quality based on current state
|
||||||
|
# This is computed at stats-collection time so offline cameras
|
||||||
|
# correctly show as unusable rather than excellent
|
||||||
|
expected_fps = config.cameras[name].detect.fps
|
||||||
|
current_fps = camera_stats.camera_fps.value
|
||||||
|
reconnects = camera_stats.reconnects_last_hour.value
|
||||||
|
stalls = camera_stats.stalls_last_hour.value
|
||||||
|
|
||||||
|
if current_fps < 0.1:
|
||||||
|
quality_str = "unusable"
|
||||||
|
elif reconnects == 0 and current_fps >= 0.9 * expected_fps and stalls < 5:
|
||||||
|
quality_str = "excellent"
|
||||||
|
elif reconnects <= 2 and current_fps >= 0.6 * expected_fps:
|
||||||
|
quality_str = "fair"
|
||||||
|
elif reconnects > 10 or current_fps < 1.0 or stalls > 100:
|
||||||
|
quality_str = "unusable"
|
||||||
|
else:
|
||||||
|
quality_str = "poor"
|
||||||
|
|
||||||
|
connection_quality = {
|
||||||
|
"connection_quality": quality_str,
|
||||||
|
"expected_fps": expected_fps,
|
||||||
|
"reconnects_last_hour": reconnects,
|
||||||
|
"stalls_last_hour": stalls,
|
||||||
|
}
|
||||||
|
|
||||||
stats["cameras"][name] = {
|
stats["cameras"][name] = {
|
||||||
"camera_fps": round(camera_stats.camera_fps.value, 2),
|
"camera_fps": round(camera_stats.camera_fps.value, 2),
|
||||||
"process_fps": round(camera_stats.process_fps.value, 2),
|
"process_fps": round(camera_stats.process_fps.value, 2),
|
||||||
@ -290,6 +316,7 @@ def stats_snapshot(
|
|||||||
"ffmpeg_pid": ffmpeg_pid,
|
"ffmpeg_pid": ffmpeg_pid,
|
||||||
"audio_rms": round(camera_stats.audio_rms.value, 4),
|
"audio_rms": round(camera_stats.audio_rms.value, 4),
|
||||||
"audio_dBFS": round(camera_stats.audio_dBFS.value, 4),
|
"audio_dBFS": round(camera_stats.audio_dBFS.value, 4),
|
||||||
|
**connection_quality,
|
||||||
}
|
}
|
||||||
|
|
||||||
stats["detectors"] = {}
|
stats["detectors"] = {}
|
||||||
|
|||||||
@ -3,6 +3,7 @@ import queue
|
|||||||
import subprocess as sp
|
import subprocess as sp
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
from collections import deque
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from multiprocessing import Queue, Value
|
from multiprocessing import Queue, Value
|
||||||
from multiprocessing.synchronize import Event as MpEvent
|
from multiprocessing.synchronize import Event as MpEvent
|
||||||
@ -108,6 +109,7 @@ def capture_frames(
|
|||||||
fps: Value,
|
fps: Value,
|
||||||
skipped_fps: Value,
|
skipped_fps: Value,
|
||||||
current_frame: Value,
|
current_frame: Value,
|
||||||
|
stalls: Value,
|
||||||
stop_event: MpEvent,
|
stop_event: MpEvent,
|
||||||
) -> None:
|
) -> None:
|
||||||
frame_size = frame_shape[0] * frame_shape[1]
|
frame_size = frame_shape[0] * frame_shape[1]
|
||||||
@ -115,6 +117,12 @@ def capture_frames(
|
|||||||
frame_rate.start()
|
frame_rate.start()
|
||||||
skipped_eps = EventsPerSecond()
|
skipped_eps = EventsPerSecond()
|
||||||
skipped_eps.start()
|
skipped_eps.start()
|
||||||
|
|
||||||
|
# Stall detection
|
||||||
|
stall_timestamps = deque()
|
||||||
|
last_frame_time = 0.0
|
||||||
|
stall_active = False
|
||||||
|
|
||||||
config_subscriber = CameraConfigUpdateSubscriber(
|
config_subscriber = CameraConfigUpdateSubscriber(
|
||||||
None, {config.name: config}, [CameraConfigUpdateEnum.enabled]
|
None, {config.name: config}, [CameraConfigUpdateEnum.enabled]
|
||||||
)
|
)
|
||||||
@ -156,6 +164,32 @@ def capture_frames(
|
|||||||
|
|
||||||
frame_rate.update()
|
frame_rate.update()
|
||||||
|
|
||||||
|
# Update stall metrics
|
||||||
|
now = datetime.now().timestamp()
|
||||||
|
|
||||||
|
if last_frame_time > 0:
|
||||||
|
delta = now - last_frame_time
|
||||||
|
# Use observed fps when available; fall back to expected
|
||||||
|
observed_fps = fps.value if fps.value > 0 else config.detect.fps
|
||||||
|
# Compute a robust threshold: 2x frame interval, but at least 1s to ignore jitter
|
||||||
|
interval = 1.0 / max(observed_fps, 0.1)
|
||||||
|
stall_threshold = max(2.0 * interval, 2.0)
|
||||||
|
|
||||||
|
# Count a single stall per continuous gap exceeding threshold
|
||||||
|
if delta > stall_threshold:
|
||||||
|
if not stall_active:
|
||||||
|
stall_timestamps.append(now)
|
||||||
|
stall_active = True
|
||||||
|
else:
|
||||||
|
stall_active = False
|
||||||
|
last_frame_time = now
|
||||||
|
|
||||||
|
while stall_timestamps and stall_timestamps[0] < now - 3600:
|
||||||
|
stall_timestamps.popleft()
|
||||||
|
|
||||||
|
if stalls:
|
||||||
|
stalls.value = len(stall_timestamps)
|
||||||
|
|
||||||
# don't lock the queue to check, just try since it should rarely be full
|
# don't lock the queue to check, just try since it should rarely be full
|
||||||
try:
|
try:
|
||||||
# add to the queue
|
# add to the queue
|
||||||
@ -179,6 +213,8 @@ class CameraWatchdog(threading.Thread):
|
|||||||
camera_fps,
|
camera_fps,
|
||||||
skipped_fps,
|
skipped_fps,
|
||||||
ffmpeg_pid,
|
ffmpeg_pid,
|
||||||
|
stalls,
|
||||||
|
reconnects,
|
||||||
stop_event,
|
stop_event,
|
||||||
):
|
):
|
||||||
threading.Thread.__init__(self)
|
threading.Thread.__init__(self)
|
||||||
@ -199,6 +235,9 @@ class CameraWatchdog(threading.Thread):
|
|||||||
self.frame_index = 0
|
self.frame_index = 0
|
||||||
self.stop_event = stop_event
|
self.stop_event = stop_event
|
||||||
self.sleeptime = self.config.ffmpeg.retry_interval
|
self.sleeptime = self.config.ffmpeg.retry_interval
|
||||||
|
self.reconnect_timestamps = deque()
|
||||||
|
self.stalls = stalls
|
||||||
|
self.reconnects = reconnects
|
||||||
|
|
||||||
self.config_subscriber = CameraConfigUpdateSubscriber(
|
self.config_subscriber = CameraConfigUpdateSubscriber(
|
||||||
None,
|
None,
|
||||||
@ -239,6 +278,14 @@ class CameraWatchdog(threading.Thread):
|
|||||||
else:
|
else:
|
||||||
self.ffmpeg_detect_process.wait()
|
self.ffmpeg_detect_process.wait()
|
||||||
|
|
||||||
|
# Update reconnects
|
||||||
|
now = datetime.now().timestamp()
|
||||||
|
self.reconnect_timestamps.append(now)
|
||||||
|
while self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600:
|
||||||
|
self.reconnect_timestamps.popleft()
|
||||||
|
if self.reconnects:
|
||||||
|
self.reconnects.value = len(self.reconnect_timestamps)
|
||||||
|
|
||||||
# Wait for old capture thread to fully exit before starting a new one
|
# Wait for old capture thread to fully exit before starting a new one
|
||||||
if self.capture_thread is not None and self.capture_thread.is_alive():
|
if self.capture_thread is not None and self.capture_thread.is_alive():
|
||||||
self.logger.info("Waiting for capture thread to exit...")
|
self.logger.info("Waiting for capture thread to exit...")
|
||||||
@ -461,6 +508,7 @@ class CameraWatchdog(threading.Thread):
|
|||||||
self.frame_queue,
|
self.frame_queue,
|
||||||
self.camera_fps,
|
self.camera_fps,
|
||||||
self.skipped_fps,
|
self.skipped_fps,
|
||||||
|
self.stalls,
|
||||||
self.stop_event,
|
self.stop_event,
|
||||||
)
|
)
|
||||||
self.capture_thread.start()
|
self.capture_thread.start()
|
||||||
@ -514,6 +562,7 @@ class CameraCaptureRunner(threading.Thread):
|
|||||||
frame_queue: Queue,
|
frame_queue: Queue,
|
||||||
fps: Value,
|
fps: Value,
|
||||||
skipped_fps: Value,
|
skipped_fps: Value,
|
||||||
|
stalls: Value,
|
||||||
stop_event: MpEvent,
|
stop_event: MpEvent,
|
||||||
):
|
):
|
||||||
threading.Thread.__init__(self)
|
threading.Thread.__init__(self)
|
||||||
@ -530,6 +579,7 @@ class CameraCaptureRunner(threading.Thread):
|
|||||||
self.ffmpeg_process = ffmpeg_process
|
self.ffmpeg_process = ffmpeg_process
|
||||||
self.current_frame = Value("d", 0.0)
|
self.current_frame = Value("d", 0.0)
|
||||||
self.last_frame = 0
|
self.last_frame = 0
|
||||||
|
self.stalls = stalls
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
capture_frames(
|
capture_frames(
|
||||||
@ -543,6 +593,7 @@ class CameraCaptureRunner(threading.Thread):
|
|||||||
self.fps,
|
self.fps,
|
||||||
self.skipped_fps,
|
self.skipped_fps,
|
||||||
self.current_frame,
|
self.current_frame,
|
||||||
|
self.stalls,
|
||||||
self.stop_event,
|
self.stop_event,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -576,6 +627,8 @@ class CameraCapture(FrigateProcess):
|
|||||||
self.camera_metrics.camera_fps,
|
self.camera_metrics.camera_fps,
|
||||||
self.camera_metrics.skipped_fps,
|
self.camera_metrics.skipped_fps,
|
||||||
self.camera_metrics.ffmpeg_pid,
|
self.camera_metrics.ffmpeg_pid,
|
||||||
|
self.camera_metrics.stalls_last_hour,
|
||||||
|
self.camera_metrics.reconnects_last_hour,
|
||||||
self.stop_event,
|
self.stop_event,
|
||||||
)
|
)
|
||||||
camera_watchdog.start()
|
camera_watchdog.start()
|
||||||
|
|||||||
@ -151,6 +151,17 @@
|
|||||||
"cameraDetectionsPerSecond": "{{camName}} detections per second",
|
"cameraDetectionsPerSecond": "{{camName}} detections per second",
|
||||||
"cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second"
|
"cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second"
|
||||||
},
|
},
|
||||||
|
"connectionQuality": {
|
||||||
|
"title": "Connection Quality",
|
||||||
|
"excellent": "Excellent",
|
||||||
|
"fair": "Fair",
|
||||||
|
"poor": "Poor",
|
||||||
|
"unusable": "Unusable",
|
||||||
|
"fps": "FPS",
|
||||||
|
"expectedFps": "Expected FPS",
|
||||||
|
"reconnectsLastHour": "Reconnects (last hour)",
|
||||||
|
"stallsLastHour": "Stalls (last hour)"
|
||||||
|
},
|
||||||
"toast": {
|
"toast": {
|
||||||
"success": {
|
"success": {
|
||||||
"copyToClipboard": "Copied probe data to clipboard."
|
"copyToClipboard": "Copied probe data to clipboard."
|
||||||
|
|||||||
76
web/src/components/camera/ConnectionQualityIndicator.tsx
Normal file
76
web/src/components/camera/ConnectionQualityIndicator.tsx
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
import { useTranslation } from "react-i18next";
|
||||||
|
import {
|
||||||
|
Tooltip,
|
||||||
|
TooltipContent,
|
||||||
|
TooltipTrigger,
|
||||||
|
} from "@/components/ui/tooltip";
|
||||||
|
import { cn } from "@/lib/utils";
|
||||||
|
|
||||||
|
type ConnectionQualityIndicatorProps = {
|
||||||
|
quality: "excellent" | "fair" | "poor" | "unusable";
|
||||||
|
expectedFps: number;
|
||||||
|
reconnects: number;
|
||||||
|
stalls: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function ConnectionQualityIndicator({
|
||||||
|
quality,
|
||||||
|
expectedFps,
|
||||||
|
reconnects,
|
||||||
|
stalls,
|
||||||
|
}: ConnectionQualityIndicatorProps) {
|
||||||
|
const { t } = useTranslation(["views/system"]);
|
||||||
|
|
||||||
|
const getColorClass = (quality: string): string => {
|
||||||
|
switch (quality) {
|
||||||
|
case "excellent":
|
||||||
|
return "bg-success";
|
||||||
|
case "fair":
|
||||||
|
return "bg-yellow-500";
|
||||||
|
case "poor":
|
||||||
|
return "bg-orange-500";
|
||||||
|
case "unusable":
|
||||||
|
return "bg-destructive";
|
||||||
|
default:
|
||||||
|
return "bg-gray-500";
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const qualityLabel = t(`cameras.connectionQuality.${quality}`);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Tooltip>
|
||||||
|
<TooltipTrigger asChild>
|
||||||
|
<div
|
||||||
|
className={cn(
|
||||||
|
"inline-block size-3 cursor-pointer rounded-full",
|
||||||
|
getColorClass(quality),
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
</TooltipTrigger>
|
||||||
|
<TooltipContent className="max-w-xs">
|
||||||
|
<div className="space-y-2">
|
||||||
|
<div className="font-semibold">
|
||||||
|
{t("cameras.connectionQuality.title")}
|
||||||
|
</div>
|
||||||
|
<div className="text-sm">
|
||||||
|
<div className="capitalize">{qualityLabel}</div>
|
||||||
|
<div className="mt-2 space-y-1 text-xs">
|
||||||
|
<div>
|
||||||
|
{t("cameras.connectionQuality.expectedFps")}:{" "}
|
||||||
|
{expectedFps.toFixed(1)} {t("cameras.connectionQuality.fps")}
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
{t("cameras.connectionQuality.reconnectsLastHour")}:{" "}
|
||||||
|
{reconnects}
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
{t("cameras.connectionQuality.stallsLastHour")}: {stalls}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</TooltipContent>
|
||||||
|
</Tooltip>
|
||||||
|
);
|
||||||
|
}
|
||||||
@ -24,6 +24,10 @@ export type CameraStats = {
|
|||||||
pid: number;
|
pid: number;
|
||||||
process_fps: number;
|
process_fps: number;
|
||||||
skipped_fps: number;
|
skipped_fps: number;
|
||||||
|
connection_quality: "excellent" | "fair" | "poor" | "unusable";
|
||||||
|
expected_fps: number;
|
||||||
|
reconnects_last_hour: number;
|
||||||
|
stalls_last_hour: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type CpuStats = {
|
export type CpuStats = {
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import { useFrigateStats } from "@/api/ws";
|
import { useFrigateStats } from "@/api/ws";
|
||||||
import { CameraLineGraph } from "@/components/graph/LineGraph";
|
import { CameraLineGraph } from "@/components/graph/LineGraph";
|
||||||
import CameraInfoDialog from "@/components/overlay/CameraInfoDialog";
|
import CameraInfoDialog from "@/components/overlay/CameraInfoDialog";
|
||||||
|
import { ConnectionQualityIndicator } from "@/components/camera/ConnectionQualityIndicator";
|
||||||
import { Skeleton } from "@/components/ui/skeleton";
|
import { Skeleton } from "@/components/ui/skeleton";
|
||||||
import { FrigateConfig } from "@/types/frigateConfig";
|
import { FrigateConfig } from "@/types/frigateConfig";
|
||||||
import { FrigateStats } from "@/types/stats";
|
import { FrigateStats } from "@/types/stats";
|
||||||
@ -282,8 +283,37 @@ export default function CameraMetrics({
|
|||||||
)}
|
)}
|
||||||
<div className="flex w-full flex-col gap-3">
|
<div className="flex w-full flex-col gap-3">
|
||||||
<div className="flex flex-row items-center justify-between">
|
<div className="flex flex-row items-center justify-between">
|
||||||
<div className="text-sm font-medium text-muted-foreground smart-capitalize">
|
<div className="flex items-center gap-2">
|
||||||
<CameraNameLabel camera={camera} />
|
<div className="text-sm font-medium text-muted-foreground smart-capitalize">
|
||||||
|
<CameraNameLabel camera={camera} />
|
||||||
|
</div>
|
||||||
|
{statsHistory.length > 0 &&
|
||||||
|
statsHistory[statsHistory.length - 1]?.cameras[
|
||||||
|
camera.name
|
||||||
|
] && (
|
||||||
|
<ConnectionQualityIndicator
|
||||||
|
quality={
|
||||||
|
statsHistory[statsHistory.length - 1]?.cameras[
|
||||||
|
camera.name
|
||||||
|
]?.connection_quality
|
||||||
|
}
|
||||||
|
expectedFps={
|
||||||
|
statsHistory[statsHistory.length - 1]?.cameras[
|
||||||
|
camera.name
|
||||||
|
]?.expected_fps || 0
|
||||||
|
}
|
||||||
|
reconnects={
|
||||||
|
statsHistory[statsHistory.length - 1]?.cameras[
|
||||||
|
camera.name
|
||||||
|
]?.reconnects_last_hour || 0
|
||||||
|
}
|
||||||
|
stalls={
|
||||||
|
statsHistory[statsHistory.length - 1]?.cameras[
|
||||||
|
camera.name
|
||||||
|
]?.stalls_last_hour || 0
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger>
|
<TooltipTrigger>
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user