Add recording keyframe analysis to camera probe dialog (#23453)

* backend: endpoint and util funcs

* tests

* frontend and i18n

* update openapi spec

* add tip to docs

frigate/api/camera.py

@@ -34,11 +34,15 @@ from frigate.config.camera.updater import (
 )
 from frigate.config.env import substitute_frigate_vars
 from frigate.models import User
-from frigate.util.builtin import clean_camera_user_pass
+from frigate.util.builtin import clean_camera_user_pass, get_record_segment_time
 from frigate.util.camera_cleanup import cleanup_camera_db, cleanup_camera_files
 from frigate.util.config import find_config_file
 from frigate.util.image import run_ffmpeg_snapshot
-from frigate.util.services import ffprobe_stream, is_restricted_go2rtc_source
+from frigate.util.services import (
+    analyze_record_keyframes,
+    ffprobe_stream,
+    is_restricted_go2rtc_source,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -362,6 +366,48 @@ def ffprobe(request: Request, paths: str = "", detailed: bool = False):
     return JSONResponse(content=output)
 
 
+@router.get("/keyframe_analysis", dependencies=[Depends(require_role(["admin"]))])
+async def keyframe_analysis(request: Request, camera: str = ""):
+    """Probe a camera's record stream and classify its keyframe spacing.
+
+    Detects smart/+ codecs and long/variable GOPs that degrade recording.
+    """
+    config: FrigateConfig = request.app.frigate_config
+
+    if camera not in config.cameras:
+        return JSONResponse(
+            content={"success": False, "message": f"{camera} is not a valid camera."},
+            status_code=404,
+        )
+
+    camera_config = config.cameras[camera]
+
+    if not camera_config.enabled:
+        return JSONResponse(
+            content={"success": False, "message": f"{camera} is not enabled."},
+            status_code=404,
+        )
+
+    # keyframe spacing only matters when this camera is recording
+    if not camera_config.record.enabled:
+        return JSONResponse(content={"severity": "record_disabled"})
+
+    # recording guarantees an input carries the record role; its index matches
+    # the "Stream N" numbering the ffprobe endpoint surfaces (same input order)
+    record_index, record_input = next(
+        (idx, i)
+        for idx, i in enumerate(camera_config.ffmpeg.inputs)
+        if "record" in i.roles
+    )
+
+    segment_time = get_record_segment_time(camera_config)
+    result = await analyze_record_keyframes(
+        config.ffmpeg, record_input.path, segment_time
+    )
+    result["stream_index"] = record_index
+    return JSONResponse(content=result)
+
+
 @router.get("/ffprobe/snapshot", dependencies=[Depends(require_role(["admin"]))])
 def ffprobe_snapshot(request: Request, url: str = "", timeout: int = 10):
     """Get a snapshot from a stream URL using ffmpeg."""

frigate/test/http_api/test_http_keyframe_analysis.py

@@ -0,0 +1,58 @@
+from unittest.mock import AsyncMock, patch
+
+from frigate.models import Event, Recordings, ReviewSegment
+from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
+
+
+class TestHttpKeyframeAnalysis(BaseTestHttp):
+    def setUp(self):
+        super().setUp([Event, Recordings, ReviewSegment])
+
+    def test_invalid_camera_returns_404(self):
+        app = super().create_app()
+        with AuthTestClient(app) as client:
+            response = client.get("/keyframe_analysis?camera=does_not_exist")
+            assert response.status_code == 404
+
+    def test_record_disabled_returns_neutral(self):
+        # default minimal_config has recording disabled
+        app = super().create_app()
+        with AuthTestClient(app) as client:
+            response = client.get("/keyframe_analysis?camera=front_door")
+            assert response.status_code == 200
+            assert response.json()["severity"] == "record_disabled"
+
+    def test_probes_record_input_and_returns_severity(self):
+        self.minimal_config["cameras"]["front_door"]["ffmpeg"]["inputs"] = [
+            {
+                "path": "rtsp://10.0.0.1:554/record",
+                "roles": ["detect", "record"],
+            }
+        ]
+        self.minimal_config["cameras"]["front_door"]["record"] = {"enabled": True}
+        app = super().create_app()
+
+        canned = {
+            "severity": "ok",
+            "keyframe_count": 5,
+            "max_gap": 1.0,
+            "mean_gap": 1.0,
+            "min_gap": 1.0,
+            "segment_time": 10,
+            "duration_observed": 4.0,
+            "thresholds": {"warning": 4.0, "error": 10},
+        }
+
+        with patch(
+            "frigate.api.camera.analyze_record_keyframes",
+            AsyncMock(return_value=canned),
+        ) as mock_probe:
+            with AuthTestClient(app) as client:
+                response = client.get("/keyframe_analysis?camera=front_door")
+
+        assert response.status_code == 200
+        assert response.json()["severity"] == "ok"
+        # index matches the input carrying the record role ("Stream 1")
+        assert response.json()["stream_index"] == 0
+        # the record-role input path was probed
+        assert mock_probe.await_args.args[1] == "rtsp://10.0.0.1:554/record"

frigate/test/test_keyframe_analysis.py

@@ -0,0 +1,111 @@
+"""Tests for keyframe-spacing analysis used to detect smart/+ codecs."""
+
+import asyncio
+import unittest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from frigate.util.services import (
+    analyze_record_keyframes,
+    classify_keyframe_gaps,
+    parse_keyframe_packets,
+)
+
+
+class TestClassifyKeyframeGaps(unittest.TestCase):
+    def test_ok_when_gaps_small(self):
+        # keyframes every ~1s
+        pts = [0.0, 1.0, 2.0, 3.0, 4.0]
+        result = classify_keyframe_gaps(pts, segment_time=10)
+        self.assertEqual(result["severity"], "ok")
+        self.assertEqual(result["max_gap"], 1.0)
+        self.assertEqual(result["keyframe_count"], 5)
+        self.assertEqual(result["thresholds"], {"warning": 4.0, "error": 10})
+
+    def test_warning_when_gap_exceeds_four_seconds(self):
+        pts = [0.0, 1.0, 6.5]  # 5.5s gap
+        result = classify_keyframe_gaps(pts, segment_time=10)
+        self.assertEqual(result["severity"], "warning")
+        self.assertEqual(result["max_gap"], 5.5)
+
+    def test_error_when_gap_exceeds_segment_time(self):
+        pts = [0.0, 12.0]  # 12s gap > 10s segment
+        result = classify_keyframe_gaps(pts, segment_time=10)
+        self.assertEqual(result["severity"], "error")
+
+    def test_error_threshold_tracks_segment_time(self):
+        pts = [0.0, 6.0]  # 6s gap, segment_time=5 -> error
+        result = classify_keyframe_gaps(pts, segment_time=5)
+        self.assertEqual(result["severity"], "error")
+
+    def test_unknown_with_single_keyframe(self):
+        result = classify_keyframe_gaps([1.0], segment_time=10)
+        self.assertEqual(result["severity"], "unknown")
+        self.assertIsNone(result["max_gap"])
+        self.assertEqual(result["keyframe_count"], 1)
+
+    def test_unknown_with_no_keyframes(self):
+        result = classify_keyframe_gaps([], segment_time=10)
+        self.assertEqual(result["severity"], "unknown")
+        self.assertEqual(result["keyframe_count"], 0)
+
+
+class TestParseKeyframePackets(unittest.TestCase):
+    def test_extracts_keyframe_pts_and_max(self):
+        output = "0.000000,K__\n0.033333,___\n1.000000,K__\n1.500000,___\n"
+        keyframe_pts, max_pts = parse_keyframe_packets(output)
+        self.assertEqual(keyframe_pts, [0.0, 1.0])
+        self.assertEqual(max_pts, 1.5)
+
+    def test_skips_unparseable_and_empty_lines(self):
+        output = "N/A,K__\n\n2.0,K__\nbad line\n"
+        keyframe_pts, max_pts = parse_keyframe_packets(output)
+        self.assertEqual(keyframe_pts, [2.0])
+        self.assertEqual(max_pts, 2.0)
+
+    def test_empty_output(self):
+        keyframe_pts, max_pts = parse_keyframe_packets("")
+        self.assertEqual(keyframe_pts, [])
+        self.assertIsNone(max_pts)
+
+
+class TestAnalyzeRecordKeyframes(unittest.IsolatedAsyncioTestCase):
+    async def test_merges_duration_and_classification(self):
+        csv = b"0.0,K__\n1.0,___\n6.0,K__\n7.0,___\n"
+        proc = MagicMock()
+        proc.communicate = AsyncMock(return_value=(csv, b""))
+        ffmpeg = MagicMock()
+        ffmpeg.ffprobe_path = "/usr/bin/ffprobe"
+
+        with patch(
+            "frigate.util.services.asyncio.create_subprocess_exec",
+            AsyncMock(return_value=proc),
+        ):
+            result = await analyze_record_keyframes(
+                ffmpeg, "rtsp://cam/stream", segment_time=10
+            )
+
+        self.assertEqual(result["severity"], "warning")  # 6s gap > 4s
+        self.assertEqual(result["max_gap"], 6.0)
+        self.assertEqual(result["duration_observed"], 7.0)
+
+    async def test_timeout_returns_unknown(self):
+        proc = MagicMock()
+        proc.communicate = AsyncMock(side_effect=asyncio.TimeoutError())
+        proc.kill = MagicMock()
+        ffmpeg = MagicMock()
+        ffmpeg.ffprobe_path = "/usr/bin/ffprobe"
+
+        with patch(
+            "frigate.util.services.asyncio.create_subprocess_exec",
+            AsyncMock(return_value=proc),
+        ):
+            result = await analyze_record_keyframes(
+                ffmpeg, "rtsp://cam/stream", segment_time=10
+            )
+
+        self.assertEqual(result["severity"], "unknown")
+        proc.kill.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/util/builtin.py

@@ -14,13 +14,16 @@ import urllib.parse
 from collections.abc import Mapping
 from multiprocessing.managers import ValueProxy
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
 
 import numpy as np
 from ruamel.yaml import YAML
 
 from frigate.const import REGEX_HTTP_CAMERA_USER_PASS, REGEX_RTSP_CAMERA_USER_PASS
 
+if TYPE_CHECKING:
+    from frigate.config import CameraConfig
+
 logger = logging.getLogger(__name__)
 
 
@@ -132,6 +135,24 @@ def get_ffmpeg_arg_list(arg: Any) -> list:
     return arg if isinstance(arg, list) else shlex.split(arg)
 
 
+# all built-in record presets use this segment_time
+DEFAULT_RECORD_SEGMENT_TIME = 10
+
+
+def get_record_segment_time(config: "CameraConfig") -> int:
+    """Extract -segment_time from the camera's record output args."""
+    record_args = get_ffmpeg_arg_list(config.ffmpeg.output_args.record)
+
+    if record_args and record_args[0].startswith("preset"):
+        return DEFAULT_RECORD_SEGMENT_TIME
+
+    try:
+        idx = record_args.index("-segment_time")
+        return int(record_args[idx + 1])
+    except (ValueError, IndexError):
+        return DEFAULT_RECORD_SEGMENT_TIME
+
+
 def load_labels(
     path: Optional[str], encoding="utf-8", prefill=91, indexed: bool | None = None
 ):

frigate/util/services.py

@@ -879,6 +879,131 @@ def ffprobe_stream(ffmpeg, path: str, detailed: bool = False) -> sp.CompletedPro
     return result
 
 
+KEYFRAME_PROBE_WINDOW_SECONDS = 20
+KEYFRAME_GAP_WARNING_SECONDS = 4.0
+
+
+def parse_keyframe_packets(output: str) -> Tuple[List[float], Optional[float]]:
+    """Parse ffprobe CSV `pts_time,flags` output.
+
+    Returns the presentation timestamps of keyframes (flags containing "K")
+    and the maximum timestamp observed across all packets.
+    """
+    keyframe_pts: List[float] = []
+    max_pts: Optional[float] = None
+
+    for line in output.splitlines():
+        parts = line.split(",")
+        if len(parts) < 2:
+            continue
+        try:
+            pts = float(parts[0])
+        except ValueError:
+            continue
+        if max_pts is None or pts > max_pts:
+            max_pts = pts
+        if "K" in parts[1]:
+            keyframe_pts.append(pts)
+
+    return keyframe_pts, max_pts
+
+
+def classify_keyframe_gaps(
+    keyframe_pts: List[float], segment_time: int
+) -> dict[str, Any]:
+    """Classify keyframe spacing for recording suitability.
+
+    A camera using a smart/+ codec or a long/variable GOP produces large or
+    irregular gaps between keyframes, which breaks time-based recording
+    segmentation. Severity:
+      - "unknown" when fewer than two keyframes were observed
+      - "error" when the longest gap exceeds the record segment length
+      - "warning" when the longest gap exceeds the warning threshold
+      - "ok" otherwise
+    """
+    thresholds = {
+        "warning": KEYFRAME_GAP_WARNING_SECONDS,
+        "error": segment_time,
+    }
+
+    if len(keyframe_pts) < 2:
+        return {
+            "keyframe_count": len(keyframe_pts),
+            "max_gap": None,
+            "mean_gap": None,
+            "min_gap": None,
+            "segment_time": segment_time,
+            "severity": "unknown",
+            "thresholds": thresholds,
+        }
+
+    gaps = [b - a for a, b in zip(keyframe_pts, keyframe_pts[1:])]
+    max_gap = max(gaps)
+
+    if max_gap > segment_time:
+        severity = "error"
+    elif max_gap > KEYFRAME_GAP_WARNING_SECONDS:
+        severity = "warning"
+    else:
+        severity = "ok"
+
+    return {
+        "keyframe_count": len(keyframe_pts),
+        "max_gap": round(max_gap, 2),
+        "mean_gap": round(sum(gaps) / len(gaps), 2),
+        "min_gap": round(min(gaps), 2),
+        "segment_time": segment_time,
+        "severity": severity,
+        "thresholds": thresholds,
+    }
+
+
+async def analyze_record_keyframes(
+    ffmpeg, url: str, segment_time: int, window: int = KEYFRAME_PROBE_WINDOW_SECONDS
+) -> dict[str, Any]:
+    """Probe a stream for ~`window` seconds and classify its keyframe spacing.
+
+    Reads video packet flags via ffprobe to find keyframes, then measures the
+    gaps between them. On timeout or failure returns an "unknown" result rather
+    than a false all-clear.
+    """
+    clean_url = escape_special_characters(url)
+    cmd = [
+        ffmpeg.ffprobe_path,
+        "-v",
+        "error",
+        "-select_streams",
+        "v:0",
+        "-read_intervals",
+        f"%+{window}",
+        "-show_entries",
+        "packet=pts_time,flags",
+        "-of",
+        "csv=p=0",
+        clean_url,
+    ]
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=window + 15)
+    except asyncio.TimeoutError:
+        logger.warning("Keyframe probe timed out for record stream")
+        proc.kill()
+        return classify_keyframe_gaps([], segment_time)
+    except OSError as err:
+        logger.error("Keyframe probe failed: %s", err)
+        return classify_keyframe_gaps([], segment_time)
+
+    keyframe_pts, max_pts = parse_keyframe_packets(stdout.decode("utf-8", "replace"))
+    result = classify_keyframe_gaps(keyframe_pts, segment_time)
+    result["duration_observed"] = round(max_pts, 2) if max_pts is not None else None
+    return result
+
+
 def vainfo_hwaccel(device_name: Optional[str] = None) -> sp.CompletedProcess:
     """Run vainfo."""
     if not device_name:

frigate/video/ffmpeg.py

@@ -24,7 +24,7 @@ from frigate.config.camera.updater import (
 )
 from frigate.const import PROCESS_PRIORITY_HIGH
 from frigate.log import LogPipe
-from frigate.util.builtin import EventsPerSecond, get_ffmpeg_arg_list
+from frigate.util.builtin import EventsPerSecond, get_record_segment_time
 from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg
 from frigate.util.image import (
     FrameManager,
@@ -34,23 +34,6 @@ from frigate.util.process import FrigateProcess
 
 logger = logging.getLogger(__name__)
 
-# all built-in record presets use this segment_time
-DEFAULT_RECORD_SEGMENT_TIME = 10
-
-
-def _get_record_segment_time(config: CameraConfig) -> int:
-    """Extract -segment_time from the camera's record output args."""
-    record_args = get_ffmpeg_arg_list(config.ffmpeg.output_args.record)
-
-    if record_args and record_args[0].startswith("preset"):
-        return DEFAULT_RECORD_SEGMENT_TIME
-
-    try:
-        idx = record_args.index("-segment_time")
-        return int(record_args[idx + 1])
-    except (ValueError, IndexError):
-        return DEFAULT_RECORD_SEGMENT_TIME
-
 
 def capture_frames(
     ffmpeg_process: sp.Popen[Any],
@@ -185,7 +168,7 @@ class CameraWatchdog(threading.Thread):
         # `valid` segments are published with the segment's start time, so the
         # gap between consecutive publishes can reach 2 * segment_time. Pad the
         # staleness threshold so it's never tighter than that worst case.
-        segment_time = _get_record_segment_time(self.config)
+        segment_time = get_record_segment_time(self.config)
         self.record_stale_threshold = max(120, 2 * segment_time + 30)
 
         # Stall tracking (based on last processed frame)