Improve History playback after camera audio dropouts (#24793)

* skip vod clips that start after their audio ends

When a camera's audio dropped out, a recording segment could end its audio early. If a playback range started past that point in the segment, nginx-vod saw no audio track in the first clip, failed the whole range with a track count mismatch, and left the player spinning for up to an hour of footage. Recording now stores each segment's audio length alongside its keyframe index, from the same ffprobe pass, and plan_clip skips a clip whose window starts at or after its audio ends when the range has other clips. A range made of only that clip still plays video-only, and legacy rows without the new column are unchanged.

* keep audio skip consistent with playback

Clips were skipped even when the range would be served video-only, and seeking into a skipped span landed on later footage. The skip now only applies when every row in the range carries audio, and seeks inside a zero-duration span land on the next clip.

frigate/test/test_maintainer.py

@@ -155,7 +155,7 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase):
         cache_path = "/tmp/cache/test_cam@20260417150000+0000.mp4"
 
         maintainer.end_time_cache = {
-            cache_path: (end_time, 10.0, None, None, None, None, None)
+            cache_path: (end_time, 10.0, None, None, None, None, None, None)
         }
         # Single processed frame well past end_time with no motion/objects.
         maintainer.object_recordings_info["test_cam"] = [(now.timestamp(), [], [], [])]
@@ -199,7 +199,7 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase):
         cache_path = "/tmp/cache/test_cam@20260417150000+0000.mp4"
 
         maintainer.end_time_cache = {
-            cache_path: (end_time, 10.0, False, None, None, None, [])
+            cache_path: (end_time, 10.0, False, None, None, None, [], None)
         }
         maintainer.object_recordings_info["test_cam"] = [(now.timestamp(), [], [], [])]
         maintainer.audio_recordings_info["test_cam"] = []
@@ -255,7 +255,7 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase):
         cache_path = "/tmp/cache/test_cam@20260417150000+0000.mp4"
 
         maintainer.end_time_cache = {
-            cache_path: (end_time, 10.0, False, None, None, None, [])
+            cache_path: (end_time, 10.0, False, None, None, None, [], None)
         }
         maintainer.object_recordings_info["test_cam"] = [(now.timestamp(), [], [], [])]
         maintainer.audio_recordings_info["test_cam"] = []
@@ -313,7 +313,7 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase):
         cache_path = "/tmp/cache/test_cam@20260417150000+0000.mp4"
 
         maintainer.end_time_cache = {
-            cache_path: (end_time, 10.0, False, None, None, None, [])
+            cache_path: (end_time, 10.0, False, None, None, None, [], None)
         }
         # Metadata has only reached partway into the segment.
         maintainer.object_recordings_info["test_cam"] = [

frigate/test/test_record_sub_maintainer.py

@@ -77,8 +77,8 @@ async def _validate_segment(
     with (
         patch("frigate.record.maintainer.get_video_properties", probe),
         patch(
-            "frigate.record.maintainer.get_keyframe_offsets",
-            AsyncMock(return_value=[0]),
+            "frigate.record.maintainer.get_playback_index",
+            AsyncMock(return_value=([0], 10.0)),
         ),
         patch("frigate.record.maintainer.os.path.getmtime", getmtime),
     ):
@@ -178,7 +178,7 @@ class TestValidateAndMoveSubSegment(unittest.IsolatedAsyncioTestCase):
         # pre-fill the end time cache so no ffprobe is attempted; the
         # audio and codec fields stay unknown without one
         maintainer.end_time_cache = {
-            cache_path: (end_time, 10.0, None, None, None, None, None)
+            cache_path: (end_time, 10.0, None, None, None, None, None, None)
         }
         # a processed frame past end_time so the segment is considered ready
         maintainer.object_recordings_info["test_cam"] = [(now.timestamp(), [], [], [])]
@@ -264,8 +264,8 @@ class TestSegmentAudioPresence(unittest.IsolatedAsyncioTestCase):
                 with (
                     patch("frigate.record.maintainer.get_video_properties", probe),
                     patch(
-                        "frigate.record.maintainer.get_keyframe_offsets",
-                        AsyncMock(return_value=[0, 2000]),
+                        "frigate.record.maintainer.get_playback_index",
+                        AsyncMock(return_value=([0, 2000], 9.5)),
                     ),
                 ):
                     await maintainer.validate_and_move_segment(
@@ -284,6 +284,7 @@ class TestSegmentAudioPresence(unittest.IsolatedAsyncioTestCase):
                 self.assertEqual(call_args[8], audio_rate)
                 self.assertEqual(call_args[9], audio_codec)
                 self.assertEqual(call_args[11], [0, 2000])
+                self.assertEqual(call_args[12], 9.5)
                 self.assertEqual(call_args[10], video_codec)
                 # the probe result is cached alongside the end time so the
                 # cached path stays as informed as the probed path
@@ -558,8 +559,8 @@ class TestSegmentStartChaining(unittest.IsolatedAsyncioTestCase):
         with (
             patch("frigate.record.maintainer.get_video_properties", probe),
             patch(
-                "frigate.record.maintainer.get_keyframe_offsets",
-                AsyncMock(return_value=[0]),
+                "frigate.record.maintainer.get_playback_index",
+                AsyncMock(return_value=([0], 10.0)),
             ),
             patch(
                 "frigate.record.maintainer.os.path.getmtime",

frigate/test/test_recording_coverage.py

@@ -10,6 +10,7 @@ from peewee import SqliteDatabase
 from frigate.api.media import _vod_response
 from frigate.const import MAX_SEGMENT_DURATION
 from frigate.models import Recordings
+from frigate.util.media import parse_playback_index
 from frigate.util.recording_coverage import (
     _rows_query,
     coverage_spans,
@@ -47,6 +48,7 @@ class CoverageDbTestCase(unittest.TestCase):
         has_audio=None,
         segment_size=0,
         keyframes=None,
+        audio_duration=None,
     ):
         Recordings.create(
             id=id,
@@ -62,6 +64,7 @@ class CoverageDbTestCase(unittest.TestCase):
             has_audio=has_audio,
             segment_size=segment_size,
             keyframes=keyframes,
+            audio_duration=audio_duration,
         )
 
 
@@ -419,3 +422,132 @@ class TestVodManifestKeyFrames(CoverageDbTestCase):
         self._insert("m1", 1000.0, 1010.0, "main")
         clips = self._clips(1000.0, 1010.0)
         assert clips[0]["keyFrameDurations"] == [10000]
+
+
+class TestVodAudioCutShort(CoverageDbTestCase):
+    """A clip whose audio ends before its clipFrom has no audio track to nginx-vod."""
+
+    KEYFRAMES = [0, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000]
+
+    def _insert_pair(self, first_audio_duration):
+        # the first row's audio dropped out; the range starts 5s into it
+        for id, start, audio_duration in (
+            ("m1", 1000.0, first_audio_duration),
+            ("m2", 1010.0, 10.0),
+        ):
+            self._insert(
+                id,
+                start,
+                start + 10.0,
+                "main",
+                has_audio=True,
+                audio_codec="aac",
+                audio_rate=16000,
+                keyframes=self.KEYFRAMES,
+                audio_duration=audio_duration,
+            )
+
+    def _clips(self, start, end):
+        response = asyncio.run(_vod_response("front_door", start, end))
+        return json.loads(response.body)["sequences"][0]["clips"]
+
+    def test_clip_past_audio_end_is_skipped(self):
+        self._insert_pair(2.112)
+        clips = self._clips(1005.0, 1020.0)
+        assert [c["path"] for c in clips] == ["/tmp/m2.mp4"]
+        timeline = realized_timeline(
+            resolve_coverage("front_door", 1005.0, 1020.0), None
+        )
+        assert [t["duration"] for t in timeline] == [0, 10000]
+
+    def test_clip_with_audio_left_is_kept(self):
+        # nginx-vod keeps the track while any sample lies past clipFrom
+        self._insert_pair(5.001)
+        clips = self._clips(1005.0, 1020.0)
+        assert clips[0]["clipFrom"] == 5000
+        assert len(clips) == 2
+
+    def test_clip_starting_on_audio_end_is_skipped(self):
+        self._insert_pair(5.0)
+        clips = self._clips(1005.0, 1020.0)
+        assert len(clips) == 1
+
+    def test_legacy_row_is_kept(self):
+        self._insert_pair(None)
+        clips = self._clips(1005.0, 1020.0)
+        assert len(clips) == 2
+
+    def test_video_only_range_keeps_clip(self):
+        # a video-only sub row serves the whole range without audio, so
+        # nginx-vod never compares audio tracks
+        self._insert(
+            "m1",
+            1000.0,
+            1010.0,
+            "main",
+            has_audio=True,
+            audio_codec="aac",
+            audio_rate=16000,
+            keyframes=self.KEYFRAMES,
+            audio_duration=2.112,
+        )
+        self._insert(
+            "s1", 1010.0, 1020.0, "sub", has_audio=False, keyframes=self.KEYFRAMES
+        )
+        clips = self._clips(1005.0, 1020.0)
+        assert [c["path"] for c in clips] == ["/tmp/m1.mp4", "/tmp/s1.mp4"]
+        assert all(c["tracks"] == "v" for c in clips)
+        timeline = realized_timeline(
+            resolve_coverage("front_door", 1005.0, 1020.0), None
+        )
+        assert timeline[0]["duration"] > 0
+
+    def test_sole_clip_is_kept(self):
+        # alone in its sequence the clip plays video-only, so keep it
+        self._insert_pair(2.112)
+        clips = self._clips(1005.0, 1009.0)
+        assert clips[0]["path"] == "/tmp/m1.mp4"
+        timeline = realized_timeline(
+            resolve_coverage("front_door", 1005.0, 1009.0), None
+        )
+        assert timeline[0]["duration"] > 0
+
+
+class TestParsePlaybackIndex(unittest.TestCase):
+    def test_splits_video_keyframes_from_audio_duration(self):
+        output = "\n".join(
+            [
+                "video,0,0.000000,0.040000,K__",
+                "audio,1,0.000000,0.064000,K__",
+                "video,0,0.040000,0.040000,___",
+                "audio,1,0.064000,0.064000,K__",
+                "video,0,1.000000,0.040000,K__",
+                "audio,1,0.128000,N/A,K__",
+            ]
+        )
+        keyframes, audio_duration = parse_playback_index(output)
+        assert keyframes == [0, 1000]
+        assert audio_duration == 0.128
+
+    def test_no_audio_packets_reports_zero(self):
+        keyframes, audio_duration = parse_playback_index(
+            "video,0,0.000000,0.040000,K__"
+        )
+        assert keyframes == [0]
+        assert audio_duration == 0.0
+
+    def test_only_first_video_and_audio_streams_count(self):
+        output = "\n".join(
+            [
+                "video,1,0.000000,0.100000,K__",
+                "video,0,0.000000,0.040000,K__",
+                "audio,3,0.000000,0.064000,K__",
+                "audio,2,0.000000,0.064000,K__",
+                "video,1,2.500000,0.100000,K__",
+                "video,0,1.000000,0.040000,K__",
+                "audio,3,0.064000,0.064000,K__",
+            ]
+        )
+        keyframes, audio_duration = parse_playback_index(output)
+        assert keyframes == [0, 1000]
+        assert audio_duration == 0.064