From 3cd4c9c55cf8560892c3505dec5c6b70d3d7595b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:32:36 -0500 Subject: [PATCH] bound recording overlap queries by segment length Clip, export, review and recording deletion, debug replay, motion search, and audio transcription found overlapping recordings with an OR clause that gave SQLite no start_time range, so every call read the camera's whole recording history (30 to 300 ms on a 30 day database). The classification and review description point lookups did the same whenever the timestamp had no recording. A shared recording_overlap helper now adds a start_time lower bound of MAX_SEGMENT_DURATION before the range, which drops no rows because segments are always shorter than that. --- frigate/api/export.py | 15 ++-------- frigate/api/media.py | 26 ++++------------- frigate/api/record.py | 7 ++--- frigate/api/review.py | 10 ++----- .../post/review_descriptions.py | 3 +- frigate/jobs/debug_replay.py | 7 ++--- frigate/jobs/motion_search.py | 16 ++--------- frigate/record/export.py | 10 ++----- frigate/test/test_recording_coverage.py | 28 +++++++++++++++++++ frigate/util/audio.py | 7 ++--- frigate/util/classification.py | 4 +-- frigate/util/recording_coverage.py | 13 +++++++++ 12 files changed, 65 insertions(+), 81 deletions(-) diff --git a/frigate/api/export.py b/frigate/api/export.py index 3dad13fd09..f0d93bc105 100644 --- a/frigate/api/export.py +++ b/frigate/api/export.py @@ -80,6 +80,7 @@ from frigate.record.export import ( validate_ffmpeg_args, ) from frigate.util.path import sanitize_contained_path +from frigate.util.recording_coverage import recording_overlap from frigate.util.time import is_current_hour logger = logging.getLogger(__name__) @@ -177,15 +178,7 @@ def _validate_export_source( ) -> str | None: if playback_source == PlaybackSourceEnum.recordings: query = Recordings.select().where( - ( - Recordings.start_time.between(start_time, end_time) - | Recordings.end_time.between(start_time, end_time) - | ( - (start_time > Recordings.start_time) - & (end_time < Recordings.end_time) - ) - ) - & (Recordings.camera == camera_name) + recording_overlap(start_time, end_time) & (Recordings.camera == camera_name) ) # a pinned export reads only that stream, so the other stream's @@ -250,9 +243,7 @@ def _get_item_recording_export_errors( query = Recordings.select(Recordings.start_time, Recordings.end_time).where( Recordings.camera == camera_name, - Recordings.start_time.between(min_start, max_end) - | Recordings.end_time.between(min_start, max_end) - | ((min_start > Recordings.start_time) & (max_end < Recordings.end_time)), + recording_overlap(min_start, max_end), ) # a pinned batch reads only that stream, so the other stream's diff --git a/frigate/api/media.py b/frigate/api/media.py index 9613549646..80443371cf 100644 --- a/frigate/api/media.py +++ b/frigate/api/media.py @@ -42,7 +42,6 @@ from frigate.config.camera.snapshots import SnapshotsConfig from frigate.const import ( CACHE_DIR, INSTALL_DIR, - MAX_SEGMENT_DURATION, PREVIEW_FRAME_TYPE, STREAM_TYPE_MAIN, STREAM_TYPE_SUB, @@ -63,6 +62,7 @@ from frigate.util.recording_coverage import ( build_spans, null_audio_glitches, plan_clip, + recording_overlap, resolve_coverage, ) from frigate.util.time import get_normalized_tz_name @@ -348,11 +348,7 @@ async def get_snapshot_from_recording( Recordings.path, Recordings.start_time, ) - .where( - (frame_time >= Recordings.start_time) - & (frame_time <= Recordings.end_time) - & (Recordings.start_time >= frame_time - MAX_SEGMENT_DURATION) - ) + .where(recording_overlap(frame_time, frame_time)) .where(Recordings.camera == camera_name) .order_by(Recordings.stream_type.asc(), Recordings.start_time.desc()) .limit(1) @@ -368,11 +364,7 @@ async def get_snapshot_from_recording( Recordings.path, Recordings.start_time, ) - .where( - (frame_time >= Recordings.start_time) - & (frame_time <= Recordings.end_time) - & (Recordings.start_time >= frame_time - MAX_SEGMENT_DURATION) - ) + .where(recording_overlap(frame_time, frame_time)) .where(Recordings.camera == camera_name) .order_by(Recordings.stream_type.asc(), Recordings.start_time.desc()) .limit(1) @@ -435,11 +427,7 @@ async def submit_recording_snapshot_to_plus( Recordings.path, Recordings.start_time, ) - .where( - (frame_time >= Recordings.start_time) - & (frame_time <= Recordings.end_time) - & (Recordings.start_time >= frame_time - MAX_SEGMENT_DURATION) - ) + .where(recording_overlap(frame_time, frame_time)) .where(Recordings.camera == camera_name) .order_by(Recordings.stream_type.asc(), Recordings.start_time.desc()) .limit(1) @@ -553,11 +541,7 @@ async def recording_clip( Recordings.start_time, Recordings.end_time, ) - .where( - (Recordings.start_time.between(start_ts, end_ts)) - | (Recordings.end_time.between(start_ts, end_ts)) - | ((start_ts > Recordings.start_time) & (end_ts < Recordings.end_time)) - ) + .where(recording_overlap(start_ts, end_ts)) .where(Recordings.camera == camera_name) .where(Recordings.stream_type == stream_type) .order_by(Recordings.start_time.asc()) diff --git a/frigate/api/record.py b/frigate/api/record.py index 2f32e6a770..6b46243259 100644 --- a/frigate/api/record.py +++ b/frigate/api/record.py @@ -36,6 +36,7 @@ from frigate.util.recording_coverage import ( coverage_spans, known_video_codecs, realized_timelines, + recording_overlap, resolve_coverage, stream_media_summary, ) @@ -537,11 +538,7 @@ async def delete_recordings( # Build query to find overlapping recordings clauses = [ - ( - Recordings.start_time.between(start, end) - | Recordings.end_time.between(start, end) - | ((start > Recordings.start_time) & (end < Recordings.end_time)) - ), + recording_overlap(start, end), (Recordings.camera << camera_list), ] diff --git a/frigate/api/review.py b/frigate/api/review.py index c2aa29bcec..14f9bb169c 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -37,6 +37,7 @@ from frigate.const import STREAM_TYPE_MAIN from frigate.embeddings import EmbeddingsContext from frigate.models import Recordings, ReviewSegment, UserReviewStatus from frigate.review.types import SeverityEnum +from frigate.util.recording_coverage import recording_overlap from frigate.util.time import get_dst_transitions logger = logging.getLogger(__name__) @@ -585,14 +586,7 @@ def delete_reviews(body: ReviewModifyMultipleBody): camera_name = review["camera"] recordings = ( Recordings.select(Recordings.id, Recordings.path) - .where( - Recordings.start_time.between(start_time, end_time) - | Recordings.end_time.between(start_time, end_time) - | ( - (start_time > Recordings.start_time) - & (end_time < Recordings.end_time) - ) - ) + .where(recording_overlap(start_time, end_time)) .where(Recordings.camera == camera_name) .dicts() .iterator() diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index a4e80cb6b3..431472e6f4 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -37,6 +37,7 @@ from frigate.genai.manager import GenAIClientManager from frigate.models import Recordings, ReviewSegment from frigate.util.builtin import EventsPerSecond, InferenceSpeed from frigate.util.image import get_image_from_recording +from frigate.util.recording_coverage import recording_overlap from ..post.api import PostProcessorApi from ..types import DataProcessorMetrics @@ -592,7 +593,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): Recordings.path, Recordings.start_time, ) - .where((ts >= Recordings.start_time) & (ts <= Recordings.end_time)) + .where(recording_overlap(ts, ts)) .where(Recordings.camera == camera) .where(Recordings.stream_type == STREAM_TYPE_MAIN) .order_by(Recordings.start_time.desc()) diff --git a/frigate/jobs/debug_replay.py b/frigate/jobs/debug_replay.py index a530392bd7..8cfe6402b3 100644 --- a/frigate/jobs/debug_replay.py +++ b/frigate/jobs/debug_replay.py @@ -27,6 +27,7 @@ from frigate.jobs.manager import job_is_running, set_current_job from frigate.models import Export, Recordings from frigate.types import JobStatusTypesEnum from frigate.util.ffmpeg import run_ffmpeg_with_progress +from frigate.util.recording_coverage import recording_overlap if TYPE_CHECKING: from frigate.debug_replay import DebugReplayManager @@ -104,11 +105,7 @@ def query_recordings(source_camera: str, start_ts: float, end_ts: float) -> Mode Recordings.start_time, Recordings.end_time, ) - .where( - Recordings.start_time.between(start_ts, end_ts) - | Recordings.end_time.between(start_ts, end_ts) - | ((start_ts > Recordings.start_time) & (end_ts < Recordings.end_time)) - ) + .where(recording_overlap(start_ts, end_ts)) .where(Recordings.camera == source_camera) .order_by(Recordings.start_time.asc()) ) diff --git a/frigate/jobs/motion_search.py b/frigate/jobs/motion_search.py index 5944a9e60b..da33f276b8 100644 --- a/frigate/jobs/motion_search.py +++ b/frigate/jobs/motion_search.py @@ -35,6 +35,7 @@ from frigate.jobs.motion_search_decode import ( ) from frigate.models import Recordings from frigate.types import JobStatusTypesEnum +from frigate.util.recording_coverage import recording_overlap logger = logging.getLogger(__name__) @@ -469,20 +470,7 @@ class MotionSearchRunner(threading.Thread): recordings = list( Recordings.select() .where( - ( - Recordings.start_time.between( - self.job.start_time_range, self.job.end_time_range - ) - ) - | ( - Recordings.end_time.between( - self.job.start_time_range, self.job.end_time_range - ) - ) - | ( - (self.job.start_time_range > Recordings.start_time) - & (self.job.end_time_range < Recordings.end_time) - ) + recording_overlap(self.job.start_time_range, self.job.end_time_range) ) .where(Recordings.camera == camera_name) .where(Recordings.stream_type == STREAM_TYPE_MAIN) diff --git a/frigate/record/export.py b/frigate/record/export.py index 2ec58d664a..5efa4914fa 100644 --- a/frigate/record/export.py +++ b/frigate/record/export.py @@ -45,6 +45,7 @@ from frigate.util.recording_coverage import ( known_video_codecs, null_audio_glitches, realized_timeline, + recording_overlap, resolve_coverage, stream_media_summary, ) @@ -773,14 +774,7 @@ class RecordingExporter(threading.Thread): Recordings.start_time, Recordings.end_time, ) - .where( - Recordings.start_time.between(self.start_time, self.end_time) - | Recordings.end_time.between(self.start_time, self.end_time) - | ( - (self.start_time > Recordings.start_time) - & (self.end_time < Recordings.end_time) - ) - ) + .where(recording_overlap(self.start_time, self.end_time)) .where( (Recordings.camera == self.camera) & (Recordings.stream_type == stream_type) diff --git a/frigate/test/test_recording_coverage.py b/frigate/test/test_recording_coverage.py index b264a31f8a..c559ec4997 100644 --- a/frigate/test/test_recording_coverage.py +++ b/frigate/test/test_recording_coverage.py @@ -16,6 +16,7 @@ from frigate.util.recording_coverage import ( null_audio_glitches, plan_clip, realized_timeline, + recording_overlap, resolve_coverage, stream_media_summary, ) @@ -276,6 +277,33 @@ class TestRecordingCoverage(CoverageDbTestCase): assert plan.duration_ms == 0 +class TestRecordingOverlap(CoverageDbTestCase): + def test_matches_unbounded_overlap(self): + spans = [(1000, 1010), (1010, 1020), (1100, 1100 + MAX_SEGMENT_DURATION - 0.5)] + for i, (start, end) in enumerate(spans): + self._insert(f"r{i}", start, end, "main") + + probes = [995, 1000, 1005, 1010, 1015, 1020, 1050, 1100, 1650, 1699.5, 1700] + for after in probes: + for before in probes: + if before < after: + continue + + with self.subTest(after=after, before=before): + unbounded = Recordings.select(Recordings.id).where( + Recordings.start_time.between(after, before) + | Recordings.end_time.between(after, before) + | ( + (after > Recordings.start_time) + & (before < Recordings.end_time) + ) + ) + bounded = Recordings.select(Recordings.id).where( + recording_overlap(after, before) + ) + self.assertEqual({r.id for r in bounded}, {r.id for r in unbounded}) + + class TestVodManifestPolicy(CoverageDbTestCase): """Discontinuity policy of the vod mapping builder.""" diff --git a/frigate/util/audio.py b/frigate/util/audio.py index e06a40a2df..1666cee139 100644 --- a/frigate/util/audio.py +++ b/frigate/util/audio.py @@ -19,6 +19,7 @@ from frigate.const import ( STREAM_TYPE_SUB, ) from frigate.models import Recordings +from frigate.util.recording_coverage import recording_overlap logger = logging.getLogger(__name__) @@ -67,11 +68,7 @@ def _get_recordings_for_range( Recordings.start_time, Recordings.end_time, ) - .where( - (Recordings.start_time.between(start_ts, end_ts)) - | (Recordings.end_time.between(start_ts, end_ts)) - | ((start_ts > Recordings.start_time) & (end_ts < Recordings.end_time)) - ) + .where(recording_overlap(start_ts, end_ts)) .where(Recordings.camera == camera_name) .where(Recordings.stream_type == stream_type) .order_by(Recordings.start_time.asc()) diff --git a/frigate/util/classification.py b/frigate/util/classification.py index 6c4daabfb8..b750451209 100644 --- a/frigate/util/classification.py +++ b/frigate/util/classification.py @@ -32,6 +32,7 @@ from frigate.util.image import ( relative_box_to_absolute, ) from frigate.util.process import FrigateProcess +from frigate.util.recording_coverage import recording_overlap BATCH_SIZE = 16 EPOCHS = 50 @@ -553,8 +554,7 @@ def _extract_keyframes( recording = ( Recordings.select() .where( - (timestamp >= Recordings.start_time) - & (timestamp <= Recordings.end_time) + recording_overlap(timestamp, timestamp) & (Recordings.camera == camera) & (Recordings.stream_type == STREAM_TYPE_MAIN) ) diff --git a/frigate/util/recording_coverage.py b/frigate/util/recording_coverage.py index 55ca2449a9..5c059811e1 100644 --- a/frigate/util/recording_coverage.py +++ b/frigate/util/recording_coverage.py @@ -23,6 +23,19 @@ class CoverageInterval: sub: Any | None +def recording_overlap(after: float, before: float) -> Any: + """Match recordings overlapping [after, before] as a bounded index range. + + Segments never reach MAX_SEGMENT_DURATION, so the start_time lower bound + drops no rows and keeps SQLite from walking the camera's whole history. + """ + return ( + (Recordings.start_time <= before) + & (Recordings.end_time >= after) + & (Recordings.start_time >= after - MAX_SEGMENT_DURATION) + ) + + def _rows_query(camera: str, after: float, before: float, stream_type: str) -> Any: return ( Recordings.select(