mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-15 01:53:02 +03:00
Improve History's seek startup time and recordings query performance (#24011)
* serve a segment startup ladder so seeks begin playing sooner nginx-vod was handed one 10s segment per recording file, so every playlist start had to download and decode a full segment before the first frame. Declare real keyframe data per clip and let nginx cut short leading segments from it. - add vod_bootstrap_segment_durations 1000/2000/4000 so each playlist starts with 1s/2s/4s segments before settling at 10s - emit real clip-relative keyFrameDurations (plus firstKeyFrameOffset when nonzero) from the recording keyframe index; rows without an index keep the whole-clip declaration, the only safe cut without keyframe knowledge - drop the manifest's segment_duration field, which was always inert: nginx-vod parses only camelCase segmentDuration - rebuild the player source at the seek target, quantized to a 10s grid, so the ladder applies to every seek and seek URLs stay repeatable for nginx's mapping and response caches - route the seek model, in-range checks, and the stale-report guard through the source window rather than the chunk range - bridge repositioning seeks (>2s from the last played timestamp) through the preview player and hold the release anchor one commit, so neither path paints a stale frame - clear a pending loading timer before replacing it; an orphaned timer escaped onPlaying's clearTimeout and flashed loading mid-playback * keep recordings queries on their indexes Several recordings queries degraded into full scans or large sorts on big databases: the planner ignored index order, or the query shape gave it nothing tight to seek on. Reshape them into bounded seeks and add the composite index the per-stream lookups need. - index recordings on (camera, stream_type, start_time DESC) and drop the (camera, stream_type) index it supersedes - walk the recordings summary day by day with EXISTS probes and per-camera MIN/MAX seeks, skipping ahead over empty gaps instead of bucketing every row for the requested cameras - run the summary endpoint on the event loop rather than the threadpool - bound the unavailable-recordings query by start_time per camera and merge the results in Python - bound the expire query's start_time so it seeks the retention window instead of scanning a camera's whole history - enumerate deleted cameras with one index seek each rather than a camera NOT IN (...) scan - compute bandwidth with segment_size filtered in a CASE projection; as a WHERE predicate it baited the planner into the (camera, segment_size) index plus a full sort of the camera's history - fall back to a 1000-segment window when the recent 100 are all zero-size, so an ingest glitch doesn't report zero bandwidth - limit the needs_refresh count instead of counting every segment - cover sub-only and sparse calendar days, midnight-spanning day attribution, multi-camera gap merging, deleted-camera expiry, and zero-size segment runs * fix mypy
This commit is contained in:
+39
-17
@@ -167,6 +167,10 @@ class RecordingCleanup(threading.Thread):
|
||||
.where(
|
||||
(Recordings.camera == config.name)
|
||||
& (Recordings.stream_type == stream_type)
|
||||
& (
|
||||
Recordings.start_time
|
||||
< max(continuous_expire_date, motion_expire_date)
|
||||
)
|
||||
& (
|
||||
(
|
||||
(Recordings.end_time < continuous_expire_date)
|
||||
@@ -333,27 +337,45 @@ class RecordingCleanup(threading.Thread):
|
||||
expire_before = (
|
||||
datetime.datetime.now() - datetime.timedelta(days=expire_days)
|
||||
).timestamp()
|
||||
no_camera_recordings = (
|
||||
Recordings.select(
|
||||
Recordings.id,
|
||||
Recordings.path,
|
||||
)
|
||||
.where(
|
||||
Recordings.camera.not_in(list(self.config.cameras.keys())), # type: ignore[call-arg, arg-type, misc]
|
||||
Recordings.end_time < expire_before,
|
||||
)
|
||||
.namedtuples()
|
||||
.iterator()
|
||||
)
|
||||
|
||||
# enumerate the distinct cameras with one index seek each
|
||||
db_cameras: list[str] = []
|
||||
last_camera: str | None = None
|
||||
while True:
|
||||
query = Recordings.select(Recordings.camera)
|
||||
if last_camera is not None:
|
||||
query = query.where(Recordings.camera > last_camera)
|
||||
next_camera = query.order_by(Recordings.camera.asc()).limit(1).scalar()
|
||||
if next_camera is None:
|
||||
break
|
||||
db_cameras.append(next_camera)
|
||||
last_camera = next_camera
|
||||
|
||||
maybe_empty_dirs = set()
|
||||
|
||||
deleted_recordings = set()
|
||||
for recording in no_camera_recordings:
|
||||
recording_path = Path(recording.path)
|
||||
recording_path.unlink(missing_ok=True)
|
||||
deleted_recordings.add(recording.id)
|
||||
maybe_empty_dirs.add(recording_path.parent)
|
||||
for camera in db_cameras:
|
||||
if camera in self.config.cameras:
|
||||
continue
|
||||
|
||||
no_camera_recordings = (
|
||||
Recordings.select(
|
||||
Recordings.id,
|
||||
Recordings.path,
|
||||
)
|
||||
.where(
|
||||
Recordings.camera == camera,
|
||||
Recordings.end_time < expire_before,
|
||||
)
|
||||
.namedtuples()
|
||||
.iterator()
|
||||
)
|
||||
|
||||
for recording in no_camera_recordings:
|
||||
recording_path = Path(recording.path)
|
||||
recording_path.unlink(missing_ok=True)
|
||||
deleted_recordings.add(recording.id)
|
||||
maybe_empty_dirs.add(recording_path.parent)
|
||||
|
||||
logger.debug(f"Expiring {len(deleted_recordings)} recordings")
|
||||
# delete up to 100,000 at a time
|
||||
|
||||
Reference in New Issue
Block a user