mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-09-27 06:48:58 +03:00
Improve History's seek startup time and recordings query performance (#24011)
* serve a segment startup ladder so seeks begin playing sooner nginx-vod was handed one 10s segment per recording file, so every playlist start had to download and decode a full segment before the first frame. Declare real keyframe data per clip and let nginx cut short leading segments from it. - add vod_bootstrap_segment_durations 1000/2000/4000 so each playlist starts with 1s/2s/4s segments before settling at 10s - emit real clip-relative keyFrameDurations (plus firstKeyFrameOffset when nonzero) from the recording keyframe index; rows without an index keep the whole-clip declaration, the only safe cut without keyframe knowledge - drop the manifest's segment_duration field, which was always inert: nginx-vod parses only camelCase segmentDuration - rebuild the player source at the seek target, quantized to a 10s grid, so the ladder applies to every seek and seek URLs stay repeatable for nginx's mapping and response caches - route the seek model, in-range checks, and the stale-report guard through the source window rather than the chunk range - bridge repositioning seeks (>2s from the last played timestamp) through the preview player and hold the release anchor one commit, so neither path paints a stale frame - clear a pending loading timer before replacing it; an orphaned timer escaped onPlaying's clearTimeout and flashed loading mid-playback * keep recordings queries on their indexes Several recordings queries degraded into full scans or large sorts on big databases: the planner ignored index order, or the query shape gave it nothing tight to seek on. Reshape them into bounded seeks and add the composite index the per-stream lookups need. - index recordings on (camera, stream_type, start_time DESC) and drop the (camera, stream_type) index it supersedes - walk the recordings summary day by day with EXISTS probes and per-camera MIN/MAX seeks, skipping ahead over empty gaps instead of bucketing every row for the requested cameras - run the summary endpoint on the event loop rather than the threadpool - bound the unavailable-recordings query by start_time per camera and merge the results in Python - bound the expire query's start_time so it seeks the retention window instead of scanning a camera's whole history - enumerate deleted cameras with one index seek each rather than a camera NOT IN (...) scan - compute bandwidth with segment_size filtered in a CASE projection; as a WHERE predicate it baited the planner into the (camera, segment_size) index plus a full sort of the camera's history - fall back to a 1000-segment window when the recent 100 are all zero-size, so an ingest glitch doesn't report zero bandwidth - limit the needs_refresh count instead of counting every segment - cover sub-only and sparse calendar days, midnight-spanning day attribution, multi-camera gap merging, deleted-camera expiry, and zero-size segment runs * fix mypy
This commit is contained in:
Josh Hawkins
committed by
Nicolas Mowen
Nicolas Mowen
parent
50e7b76eb5
commit
e77155a4f1
+12
-3
@@ -598,7 +598,15 @@ def _build_vod_clip(
|
||||
clip: dict[str, Any] = {"type": "source", "path": row.path}
|
||||
if plan.clip_from_ms is not None:
|
||||
clip["clipFrom"] = plan.clip_from_ms
|
||||
clip["keyFrameDurations"] = [plan.duration_ms]
|
||||
if plan.key_frame_durations is not None:
|
||||
# real gaps enable keyframe-aligned sub-file segments (bootstrap
|
||||
# ladder); the whole-clip fallback keeps one segment per file,
|
||||
# the only safe cut without an index
|
||||
if plan.first_key_frame_offset_ms > 0:
|
||||
clip["firstKeyFrameOffset"] = plan.first_key_frame_offset_ms
|
||||
clip["keyFrameDurations"] = plan.key_frame_durations
|
||||
else:
|
||||
clip["keyFrameDurations"] = [plan.duration_ms]
|
||||
logger.debug(
|
||||
"VOD: added clip %s duration_ms=%s clipFrom=%s",
|
||||
row.path,
|
||||
@@ -740,14 +748,15 @@ async def _vod_response(
|
||||
NGINX_VOD_MAX_CLIPS,
|
||||
)
|
||||
|
||||
# segmentation comes from the vod_* nginx directives plus per-clip
|
||||
# keyFrameDurations; a segment_duration field here was always ignored
|
||||
# (nginx-vod parses only camelCase segmentDuration)
|
||||
hour_ago = datetime.now() - timedelta(hours=1)
|
||||
content = {
|
||||
"cache": hour_ago.timestamp() > start_ts,
|
||||
"discontinuity": force_discontinuity or use_discontinuity,
|
||||
"consistentSequenceMediaInfo": True,
|
||||
"durations": durations,
|
||||
# aligns segments to recording file boundaries
|
||||
"segment_duration": max(durations),
|
||||
"sequences": [{"clips": clips}],
|
||||
}
|
||||
if use_discontinuity:
|
||||
|
||||
+83
-40
@@ -71,7 +71,7 @@ def get_recordings_storage_usage(request: Request):
|
||||
|
||||
|
||||
@router.get("/recordings/summary", dependencies=[Depends(allow_any_authenticated())])
|
||||
def all_recordings_summary(
|
||||
async def all_recordings_summary(
|
||||
request: Request,
|
||||
params: MediaRecordingsSummaryQueryParams = Depends(),
|
||||
allowed_cameras: list[str] = Depends(get_allowed_cameras_for_filter),
|
||||
@@ -88,18 +88,23 @@ def all_recordings_summary(
|
||||
else:
|
||||
camera_list = allowed_cameras
|
||||
|
||||
time_range_query = (
|
||||
Recordings.select(
|
||||
fn.MIN(Recordings.start_time).alias("min_time"),
|
||||
fn.MAX(Recordings.start_time).alias("max_time"),
|
||||
min_time: float | None = None
|
||||
max_time: float | None = None
|
||||
for camera in camera_list:
|
||||
cam_min = (
|
||||
Recordings.select(fn.MIN(Recordings.start_time))
|
||||
.where(Recordings.camera == camera)
|
||||
.scalar()
|
||||
)
|
||||
.where(Recordings.camera << camera_list)
|
||||
.dicts()
|
||||
.get()
|
||||
)
|
||||
|
||||
min_time = time_range_query.get("min_time")
|
||||
max_time = time_range_query.get("max_time")
|
||||
if cam_min is None:
|
||||
continue
|
||||
cam_max = (
|
||||
Recordings.select(fn.MAX(Recordings.start_time))
|
||||
.where(Recordings.camera == camera)
|
||||
.scalar()
|
||||
)
|
||||
min_time = cam_min if min_time is None else min(min_time, cam_min)
|
||||
max_time = cam_max if max_time is None else max(max_time, cam_max)
|
||||
|
||||
if min_time is None or max_time is None:
|
||||
return JSONResponse(content={})
|
||||
@@ -109,22 +114,60 @@ def all_recordings_summary(
|
||||
days: dict[str, bool] = {}
|
||||
|
||||
for period_start, period_end, period_offset in dst_periods:
|
||||
day_expr = ((Recordings.start_time + period_offset) / 86400).cast("int")
|
||||
first_start = max(min_time, period_start - MAX_SEGMENT_DURATION)
|
||||
first_day = int((first_start + period_offset) // 86400)
|
||||
last_day = int((min(max_time, period_end) + period_offset) // 86400)
|
||||
|
||||
period_query = (
|
||||
Recordings.select(day_expr.alias("day_idx"))
|
||||
.where(
|
||||
(Recordings.camera << camera_list)
|
||||
& (Recordings.end_time >= period_start)
|
||||
& (Recordings.start_time <= period_end)
|
||||
day_idx = first_day
|
||||
while day_idx <= last_day:
|
||||
day_str = (dt.date(1970, 1, 1) + dt.timedelta(days=day_idx)).isoformat()
|
||||
day_start = day_idx * 86400 - period_offset
|
||||
day_end = day_start + 86400
|
||||
|
||||
if day_str in days:
|
||||
day_idx += 1
|
||||
continue
|
||||
|
||||
if day_end <= period_end:
|
||||
upper = Recordings.start_time < day_end
|
||||
else:
|
||||
upper = Recordings.start_time <= period_end
|
||||
|
||||
has_recordings = (
|
||||
Recordings.select(Recordings.id)
|
||||
.where(
|
||||
(Recordings.camera << camera_list)
|
||||
& (Recordings.end_time >= period_start)
|
||||
& (Recordings.start_time >= day_start)
|
||||
& upper
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
.distinct()
|
||||
.namedtuples()
|
||||
)
|
||||
if has_recordings:
|
||||
days[day_str] = True
|
||||
day_idx += 1
|
||||
continue
|
||||
|
||||
for g in period_query:
|
||||
day_str = (dt.date(1970, 1, 1) + dt.timedelta(days=g.day_idx)).isoformat()
|
||||
days[day_str] = True
|
||||
# empty day
|
||||
next_start: float | None = None
|
||||
for camera in camera_list:
|
||||
cam_next = (
|
||||
Recordings.select(fn.MIN(Recordings.start_time))
|
||||
.where(
|
||||
Recordings.camera == camera,
|
||||
Recordings.start_time >= day_end,
|
||||
Recordings.start_time <= period_end,
|
||||
)
|
||||
.scalar()
|
||||
)
|
||||
if cam_next is not None and (
|
||||
next_start is None or cam_next < next_start
|
||||
):
|
||||
next_start = cam_next
|
||||
|
||||
if next_start is None:
|
||||
break
|
||||
day_idx = max(day_idx + 1, int((next_start + period_offset) // 86400))
|
||||
|
||||
return JSONResponse(content=dict(sorted(days.items())))
|
||||
|
||||
@@ -373,22 +416,22 @@ async def no_recordings(
|
||||
)
|
||||
scale = params.scale
|
||||
|
||||
clauses = [
|
||||
(Recordings.end_time >= after) & (Recordings.start_time <= before),
|
||||
(Recordings.camera << camera_list),
|
||||
]
|
||||
recordings: list[tuple[float, float]] = []
|
||||
for camera in camera_list:
|
||||
recordings.extend(
|
||||
Recordings.select(Recordings.start_time, Recordings.end_time)
|
||||
.where(
|
||||
Recordings.camera == camera,
|
||||
Recordings.start_time >= after - MAX_SEGMENT_DURATION,
|
||||
Recordings.end_time >= after,
|
||||
Recordings.start_time <= before,
|
||||
)
|
||||
.tuples()
|
||||
.iterator()
|
||||
)
|
||||
|
||||
# Get recording start times
|
||||
data: list[Recordings] = (
|
||||
Recordings.select(Recordings.start_time, Recordings.end_time)
|
||||
.where(reduce(operator.and_, clauses))
|
||||
.order_by(Recordings.start_time.asc())
|
||||
.dicts()
|
||||
.iterator()
|
||||
)
|
||||
|
||||
# Convert recordings to list of (start, end) tuples, ordered by start_time
|
||||
recordings = [(r["start_time"], r["end_time"]) for r in data]
|
||||
# the merge pass below expects a single start-ordered timeline
|
||||
recordings.sort()
|
||||
|
||||
# Merge overlapping/adjacent recordings into covered intervals. The query
|
||||
# orders by start_time, so a single pass merges them
|
||||
|
||||
Reference in New Issue
Block a user