administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-09-27 20:58:59 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Add sub stream recording with adaptive quality playback (#24009)

Browse Source 
* add sub stream recording with adaptive quality playback

Optionally record a second, lower bitrate stream alongside the main
recording stream via a `record_sub` input role and `record.sub` config block, with its own retention windows.
Recordings rows now carry the stream type plus the media details needed to serve both streams from one manifest: video codec, audio presence, audio codec and rate, and a record-time keyframe index.

Playback resolves coverage across both streams and merges them into a single VOD sequence, falling back to a discontinuity manifest with per-clip init segments when the media signatures differ. The player exposes a quality selector, and an auto governor picks the stream from stall time, bandwidth, codec support, and the save-data hint.

* fix tests and i18n
This commit is contained in:
Josh Hawkins
2026-09-12 07:30:04 -06:00
committed by Nicolas Mowen
parent f7c5500ea8
commit 1498231eb9
68 changed files with 6794 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frigate/api/media.py
+243 -125
View File
@@ -8,6 +8,7 @@ import os
import subprocess as sp
import time
from datetime import UTC, datetime, timedelta
from enum import Enum
from pathlib import Path as FilePath
from typing import Any
from urllib.parse import unquote
@@ -39,8 +40,9 @@ from frigate.config.camera.snapshots import SnapshotsConfig
from frigate.const import (
CACHE_DIR,
INSTALL_DIR,
MAX_SEGMENT_DURATION,
PREVIEW_FRAME_TYPE,
STREAM_TYPE_MAIN,
STREAM_TYPE_SUB,
)
from frigate.models import Event, Previews, Recordings, Regions, ReviewSegment
from frigate.output.preview import get_most_recent_preview_frame
@@ -52,12 +54,34 @@ from frigate.util.file import (
load_event_snapshot_image,
)
from frigate.util.image import get_image_from_recording, get_image_quality_params
from frigate.util.media import get_keyframe_before
from frigate.util.object import create_empty_regions_grid
from frigate.util.recording_coverage import (
build_spans,
null_audio_glitches,
plan_clip,
resolve_coverage,
stream_has_audio,
)
logger = logging.getLogger(__name__)
# must match the patched MAX_CLIPS in docker/main/build_nginx.sh; a
# normal hour needs ~360, one clip per recording file
NGINX_VOD_MAX_CLIPS = 1080
class VodStreamPreference(str, Enum):
"""Stream pin for the path-segment VOD route.
nginx-vod derives its mapping fetch URI from the playlist URL path
(query params are dropped), so the preference must be a path segment.
"""
main = STREAM_TYPE_MAIN
sub = STREAM_TYPE_SUB
router = APIRouter(tags=[Tags.media])
@@ -319,7 +343,7 @@ async def get_snapshot_from_recording(
& (frame_time <= Recordings.end_time)
)
.where(Recordings.camera == camera_name)
.order_by(Recordings.start_time.desc())
.order_by(Recordings.stream_type.asc(), Recordings.start_time.desc())
.limit(1)
.get()
)
@@ -338,7 +362,7 @@ async def get_snapshot_from_recording(
& (frame_time <= Recordings.end_time)
)
.where(Recordings.camera == camera_name)
.order_by(Recordings.start_time.desc())
.order_by(Recordings.stream_type.asc(), Recordings.start_time.desc())
.limit(1)
.get()
)
@@ -398,7 +422,7 @@ async def submit_recording_snapshot_to_plus(
(frame_time >= Recordings.start_time) & (frame_time <= Recordings.end_time)
)
.where(Recordings.camera == camera_name)
.order_by(Recordings.start_time.desc())
.order_by(Recordings.stream_type.asc(), Recordings.start_time.desc())
.limit(1)
)
@@ -472,20 +496,29 @@ async def recording_clip(
FilePath(file_path).unlink(missing_ok=True)
break
recordings = (
Recordings.select(
Recordings.path,
Recordings.start_time,
Recordings.end_time,
def get_clip_query(stream_type: str):
return (
Recordings.select(
Recordings.path,
Recordings.start_time,
Recordings.end_time,
)
.where(
(Recordings.start_time.between(start_ts, end_ts))
| (Recordings.end_time.between(start_ts, end_ts))
| ((start_ts > Recordings.start_time) & (end_ts < Recordings.end_time))
)
.where(Recordings.camera == camera_name)
.where(Recordings.stream_type == stream_type)
.order_by(Recordings.start_time.asc())
)
.where(
(Recordings.start_time.between(start_ts, end_ts))
| (Recordings.end_time.between(start_ts, end_ts))
| ((start_ts > Recordings.start_time) & (end_ts < Recordings.end_time))
)
.where(Recordings.camera == camera_name)
.order_by(Recordings.start_time.asc())
)
# never mix streams in one concat; use main when available and
# fall back to sub for expired-main history
recordings = get_clip_query(STREAM_TYPE_MAIN)
if recordings.count() == 0:
recordings = get_clip_query(STREAM_TYPE_SUB)
if recordings.count() == 0:
return JSONResponse(
@@ -549,17 +582,52 @@ async def recording_clip(
)
@router.get(
"/vod/{camera_name}/start/{start_ts}/end/{end_ts}",
dependencies=[Depends(require_camera_access)],
description="Returns an HLS playlist for the specified timestamp-range on the specified camera. Append /master.m3u8 or /index.m3u8 for HLS playback.",
)
async def vod_ts(
def _build_vod_clip(
row: Any, start: float, end: float
) -> tuple[dict[str, Any], int] | None:
"""Build one nginx-vod clip dict + duration (ms) for a recording row trimmed to [start, end).
Realization comes entirely from the shared plan_clip, so the coverage
endpoint's realized timelines match this manifest by construction.
"""
plan = plan_clip(row, start, end)
if plan.skipped:
return None
clip: dict[str, Any] = {"type": "source", "path": row.path}
if plan.clip_from_ms is not None:
clip["clipFrom"] = plan.clip_from_ms
clip["keyFrameDurations"] = [plan.duration_ms]
logger.debug(
"VOD: added clip %s duration_ms=%s clipFrom=%s",
row.path,
plan.duration_ms,
clip.get("clipFrom"),
)
return clip, plan.duration_ms
async def _vod_response(
camera_name: str,
start_ts: float,
end_ts: float,
force_discontinuity: bool = False,
):
stream_preference: str | None = None,
) -> JSONResponse:
"""Build an nginx-vod mapping JSON for a camera over a timestamp range.
Always a single-sequence mapping; quality selection happens in the
frontend by choosing between this route and the stream-pinned routes.
Args:
camera_name: The camera to build the mapping for
start_ts: Range start as a unix timestamp
end_ts: Range end as a unix timestamp
force_discontinuity: Emit HLS discontinuity markers between clips
stream_preference: Pin the manifest to one stream type ("main" or
"sub"), serving only that stream's recordings
"""
logger.debug(
"VOD: Generating VOD for %s from %s to %s with force_discontinuity=%s",
camera_name,
@@ -567,104 +635,85 @@ async def vod_ts(
end_ts,
force_discontinuity,
)
recordings = (
Recordings.select(
Recordings.path,
Recordings.duration,
Recordings.end_time,
Recordings.start_time,
)
.where(
Recordings.camera == camera_name,
Recordings.start_time >= start_ts - MAX_SEGMENT_DURATION,
Recordings.start_time <= end_ts,
Recordings.end_time >= start_ts,
)
.order_by(Recordings.start_time.asc())
.iterator()
intervals = resolve_coverage(camera_name, start_ts, end_ts)
# rows contradicting their stream's audio composition are
# truncated-shutdown glitches
main_audio = stream_has_audio(intervals, main=True)
sub_audio = stream_has_audio(intervals, main=False)
spans = build_spans(
null_audio_glitches(intervals, main_audio, sub_audio),
stream_preference,
)
clips = []
durations = []
min_duration_ms = 100 # Minimum 100ms to ensure at least one video frame
max_duration_ms = MAX_SEGMENT_DURATION * 1000
recording: Recordings
for recording in recordings:
durations: list[int] = []
clips: list[dict[str, Any]] = []
# gathered after glitch-nulling and span building, so the policy
# decisions below reflect the manifest's real contents
video_codecs: set[str] = set()
audio_presence: set[bool] = set()
audio_params: set[tuple[str | None, int | None]] = set()
span_streams: set[bool] = set()
for row, span_start, span_end, span_is_main in spans:
logger.debug(
"VOD: processing recording: %s start=%s end=%s duration=%s",
recording.path,
recording.start_time,
recording.end_time,
recording.duration,
row.path,
row.start_time,
row.end_time,
row.duration,
)
built = _build_vod_clip(row, span_start, span_end)
clip = {"type": "source", "path": recording.path}
duration = int(recording.duration * 1000)
# adjust start offset if start_ts is after recording.start_time
if start_ts > recording.start_time:
inpoint = int((start_ts - recording.start_time) * 1000)
clip["clipFrom"] = inpoint
duration -= inpoint
logger.debug(
"VOD: applied clipFrom %sms to %s",
inpoint,
recording.path,
)
# adjust end if recording.end_time is after end_ts
if recording.end_time > end_ts:
duration -= int((recording.end_time - end_ts) * 1000)
# nginx-vod-module pushes clipFrom forward to the next keyframe,
# which can leave too few frames and produce an empty/unplayable
# segment. Snap clipFrom back to the preceding keyframe so the
# segment always starts with a decodable frame.
if "clipFrom" in clip:
keyframe_ms = get_keyframe_before(recording.path, clip["clipFrom"])
if keyframe_ms is not None:
gained = clip["clipFrom"] - keyframe_ms
clip["clipFrom"] = keyframe_ms
duration += gained
logger.debug(
"VOD: snapped clipFrom to keyframe at %sms for %s, duration now %sms",
keyframe_ms,
recording.path,
duration,
)
else:
# could not read keyframes, remove clipFrom to use full recording
logger.debug(
"VOD: no keyframe info for %s, removing clipFrom to use full recording",
recording.path,
)
del clip["clipFrom"]
duration = int(recording.duration * 1000)
if recording.end_time > end_ts:
duration -= int((recording.end_time - end_ts) * 1000)
if duration < min_duration_ms:
# skip if the clip has no valid duration (too short to contain frames)
logger.debug(
"VOD: skipping recording %s - resulting duration %sms too short",
recording.path,
duration,
)
if built is None:
continue
if min_duration_ms <= duration < max_duration_ms:
clip["keyFrameDurations"] = [duration]
clips.append(clip)
durations.append(duration)
logger.debug(
"VOD: added clip %s duration_ms=%s clipFrom=%s",
recording.path,
duration,
clip.get("clipFrom"),
)
else:
logger.warning(f"Recording clip is missing or empty: {recording.path}")
clips.append(built[0])
durations.append(built[1])
span_streams.add(span_is_main)
if row.video_codec is not None:
video_codecs.add(row.video_codec)
audio_presence.add(row.has_audio is not False)
# legacy rows contribute no signature, so uniformly-unknown
# history keeps the legacy shape
if row.has_audio is not False and (
row.audio_codec is not None or row.audio_rate is not None
):
audio_params.add((row.audio_codec, row.audio_rate))
# nginx-vod requires a uniform track count per sequence, and adding or
# removing an audio track across an MSE discontinuity is unproven
if len(audio_presence) > 1:
logger.debug(
"VOD: %s mixes audio-bearing and audio-less recordings between "
"%s and %s; serving the range without audio",
camera_name,
start_ts,
end_ts,
)
for clip in clips:
clip["tracks"] = "v"
# discontinuity mode emits per-clip init segments, letting the decoder
# reconfigure at each boundary. Stream type counts as a signature of
# its own: the two encoders differ in SPS/PPS even when codec name and
# audio params match, and a single-init manifest then decode-fails on
# players that only configure from the init segment (iOS)
use_discontinuity = (
len(video_codecs) > 1 or len(audio_params) > 1 or len(span_streams) > 1
)
if use_discontinuity:
logger.debug(
"VOD: %s mixes media signatures between %s and %s (video codecs "
"%s, audio params %s, streams %s); serving a discontinuity "
"manifest with per-clip init segments",
camera_name,
start_ts,
end_ts,
sorted(video_codecs),
sorted(audio_params, key=str),
sorted(span_streams),
)
if not clips:
logger.error(
@@ -678,16 +727,49 @@ async def vod_ts(
status_code=404,
)
if len(clips) > NGINX_VOD_MAX_CLIPS:
logger.warning(
"VOD: %s needs %d clips between %s and %s, exceeding nginx's "
"limit of %d; playback of this range will fail. This usually "
"means the camera produced abnormally short recording segments "
"(check the stream's timestamps)",
camera_name,
len(clips),
start_ts,
end_ts,
NGINX_VOD_MAX_CLIPS,
)
hour_ago = datetime.now() - timedelta(hours=1)
return JSONResponse(
content={
"cache": hour_ago.timestamp() > start_ts,
"discontinuity": force_discontinuity,
"consistentSequenceMediaInfo": True,
"durations": durations,
"segment_duration": max(durations),
"sequences": [{"clips": clips}],
}
content = {
"cache": hour_ago.timestamp() > start_ts,
"discontinuity": force_discontinuity or use_discontinuity,
"consistentSequenceMediaInfo": True,
"durations": durations,
# aligns segments to recording file boundaries
"segment_duration": max(durations),
"sequences": [{"clips": clips}],
}
if use_discontinuity:
# clip-indexed naming is what makes nginx-vod emit per-clip
# EXT-X-MAP outside of its live mode
content["initialClipIndex"] = 1
return JSONResponse(content=content)
@router.get(
"/vod/{camera_name}/start/{start_ts}/end/{end_ts}",
dependencies=[Depends(require_camera_access)],
description="Returns an HLS playlist for the specified timestamp-range on the specified camera. Append /master.m3u8 or /index.m3u8 for HLS playback.",
)
async def vod_ts(
camera_name: str,
start_ts: float,
end_ts: float,
force_discontinuity: bool = False,
):
return await _vod_response(
camera_name, start_ts, end_ts, force_discontinuity=force_discontinuity
)
@@ -776,7 +858,43 @@ async def vod_clip(
start_ts: float,
end_ts: float,
):
return await vod_ts(camera_name, start_ts, end_ts, force_discontinuity=True)
# the tracking-details player corrects its timeline from
# sequences[0].clips[0].clipFrom
return await _vod_response(
camera_name,
start_ts,
end_ts,
force_discontinuity=True,
)
# registered after /vod/clip/... on purpose: both routes are six path
# segments, Starlette matches structurally in registration order, and the
# enum validation on {stream} would otherwise 422 every /vod/clip request
@router.get(
"/vod/{camera_name}/{stream}/start/{start_ts}/end/{end_ts}",
dependencies=[Depends(require_camera_access)],
description="Returns an HLS playlist pinned to one stream type (main or sub) for the specified timestamp-range on the specified camera. Append /master.m3u8 or /index.m3u8 for HLS playback.",
)
async def vod_ts_stream(
camera_name: str,
stream: VodStreamPreference,
start_ts: float,
end_ts: float,
force_discontinuity: bool = False,
):
"""VOD for a timestamp range pinned to one stream type.
How the frontend selects quality, now that mappings are always
single-sequence.
"""
return await _vod_response(
camera_name,
start_ts,
end_ts,
force_discontinuity=force_discontinuity,
stream_preference=stream.value,
)
@router.get(
frigate/api/record.py
+106 -16
View File
@@ -25,8 +25,20 @@ from frigate.api.defs.query.recordings_query_parameters import (
)
from frigate.api.defs.response.generic_response import GenericResponse
from frigate.api.defs.tags import Tags
from frigate.const import MAX_SEGMENT_DURATION, RECORD_DIR
from frigate.const import (
MAX_SEGMENT_DURATION,
RECORD_DIR,
STREAM_TYPE_MAIN,
STREAM_TYPE_SUB,
)
from frigate.models import Event, Recordings
from frigate.util.recording_coverage import (
coverage_spans,
known_video_codecs,
realized_timelines,
resolve_coverage,
stream_media_summary,
)
from frigate.util.time import get_dst_transitions
logger = logging.getLogger(__name__)
@@ -149,23 +161,28 @@ async def recordings_summary(camera_name: str, timezone: str = "utc"):
period_hour_modifier = f"{hours_offset} hour"
period_minute_modifier = f"{minutes_offset} minute"
hour_expression = fn.strftime(
"%Y-%m-%d %H",
fn.datetime(
Recordings.start_time,
"unixepoch",
period_hour_modifier,
period_minute_modifier,
),
)
# sub rows duplicate the camera's motion/object stats, so
# aggregating them too would double-count
recording_groups = (
Recordings.select(
fn.strftime(
"%Y-%m-%d %H",
fn.datetime(
Recordings.start_time,
"unixepoch",
period_hour_modifier,
period_minute_modifier,
),
).alias("hour"),
hour_expression.alias("hour"),
fn.SUM(Recordings.duration).alias("duration"),
fn.SUM(Recordings.motion).alias("motion"),
fn.SUM(Recordings.objects).alias("objects"),
)
.where(
(Recordings.camera == camera_name)
& (Recordings.stream_type == STREAM_TYPE_MAIN)
& (Recordings.end_time >= period_start)
& (Recordings.start_time <= period_end)
)
@@ -174,6 +191,23 @@ async def recordings_summary(camera_name: str, timezone: str = "utc"):
.namedtuples()
)
# sub recordings can outlive main, so hours covered only by sub
# rows are reported too, flagged as sub_only
sub_groups = (
Recordings.select(
hour_expression.alias("hour"),
fn.SUM(Recordings.duration).alias("duration"),
)
.where(
(Recordings.camera == camera_name)
& (Recordings.stream_type == STREAM_TYPE_SUB)
& (Recordings.end_time >= period_start)
& (Recordings.start_time <= period_end)
)
.group_by((Recordings.start_time + period_offset).cast("int") / 3600)
.namedtuples()
)
event_groups = (
Event.select(
fn.strftime(
@@ -197,17 +231,43 @@ async def recordings_summary(camera_name: str, timezone: str = "utc"):
event_map = {g.hour: g.count for g in event_groups}
for recording_group in recording_groups:
parts = recording_group.hour.split()
hour_stats = [
(
g.hour,
{
"motion": g.motion,
"objects": g.objects,
"duration": round(g.duration),
},
)
for g in recording_groups
]
main_hours = {group_hour for group_hour, _ in hour_stats}
hour_stats.extend(
(
g.hour,
{
"motion": 0,
"objects": 0,
"duration": round(g.duration),
"sub_only": True,
},
)
for g in sub_groups
if g.hour not in main_hours
)
# restore the most-recent-first ordering after merging in sub hours
hour_stats.sort(key=lambda entry: entry[0], reverse=True)
for group_hour, stats in hour_stats:
parts = group_hour.split()
hour = parts[1]
day = parts[0]
events_count = event_map.get(recording_group.hour, 0)
events_count = event_map.get(group_hour, 0)
hour_data = {
"hour": hour,
"events": events_count,
"motion": recording_group.motion,
"objects": recording_group.objects,
"duration": round(recording_group.duration),
**stats,
}
if day in days:
# merge counts if already present (edge-case at DST boundary)
@@ -223,6 +283,35 @@ async def recordings_summary(camera_name: str, timezone: str = "utc"):
return JSONResponse(content=list(days.values()))
@router.get(
"/{camera_name}/recordings/coverage",
dependencies=[Depends(require_camera_access)],
)
async def recordings_coverage(
camera_name: str, after: float, before: float, timelines: bool = False
):
"""Returns merged recording coverage spans plus codec compatibility.
codecs_compatible is false only when more than one known video codec
appears across the range's rows, the case where the merged vod route
degrades to a single-stream manifest.
"""
intervals = resolve_coverage(camera_name, after, before)
content = {
"spans": coverage_spans(intervals),
"codecs_compatible": len(known_video_codecs(intervals)) <= 1,
"streams": stream_media_summary(intervals),
}
# pure computation (shared plan_clip, record-time keyframe index), but
# opt-in for payload hygiene: day-level requests need only the spans
if timelines:
content["timelines"] = realized_timelines(intervals)
return JSONResponse(content=content)
@router.get("/{camera_name}/recordings", dependencies=[Depends(require_camera_access)])
async def recordings(
camera_name: str,
@@ -243,6 +332,7 @@ async def recordings(
)
.where(
Recordings.camera == camera_name,
Recordings.stream_type == STREAM_TYPE_MAIN,
Recordings.start_time >= after - MAX_SEGMENT_DURATION,
Recordings.end_time >= after,
Recordings.start_time <= before,
frigate/api/review.py
+3
View File
@@ -33,6 +33,7 @@ from frigate.api.defs.response.review_response import (
ReviewSummaryResponse,
)
from frigate.api.defs.tags import Tags
from frigate.const import STREAM_TYPE_MAIN
from frigate.embeddings import EmbeddingsContext
from frigate.models import Recordings, ReviewSegment, UserReviewStatus
from frigate.review.types import SeverityEnum
@@ -598,6 +599,8 @@ def motion_activity(
clauses = [(Recordings.start_time > after) & (Recordings.end_time < before)]
clauses.append(Recordings.motion > 0)
# sub rows duplicate the camera's motion stats, so only count main rows
clauses.append(Recordings.stream_type == STREAM_TYPE_MAIN)
if cameras != "all":
requested = set(cameras.split(","))