administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-09-26 19:58:58 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Add sub stream recording with adaptive quality playback (#24009)

Browse Source 
* add sub stream recording with adaptive quality playback

Optionally record a second, lower bitrate stream alongside the main
recording stream via a `record_sub` input role and `record.sub` config block, with its own retention windows.
Recordings rows now carry the stream type plus the media details needed to serve both streams from one manifest: video codec, audio presence, audio codec and rate, and a record-time keyframe index.

Playback resolves coverage across both streams and merges them into a single VOD sequence, falling back to a discontinuity manifest with per-clip init segments when the media signatures differ. The player exposes a quality selector, and an auto governor picks the stream from stall time, bandwidth, codec support, and the save-data hint.

* fix tests and i18n
This commit is contained in:
Josh Hawkins
2026-09-12 07:30:04 -06:00
committed by Nicolas Mowen
parent f7c5500ea8
commit 1498231eb9
68 changed files with 6794 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/src/components/config-form/section-configs/ffmpeg.ts
+13
View File
@@ -20,6 +20,13 @@ const ffmpegArgsWidget = (
},
});
const recordSubArgsWidget = () =>
ffmpegArgsWidget("output_args.record_sub", {
allowInherit: true,
forceSplitLayout: true,
unsetLabelKey: "configForm.ffmpegArgs.sameAsRecord",
});
const ffmpeg: SectionConfigOverrides = {
base: {
sectionDocs: "/configuration/ffmpeg_presets",
@@ -75,6 +82,8 @@ const ffmpeg: SectionConfigOverrides = {
output_args: "/configuration/ffmpeg_presets#output-args-presets",
"inputs.output_args": "/configuration/ffmpeg_presets#output-args-presets",
"output_args.record": "/configuration/ffmpeg_presets#output-args-presets",
"output_args.record_sub":
"/configuration/ffmpeg_presets#output-args-presets",
"inputs.roles": "/configuration/cameras/#setting-up-camera-inputs",
apple_compatibility:
"/configuration/camera_specific#h265-cameras-via-safari",
@@ -112,9 +121,11 @@ const ffmpeg: SectionConfigOverrides = {
output_args: {
detect: arrayAsTextWidget,
record: ffmpegArgsWidget("output_args.record"),
record_sub: recordSubArgsWidget(),
items: {
detect: arrayAsTextWidget,
record: ffmpegArgsWidget("output_args.record"),
record_sub: recordSubArgsWidget(),
},
},
inputs: {
@@ -148,6 +159,7 @@ const ffmpeg: SectionConfigOverrides = {
items: {
detect: arrayAsTextWidget,
record: ffmpegArgsWidget("output_args.record"),
record_sub: recordSubArgsWidget(),
},
},
},
@@ -176,6 +188,7 @@ const ffmpeg: SectionConfigOverrides = {
output_args: {
detect: arrayAsTextWidget,
record: ffmpegArgsWidget("output_args.record"),
record_sub: recordSubArgsWidget(),
},
},
},
web/src/components/config-form/section-configs/record.ts
+20
View File
@@ -15,6 +15,19 @@ const record: SectionConfigOverrides = {
);
},
},
{
key: "no-record-sub-role",
messageKey: "configMessages.record.noRecordSubRole",
severity: "warning",
condition: (ctx) => {
if (ctx.level !== "camera" || !ctx.fullCameraConfig) return false;
const sub = ctx.formData?.sub as Record<string, unknown> | undefined;
if (!sub?.enabled) return false;
return !ctx.fullCameraConfig.ffmpeg?.inputs?.some((i) =>
i.roles?.includes("record_sub"),
);
},
},
],
fieldDocs: {
"alerts.pre_capture":
@@ -34,6 +47,7 @@ const record: SectionConfigOverrides = {
"motion",
"alerts",
"detections",
"sub",
"preview",
"export",
],
@@ -67,6 +81,12 @@ const record: SectionConfigOverrides = {
"detections.retain.mode": {
"ui:options": { enumI18nPrefix: "retainMode" },
},
"sub.alerts.mode": {
"ui:options": { enumI18nPrefix: "retainMode" },
},
"sub.detections.mode": {
"ui:options": { enumI18nPrefix: "retainMode" },
},
preview: {
"ui:options": { defaultOpen: true, disableCollapsible: true },
quality: {
web/src/components/config-form/theme/fields/CameraInputsField.tsx
+29 -1
View File
@@ -193,6 +193,25 @@ export function CameraInputsField(props: FieldProps) {
}
}, [fieldPathId.path, inputs, onChange]);
const getRolesUsedByOtherInputs = useCallback(
(index: number): string[] => {
const used = new Set<string>();
inputs.forEach((input, currentIndex) => {
if (currentIndex === index || !Array.isArray(input.roles)) {
return;
}
input.roles.forEach((role) => {
if (typeof role === "string") {
used.add(role);
}
});
});
return [...used];
},
[inputs],
);
const handleFieldValueChange = useCallback(
(index: number, fieldName: string, nextValue: unknown) => {
const nextInputs = cloneDeep(inputs);
@@ -466,7 +485,16 @@ export function CameraInputsField(props: FieldProps) {
/>
</div>
<div className="w-full">{renderField(index, "roles")}</div>
<div className="w-full">
{renderField(index, "roles", {
extraUiSchema: {
"ui:options": {
rolesUsedByOtherInputs:
getRolesUsedByOtherInputs(index),
},
},
})}
</div>
{renderField(index, "input_args")}
web/src/components/config-form/theme/widgets/FfmpegArgsWidget.tsx
+18 -3
View File
@@ -30,6 +30,7 @@ type PresetField =
| "hwaccel_args"
| "input_args"
| "output_args.record"
| "output_args.record_sub"
| "output_args.detect";
const getPresetOptions = (
@@ -49,7 +50,10 @@ const getPresetOptions = (
}
if (field.startsWith("output_args.")) {
const key = field.split(".")[1] as "record" | "detect";
const key =
field === "output_args.record_sub"
? "record"
: (field.split(".")[1] as "record" | "detect");
return data.output_args?.[key] ?? [];
}
@@ -127,6 +131,7 @@ export function FfmpegArgsWidget(props: WidgetProps) {
const globalFieldPath =
(options?.ffmpegGlobalFieldPath as string | undefined) ?? presetField;
const allowInherit = options?.allowInherit === true;
const unsetLabelKey = options?.unsetLabelKey as string | undefined;
const hideDescription = options?.hideDescription === true;
const useSplitLayout = options?.splitLayout !== false;
@@ -287,6 +292,12 @@ export function FfmpegArgsWidget(props: WidgetProps) {
: "ffmpeg.output_args.record.description";
}
if (presetField === "output_args.record_sub") {
return isInputScoped
? "ffmpeg.inputs.output_args.record_sub.description"
: "ffmpeg.output_args.record_sub.description";
}
if (presetField === "output_args.detect") {
return isInputScoped
? "ffmpeg.inputs.output_args.detect.description"
@@ -345,7 +356,9 @@ export function FfmpegArgsWidget(props: WidgetProps) {
}
/>
<label htmlFor={`${id}-inherit`} className="cursor-pointer text-sm">
{t("configForm.ffmpegArgs.inherit", { ns: "views/settings" })}
{t(unsetLabelKey ?? "configForm.ffmpegArgs.inherit", {
ns: "views/settings",
})}
</label>
</div>
) : (
@@ -361,7 +374,9 @@ export function FfmpegArgsWidget(props: WidgetProps) {
}
/>
<label htmlFor={`${id}-none`} className="cursor-pointer text-sm">
{t("configForm.ffmpegArgs.none", { ns: "views/settings" })}
{t(unsetLabelKey ?? "configForm.ffmpegArgs.none", {
ns: "views/settings",
})}
</label>
</div>
)}
web/src/components/config-form/theme/widgets/InputRolesWidget.tsx
+41 -6
View File
@@ -3,7 +3,14 @@ import { useMemo } from "react";
import { useTranslation } from "react-i18next";
import { Switch } from "@/components/ui/switch";
const INPUT_ROLES = ["detect", "record", "audio"] as const;
const INPUT_ROLES = ["detect", "record", "record_sub", "audio"] as const;
// Recording the sub stream from the same input as record would just
// re-record the main stream, so the two roles are mutually exclusive.
const CONFLICTING_ROLES: Partial<Record<string, string>> = {
record: "record_sub",
record_sub: "record",
};
function normalizeValue(value: unknown): string[] {
if (Array.isArray(value)) {
@@ -18,11 +25,18 @@ function normalizeValue(value: unknown): string[] {
}
export function InputRolesWidget(props: WidgetProps) {
const { id, value, disabled, readonly, onChange } = props;
const { id, value, disabled, readonly, onChange, options } = props;
const { t } = useTranslation(["views/settings"]);
const selectedRoles = useMemo(() => normalizeValue(value), [value]);
// Each role may only be assigned to a single input, so roles already
// used by sibling inputs are locked.
const rolesUsedByOtherInputs = useMemo(
() => normalizeValue(options?.rolesUsedByOtherInputs),
[options],
);
const toggleRole = (role: string, enabled: boolean) => {
if (enabled) {
if (!selectedRoles.includes(role)) {
@@ -39,6 +53,20 @@ export function InputRolesWidget(props: WidgetProps) {
<div className="grid gap-2">
{INPUT_ROLES.map((role) => {
const checked = selectedRoles.includes(role);
const usedByOtherInput =
!checked && rolesUsedByOtherInputs.includes(role);
const conflictingRole = CONFLICTING_ROLES[role];
const hasConflict =
!checked &&
conflictingRole !== undefined &&
selectedRoles.includes(conflictingRole);
const hint = usedByOtherInput
? t("configForm.inputRoles.roleInUse", { ns: "views/settings" })
: hasConflict
? t("configForm.inputRoles.recordSubConflict", {
ns: "views/settings",
})
: undefined;
const label = t(`configForm.inputRoles.options.${role}`, {
ns: "views/settings",
defaultValue: role,
@@ -49,13 +77,20 @@ export function InputRolesWidget(props: WidgetProps) {
key={role}
className="flex items-center justify-between rounded-md px-3 py-0"
>
<label htmlFor={`${id}-${role}`} className="text-sm">
{label}
</label>
<div className="flex flex-col">
<label htmlFor={`${id}-${role}`} className="text-sm">
{label}
</label>
{hint ? (
<span className="text-xs text-muted-foreground">{hint}</span>
) : null}
</div>
<Switch
id={`${id}-${role}`}
checked={checked}
disabled={disabled || readonly}
disabled={
disabled || readonly || usedByOtherInput || hasConflict
}
onCheckedChange={(enabled) => toggleRole(role, !!enabled)}
/>
</div>
web/src/components/filter/ReviewFilterGroup.tsx
+2 -4
View File
@@ -14,12 +14,10 @@ import { FaCheckCircle, FaFilter, FaRunning } from "react-icons/fa";
import { isDesktop, isMobile } from "react-device-detect";
import { Switch } from "../ui/switch";
import { Label } from "../ui/label";
import MobileReviewSettingsDrawer, {
DrawerFeatures,
} from "../overlay/MobileReviewSettingsDrawer";
import MobileReviewSettingsDrawer from "../overlay/MobileReviewSettingsDrawer";
import useOptimisticState from "@/hooks/use-optimistic-state";
import FilterSwitch from "./FilterSwitch";
import { FilterList, GeneralFilter } from "@/types/filter";
import { DrawerFeatures, FilterList, GeneralFilter } from "@/types/filter";
import CalendarFilterButton from "./CalendarFilterButton";
import { CamerasFilterButton } from "./CamerasFilterButton";
import PlatformAwareDialog from "../overlay/dialog/PlatformAwareDialog";
web/src/components/overlay/MobileReviewSettingsDrawer.tsx
+71 -20
View File
@@ -10,7 +10,12 @@ import {
DebugReplayContent,
SaveDebugReplayOverlay,
} from "./DebugReplayDialog";
import { ExportMode, GeneralFilter } from "@/types/filter";
import {
DEFAULT_DRAWER_FEATURES,
DrawerFeatures,
ExportMode,
GeneralFilter,
} from "@/types/filter";
import ReviewActivityCalendar from "./ReviewActivityCalendar";
import { SelectSeparator } from "../ui/select";
import {
@@ -31,6 +36,14 @@ import { StartExportResponse } from "@/types/export";
import { ShareTimestampContent } from "./ShareTimestampDialog";
import { useIsAdmin } from "@/hooks/use-is-admin";
import { cn } from "@/lib/utils";
import { FaTriangleExclamation } from "react-icons/fa6";
import { MdHighQuality } from "react-icons/md";
import { QualitySelectorContent } from "../player/QualitySelector";
import {
AutoQualityReason,
PlaybackQuality,
RecordingCoverage,
} from "@/types/record";
type DrawerMode =
| "none"
@@ -39,25 +52,8 @@ type DrawerMode =
| "calendar"
| "filter"
| "debug-replay"
| "share-timestamp";
const DRAWER_FEATURES = [
"export",
"calendar",
"filter",
"debug-replay",
"share-timestamp",
"motion-search",
] as const;
export type DrawerFeatures = (typeof DRAWER_FEATURES)[number];
const DEFAULT_DRAWER_FEATURES: DrawerFeatures[] = [
"export",
"calendar",
"filter",
"debug-replay",
"share-timestamp",
"motion-search",
];
| "share-timestamp"
| "quality";
type MobileReviewSettingsDrawerProps = {
features?: DrawerFeatures[];
@@ -84,6 +80,12 @@ type MobileReviewSettingsDrawerProps = {
setRange: (range: TimeRange | undefined) => void;
setMode: (mode: ExportMode) => void;
setShowExportPreview: (showPreview: boolean) => void;
quality?: PlaybackQuality;
onSetQuality?: (quality: PlaybackQuality) => void;
qualityStreams?: RecordingCoverage["streams"];
qualityAutoLow?: boolean;
qualityAutoLowReason?: AutoQualityReason;
qualityMainUnsupported?: boolean;
};
export default function MobileReviewSettingsDrawer({
features = DEFAULT_DRAWER_FEATURES,
@@ -110,12 +112,19 @@ export default function MobileReviewSettingsDrawer({
setRange,
setMode,
setShowExportPreview,
quality,
onSetQuality,
qualityStreams,
qualityAutoLow,
qualityAutoLowReason,
qualityMainUnsupported,
}: MobileReviewSettingsDrawerProps) {
const { t } = useTranslation([
"views/recording",
"components/dialog",
"views/replay",
"views/events",
"components/player",
"common",
]);
const isAdmin = useIsAdmin();
@@ -395,6 +404,21 @@ export default function MobileReviewSettingsDrawer({
{t("filter")}
</Button>
)}
{features.includes("quality") && onSetQuality && (
<Button
className="flex w-full items-center justify-center gap-2"
aria-label={t("quality.label", { ns: "components/player" })}
onClick={() => setDrawerMode("quality")}
>
<div className="relative">
<MdHighQuality className="size-5 rounded-md bg-secondary-foreground fill-secondary p-1" />
{qualityAutoLow && (
<FaTriangleExclamation className="absolute -bottom-1 -right-1 size-2.5 text-danger" />
)}
</div>
{t("quality.label", { ns: "components/player" })}
</Button>
)}
{features.includes("share-timestamp") && (
<Button
className="flex w-full items-center justify-center gap-2"
@@ -623,6 +647,33 @@ export default function MobileReviewSettingsDrawer({
}}
/>
);
} else if (drawerMode == "quality") {
content = (
<div className="flex w-full flex-col">
<div className="relative mb-2 h-8 w-full">
<div
className="absolute left-0 text-selected"
onClick={() => setDrawerMode("select")}
>
{t("button.back", { ns: "common" })}
</div>
<div className="absolute left-1/2 -translate-x-1/2 text-muted-foreground">
{t("quality.label", { ns: "components/player" })}
</div>
</div>
<QualitySelectorContent
quality={quality ?? "auto"}
onSetQuality={(newQuality) => {
onSetQuality?.(newQuality);
setDrawerMode("none");
}}
streams={qualityStreams}
autoLow={qualityAutoLow}
autoLowReason={qualityAutoLowReason}
mainUnsupported={qualityMainUnsupported}
/>
</div>
);
} else if (drawerMode == "share-timestamp") {
content = (
<div className="w-full">
web/src/components/player/HlsVideoPlayer.tsx
+189 -40
View File
@@ -26,7 +26,7 @@ import { useIsAdmin } from "@/hooks/use-is-admin";
// Android native hls does not seek correctly
const USE_NATIVE_HLS = false;
const HLS_MIME_TYPE = "application/vnd.apple.mpegurl" as const;
const unsupportedErrorCodes = [
const unsupportedErrorCodes: number[] = [
MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED,
MediaError.MEDIA_ERR_DECODE,
];
@@ -58,6 +58,14 @@ type HlsVideoPlayerProps = {
onSnapshot?: (playTime: number) => Promise<void> | void;
toggleFullscreen?: () => void;
onError?: (error: RecordingPlayerError) => void;
onStallStart?: () => void;
onStallEnd?: () => void;
onSeekStart?: () => void;
onBandwidthSample?: (estimateBps: number, levelBitrateBps?: number) => void;
onFatalNetworkError?: () => boolean;
onFatalCodecError?: () => boolean;
initialBandwidthEstimate?: number;
bufferLength?: number;
isDetailMode?: boolean;
camera?: string;
currentTimeOverride?: number;
@@ -86,6 +94,14 @@ export default function HlsVideoPlayer({
onSnapshot,
toggleFullscreen,
onError,
onStallStart,
onStallEnd,
onSeekStart,
onBandwidthSample,
onFatalNetworkError,
onFatalCodecError,
initialBandwidthEstimate,
bufferLength,
isDetailMode = false,
camera,
currentTimeOverride,
@@ -101,9 +117,37 @@ export default function HlsVideoPlayer({
// playback
const hlsRef = useRef<Hls>(undefined);
const [useHlsCompat, setUseHlsCompat] = useState(false);
// kept in a ref so changing callback identities do not recreate the
// Hls instance; the setup effect must only re-run on source changes
const qualitySignalsRef = useRef({
onStallStart,
onStallEnd,
onSeekStart,
onBandwidthSample,
onFatalNetworkError,
onFatalCodecError,
initialBandwidthEstimate,
});
// must resolve before the first render: a mount-effect flip would run
// the first source effect in native mode, briefly handing iOS a native
// HLS src that hls.js then tears away mid-load
const [useHlsCompat, setUseHlsCompat] = useState(() => {
if (
USE_NATIVE_HLS &&
document.createElement("video").canPlayType(HLS_MIME_TYPE)
) {
return false;
}
return Hls.isSupported();
});
const [loadedMetadata, setLoadedMetadata] = useState(false);
const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
// native HLS playback has no MSE, so it recovers from pipeline errors
// by reloading the source; one attempt per source
const nativeRetryRef = useRef(0);
// a ref rather than an effect-scoped counter so the element error
// handler can hold its toast while a recovery is still possible
const mediaRecoveryBudgetRef = useRef(0);
const applyVideoDimensions = useCallback(
(width: number, height: number) => {
@@ -153,27 +197,38 @@ export default function HlsVideoPlayer({
}, [videoRef, applyVideoDimensions]);
useEffect(() => {
if (!videoRef.current) {
return;
}
if (USE_NATIVE_HLS && videoRef.current.canPlayType(HLS_MIME_TYPE)) {
return;
} else if (Hls.isSupported()) {
setUseHlsCompat(true);
}
}, [videoRef]);
qualitySignalsRef.current = {
onStallStart,
onStallEnd,
onSeekStart,
onBandwidthSample,
onFatalNetworkError,
onFatalCodecError,
initialBandwidthEstimate,
};
}, [
onStallStart,
onStallEnd,
onSeekStart,
onBandwidthSample,
onFatalNetworkError,
onFatalCodecError,
initialBandwidthEstimate,
]);
useEffect(() => {
if (!videoRef.current) {
return;
}
setLoadedMetadata(false);
// loadedMetadata is intentionally NOT reset here: on a source swap
// the element already holds a decoded frame, and keeping it visible
// bridges the gap while the new source loads
const currentPlaybackRate = videoRef.current.playbackRate;
if (!useHlsCompat) {
nativeRetryRef.current = 0;
mediaRecoveryBudgetRef.current = 0;
videoRef.current.src = currentSource.playlist;
videoRef.current.load();
return;
@@ -181,14 +236,68 @@ export default function HlsVideoPlayer({
// Base HLS configuration
const hlsConfig: Partial<HlsConfig> = {
maxBufferLength: 10,
maxBufferLength: bufferLength ?? 10,
maxBufferSize: 20 * 1000 * 1000,
startPosition: currentSource.startPosition,
};
hlsRef.current = new Hls(hlsConfig);
hlsRef.current.attachMedia(videoRef.current);
hlsRef.current.loadSource(currentSource.playlist);
// every quality switch and chunk change recreates the instance, so
// seed it to keep measured throughput across source swaps
const seedEstimate = qualitySignalsRef.current.initialBandwidthEstimate;
if (seedEstimate !== undefined && seedEstimate > 0) {
hlsConfig.abrEwmaDefaultEstimate = seedEstimate;
}
const hls = new Hls(hlsConfig);
hlsRef.current = hls;
let networkRecoveryAttempts = 0;
mediaRecoveryBudgetRef.current = 1;
hls.on(Hls.Events.ERROR, (_event, data) => {
if (data.fatal) {
if (data.type === Hls.ErrorTypes.NETWORK_ERROR) {
// prefer a quality downswitch; fall back to restarting loading
const handled =
qualitySignalsRef.current.onFatalNetworkError?.() ?? false;
if (!handled && networkRecoveryAttempts < 2) {
networkRecoveryAttempts += 1;
hls.startLoad();
}
} else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) {
// retrying the same codec cannot succeed, so a codec error
// prefers a quality downswitch over recovery
const isCodecError =
data.details ===
Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR ||
data.details === Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR;
if (isCodecError && qualitySignalsRef.current.onFatalCodecError?.()) {
return;
}
if (!isCodecError && mediaRecoveryBudgetRef.current > 0) {
mediaRecoveryBudgetRef.current -= 1;
hls.recoverMediaError();
}
}
return;
}
// hls.js reports each stall episode only once, so STALL_RESOLVED
// below is what closes it
if (data.details === Hls.ErrorDetails.BUFFER_STALLED_ERROR) {
qualitySignalsRef.current.onStallStart?.();
}
});
hls.on(Hls.Events.STALL_RESOLVED, () => {
qualitySignalsRef.current.onStallEnd?.();
});
hls.on(Hls.Events.FRAG_LOADED, () => {
// manifests are single-variant, so the bitrate is always level 0
qualitySignalsRef.current.onBandwidthSample?.(
hls.bandwidthEstimate,
hls.levels?.[0]?.bitrate || undefined,
);
});
hls.attachMedia(videoRef.current);
hls.loadSource(currentSource.playlist);
videoRef.current.playbackRate = currentPlaybackRate;
return () => {
@@ -199,7 +308,7 @@ export default function HlsVideoPlayer({
hlsRef.current.destroy();
}
};
}, [videoRef, hlsRef, useHlsCompat, currentSource]);
}, [videoRef, hlsRef, useHlsCompat, currentSource, bufferLength]);
// state handling
@@ -481,11 +590,17 @@ export default function HlsVideoPlayer({
);
}
}}
onPlaying={onPlaying}
onPlaying={() => {
qualitySignalsRef.current.onStallEnd?.();
onPlaying?.();
}}
onPause={() => {
setIsPlaying(false);
clearTimeout(bufferTimeout);
// paused time must never count as stall time
qualitySignalsRef.current.onStallEnd?.();
if (isMobile && mobileCtrlTimeout) {
clearTimeout(mobileCtrlTimeout);
}
@@ -495,13 +610,18 @@ export default function HlsVideoPlayer({
// while paused and never resumes it on seek, so a seek
// into unbuffered media would never complete
hlsRef.current?.resumeBuffering();
qualitySignalsRef.current.onSeekStart?.();
}}
onWaiting={() => {
if (onError != undefined) {
if (videoRef.current?.paused) {
return;
}
if (videoRef.current?.paused) {
return;
}
// the only stall signal under native HLS playback, which
// emits no hls.js events
qualitySignalsRef.current.onStallStart?.();
if (onError != undefined) {
setBufferTimeout(
setTimeout(() => {
if (
@@ -557,23 +677,52 @@ export default function HlsVideoPlayer({
}
}}
onError={(e) => {
if (
!hlsRef.current &&
// @ts-expect-error code does exist
unsupportedErrorCodes.includes(e.target.error.code) &&
videoRef.current
) {
setLoadedMetadata(false);
setUseHlsCompat(true);
} else {
toast.error(
// @ts-expect-error code does exist
`Failed to play recordings (error ${e.target.error.code}): ${e.target.error.message}`,
{
position: "top-center",
},
);
const mediaError = (e.target as HTMLVideoElement).error;
if (!mediaError) {
return;
}
// an intentional source swap aborts the in-flight load;
// that abort is not an error the user can act on
if (mediaError.code === MediaError.MEDIA_ERR_ABORTED) {
return;
}
// hold the toast while the fatal handler still has a retry
// left; a failed recovery raises a second element error
if (hlsRef.current && mediaRecoveryBudgetRef.current > 0) {
return;
}
if (!hlsRef.current && videoRef.current) {
if (
unsupportedErrorCodes.includes(mediaError.code) &&
Hls.isSupported()
) {
setLoadedMetadata(false);
setUseHlsCompat(true);
return;
}
// native pipeline errors around source swaps are usually
// transient, and hls.js is no fallback without MSE
if (nativeRetryRef.current < 1) {
nativeRetryRef.current += 1;
videoRef.current.load();
return;
}
}
toast.error(
t("toast.error.playRecordingsFailed", {
code: mediaError.code,
message: mediaError.message,
}),
{
position: "top-center",
},
);
}}
/>
</div>
web/src/components/player/QualitySelector.tsx
+245
View File
@@ -0,0 +1,245 @@
import { useCallback, useMemo } from "react";
import { useTranslation } from "react-i18next";
import { isDesktop } from "react-device-detect";
import { FaTriangleExclamation } from "react-icons/fa6";
import { MdHighQuality } from "react-icons/md";
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuRadioGroup,
DropdownMenuRadioItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import {
AutoQualityReason,
PLAYBACK_QUALITIES,
PlaybackQuality,
RecordingCoverage,
StreamMediaSummary,
} from "@/types/record";
const CODEC_DISPLAY_NAMES: Record<string, string> = {
h264: "H.264",
hevc: "H.265",
h265: "H.265",
av1: "AV1",
};
const AUDIO_CODEC_DISPLAY_NAMES: Record<string, string> = {
aac: "AAC",
pcm_alaw: "PCM-A",
pcm_mulaw: "PCM-U",
opus: "Opus",
mp3: "MP3",
};
type QualitySubtitleProps = {
streams?: RecordingCoverage["streams"];
autoLow?: boolean;
autoLowReason?: AutoQualityReason;
mainUnsupported?: boolean;
};
type QualitySelectorProps = QualitySubtitleProps & {
quality: PlaybackQuality;
onSetQuality: (quality: PlaybackQuality) => void;
setControlsOpen?: (open: boolean) => void;
containerRef?: React.MutableRefObject<HTMLDivElement | null>;
};
function useQualitySubtitles({
streams,
autoLow,
autoLowReason,
mainUnsupported,
}: QualitySubtitleProps) {
const { t } = useTranslation(["components/player"]);
const streamSubtitle = useCallback(
(summary?: StreamMediaSummary) => {
if (!summary) {
return undefined;
}
const parts: string[] = [];
if (summary.video_codec != null) {
parts.push(
CODEC_DISPLAY_NAMES[summary.video_codec] ??
summary.video_codec.toUpperCase(),
);
}
// the codec decides whether the browser plays sound at all (AAC
// decodes, G.711 does not), so lead with it when known
const audioCodec =
summary.audio_codec != null
? (AUDIO_CODEC_DISPLAY_NAMES[summary.audio_codec] ??
summary.audio_codec.toUpperCase())
: null;
if (summary.has_audio === false) {
parts.push(t("quality.noAudio"));
} else if (audioCodec != null && summary.audio_rate != null) {
parts.push(
t("quality.audioCodecRate", {
codec: audioCodec,
rate: summary.audio_rate / 1000,
}),
);
} else if (audioCodec != null) {
parts.push(audioCodec);
} else if (summary.audio_rate != null) {
parts.push(t("quality.audioRate", { rate: summary.audio_rate / 1000 }));
}
return parts.length ? parts.join(" · ") : undefined;
},
[t],
);
const subtitles = useMemo<Partial<Record<PlaybackQuality, string>>>(
() => ({
auto: autoLow
? t(
autoLowReason === "codec"
? "quality.autoLowCodec"
: autoLowReason === "saveData"
? "quality.autoLowSaveData"
: "quality.autoLow",
)
: undefined,
// a stream absent from the summary has no footage in this range,
// and a pin is never silently substituted
main:
streams && !streams.main
? t("quality.noRecordings")
: mainUnsupported
? t("quality.notSupportedBrowser")
: streamSubtitle(streams?.main),
sub:
streams && !streams.sub
? t("quality.noRecordings")
: streamSubtitle(streams?.sub),
}),
[autoLow, autoLowReason, mainUnsupported, streamSubtitle, streams, t],
);
return subtitles;
}
export default function QualitySelector({
quality,
onSetQuality,
setControlsOpen,
containerRef,
streams,
autoLow,
autoLowReason,
mainUnsupported,
}: QualitySelectorProps) {
const { t } = useTranslation(["components/player"]);
const subtitles = useQualitySubtitles({
streams,
autoLow,
autoLowReason,
mainUnsupported,
});
const itemContent = useCallback(
(q: PlaybackQuality) => (
<div className="flex flex-col">
<span>{t(`quality.${q}`)}</span>
{subtitles[q] && (
<span className="text-xs text-muted-foreground">{subtitles[q]}</span>
)}
</div>
),
[subtitles, t],
);
const trigger = (
<Button
className="flex items-center gap-2.5 rounded-lg"
aria-label={t("quality.label")}
size="sm"
>
<div className="relative">
<MdHighQuality className="size-5 text-secondary-foreground" />
{autoLow && (
<FaTriangleExclamation className="absolute -bottom-0.5 -right-1 size-3 text-danger" />
)}
</div>
{isDesktop && <div className="text-primary">{t("quality.label")}</div>}
</Button>
);
return (
<DropdownMenu
onOpenChange={(open) => {
if (setControlsOpen) {
setControlsOpen(open);
}
}}
>
<DropdownMenuTrigger asChild>{trigger}</DropdownMenuTrigger>
<DropdownMenuContent
portalProps={{
container: containerRef?.current,
}}
>
<DropdownMenuRadioGroup
value={quality}
onValueChange={(value) => onSetQuality(value as PlaybackQuality)}
>
{PLAYBACK_QUALITIES.map((q) => (
<DropdownMenuRadioItem key={q} value={q} className="cursor-pointer">
{itemContent(q)}
</DropdownMenuRadioItem>
))}
</DropdownMenuRadioGroup>
</DropdownMenuContent>
</DropdownMenu>
);
}
type QualitySelectorContentProps = QualitySubtitleProps & {
quality: PlaybackQuality;
onSetQuality: (quality: PlaybackQuality) => void;
};
// drawer-friendly variant of the selector for the mobile settings drawer
export function QualitySelectorContent({
quality,
onSetQuality,
streams,
autoLow,
autoLowReason,
mainUnsupported,
}: QualitySelectorContentProps) {
const { t } = useTranslation(["components/player"]);
const subtitles = useQualitySubtitles({
streams,
autoLow,
autoLowReason,
mainUnsupported,
});
return (
<div className="flex w-full flex-col items-center gap-2">
{PLAYBACK_QUALITIES.map((q) => (
<div
key={q}
className={`w-full cursor-pointer rounded-lg py-2 text-center ${quality == q ? "bg-secondary" : ""}`}
onClick={() => onSetQuality(q)}
>
<div className="smart-capitalize">{t(`quality.${q}`)}</div>
{subtitles[q] && (
<div className="text-xs text-muted-foreground">{subtitles[q]}</div>
)}
</div>
))}
</div>
);
}
web/src/components/player/dynamic/AutoQualityGovernor.ts
+372
View File
@@ -0,0 +1,372 @@
/**
* Policy engine for auto playback quality downswitching.
*
* Stall time is measured rather than counted: hls.js reports
* BUFFER_STALLED_ERROR only once per episode (the flag resets only when
* playback resumes), so counting events makes the worst networks, where
* one freeze never resolves, the least likely to ever downswitch.
*/
export type DownswitchReason =
| "stall"
| "bandwidth"
| "fatal-error"
| "startup"
| "codec";
// stalls just after a seek are expected on any network (the target
// position is rarely buffered), so they get a longer budget and are
// kept out of the cumulative window
const SEEK_GRACE_MS = 2000;
// a single unresolved stall episode this long triggers a downswitch
const SINGLE_STALL_DOWNSWITCH_MS = 4000;
// seek-adjacent episodes only trigger once clearly beyond load latency
const GRACED_STALL_DOWNSWITCH_MS = 10000;
// total (non-graced) stall time within the rolling window that triggers
const CUMULATIVE_STALL_DOWNSWITCH_MS = 7000;
// rolling window for cumulative stall accounting; long enough to catch
// chronic short stalls, short enough that ancient history ages out
const STALL_WINDOW_MS = 60000;
// a throughput sample below bitrate * margin counts as evidence the
// connection cannot sustain the stream
const PREDICTIVE_BANDWIDTH_MARGIN = 1.1;
// consecutive low samples required for a predictive (pre-stall) downswitch
const PREDICTIVE_SAMPLE_COUNT = 3;
// measured throughput must clear the original stream's bitrate by this
// margin before a downswitched player retries full quality
const RETRY_BANDWIDTH_MARGIN = 1.5;
// the stall clock is blind before playback starts (the player is still
// paused), so the initial load needs its own budget
const STARTUP_DOWNSWITCH_MS = 10000;
// no realistic original recording stream plays comfortably below this,
// so a camera whose bitrate is not yet known starts low
const KNOWN_SLOW_START_FLOOR_BPS = 3_000_000;
// the first sample is biased toward the seeded default estimate
const PROBE_MIN_SUB_SAMPLES = 2;
type StallEpisode = {
start: number;
end: number;
};
export class AutoQualityGovernor {
// returns false when quality is pinned, sub is unavailable, or the
// player is already low
private requestDownswitch: (reason: DownswitchReason) => boolean;
private requestUpswitch: (() => void) | undefined;
private episodes: StallEpisode[] = [];
private openEpisode: { start: number; graced: boolean } | null = null;
private stallTimer: ReturnType<typeof setTimeout> | undefined;
private startupTimer: ReturnType<typeof setTimeout> | undefined;
private lastSeekTs = 0;
private consecutiveLowSamples = 0;
private upswitchProbeArmed = false;
private probeSampleCount = 0;
private mainUnplayable = false;
private holdLow = false;
// network facts survive stall-history resets: a manual pin or camera
// switch does not change what the connection can carry
private bandwidthEstimateBps: number | undefined;
private mainBitrateBps: number | undefined;
constructor(
requestDownswitch: (reason: DownswitchReason) => boolean,
requestUpswitch?: () => void,
) {
this.requestDownswitch = requestDownswitch;
this.requestUpswitch = requestUpswitch;
}
get bandwidthEstimate(): number | undefined {
return this.bandwidthEstimateBps;
}
/** Seed the connection estimate persisted from earlier sessions. */
seed(bandwidthEstimateBps: number | undefined) {
if (this.bandwidthEstimateBps === undefined) {
this.bandwidthEstimateBps = bandwidthEstimateBps;
}
}
/**
* Suppresses every path that would route playback back onto the
* original stream.
*/
markMainUnplayable() {
this.mainUnplayable = true;
}
get isMainUnplayable(): boolean {
return this.mainUnplayable;
}
/**
* Hold playback on the low stream regardless of measured headroom
* (user preference such as data saver, not a bandwidth fact).
*/
setHoldLow(hold: boolean) {
this.holdLow = hold;
}
/**
* Record the original stream's advertised bitrate learned outside of
* playback (e.g. parsed from its master playlist). Live measurements
* take precedence.
*/
learnMainBitrate(bitrateBps: number) {
if (this.mainBitrateBps === undefined && bitrateBps > 0) {
this.mainBitrateBps = bitrateBps;
}
}
/**
* Starts the time-to-first-frame budget: no stall episode can exist
* before playback starts, so a first segment too large for the
* connection would otherwise spin forever.
*/
sourceLoadStarted() {
clearTimeout(this.startupTimer);
this.startupTimer = setTimeout(
() => this.triggerDownswitch("startup"),
STARTUP_DOWNSWITCH_MS,
);
}
sourceLoadEnded() {
clearTimeout(this.startupTimer);
this.startupTimer = undefined;
}
/**
* Arm the one-shot upswitch probe after a conservative low start.
* Stays armed until it fires or a manual pin resets it, so a
* connection that improves later still recovers mid-chunk.
*/
armUpswitchProbe() {
this.upswitchProbeArmed = true;
this.probeSampleCount = 0;
}
noteSeek() {
this.lastSeekTs = Date.now();
}
/**
* A stall episode began (hls.js BUFFER_STALLED_ERROR or a video
* element waiting event). Idempotent while an episode is open, so the
* two signal sources need no cross-coordination.
*/
stallStarted() {
if (this.openEpisode) {
return;
}
const now = Date.now();
const graced = now - this.lastSeekTs < SEEK_GRACE_MS;
this.openEpisode = { start: now, graced };
// fire mid-stall: either this episode alone exceeds its budget, or
// it pushes the window's cumulative stall time over the threshold
const singleBudget = graced
? GRACED_STALL_DOWNSWITCH_MS
: SINGLE_STALL_DOWNSWITCH_MS;
const cumulativeBudget = graced
? Number.POSITIVE_INFINITY
: Math.max(0, CUMULATIVE_STALL_DOWNSWITCH_MS - this.windowStallMs(now));
this.stallTimer = setTimeout(
() => this.triggerDownswitch("stall"),
Math.min(singleBudget, cumulativeBudget),
);
}
/**
* Playback resumed (STALL_RESOLVED, playing, timeupdate) or paused.
* Closes any open episode; graced episodes never enter the window.
*/
stallEnded() {
if (!this.openEpisode) {
return;
}
clearTimeout(this.stallTimer);
this.stallTimer = undefined;
const now = Date.now();
if (!this.openEpisode.graced && now > this.openEpisode.start) {
this.episodes.push({ start: this.openEpisode.start, end: now });
}
this.openEpisode = null;
this.pruneEpisodes(now);
}
/**
* A segment finished loading. Records throughput, refreshes the
* original stream's bitrate while playing it, and downswitches
* predictively when sustained throughput cannot carry the stream.
*/
bandwidthSample(
estimateBps: number,
levelBitrateBps: number | undefined,
playingMain: boolean,
) {
if (!Number.isFinite(estimateBps) || estimateBps <= 0) {
return;
}
this.bandwidthEstimateBps = estimateBps;
if (!playingMain) {
this.consecutiveLowSamples = 0;
this.probeSampleCount += 1;
if (
this.upswitchProbeArmed &&
!this.mainUnplayable &&
!this.holdLow &&
this.probeSampleCount >= PROBE_MIN_SUB_SAMPLES &&
this.mainBitrateBps !== undefined &&
estimateBps > this.mainBitrateBps * RETRY_BANDWIDTH_MARGIN
) {
this.upswitchProbeArmed = false;
this.requestUpswitch?.();
}
return;
}
if (levelBitrateBps === undefined || levelBitrateBps <= 0) {
return;
}
this.mainBitrateBps = levelBitrateBps;
if (estimateBps < levelBitrateBps * PREDICTIVE_BANDWIDTH_MARGIN) {
this.consecutiveLowSamples += 1;
if (this.consecutiveLowSamples >= PREDICTIVE_SAMPLE_COUNT) {
this.consecutiveLowSamples = 0;
this.triggerDownswitch("bandwidth");
}
} else {
this.consecutiveLowSamples = 0;
}
}
/**
* hls.js gave up loading (retries exhausted). Returns whether a
* downswitch happened so the player knows to attempt recovery instead.
*/
fatalNetworkError(): boolean {
return this.triggerDownswitch("fatal-error");
}
/**
* Unlike bandwidth signals a codec failure is proof, so the original
* stream is marked unplayable before the downswitch. Returns whether
* a downswitch happened.
*/
fatalCodecError(): boolean {
this.mainUnplayable = true;
return this.triggerDownswitch("codec");
}
/**
* Whether a downswitched player should retry full quality at the next
* chunk boundary. Native HLS playback reports no segment stats, so
* without bandwidth evidence this falls back to a clean stall window.
*/
shouldRetryMain(): boolean {
if (this.mainUnplayable || this.holdLow) {
return false;
}
if (
this.bandwidthEstimateBps !== undefined &&
this.mainBitrateBps !== undefined
) {
return (
this.bandwidthEstimateBps > this.mainBitrateBps * RETRY_BANDWIDTH_MARGIN
);
}
return this.windowStallMs(Date.now()) === 0;
}
/**
* Whether playback should begin on the low quality stream based on
* persisted network knowledge. A fully-cold device returns false; the
* owner handles that case with a conservative start plus the probe.
*/
shouldStartLow(): boolean {
if (this.bandwidthEstimateBps === undefined) {
return false;
}
if (this.mainBitrateBps !== undefined) {
return (
this.bandwidthEstimateBps <
this.mainBitrateBps * PREDICTIVE_BANDWIDTH_MARGIN
);
}
// unknown camera bitrate: above the floor, start on the original
// and let the startup budget correct a wrong guess
return this.bandwidthEstimateBps < KNOWN_SLOW_START_FLOOR_BPS;
}
/** A manual pin invalidates stall history but not network facts. */
resetStallHistory() {
clearTimeout(this.stallTimer);
this.stallTimer = undefined;
clearTimeout(this.startupTimer);
this.startupTimer = undefined;
this.openEpisode = null;
this.episodes = [];
this.consecutiveLowSamples = 0;
this.upswitchProbeArmed = false;
this.probeSampleCount = 0;
}
/**
* A camera switch additionally invalidates the per-camera facts: the
* stream bitrate and codec playability. The holdLow preference is
* device-level and survives.
*/
resetForCamera() {
this.resetStallHistory();
this.mainBitrateBps = undefined;
this.mainUnplayable = false;
}
destroy() {
this.resetStallHistory();
}
private triggerDownswitch(reason: DownswitchReason): boolean {
const handled = this.requestDownswitch(reason);
if (handled) {
// the low stream starts with a clean record
this.resetStallHistory();
}
return handled;
}
private windowStallMs(now: number): number {
this.pruneEpisodes(now);
const windowStart = now - STALL_WINDOW_MS;
let total = 0;
for (const episode of this.episodes) {
total += episode.end - Math.max(episode.start, windowStart);
}
if (this.openEpisode && !this.openEpisode.graced) {
total += now - Math.max(this.openEpisode.start, windowStart);
}
return total;
}
private pruneEpisodes(now: number) {
const windowStart = now - STALL_WINDOW_MS;
this.episodes = this.episodes.filter(
(episode) => episode.end > windowStart,
);
}
}
web/src/components/player/dynamic/DynamicVideoController.ts
+37 -23
View File
@@ -28,7 +28,6 @@ export class DynamicVideoController {
private timeRange: TimeRange = { after: 0, before: 0 };
private inpointOffset: number = 0;
private annotationOffset: number;
private timeToStart: number | undefined = undefined;
constructor(
camera: string,
@@ -55,11 +54,6 @@ export class DynamicVideoController {
this.timeRange.after,
this.recordings[0],
);
if (this.timeToStart) {
this.seekToTimestamp(this.timeToStart);
this.timeToStart = undefined;
}
}
play() {
@@ -75,8 +69,11 @@ export class DynamicVideoController {
}
seekToTimestamp(time: number, play: boolean = false) {
// a seek outside the current playback window is a no-op: the view
// moves its anchor and chunk on such seeks, and the rebuilt source
// resumes at the anchor (startPosition plus the post-load seek).
// Seeking here would only reposition the outgoing source's media
if (time < this.timeRange.after || time > this.timeRange.before) {
this.timeToStart = time;
return;
}
@@ -95,16 +92,24 @@ export class DynamicVideoController {
return;
}
if (seekSeconds != 0) {
this.playerController.currentTime = seekSeconds;
if (this.playerController.currentTime === seekSeconds) {
// seeking to the current position fires no seeked event, so apply
// the play intent directly (this includes position 0, which the
// player sits at before its first seek)
if (play) {
this.waitAndPlay();
playWithTemporaryMuteFallback(this.playerController);
} else {
this.playerController.pause();
}
return;
}
this.playerController.currentTime = seekSeconds;
if (play) {
this.waitAndPlay();
} else {
// no op
this.playerController.pause();
}
}
@@ -139,20 +144,26 @@ export class DynamicVideoController {
getProgress(playerTime: number): number {
// take a player time in seconds and convert to timestamp in timeline
let timestamp = 0;
const recordings = this.recordings || [];
let totalTime = 0;
(this.recordings || []).every((segment) => {
for (const segment of recordings) {
if (totalTime + segment.duration > playerTime) {
// segment is here
timestamp = segment.start_time + (playerTime - totalTime);
return false;
} else {
totalTime += segment.duration;
return true;
// playlist media from before the span's wall start (keyframe
// back-snap lead-in) clamps to the span start
const wallLength = segment.end_time - segment.start_time;
const leadIn = Math.max(0, segment.duration - wallLength);
return (
segment.start_time + Math.max(0, playerTime - totalTime - leadIn)
);
}
});
totalTime += segment.duration;
}
return timestamp;
// past the modeled total: clamp to the covered end rather than
// reporting wall-clock zero
return recordings.length > 0
? recordings[recordings.length - 1].end_time
: 0;
}
scrubToTimestamp(time: number, saveIfNotReady: boolean = false) {
@@ -162,7 +173,10 @@ export class DynamicVideoController {
this.previewController.setNewPreviewStartTime(time);
}
if (scrubResult && this.playerMode != "scrubbing") {
// pause even when no preview can render this range: a hidden player
// left running reports stale times once the drag releases, bouncing
// the handlebar back and sometimes swallowing the release seek
if (this.playerMode != "scrubbing") {
this.playerMode = "scrubbing";
this.playerController.pause();
}
web/src/components/player/dynamic/DynamicVideoPlayer.tsx
+428 -21
View File
@@ -9,7 +9,12 @@ import {
import { useApiHost } from "@/api";
import useSWR from "swr";
import { FrigateConfig } from "@/types/frigateConfig";
import { Recording } from "@/types/record";
import {
AutoQualityReason,
PlaybackQuality,
Recording,
RecordingCoverage,
} from "@/types/record";
import { Preview } from "@/types/preview";
import PreviewPlayer, { PreviewController } from "../PreviewPlayer";
import { DynamicVideoController } from "./DynamicVideoController";
@@ -32,6 +37,13 @@ import {
grabVideoSnapshot,
} from "@/utils/snapshotUtil";
import { isFirefox } from "react-device-detect";
import { AutoQualityGovernor } from "./AutoQualityGovernor";
import { isCodecFamilySupported } from "@/utils/codecSupport";
import { useUserPersistence } from "@/hooks/use-user-persistence";
// forward buffer while playing the low quality stream; low bitrate makes
// a longer buffer cheap and it rides out connection variance better
const SUB_STREAM_BUFFER_LENGTH_S = 30;
/**
* Dynamically switches between video playback and scrubbing preview player.
@@ -55,6 +67,11 @@ type DynamicVideoPlayerProps = {
toggleFullscreen: () => void;
containerRef?: React.MutableRefObject<HTMLDivElement | null>;
transformedOverlay?: ReactNode;
quality?: PlaybackQuality;
onAutoQualityChange?: (
lowQuality: boolean,
reason: AutoQualityReason | undefined,
) => void;
};
export default function DynamicVideoPlayer({
className,
@@ -75,6 +92,8 @@ export default function DynamicVideoPlayer({
toggleFullscreen,
containerRef,
transformedOverlay,
quality,
onAutoQualityChange,
}: DynamicVideoPlayerProps) {
const { t } = useTranslation(["components/player", "views/live"]);
const apiHost = useApiHost();
@@ -128,7 +147,7 @@ export default function DynamicVideoPlayer({
const [isLoading, setIsLoading] = useState(false);
const [isBuffering, setIsBuffering] = useState(false);
const [loadingTimeout, setLoadingTimeout] = useState<NodeJS.Timeout>();
const loadingTimeoutRef = useRef<NodeJS.Timeout | undefined>(undefined);
// Don't set source until recordings load - we need accurate startPosition
// to avoid hls.js clamping to video end when startPosition exceeds duration
@@ -138,32 +157,80 @@ export default function DynamicVideoPlayer({
useEffect(() => {
if (!isScrubbing) {
setLoadingTimeout(setTimeout(() => setIsLoading(true), 1000));
loadingTimeoutRef.current = setTimeout(() => setIsLoading(true), 1000);
}
return () => {
if (loadingTimeout) {
clearTimeout(loadingTimeout);
if (loadingTimeoutRef.current) {
clearTimeout(loadingTimeoutRef.current);
}
};
// we only want trigger when scrubbing state changes
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [camera, isScrubbing]);
// wall-clock position to resume from once the current source finishes
// loading. A seek landing mid-load must win over the position the
// source was built around, or the post-load seek drags playback back
const sourceAnchorRef = useRef<number | undefined>(undefined);
useEffect(() => {
sourceAnchorRef.current = startTimestamp;
}, [startTimestamp]);
// a recordings change refined the seek model without changing the
// playlist, so the playback effect skips its loading indicator
const modelOnlyUpdateRef = useRef(false);
const onPlayerLoaded = useCallback(() => {
if (!controller || !startTimestamp) {
sourceLoadedRef.current = true;
governorRef.current?.sourceLoadEnded();
const anchor = sourceAnchorRef.current;
if (!controller || !anchor) {
return;
}
controller.seekToTimestamp(startTimestamp, true);
}, [startTimestamp, controller]);
// an anchor outside this chunk is stale (e.g. a natural clip
// advance); the playlist already starts where playback should
if (anchor < timeRange.after || anchor > timeRange.before) {
return;
}
// while the handlebar is down only position the hidden player, never
// start it: a mid-drag chunk prefetch can audibly blip before
// onPlaying pauses it. The release seek starts playback
controller.seekToTimestamp(anchor, !isScrubbing);
}, [controller, timeRange, isScrubbing]);
// used to re-anchor the source when an auto quality switch rebuilds
// the playlist mid-playback
const lastPlayedTimestampRef = useRef<number | undefined>(undefined);
// the range the controller's playback model was last built for; while
// a chunk change awaits its coverage, the outgoing source reports
// times that would map through the stale model
const modelTimeRangeRef = useRef<TimeRange | undefined>(undefined);
const onTimeUpdate = useCallback(
(time: number) => {
// safety net for stall or startup signals the player missed
governorRef.current?.stallEnded();
if (!sourceLoadedRef.current) {
sourceLoadedRef.current = true;
governorRef.current?.sourceLoadEnded();
}
if (isScrubbing || !controller || !onTimestampUpdate || time == 0) {
return;
}
// drop reports until the controller's model matches this chunk
if (
modelTimeRangeRef.current?.after !== timeRange.after ||
modelTimeRangeRef.current?.before !== timeRange.before
) {
return;
}
if (isLoading) {
setIsLoading(false);
}
@@ -172,9 +239,18 @@ export default function DynamicVideoPlayer({
setIsBuffering(false);
}
onTimestampUpdate(controller.getProgress(time));
const progress = controller.getProgress(time);
lastPlayedTimestampRef.current = progress;
onTimestampUpdate(progress);
},
[controller, onTimestampUpdate, isBuffering, isLoading, isScrubbing],
[
controller,
onTimestampUpdate,
isBuffering,
isLoading,
isScrubbing,
timeRange,
],
);
const onUploadFrameToPlus = useCallback(
@@ -238,45 +314,350 @@ export default function DynamicVideoPlayer({
() => ({
before: timeRange.before,
after: timeRange.after,
timelines: true,
}),
[timeRange],
);
const { data: recordings } = useSWR<Recording[]>(
[`${camera}/recordings`, recordingParams],
const { data: coverage } = useSWR<RecordingCoverage>(
[`${camera}/recordings/coverage`, recordingParams],
{ revalidateOnFocus: false },
);
// auto quality plays the default route until the governor downswitches
// to the pinned sub route; manual pins bypass this entirely
const [autoLowQuality, setAutoLowQuality] = useState(false);
const [autoLowReason, setAutoLowReason] = useState<
AutoQualityReason | undefined
>(undefined);
const autoLowQualityRef = useRef(false);
const subAvailable = useMemo(
() =>
coverage?.spans?.some((span) => span.streams.includes("sub")) ?? false,
[coverage],
);
const resolvedQuality = quality ?? "auto";
// the ref indirection keeps these reading fresh state while the
// governor stays a single instance for the component's lifetime
const tryDownswitchRef = useRef<(reason: string) => boolean>(() => false);
const tryUpswitchRef = useRef<() => void>(() => {});
const governorRef = useRef<AutoQualityGovernor | null>(null);
if (governorRef.current === null) {
governorRef.current = new AutoQualityGovernor(
(reason) => tryDownswitchRef.current(reason),
() => tryUpswitchRef.current(),
);
}
const governor = governorRef.current;
// callers pass an inline callback, so keeping it out of the notify
// effect's deps stops the notification's re-render from re-firing it
const onAutoQualityChangeRef = useRef(onAutoQualityChange);
useEffect(() => {
onAutoQualityChangeRef.current = onAutoQualityChange;
}, [onAutoQualityChange]);
useEffect(() => {
autoLowQualityRef.current = autoLowQuality;
onAutoQualityChangeRef.current?.(
autoLowQuality,
autoLowQuality ? autoLowReason : undefined,
);
}, [autoLowQuality, autoLowReason]);
useEffect(() => {
tryDownswitchRef.current = (reason: string) => {
if (
resolvedQuality !== "auto" ||
!subAvailable ||
autoLowQualityRef.current
) {
return false;
}
setAutoLowQuality(true);
setAutoLowReason(reason === "codec" ? "codec" : "bandwidth");
// so a recovered connection (or a wrong downswitch) returns to
// full quality mid-chunk rather than at the next boundary
governor.armUpswitchProbe();
return true;
};
tryUpswitchRef.current = () => {
if (resolvedQuality === "auto" && autoLowQualityRef.current) {
setAutoLowQuality(false);
setAutoLowReason(undefined);
}
};
}, [resolvedQuality, subAvailable, governor]);
// persisted across sessions so a device on a known-slow connection
// starts low instead of paying the first stall to find out
const [persistedEstimate, setPersistedEstimate, estimateLoaded] =
useUserPersistence<number>("playbackBandwidthEstimate");
const persistGovernor = useCallback(() => {
const estimate = governor.bandwidthEstimate;
if (estimate !== undefined) {
setPersistedEstimate(Math.round(estimate));
}
}, [governor, setPersistedEstimate]);
const persistGovernorRef = useRef(persistGovernor);
useEffect(() => {
persistGovernorRef.current = persistGovernor;
}, [persistGovernor]);
useEffect(() => {
// returning to auto starts fresh on the default route, except when
// this browser already proved it cannot decode the original stream
governor.resetStallHistory();
setAutoLowQuality(governor.isMainUnplayable);
setAutoLowReason(governor.isMainUnplayable ? "codec" : undefined);
// we only want to reset when the pinned quality changes
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [quality]);
useEffect(() => {
// measured connection throughput carries over across cameras
governor.resetForCamera();
setAutoLowQuality(false);
setAutoLowReason(undefined);
// we only want to reset when the camera changes
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [camera]);
// seed the governor once per camera, then decide the starting quality
const seededCameraRef = useRef<string | null>(null);
useEffect(() => {
if (seededCameraRef.current === camera || !estimateLoaded || !coverage) {
return;
}
seededCameraRef.current = camera;
const mainSummary = coverage.streams?.main;
if (mainSummary?.bitrate) {
governor.learnMainBitrate(mainSummary.bitrate);
}
governor.seed(persistedEstimate);
if (resolvedQuality !== "auto" || !subAvailable) {
return;
}
// data saver is a user preference, not a bandwidth fact: hold the
// low stream and never auto-upswitch against it (a manual pin to
// Original still wins as an explicit action)
const saveData =
(navigator as Navigator & { connection?: { saveData?: boolean } })
.connection?.saveData === true;
if (saveData) {
governor.setHoldLow(true);
}
// a browser that cannot decode the original codec can never play
// the merged route. This probe fails open (unknown codecs count as
// supported); the reactive fatal-codec path is the real authority
const mainSupported = isCodecFamilySupported(mainSummary?.video_codec);
const subSupported = isCodecFamilySupported(
coverage.streams?.sub?.video_codec,
);
if (!mainSupported && subSupported) {
governor.markMainUnplayable();
setAutoLowQuality(true);
setAutoLowReason("codec");
return;
}
if (saveData) {
setAutoLowQuality(true);
setAutoLowReason("saveData");
return;
}
// a fully cold device also starts low: the conservative start shows
// a first frame in seconds and the armed probe recovers full
// quality within a few segment loads on connections that allow it
const coldStart = governor.bandwidthEstimate === undefined;
if (!coldStart && !governor.shouldStartLow()) {
return;
}
setAutoLowQuality(true);
setAutoLowReason("bandwidth");
governor.armUpswitchProbe();
}, [
camera,
coverage,
estimateLoaded,
persistedEstimate,
resolvedQuality,
subAvailable,
governor,
]);
// time-to-first-frame budget; the stall clock is blind before
// playback starts, so an oversized first segment would spin forever
const sourceLoadedRef = useRef(false);
useEffect(() => {
sourceLoadedRef.current = false;
}, [source]);
useEffect(() => {
if (!source || isScrubbing || sourceLoadedRef.current) {
governor.sourceLoadEnded();
return;
}
governor.sourceLoadStarted();
}, [source, isScrubbing, governor]);
useEffect(() => {
// a chunk boundary is where full quality may be retried, and a
// natural point to persist what the governor has learned
setAutoLowQuality((prev) => prev && !governor.shouldRetryMain());
persistGovernorRef.current();
// we only want to re-evaluate when the playback chunk changes
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [timeRange]);
useEffect(() => {
return () => {
persistGovernorRef.current();
governor.destroy();
};
// governor is a stable per-mount instance
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const effectiveQuality: PlaybackQuality =
resolvedQuality === "auto" && autoLowQuality ? "sub" : resolvedQuality;
const onStallStart = useCallback(() => governor.stallStarted(), [governor]);
const onStallEnd = useCallback(() => governor.stallEnded(), [governor]);
const onSeekStart = useCallback(() => governor.noteSeek(), [governor]);
const onFatalNetworkError = useCallback(
() => governor.fatalNetworkError(),
[governor],
);
const onFatalCodecError = useCallback(
() => governor.fatalCodecError(),
[governor],
);
const onBandwidthSample = useCallback(
(estimateBps: number, levelBitrateBps?: number) =>
governor.bandwidthSample(
estimateBps,
levelBitrateBps,
// the merged default route leads with the original stream, so
// its samples measure original-quality sustainability
effectiveQuality !== "sub",
),
[governor, effectiveQuality],
);
// the realized timelines mirror the vod manifests exactly, including
// keyframe back-snap lead-in at cross-stream hand-offs. Walking wall
// lengths instead drifts ~0.5s per hand-off, since the playlist
// contains lead-in media the model never knew about
const recordings = useMemo<Recording[] | undefined>(() => {
const timeline =
coverage?.timelines?.[
effectiveQuality === "main" || effectiveQuality === "sub"
? effectiveQuality
: "auto"
];
if (!timeline) {
return undefined;
}
return timeline.map((span) => ({
start_time: span.start_time,
end_time: span.end_time,
duration: span.duration / 1000,
})) as Recording[];
}, [coverage, effectiveQuality]);
// lets the effect below tell quality rebuilds apart from chunk changes
const prevEffectiveQualityRef = useRef(effectiveQuality);
useEffect(() => {
const qualityChanged = prevEffectiveQualityRef.current !== effectiveQuality;
prevEffectiveQualityRef.current = effectiveQuality;
if (!recordings?.length) {
if (recordings?.length == 0) {
// drop any stale source so the previous playlist unmounts
// instead of playing under the no-recording state
setSource(undefined);
setNoRecording(true);
// with no source nothing will play to clear a pending
// camera-switch load, hiding the message behind a preview frame
if (loadingTimeoutRef.current) {
clearTimeout(loadingTimeoutRef.current);
}
setIsLoading(false);
}
return;
}
// an identical playlist means coverage only refined the seek model;
// skip the rebuild so the player is not torn down
const streamPath =
effectiveQuality === "main" || effectiveQuality === "sub"
? `/${effectiveQuality}`
: "";
const playlist = `${apiHost}vod/${camera}${streamPath}/start/${recordingParams.after}/end/${recordingParams.before}/master.m3u8`;
if (!qualityChanged && source?.playlist === playlist) {
modelOnlyUpdateRef.current = true;
return;
}
// a quality switch rebuilds mid-playback, so anchor to the live
// playhead rather than the chunk-stale startTimestamp prop. The
// controller still holds the OUTGOING timeline here (newPlayback
// runs in a later effect), and the timeupdate-throttled lastPlayed
// ref lags the frame on screen by up to ~250ms
const liveTime = playerRef.current?.currentTime;
const livePlayed =
qualityChanged && controller && liveTime !== undefined && liveTime > 0
? controller.getProgress(liveTime)
: undefined;
const lastPlayed = livePlayed ?? lastPlayedTimestampRef.current;
const anchorTimestamp =
qualityChanged &&
lastPlayed !== undefined &&
lastPlayed >= timeRange.after &&
lastPlayed <= timeRange.before
? lastPlayed
: startTimestamp;
sourceAnchorRef.current = anchorTimestamp;
let startPosition = undefined;
if (startTimestamp) {
if (anchorTimestamp) {
const inpointOffset = calculateInpointOffset(
recordingParams.after,
(recordings || [])[0],
);
startPosition = calculateSeekPosition(
startTimestamp,
anchorTimestamp,
recordings,
inpointOffset,
);
}
setSource({
playlist: `${apiHost}vod/${camera}/start/${recordingParams.after}/end/${recordingParams.before}/master.m3u8`,
playlist,
startPosition,
});
// we only want to rebuild the source when the playlist itself changes;
// startTimestamp, timeRange, and the anchor refs are read as-of-rebuild
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [recordings]);
}, [recordings, effectiveQuality]);
useEffect(() => {
if (!controller || !recordings?.length) {
@@ -287,12 +668,28 @@ export default function DynamicVideoPlayer({
playerRef.current.autoplay = !isScrubbing;
}
setLoadingTimeout(setTimeout(() => setIsLoading(true), 1000));
const modelOnlyUpdate = modelOnlyUpdateRef.current;
modelOnlyUpdateRef.current = false;
// on a source swap the element already has a decoded frame; keep it
// visible under the buffering indicator rather than hiding it
// behind the preview player like the initial load does
const hasDecodedFrame =
(playerRef.current?.readyState ?? 0) >=
HTMLMediaElement.HAVE_CURRENT_DATA;
if (!modelOnlyUpdate) {
loadingTimeoutRef.current = setTimeout(
() => (hasDecodedFrame ? setIsBuffering(true) : setIsLoading(true)),
1000,
);
}
controller.newPlayback({
recordings: recordings ?? [],
timeRange,
});
modelTimeRangeRef.current = timeRange;
// we only want this to change when controller or recordings update
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -356,8 +753,8 @@ export default function DynamicVideoPlayer({
playerRef.current?.pause();
}
if (loadingTimeout) {
clearTimeout(loadingTimeout);
if (loadingTimeoutRef.current) {
clearTimeout(loadingTimeoutRef.current);
}
setNoRecording(false);
@@ -372,6 +769,16 @@ export default function DynamicVideoPlayer({
setIsBuffering(true);
}
}}
onStallStart={onStallStart}
onStallEnd={onStallEnd}
onSeekStart={onSeekStart}
onBandwidthSample={onBandwidthSample}
onFatalNetworkError={onFatalNetworkError}
onFatalCodecError={onFatalCodecError}
initialBandwidthEstimate={governor.bandwidthEstimate}
bufferLength={
effectiveQuality === "sub" ? SUB_STREAM_BUFFER_LENGTH_S : undefined
}
isDetailMode={isDetailMode}
camera={contextCamera || camera}
currentTimeOverride={currentTime}
web/src/components/settings/wizard/Step3StreamConfig.tsx
+45 -21
View File
@@ -45,6 +45,13 @@ import {
CommandList,
} from "@/components/ui/command";
// Recording the sub stream from the same stream as record would just
// re-record the main stream, so the two roles are mutually exclusive.
const CONFLICTING_ROLES: Partial<Record<StreamRole, StreamRole>> = {
record: "record_sub",
record_sub: "record",
};
type Step3StreamConfigProps = {
wizardData: Partial<WizardFormData>;
onUpdate: (data: Partial<WizardFormData>) => void;
@@ -163,9 +170,12 @@ export default function Step3StreamConfig({
const newRoles = stream.roles.filter((r) => r !== role);
updateStream(streamId, { roles: newRoles });
} else {
// Check if role is already used in another stream
const usedRoles = getUsedRolesExcludingStream(streamId);
if (!usedRoles.has(role)) {
const conflictingRole = CONFLICTING_ROLES[role];
const hasConflict = conflictingRole
? stream.roles.includes(conflictingRole)
: false;
if (!usedRoles.has(role) && !hasConflict) {
// Allow adding the role
const newRoles = [...stream.roles, role];
updateStream(streamId, { roles: newRoles });
@@ -617,6 +627,10 @@ export default function Step3StreamConfig({
<strong>record</strong> -{" "}
{t("cameraWizard.step3.rolesPopover.record")}
</div>
<div>
<strong>record_sub</strong> -{" "}
{t("cameraWizard.step3.rolesPopover.record_sub")}
</div>
<div>
<strong>audio</strong> -{" "}
{t("cameraWizard.step3.rolesPopover.audio")}
@@ -639,25 +653,35 @@ export default function Step3StreamConfig({
</div>
<div className="rounded-lg bg-background p-3">
<div className="flex flex-wrap gap-2">
{(["detect", "record", "audio"] as const).map((role) => {
const isUsedElsewhere = getUsedRolesExcludingStream(
stream.id,
).has(role);
const isChecked = stream.roles.includes(role);
return (
<div
key={role}
className="flex w-full items-center justify-between"
>
<span className="text-sm capitalize">{role}</span>
<Switch
checked={isChecked}
onCheckedChange={() => toggleRole(stream.id, role)}
disabled={!isChecked && isUsedElsewhere}
/>
</div>
);
})}
{(["detect", "record", "record_sub", "audio"] as const).map(
(role) => {
const isUsedElsewhere = getUsedRolesExcludingStream(
stream.id,
).has(role);
const conflictingRole = CONFLICTING_ROLES[role];
const hasConflict = conflictingRole
? stream.roles.includes(conflictingRole)
: false;
const isChecked = stream.roles.includes(role);
return (
<div
key={role}
className="flex w-full items-center justify-between"
>
<span className="text-sm capitalize">{role}</span>
<Switch
checked={isChecked}
onCheckedChange={() =>
toggleRole(stream.id, role)
}
disabled={
!isChecked && (isUsedElsewhere || hasConflict)
}
/>
</div>
);
},
)}
</div>
</div>
</div>
web/src/components/timeline/MotionReviewTimeline.tsx
+14
View File
@@ -42,6 +42,7 @@ export type MotionReviewTimelineProps = {
events: ReviewSegment[];
motion_events: MotionData[];
noRecordingRanges?: RecordingSegment[];
subOnlyRanges?: Pick<RecordingSegment, "start_time" | "end_time">[];
contentRef: RefObject<HTMLDivElement | null>;
timelineRef?: RefObject<HTMLDivElement | null>;
onHandlebarDraggingChange?: (isDragging: boolean) => void;
@@ -76,6 +77,7 @@ export function MotionReviewTimeline({
events,
motion_events,
noRecordingRanges,
subOnlyRanges,
contentRef,
timelineRef,
onHandlebarDraggingChange,
@@ -122,6 +124,17 @@ export function MotionReviewTimeline({
[noRecordingRanges],
);
const getIsSubOnly = useCallback(
(time: number): boolean => {
if (subOnlyRanges == undefined) return false;
return subOnlyRanges.some(
(range) => time >= range.start_time && time < range.end_time,
);
},
[subOnlyRanges],
);
const segmentTimes = useMemo(() => {
const segments = [];
let segmentTime = timelineStartAligned;
@@ -245,6 +258,7 @@ export function MotionReviewTimeline({
motionOnly={motionOnly}
getMotionSegmentValue={getMotionSegmentValue}
getRecordingAvailability={getRecordingAvailability}
getIsSubOnly={getIsSubOnly}
alwaysShowMotionLine={alwaysShowMotionLine}
/>
</ReviewTimeline>
web/src/components/timeline/MotionSegment.tsx
+6
View File
@@ -5,6 +5,7 @@ import React, { useCallback, useEffect, useMemo, useRef } from "react";
import { MinimapBounds, Tick, Timestamp } from "./segment-metadata";
import { useMotionSegmentUtils } from "@/hooks/use-motion-segment-utils";
import { isMobile } from "react-device-detect";
import { useTranslation } from "react-i18next";
import useTapUtils from "@/hooks/use-tap-utils";
import { cn } from "@/lib/utils";
@@ -16,6 +17,7 @@ type MotionSegmentProps = {
firstHalfMotionValue: number;
secondHalfMotionValue: number;
hasRecording?: boolean;
isSubOnly?: boolean;
prevIsNoRecording?: boolean;
nextIsNoRecording?: boolean;
motionOnly: boolean;
@@ -36,6 +38,7 @@ export function MotionSegment({
firstHalfMotionValue,
secondHalfMotionValue,
hasRecording,
isSubOnly,
prevIsNoRecording,
nextIsNoRecording,
motionOnly,
@@ -47,6 +50,7 @@ export function MotionSegment({
dense,
alwaysShowMotionLine = false,
}: MotionSegmentProps) {
const { t } = useTranslation("views/events");
const severityType = "all";
const { getSeverity, getReviewed, displaySeverityType } =
useEventSegmentUtils(segmentDuration, events, severityType);
@@ -194,8 +198,10 @@ export function MotionSegment({
segmentClasses,
severity[0] && "bg-gradient-to-r",
severity[0] && severityColorsBg[severity[0]],
isSubOnly && "bg-background/50",
hasRecording == false && "bg-background",
)}
title={isSubOnly ? t("subOnlyQuality") : undefined}
onClick={segmentClick}
onTouchEnd={(event) => handleTouchStart(event, segmentClick)}
>
web/src/components/timeline/VirtualizedMotionSegments.tsx
+5
View File
@@ -25,6 +25,7 @@ type VirtualizedMotionSegmentsProps = {
motionOnly: boolean;
getMotionSegmentValue: (timestamp: number) => number;
getRecordingAvailability: (timestamp: number) => boolean | undefined;
getIsSubOnly: (timestamp: number) => boolean;
alwaysShowMotionLine: boolean;
};
@@ -58,6 +59,7 @@ export const VirtualizedMotionSegments = forwardRef<
motionOnly,
getMotionSegmentValue,
getRecordingAvailability,
getIsSubOnly,
alwaysShowMotionLine,
},
ref,
@@ -161,6 +163,7 @@ export const VirtualizedMotionSegments = forwardRef<
);
const hasRecording = getRecordingAvailability(segmentTime);
const isSubOnly = getIsSubOnly(segmentTime);
// Check if previous and next segments have recordings
// This is important because in motionOnly mode, the segments array is filtered
@@ -195,6 +198,7 @@ export const VirtualizedMotionSegments = forwardRef<
firstHalfMotionValue={firstHalfMotionValue}
secondHalfMotionValue={secondHalfMotionValue}
hasRecording={hasRecording}
isSubOnly={isSubOnly}
prevIsNoRecording={prevIsNoRecording}
nextIsNoRecording={nextIsNoRecording}
segmentDuration={segmentDuration}
@@ -216,6 +220,7 @@ export const VirtualizedMotionSegments = forwardRef<
events,
getMotionSegmentValue,
getRecordingAvailability,
getIsSubOnly,
motionOnly,
segmentDuration,
showMinimap,