Add Auto live view option with transcoding options (#24519)

* implement auto mode for single camera live view

* add transcoded live streams and stream ordering for auto mode

Cameras can now add lower quality live streams that go2rtc transcodes to H.264 on demand. `live.transcode` takes a source stream and a list of heights and bitrates, and each quality becomes a `{camera}_transcode_{height}p` stream. The config validator adds them to `live.streams` without moving any the user already placed, and drops them when transcoding is disabled or a height changes. `create_config.py` writes them into go2rtc's generated config at startup, and saving the config or deleting a camera syncs them through go2rtc's API, so no restart is needed. They use `#hardware`, so go2rtc picks a hardware encoder and falls back to the CPU when there isn't one.

The Live playback settings stream list can be reordered by drag, since its order is the auto ladder. Auto order sorts it by bitrate, measuring native streams through a new admin-only `/go2rtc/streams/{name}/bitrate` endpoint and using the configured bitrate for transcoded ones. A pure reorder wasn't saved before because RJSF, the settings form, and `update_yaml` all ignore map key order. Sections can now mark a map with `orderedMaps`, which sends the whole map with `replace_paths` so `config_set` rewrites it in order.

Transcoded streams aren't in `go2rtc.streams`, which only lists yaml streams, so the frontend treated them as not restreamed and fell back to jsmpeg. Every restream check now goes through `isRestreamedStream`.

Auto treated any stall with no bytes in the last 2 seconds as a dead camera and handed it to the error fallback, which went straight to jsmpeg. Heavy congestion can stop delivery completely, so congested viewers skipped every lower stream. Auto now declines only when stats show the camera offline, and a stall on a live camera steps down. The stream picker also has a Try highest quality button that sends auto back to the top stream.

* fixes

web/src/views/live/LiveCameraView.tsx

@@ -13,6 +13,8 @@ import CameraFeatureToggle from "@/components/dynamic/CameraFeatureToggle";
 import FilterSwitch from "@/components/filter/FilterSwitch";
 import LivePlayer from "@/components/player/LivePlayer";
 import StreamTechnologySelect from "@/components/player/StreamTechnologySelect";
+import LiveStreamSelect from "@/components/player/LiveStreamSelect";
+import { useAutoLiveStream } from "@/hooks/use-auto-live-stream";
 import { Button } from "@/components/ui/button";
 import { Drawer, DrawerContent, DrawerTrigger } from "@/components/ui/drawer";
 import {
@@ -30,6 +32,7 @@ import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { useWebRTCAvailableForStream } from "@/hooks/use-webrtc-availability";
 import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
 import {
+  LiveAutoReason,
   LivePlayerError,
   TwoWayTalkError,
   LivePlayerMode,
@@ -38,13 +41,7 @@ import {
   WebRTCUnavailableReason,
 } from "@/types/live";
 import { RecordingStartingPoint } from "@/types/record";
-import React, {
-  useCallback,
-  useEffect,
-  useMemo,
-  useRef,
-  useState,
-} from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import {
   isDesktop,
   isFirefox,
@@ -98,14 +95,6 @@ import useSWR from "swr";
 import { cn } from "@/lib/utils";
 import { useSessionPersistence } from "@/hooks/use-session-persistence";
 
-import {
-  Select,
-  SelectContent,
-  SelectGroup,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from "@/components/ui/select";
 import { useUserPersistence } from "@/hooks/use-user-persistence";
 import { Label } from "@/components/ui/label";
 import { Switch } from "@/components/ui/switch";
@@ -116,6 +105,7 @@ import { useIsAdmin } from "@/hooks/use-is-admin";
 import { useTranslation } from "react-i18next";
 import { useDocDomain } from "@/hooks/use-doc-domain";
 import { detectCameraAudioFeatures } from "@/utils/cameraUtil";
+import { isRestreamedStream } from "@/utils/liveTranscode";
 import PtzControlPanel from "@/components/overlay/PtzControlPanel";
 import ObjectSettingsView from "../settings/ObjectSettingsView";
 import { useSearchEffect } from "@/hooks/use-overlay-state";
@@ -157,11 +147,9 @@ export default function LiveCameraView({
 
   // supported features
 
-  const [streamName, setStreamName, streamNameLoaded] =
-    useUserPersistence<string>(
-      `${camera.name}-stream`,
-      Object.values(camera.live.streams)[0],
-    );
+  // an absent saved stream means auto
+  const [pinnedStream, setPinnedStream, streamNameLoaded, clearPinnedStream] =
+    useUserPersistence<string>(`${camera.name}-stream`);
 
   const [
     userPreferredLiveMode,
@@ -180,25 +168,67 @@ export default function LiveCameraView({
   const preferencesLoaded =
     streamNameLoaded && userPreferredLiveModeLoaded && forceLowBandwidthLoaded;
 
+  const [mic, setMic] = useState(false);
+  const [debug, setDebug] = useState(false);
+  // error fallback latches, layered on top in preferredLiveMode
+  const [webRTC, setWebRTC] = useState(false);
+  const [lowBandwidth, setLowBandwidth] = useState(false);
+
+  const liveStreams = useMemo(
+    () => Object.values(camera.live.streams || {}),
+    [camera.live.streams],
+  );
+  const autoAvailable = liveStreams.length > 1;
+  const autoSelected = autoAvailable && pinnedStream == undefined;
+
+  const autoStream = useAutoLiveStream({
+    cameraName: camera.name,
+    streams: liveStreams,
+    selected: autoSelected,
+    paused:
+      !preferencesLoaded ||
+      mic ||
+      debug ||
+      webRTC ||
+      lowBandwidth ||
+      (forceLowBandwidth ?? false),
+  });
+  const {
+    handleError: autoHandleError,
+    reset: resetAutoStream,
+    onHealthSample,
+  } = autoStream;
+
+  // the stream actually playing; everything downstream keys off it
+  const streamName = autoSelected
+    ? autoStream.streamName
+    : (pinnedStream ?? liveStreams[0]);
+
+  const selectStream = useCallback(
+    (stream: string | undefined) => {
+      if (stream == undefined) {
+        clearPinnedStream();
+      } else {
+        setPinnedStream(stream);
+      }
+    },
+    [clearPinnedStream, setPinnedStream],
+  );
+
   const isRestreamed = useMemo(
-    () =>
-      config &&
-      Object.keys(config.go2rtc.streams || {}).includes(streamName ?? ""),
+    () => isRestreamedStream(config, streamName),
     [config, streamName],
   );
 
-  // validate stored stream name and reset if now invalid
+  // a pin to a stream no longer in config falls back to auto
 
   useEffect(() => {
-    if (!streamNameLoaded) return;
+    if (!streamNameLoaded || pinnedStream == undefined) return;
 
-    const available = Object.values(camera.live.streams || {});
-    if (available.length === 0) return;
-
-    if (streamName != null && !available.includes(streamName)) {
-      setStreamName(available[0]);
+    if (liveStreams.length > 0 && !liveStreams.includes(pinnedStream)) {
+      clearPinnedStream();
     }
-  }, [streamNameLoaded, camera.live.streams, streamName, setStreamName]);
+  }, [streamNameLoaded, liveStreams, pinnedStream, clearPinnedStream]);
 
   const { data: cameraMetadata } = useSWR<LiveStreamMetadata>(
     isRestreamed ? `go2rtc/streams/${streamName}` : null,
@@ -395,7 +425,6 @@ export default function LiveCameraView({
   // playback state
 
   const [audio, setAudio] = useSessionPersistence("liveAudio", false);
-  const [mic, setMic] = useState(false);
 
   // the mic only connects through the WebRTC player, so it can't stay on
   // for a stream that has lost it
@@ -404,9 +433,8 @@ export default function LiveCameraView({
       setMic(false);
     }
   }, [talkAvailable]);
-  const [webRTC, setWebRTC] = useState(false);
+
   const [pip, setPip] = useState(false);
-  const [lowBandwidth, setLowBandwidth] = useState(false);
 
   const [playInBackground, setPlayInBackground] = useUserPersistence<boolean>(
     `${camera.name}-background-play`,
@@ -414,7 +442,6 @@ export default function LiveCameraView({
   );
 
   const [showStats, setShowStats] = useState(false);
-  const [debug, setDebug] = useState(false);
 
   useSearchEffect("debug", (value: string) => {
     if (value === "true") {
@@ -434,7 +461,7 @@ export default function LiveCameraView({
       return "webrtc";
     }
 
-    if (forceLowBandwidth) {
+    if (forceLowBandwidth || autoStream.atFloor) {
       return "jsmpeg";
     }
 
@@ -467,6 +494,7 @@ export default function LiveCameraView({
     isRestreamed,
     resolvedUserMode,
     isWebRTCAvailable,
+    autoStream.atFloor,
   ]);
 
   // A latched error fallback would keep overriding the user's new choice.
@@ -593,6 +621,11 @@ export default function LiveCameraView({
   const handleError = useCallback(
     (e: LivePlayerError) => {
       if (e) {
+        // auto answers congestion and codec failures with a lower stream
+        if (autoHandleError(e)) {
+          return;
+        }
+
         // WebRTC can now be the user's own choice, so the fallback flag no
         // longer implies untried: hopping to the mode that just failed sticks.
         if (preferredLiveMode !== "webrtc" && webRTCUsable) {
@@ -603,9 +636,14 @@ export default function LiveCameraView({
         }
       }
     },
-    [preferredLiveMode, webRTCUsable],
+    [autoHandleError, preferredLiveMode, webRTCUsable],
   );
 
+  const resetStream = useCallback(() => {
+    setLowBandwidth(false);
+    resetAutoStream();
+  }, [resetAutoStream]);
+
   const handleMicrophoneError = useCallback(
     (error: TwoWayTalkError) => {
       setMic(false);
@@ -787,8 +825,11 @@ export default function LiveCameraView({
                 camera.audio_transcription.enabled_in_config
               }
               fullscreen={fullscreen}
-              streamName={streamName ?? ""}
-              setStreamName={setStreamName}
+              pinnedStream={pinnedStream}
+              playingStream={streamName ?? ""}
+              autoAvailable={autoAvailable}
+              autoReason={autoStream.reason}
+              onSelectStream={selectStream}
               userPreferredLiveMode={resolvedUserMode}
               setUserPreferredLiveMode={setUserPreferredLiveMode}
               forceLowBandwidth={forceLowBandwidth ?? false}
@@ -801,7 +842,7 @@ export default function LiveCameraView({
               isRestreamed={isRestreamed ?? false}
               isWebRTCAvailable={isWebRTCAvailable}
               webRTCUnavailableReason={webRTCAvailability.reason}
-              setLowBandwidth={setLowBandwidth}
+              onResetStream={resetStream}
               supportsAudioOutput={supportsAudioOutput}
               supports2WayTalk={supports2WayTalk}
               cameraEnabled={cameraEnabled}
@@ -889,6 +930,8 @@ export default function LiveCameraView({
                   containerRef={containerRef}
                   setFullResolution={setFullResolution}
                   onError={handleError}
+                  onHealthSample={autoSelected ? onHealthSample : undefined}
+                  streamAuto={autoSelected}
                   onMicrophoneError={handleMicrophoneError}
                 />
               </div>
@@ -942,8 +985,11 @@ type FrigateCameraFeaturesProps = {
   autotrackingEnabled: boolean;
   transcriptionEnabled: boolean;
   fullscreen: boolean;
-  streamName: string;
-  setStreamName?: (value: string | undefined) => void;
+  pinnedStream: string | undefined;
+  playingStream: string;
+  autoAvailable: boolean;
+  autoReason?: LiveAutoReason;
+  onSelectStream: (stream: string | undefined) => void;
   userPreferredLiveMode: LivePlayerMode;
   setUserPreferredLiveMode: (value: LivePlayerMode | undefined) => void;
   forceLowBandwidth: boolean;
@@ -956,7 +1002,7 @@ type FrigateCameraFeaturesProps = {
   isRestreamed: boolean;
   isWebRTCAvailable: boolean;
   webRTCUnavailableReason?: WebRTCUnavailableReason;
-  setLowBandwidth: React.Dispatch<React.SetStateAction<boolean>>;
+  onResetStream: () => void;
   supportsAudioOutput: boolean;
   supports2WayTalk: boolean;
   cameraEnabled: boolean;
@@ -970,8 +1016,11 @@ function FrigateCameraFeatures({
   autotrackingEnabled,
   transcriptionEnabled,
   fullscreen,
-  streamName,
-  setStreamName,
+  pinnedStream,
+  playingStream,
+  autoAvailable,
+  autoReason,
+  onSelectStream,
   userPreferredLiveMode,
   setUserPreferredLiveMode,
   forceLowBandwidth,
@@ -984,7 +1033,7 @@ function FrigateCameraFeatures({
   isRestreamed,
   isWebRTCAvailable,
   webRTCUnavailableReason,
-  setLowBandwidth,
+  onResetStream,
   supportsAudioOutput,
   supports2WayTalk,
   cameraEnabled,
@@ -1016,10 +1065,6 @@ function FrigateCameraFeatures({
 
   const isAdmin = useIsAdmin();
 
-  const streamSelectLabel = Object.keys(camera.live.streams).find(
-    (key) => camera.live.streams[key] === streamName,
-  );
-
   // manual event
 
   const recordingEventIdRef = useRef<string | null>(null);
@@ -1363,33 +1408,16 @@ function FrigateCameraFeatures({
                       <Label htmlFor="streaming-method">
                         {t("stream.title")}
                       </Label>
-                      <Select
-                        value={streamName}
+                      <LiveStreamSelect
+                        streams={camera.live.streams}
+                        pinnedStream={pinnedStream}
+                        playingStream={playingStream}
+                        autoAvailable={autoAvailable}
+                        autoReason={autoReason}
+                        onSelect={onSelectStream}
+                        onRetry={onResetStream}
                         disabled={debug || forceLowBandwidth}
-                        onValueChange={(value) => {
-                          setStreamName?.(value);
-                        }}
-                      >
-                        <SelectTrigger className="w-full">
-                          <SelectValue>{streamSelectLabel}</SelectValue>
-                        </SelectTrigger>
-
-                        <SelectContent>
-                          <SelectGroup>
-                            {Object.entries(camera.live.streams).map(
-                              ([stream, name]) => (
-                                <SelectItem
-                                  key={stream}
-                                  className="cursor-pointer"
-                                  value={name}
-                                >
-                                  {stream}
-                                </SelectItem>
-                              ),
-                            )}
-                          </SelectGroup>
-                        </SelectContent>
-                      </Select>
+                      />
 
                       {debug && (
                         <div className="flex flex-row items-center gap-1 text-sm text-muted-foreground">
@@ -1510,7 +1538,7 @@ function FrigateCameraFeatures({
                               aria-label={t("stream.lowBandwidth.resetStream")}
                               variant="outline"
                               size="sm"
-                              onClick={() => setLowBandwidth(false)}
+                              onClick={onResetStream}
                             >
                               <MdOutlineRestartAlt className="size-5 text-primary-variant" />
                               <div className="text-primary-variant">
@@ -1772,33 +1800,16 @@ function FrigateCameraFeatures({
                 Object.values(camera.live.streams).length > 0 && (
                   <div className="mt-1 p-2">
                     <div className="mb-1 text-sm">{t("stream.title")}</div>
-                    <Select
-                      value={streamName}
-                      onValueChange={(value) => {
-                        setStreamName?.(value);
-                      }}
+                    <LiveStreamSelect
+                      streams={camera.live.streams}
+                      pinnedStream={pinnedStream}
+                      playingStream={playingStream}
+                      autoAvailable={autoAvailable}
+                      autoReason={autoReason}
+                      onSelect={onSelectStream}
+                      onRetry={onResetStream}
                       disabled={debug || forceLowBandwidth}
-                    >
-                      <SelectTrigger className="w-full">
-                        <SelectValue>{streamSelectLabel}</SelectValue>
-                      </SelectTrigger>
-
-                      <SelectContent>
-                        <SelectGroup>
-                          {Object.entries(camera.live.streams).map(
-                            ([stream, name]) => (
-                              <SelectItem
-                                key={stream}
-                                className="cursor-pointer"
-                                value={name}
-                              >
-                                {stream}
-                              </SelectItem>
-                            ),
-                          )}
-                        </SelectGroup>
-                      </SelectContent>
-                    </Select>
+                    />
 
                     {debug && (
                       <div className="flex flex-row items-center gap-1 text-sm text-muted-foreground">
@@ -1915,7 +1926,7 @@ function FrigateCameraFeatures({
                             variant="outline"
                             size="sm"
                             disabled={debug}
-                            onClick={() => setLowBandwidth(false)}
+                            onClick={onResetStream}
                           >
                             <MdOutlineRestartAlt className="size-5 text-primary-variant" />
                             <div className="text-primary-variant">