implement auto mode for single camera live view

docs/docs/configuration/live.md

@@ -92,7 +92,7 @@ go2rtc:
 
 ### Setting Streams For Live UI
 
-You can configure Frigate to allow manual selection of the stream you want to view in the Live UI. For example, you may want to view your camera's substream on mobile devices, but the full resolution stream on desktop devices. Setting the streams list will populate a dropdown in the UI's Live view that allows you to choose between the streams. This stream setting is _per device_ and is saved in your browser's local storage.
+You can configure Frigate to allow manual selection of the stream you want to view in the Live UI. For example, you may want to view your camera's substream on mobile devices, but the full resolution stream on desktop devices. Setting the streams list will populate a dropdown in the UI's Live view that allows you to choose between the streams. This stream setting is _per device_ and is saved in your browser's local storage. When a camera has more than one stream, the dropdown also offers **Auto**, which is used until you pick a specific stream. Auto starts on the first stream, steps down the list when your connection can't keep up, and steps back up when it recovers. List streams from highest to lowest quality, and avoid names that are plain numbers (such as `720`), which the browser sorts ahead of the others.
 
 Additionally, when creating and editing camera groups in the UI, you can choose the stream you want to use for your camera group's Live dashboard.
 
@@ -363,6 +363,13 @@ When your browser runs into problems playing back your camera streams, it will l
     - `Safari reported InvalidStateError.`
     - `Safari reported decoding errors.`
 
+- **mse-codec**
+  - What it means: go2rtc has no codec for this stream that the browser can play.
+  - What to try: Pick a stream with a codec the browser supports (H.264 is the most compatible), or use a browser that supports the stream's codec. In Auto, Frigate skips this stream for the rest of the session.
+
+  - Possible console messages from the player code:
+    - `mse: streams: codecs not matched: ...`
+
 - **stalled**
   - What it means: Playback has stalled because the player has fallen too far behind live (extended buffering or no data arriving).
   - What to try: This is usually indicative of the browser struggling to decode too many high-resolution streams at once. Try selecting a lower-bandwidth stream (substream), reduce the number of live streams open, improve the network connection, or lower the camera resolution. Also check your camera's keyframe (I-frame) interval: shorter intervals make playback start and recover faster. You can also try increasing the timeout value in <NavPath path="Settings > UI" /> .

web/e2e/specs/live-auto-stream.spec.ts

@@ -0,0 +1,263 @@
+/**
+ * Auto live stream selection in the single-camera view.
+ *
+ * go2rtc is mocked per stream. A failing stream answers MSE codec
+ * negotiation with go2rtc's "codecs not matched" error, the one ladder
+ * trigger Chromium reproduces without real media. A healthy stream
+ * answers negotiation and then sends nothing, so it waits quietly.
+ */
+import type { Locator, Page } from "@playwright/test";
+import { test, expect } from "../fixtures/frigate-test";
+import { LivePage } from "../pages/live.page";
+
+// the mocked profile is admin, so useUserPersistence keys are namespaced
+const STREAM_KEY = "front_door-stream:admin";
+const CODEC_ERROR = "mse: streams: codecs not matched: H265 => H264";
+
+const CONFIG = {
+  go2rtc: {
+    streams: {
+      front_door: ["rtsp://127.0.0.1:8554/front_door"],
+      front_door_sub: ["rtsp://127.0.0.1:8554/front_door_sub"],
+      backyard: ["rtsp://127.0.0.1:8554/backyard"],
+    },
+    webrtc: { candidates: [], ice_servers: [] },
+  },
+  cameras: {
+    front_door: {
+      live: { streams: { Sub: "front_door_sub" } },
+    },
+  },
+};
+
+test.use({ expectedErrors: [/MSE error 'mse-codec'/] });
+
+async function writeIdb(page: Page, entries: Record<string, unknown>) {
+  await page.evaluate(async (data) => {
+    await new Promise<void>((resolve, reject) => {
+      const request = indexedDB.open("keyval-store", 1);
+      request.onupgradeneeded = () =>
+        request.result.createObjectStore("keyval");
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => {
+        const tx = request.result.transaction("keyval", "readwrite");
+        const store = tx.objectStore("keyval");
+        Object.entries(data).forEach(([key, value]) => store.put(value, key));
+        tx.oncomplete = () => resolve();
+        tx.onerror = () => reject(tx.error);
+      };
+    });
+  }, entries);
+}
+
+async function readIdb(page: Page, key: string) {
+  return page.evaluate(async (target) => {
+    return new Promise((resolve, reject) => {
+      const request = indexedDB.open("keyval-store", 1);
+      request.onupgradeneeded = () =>
+        request.result.createObjectStore("keyval");
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => {
+        const tx = request.result.transaction("keyval", "readonly");
+        const get = tx.objectStore("keyval").get(target);
+        get.onsuccess = () => resolve(get.result ?? null);
+        get.onerror = () => reject(get.error);
+      };
+    });
+  }, key);
+}
+
+// records the src of every MSE socket in open order
+async function mockGo2rtc(page: Page, failing: string[] = []) {
+  const opened: string[] = [];
+
+  await page.routeWebSocket("**/live/mse/api/ws**", (ws) => {
+    const src = new URL(ws.url()).searchParams.get("src") ?? "";
+    opened.push(src);
+
+    ws.onMessage((message) => {
+      const msg = JSON.parse(message.toString());
+      if (msg.type !== "mse") {
+        return;
+      }
+
+      ws.send(
+        JSON.stringify(
+          failing.includes(src)
+            ? { type: "error", value: CODEC_ERROR }
+            : { type: "mse", value: 'video/mp4; codecs="avc1.640029"' },
+        ),
+      );
+    });
+  });
+
+  await page.route("**/api/go2rtc/streams/**", (route) =>
+    route.fulfill({
+      json: {
+        producers: [{ medias: ["video, recvonly, H264"] }],
+        consumers: [],
+      },
+    }),
+  );
+
+  return opened;
+}
+
+async function openDesktopSettings(page: Page) {
+  const live = new LivePage(page, true);
+  await expect(live.backButton).toBeVisible({ timeout: 10_000 });
+  await page.locator("button:has(svg)").last().click();
+  const menu = page.locator('[role="menu"], [data-radix-menu-content]').first();
+  await expect(menu).toBeVisible({ timeout: 3_000 });
+  return menu;
+}
+
+function streamTrigger(menu: Locator) {
+  return menu
+    .locator('div:has(> label[for="streaming-method"]) [role="combobox"]')
+    .first();
+}
+
+test.describe("Auto live stream @mobile", () => {
+  test("a codec error on the main stream moves Auto to the sub stream", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, ["front_door"]);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_sub");
+    expect(opened[0]).toBe("front_door");
+  });
+});
+
+test.describe("Auto live stream selector", () => {
+  test.beforeEach(({ frigateApp }) => {
+    test.skip(frigateApp.isMobile, "Desktop dropdown only");
+  });
+
+  test("no saved stream opens Auto on the first stream", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect.poll(() => opened[0], { timeout: 10_000 }).toBe("front_door");
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Auto · front_door");
+  });
+
+  test("a codec error shows the codec reason", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, ["front_door"]);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_sub");
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Auto · Sub");
+    await expect(
+      menu.getByText(/because this browser can't play a higher-quality stream/),
+    ).toBeVisible();
+  });
+
+  test("a pinned main takes the legacy fallback instead of stepping down", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, ["front_door"]);
+
+    await frigateApp.goto("/");
+    await writeIdb(frigateApp.page, { [STREAM_KEY]: "front_door" });
+    await frigateApp.goto("/#front_door");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(
+      menu.getByText(/Live view is in low-bandwidth mode/),
+    ).toBeVisible({ timeout: 10_000 });
+    expect(opened).not.toContain("front_door_sub");
+  });
+
+  test("selecting Auto clears the saved stream", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/");
+    await writeIdb(frigateApp.page, { [STREAM_KEY]: "front_door_sub" });
+    await frigateApp.goto("/#front_door");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Sub");
+    await streamTrigger(menu).click();
+    await frigateApp.page.getByRole("option", { name: /^Auto/ }).click();
+
+    await expect.poll(() => readIdb(frigateApp.page, STREAM_KEY)).toBeNull();
+  });
+
+  test("a stale pin falls back to Auto", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/");
+    await writeIdb(frigateApp.page, { [STREAM_KEY]: "removed_stream" });
+    await frigateApp.goto("/#front_door");
+
+    await expect.poll(() => readIdb(frigateApp.page, STREAM_KEY)).toBeNull();
+    await expect.poll(() => opened[0], { timeout: 10_000 }).toBe("front_door");
+  });
+
+  test("a single-stream camera offers no Auto", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/#backyard");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await streamTrigger(menu).click();
+    await expect(
+      frigateApp.page.getByRole("option", { name: "backyard" }),
+    ).toBeVisible();
+    await expect(
+      frigateApp.page.getByRole("option", { name: /^Auto/ }),
+    ).toHaveCount(0);
+  });
+
+  test("Reset stream at the floor returns to the top stream", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, [
+      "front_door",
+      "front_door_sub",
+    ]);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_sub");
+    expect(opened[0]).toBe("front_door");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Auto · Low bandwidth");
+    await expect(
+      menu.getByText(/Playing the low-bandwidth feed/),
+    ).toBeVisible();
+
+    const beforeReset = opened.length;
+    await menu.getByRole("button", { name: "Reset stream" }).click();
+
+    // leaving the floor changes the stream and the mode in one render,
+    // which must still open each stream exactly once
+    await expect
+      .poll(() => opened.slice(beforeReset), { timeout: 10_000 })
+      .toEqual(["front_door", "front_door_sub"]);
+  });
+});

web/public/locales/en/components/player.json

@@ -45,7 +45,9 @@
       }
     },
     "decodedFrames": "Decoded Frames:",
-    "droppedFrameRate": "Dropped Frame Rate:"
+    "droppedFrameRate": "Dropped Frame Rate:",
+    "stream": "Stream:",
+    "streamAuto": "{{stream}} (auto)"
   },
   "toast": {
     "success": {

web/public/locales/en/views/live.json

@@ -132,6 +132,18 @@
   },
   "stream": {
     "title": "Stream",
+    "auto": {
+      "label": "Auto",
+      "tips": "Switches streams to match your connection",
+      "selected": "Auto · {{stream}}",
+      "floorLabel": "Low bandwidth",
+      "reason": {
+        "bandwidth": "Playing {{stream}} because the connection couldn't keep up with a higher-quality stream",
+        "codec": "Playing {{stream}} because this browser can't play a higher-quality stream",
+        "saveData": "Playing {{stream}} because Data Saver is on",
+        "floor": "Playing the low-bandwidth feed because no stream could keep up"
+      }
+    },
     "audio": {
       "tips": {
         "title": "Audio must be output from your camera and configured in go2rtc for this stream."

web/src/components/player/LivePlayer.tsx

@@ -10,6 +10,7 @@ import { MdCircle } from "react-icons/md";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import { useCameraActivity } from "@/hooks/use-camera-activity";
 import {
+  LiveHealthSample,
   LivePlayerError,
   TwoWayTalkError,
   LivePlayerMode,
@@ -52,6 +53,8 @@ type LivePlayerProps = {
   onClick?: () => void;
   setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
   onError?: (error: LivePlayerError) => void;
+  onHealthSample?: (sample: LiveHealthSample) => void;
+  streamAuto?: boolean;
   onMicrophoneError?: (error: TwoWayTalkError) => void;
   onResetLiveMode?: () => void;
   onLiveAspectChange?: (aspectRatio: number | undefined) => void;
@@ -79,6 +82,8 @@ export default function LivePlayer({
   onClick,
   setFullResolution,
   onError,
+  onHealthSample,
+  streamAuto = false,
   onMicrophoneError,
   onResetLiveMode,
   onLiveAspectChange,
@@ -109,6 +114,14 @@ export default function LivePlayer({
     droppedFrameRate: 0, // percentage
   });
 
+  const streamLabel = useMemo(
+    () =>
+      Object.keys(cameraConfig.live.streams).find(
+        (label) => cameraConfig.live.streams[label] === streamName,
+      ),
+    [cameraConfig.live.streams, streamName],
+  );
+
   // camera activity
 
   const {
@@ -230,21 +243,17 @@ export default function LivePlayer({
   }, [preferredLiveMode]);
 
   const [key, setKey] = useState(0);
-  const prevStreamNameRef = useRef(streamName);
 
-  const resetPlayer = () => {
+  // the stream is part of the MSE and WebRTC keys, so a stream change
-    setLiveReady(false);
+  // remounts them in the same render and the new player is hidden until it
-    setKey((prevKey) => prevKey + 1);
+  // plays. jsmpeg plays the camera, not the stream, and keeps playing
-  };
+  const [renderedStream, setRenderedStream] = useState(streamName);
-
+  if (renderedStream !== streamName) {
-  useEffect(() => {
+    setRenderedStream(streamName);
-    if (prevStreamNameRef.current !== streamName) {
+    if (preferredLiveMode !== "jsmpeg") {
-      prevStreamNameRef.current = streamName;
+      setLiveReady(false);
-      if (streamName) {
-        resetPlayer();
-      }
     }
-  }, [streamName]);
+  }
 
   useEffect(() => {
     if (showStillWithoutActivity && !autoLive) {
@@ -294,7 +303,7 @@ export default function LivePlayer({
   } else if (preferredLiveMode == "webrtc") {
     player = (
       <WebRtcPlayer
-        key={"webrtc_" + key}
+        key={`webrtc_${streamName}_${key}`}
         className={`size-full ${liveReady ? "" : "hidden"}`}
         camera={streamName}
         playbackEnabled={cameraActive || liveReady}
@@ -308,6 +317,7 @@ export default function LivePlayer({
         onPlaying={playerIsPlaying}
         pip={pip}
         onError={onError}
+        onHealthSample={onHealthSample}
         onMicrophoneError={onMicrophoneError}
       />
     );
@@ -315,7 +325,7 @@ export default function LivePlayer({
     if ("MediaSource" in window || "ManagedMediaSource" in window) {
       player = (
         <MSEPlayer
-          key={"mse_" + key}
+          key={`mse_${streamName}_${key}`}
           className={`size-full ${liveReady ? "" : "hidden"}`}
           camera={streamName}
           playbackEnabled={cameraActive || liveReady}
@@ -328,6 +338,7 @@ export default function LivePlayer({
           pip={pip}
           setFullResolution={handleFullResolution}
           onError={onError}
+          onHealthSample={onHealthSample}
         />
       );
     } else {
@@ -561,7 +572,12 @@ export default function LivePlayer({
           )}
       </div>
       {showStats && (
-        <PlayerStats stats={stats} minimal={cameraRef !== undefined} />
+        <PlayerStats
+          stats={stats}
+          minimal={cameraRef !== undefined}
+          streamLabel={preferredLiveMode === "jsmpeg" ? undefined : streamLabel}
+          streamAuto={streamAuto}
+        />
       )}
     </div>
   );

web/src/components/player/LiveStreamGovernor.ts

@@ -0,0 +1,291 @@
+/**
+ * Policy engine for auto live stream selection.
+ *
+ * Rungs index the camera's live streams in config order, followed by
+ * the jsmpeg floor. Evidence is media delivery falling behind wall
+ * time while bytes still arrive, which separates congestion from a
+ * dead camera or a go2rtc restart. No browser API reports spare
+ * bandwidth, so upswitching is try-and-see with backoff.
+ *
+ * Time comes from an injected clock and the owner calls tick() once a
+ * second, so the class holds no timers.
+ */
+
+import type {
+  LiveAutoReason,
+  LiveHealthSample,
+  LivePlayerError,
+} from "@/types/live";
+
+// go2rtc bursts its GOP cache when a consumer attaches
+const WARMUP_MS = 3000;
+// bytes this recent mean a stall is congestion rather than a dead source
+const BYTES_RECENT_MS = 2000;
+// media per wall second below this is a low sample. Low samples reset
+// the upswitch clock and feed the chronic window
+const LOW_RATE = 0.85;
+// net shortfall within the window that downswitches on sustained
+// under-delivery. Keyframes make per-second delivery bursty, so the
+// window lets a fast second offset a slow one
+const SUSTAINED_DEFICIT_S = 2;
+const SUSTAINED_WINDOW_MS = 10000;
+// low-sample deficit within the window that downswitches on repeated
+// short stalls
+const CHRONIC_DEFICIT_S = 6;
+const CHRONIC_WINDOW_MS = 60000;
+// clean time before trying one rung up, doubled after each failed try
+const UPSWITCH_WAIT_MS = 60000;
+const UPSWITCH_WAIT_MAX_MS = 600000;
+// a downswitch this soon after an upswitch counts as a failed try
+const PROBATION_MS = 30000;
+
+export type LiveGovernorSnapshot = {
+  rung: number;
+  reason: LiveAutoReason | undefined;
+};
+
+type Shortfall = {
+  ts: number;
+  seconds: number;
+};
+
+const total = (entries: Shortfall[]) =>
+  entries.reduce((sum, entry) => sum + entry.seconds, 0);
+
+export class LiveStreamGovernor {
+  private readonly floor: number;
+  private readonly now: () => number;
+  private readonly listeners = new Set<() => void>();
+  private readonly unplayable = new Set<number>();
+
+  private snapshot: LiveGovernorSnapshot = { rung: 0, reason: undefined };
+  private holdLow = false;
+
+  private rungStartTs: number;
+  private cleanSinceTs: number;
+  private lastBytesTs = Number.NEGATIVE_INFINITY;
+  // every sample, signed, for the sustained window
+  private shortfalls: Shortfall[] = [];
+  // low samples only, for the chronic window
+  private deficits: Shortfall[] = [];
+  private upswitchWaitMs = UPSWITCH_WAIT_MS;
+  private probationUntil = 0;
+
+  /** rungCount includes the jsmpeg floor. */
+  constructor(rungCount: number, now: () => number = Date.now) {
+    this.floor = rungCount - 1;
+    this.now = now;
+    this.rungStartTs = now();
+    this.cleanSinceTs = this.rungStartTs;
+  }
+
+  subscribe = (listener: () => void) => {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  };
+
+  getSnapshot = (): LiveGovernorSnapshot => this.snapshot;
+
+  /**
+   * One health sample from the playing transport. Zero-byte samples
+   * belong to the stall watchdog, and warmup samples only prove that
+   * bytes are arriving.
+   */
+  sample({ bytes, mediaSeconds, wallSeconds }: LiveHealthSample) {
+    if (bytes <= 0 || wallSeconds <= 0) {
+      return;
+    }
+
+    const now = this.now();
+    this.lastBytesTs = now;
+
+    if (now - this.rungStartTs < WARMUP_MS) {
+      return;
+    }
+
+    // WebRTC reports a finished freeze in one sample, so media can be
+    // negative and one sample can carry more shortfall than its wall time
+    const shortfall = { ts: now, seconds: wallSeconds - mediaSeconds };
+    this.shortfalls.push(shortfall);
+    if (mediaSeconds / wallSeconds < LOW_RATE) {
+      this.cleanSinceTs = now;
+      this.deficits.push(shortfall);
+    }
+
+    this.shortfalls = this.shortfalls.filter(
+      (entry) => entry.ts > now - SUSTAINED_WINDOW_MS,
+    );
+    this.deficits = this.deficits.filter(
+      (entry) => entry.ts > now - CHRONIC_WINDOW_MS,
+    );
+
+    if (
+      total(this.shortfalls) >= SUSTAINED_DEFICIT_S ||
+      total(this.deficits) >= CHRONIC_DEFICIT_S
+    ) {
+      this.stepDown();
+    }
+  }
+
+  /**
+   * A player error. Returns true when auto handled it by stepping down,
+   * false to hand it to the error fallback.
+   */
+  playerError(error: LivePlayerError): boolean {
+    const { rung } = this.snapshot;
+    if (rung === this.floor) {
+      return false;
+    }
+
+    // go2rtc's codec reply is proof, so the rung is not retried this session
+    if (error === "mse-codec") {
+      this.unplayable.add(rung);
+      return this.stepDown();
+    }
+
+    // the rung below played a moment ago, so a failed upswitch returns to it
+    if (this.now() < this.probationUntil) {
+      return this.stepDown();
+    }
+
+    // without recent bytes the source is dead rather than congested,
+    // and a lower stream from the same camera would fail the same way
+    if (this.now() - this.lastBytesTs > BYTES_RECENT_MS) {
+      return false;
+    }
+
+    return this.stepDown();
+  }
+
+  /** Called once a second while auto is active and the page is visible. */
+  tick() {
+    const now = this.now();
+
+    // the higher rung survived probation, so backoff starts over
+    if (this.probationUntil > 0 && now >= this.probationUntil) {
+      this.probationUntil = 0;
+      this.upswitchWaitMs = UPSWITCH_WAIT_MS;
+    }
+
+    if (now - this.cleanSinceTs < this.upswitchWaitMs) {
+      return;
+    }
+
+    // data saver may climb out of the floor, but never past its stream
+    const target = this.nextPlayableAbove(this.snapshot.rung);
+    if (
+      target === undefined ||
+      (this.holdLow && target < this.lowestPlayable())
+    ) {
+      return;
+    }
+
+    this.probationUntil = now + PROBATION_MS;
+    this.moveTo(target);
+  }
+
+  /**
+   * Data saver holds the lowest playable stream and never upswitches past
+   * it, whatever the connection could carry.
+   */
+  setHoldLow(hold: boolean) {
+    this.holdLow = hold;
+    this.moveTo(
+      hold
+        ? Math.max(this.snapshot.rung, this.lowestPlayable())
+        : this.snapshot.rung,
+    );
+  }
+
+  /** A remount or unpause on the same rung starts with clean evidence. */
+  resetHistory() {
+    const now = this.now();
+    this.rungStartTs = now;
+    this.cleanSinceTs = now;
+    this.lastBytesTs = Number.NEGATIVE_INFINITY;
+    this.shortfalls = [];
+    this.deficits = [];
+  }
+
+  /** Returns to the top rung and clears every session fact. */
+  reset() {
+    this.unplayable.clear();
+    this.upswitchWaitMs = UPSWITCH_WAIT_MS;
+    this.probationUntil = 0;
+    this.moveTo(this.holdLow ? this.lowestPlayable() : 0);
+  }
+
+  private stepDown(): boolean {
+    const target = this.nextPlayableBelow(this.snapshot.rung);
+    if (target === undefined) {
+      return false;
+    }
+
+    if (this.now() < this.probationUntil) {
+      this.upswitchWaitMs = Math.min(
+        this.upswitchWaitMs * 2,
+        UPSWITCH_WAIT_MAX_MS,
+      );
+    }
+    this.probationUntil = 0;
+
+    this.moveTo(target);
+    return true;
+  }
+
+  private moveTo(rung: number) {
+    this.resetHistory();
+
+    const reason = this.reasonAt(rung);
+    if (rung === this.snapshot.rung && reason === this.snapshot.reason) {
+      return;
+    }
+
+    this.snapshot = { rung, reason };
+    this.listeners.forEach((listener) => listener());
+  }
+
+  private reasonAt(rung: number): LiveAutoReason | undefined {
+    if (rung === this.floor) {
+      return "floor";
+    }
+
+    if (this.nextPlayableAbove(rung) === undefined) {
+      return [...this.unplayable].some((unplayable) => unplayable < rung)
+        ? "codec"
+        : undefined;
+    }
+
+    return this.holdLow ? "saveData" : "bandwidth";
+  }
+
+  private nextPlayableAbove(rung: number): number | undefined {
+    for (let candidate = rung - 1; candidate >= 0; candidate--) {
+      if (!this.unplayable.has(candidate)) {
+        return candidate;
+      }
+    }
+    return undefined;
+  }
+
+  // the floor is always playable
+  private nextPlayableBelow(rung: number): number | undefined {
+    for (let candidate = rung + 1; candidate <= this.floor; candidate++) {
+      if (candidate === this.floor || !this.unplayable.has(candidate)) {
+        return candidate;
+      }
+    }
+    return undefined;
+  }
+
+  private lowestPlayable(): number {
+    for (let candidate = this.floor - 1; candidate >= 0; candidate--) {
+      if (!this.unplayable.has(candidate)) {
+        return candidate;
+      }
+    }
+    return this.floor;
+  }
+}

web/src/components/player/LiveStreamSelect.tsx

@@ -0,0 +1,105 @@
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { LiveAutoReason } from "@/types/live";
+import { useCallback } from "react";
+import { useTranslation } from "react-i18next";
+import { LuInfo } from "react-icons/lu";
+
+// Radix Select needs a non-empty value. Auto itself is stored as the
+// absence of a pinned stream
+const AUTO_VALUE = "__frigate_auto__";
+
+type LiveStreamSelectProps = {
+  // friendly name to go2rtc stream name, in ladder order
+  streams: Record<string, string>;
+  pinnedStream: string | undefined;
+  playingStream: string;
+  autoAvailable: boolean;
+  autoReason?: LiveAutoReason;
+  // undefined selects auto
+  onSelect: (stream: string | undefined) => void;
+  disabled: boolean;
+};
+
+export default function LiveStreamSelect({
+  streams,
+  pinnedStream,
+  playingStream,
+  autoAvailable,
+  autoReason,
+  onSelect,
+  disabled,
+}: LiveStreamSelectProps) {
+  const { t } = useTranslation(["views/live"]);
+
+  const labelFor = useCallback(
+    (name: string) =>
+      Object.keys(streams).find((label) => streams[label] === name) ?? name,
+    [streams],
+  );
+
+  const auto = autoAvailable && pinnedStream == undefined;
+  const value = auto ? AUTO_VALUE : (pinnedStream ?? playingStream);
+
+  return (
+    <>
+      <Select
+        value={value}
+        disabled={disabled}
+        onValueChange={(next) =>
+          onSelect(next === AUTO_VALUE ? undefined : next)
+        }
+      >
+        <SelectTrigger className="w-full">
+          <SelectValue>
+            {auto
+              ? t("stream.auto.selected", {
+                  stream:
+                    autoReason === "floor"
+                      ? t("stream.auto.floorLabel")
+                      : labelFor(playingStream),
+                })
+              : labelFor(value)}
+          </SelectValue>
+        </SelectTrigger>
+
+        <SelectContent>
+          <SelectGroup>
+            {autoAvailable && (
+              <SelectItem className="cursor-pointer" value={AUTO_VALUE}>
+                <span className="flex flex-col gap-0.5 whitespace-normal">
+                  <span>{t("stream.auto.label")}</span>
+                  <span className="text-xs text-muted-foreground">
+                    {t("stream.auto.tips")}
+                  </span>
+                </span>
+              </SelectItem>
+            )}
+            {Object.entries(streams).map(([label, name]) => (
+              <SelectItem key={label} className="cursor-pointer" value={name}>
+                {label}
+              </SelectItem>
+            ))}
+          </SelectGroup>
+        </SelectContent>
+      </Select>
+
+      {auto && autoReason && !disabled && (
+        <div className="flex flex-row items-center gap-1 text-sm text-muted-foreground">
+          <LuInfo className="size-4 shrink-0" />
+          <div>
+            {t(`stream.auto.reason.${autoReason}`, {
+              stream: labelFor(playingStream),
+            })}
+          </div>
+        </div>
+      )}
+    </>
+  );
+}

web/src/components/player/MsePlayer.tsx

@@ -1,6 +1,7 @@
 import { baseUrl } from "@/api/baseUrl";
 import { useUserPersistence } from "@/hooks/use-user-persistence";
 import {
+  LiveHealthSample,
   LivePlayerError,
   PlayerStatsType,
   VideoResolutionType,
@@ -28,6 +29,7 @@ type MSEPlayerProps = {
   onPlaying?: () => void;
   setFullResolution?: React.Dispatch<SetStateAction<VideoResolutionType>>;
   onError?: (error: LivePlayerError) => void;
+  onHealthSample?: (sample: LiveHealthSample) => void;
 };
 
 function MSEPlayer({
@@ -43,6 +45,7 @@ function MSEPlayer({
   onPlaying,
   setFullResolution,
   onError,
+  onHealthSample,
 }: MSEPlayerProps) {
   const RECONNECT_TIMEOUT: number = 10000;
   const BUFFERING_COOLDOWN_TIMEOUT: number = 5000;
@@ -71,6 +74,7 @@ function MSEPlayer({
   const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
   const [errorCount, setErrorCount] = useState<number>(0);
   const totalBytesLoaded = useRef(0);
+  const appendingRef = useRef(false);
 
   const [fallbackTimeout] = useUserPersistence<number>(
     "liveFallbackTimeout",
@@ -95,6 +99,13 @@ function MSEPlayer({
     return `${baseUrl.replace(/^http/, "ws")}live/mse/api/ws?src=${camera}`;
   }, [camera]);
 
+  // socket handlers are bound once per connection, so they reach the
+  // current onError through a ref
+  const onErrorRef = useRef(onError);
+  useEffect(() => {
+    onErrorRef.current = onError;
+  }, [onError]);
+
   const handleError = useCallback(
     (error: LivePlayerError, description: string = "Unknown error") => {
       // eslint-disable-next-line no-console
@@ -110,11 +121,11 @@ function MSEPlayer({
         // eslint-disable-next-line no-console
         console.error(`${camera} - Supported codecs: ${CODECS.join(", ")}`);
       }
-      onError?.(error);
+      onErrorRef.current?.(error);
     },
     // we know that these deps are correct
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    [camera, onError],
+    [camera],
   );
 
   const handleLoadedMetadata = useCallback(() => {
@@ -470,7 +481,12 @@ function MSEPlayer({
             const data = buf.slice(0, bufLen);
             bufLen = 0;
             sb.appendBuffer(data);
-          } else if (sb.buffered && sb.buffered.length) {
+            return;
+          }
+
+          appendingRef.current = false;
+
+          if (sb.buffered && sb.buffered.length) {
             const end = sb.buffered.end(sb.buffered.length - 1) - 15;
             const start = sb.buffered.start(0);
             if (end > start) {
@@ -488,6 +504,7 @@ function MSEPlayer({
 
       ondataRef.current = (data) => {
         totalBytesLoaded.current += data.byteLength;
+        appendingRef.current = true;
 
         if (sb?.updating || bufLen > 0) {
           const b = new Uint8Array(data);
@@ -503,6 +520,19 @@ function MSEPlayer({
         }
       };
     };
+
+    // go2rtc answers a codec it cannot provide with an error message
+    // instead of a negotiation reply
+    onmessageRef.current["error"] = (msg) => {
+      if (msg.type !== "error" || !msg.value?.includes("codecs not matched")) {
+        return;
+      }
+
+      if (wsRef.current) {
+        onDisconnect();
+      }
+      handleError("mse-codec", msg.value);
+    };
   };
 
   const getBufferedTime = (video: HTMLVideoElement | null) => {
@@ -740,19 +770,38 @@ function MSEPlayer({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [playbackEnabled]);
 
-  // stats
+  // stats and health samples
+
+  const onHealthSampleRef = useRef(onHealthSample);
+  useEffect(() => {
+    onHealthSampleRef.current = onHealthSample;
+  }, [onHealthSample]);
+  const sampleHealth = onHealthSample !== undefined;
 
   useEffect(() => {
     const video = videoRef.current;
     let lastLoadedBytes = totalBytesLoaded.current;
     let lastTimestamp = Date.now();
+    let healthBytes = lastLoadedBytes;
+    let healthTimestamp = lastTimestamp;
+    // a restart mid-stream must not count the existing buffer as new media
+    let lastBufferedEnd =
+      video && video.buffered.length > 0
+        ? video.buffered.end(video.buffered.length - 1)
+        : 0;
+    let deferred = false;
 
-    if (!getStats) return;
+    if (!getStats && !sampleHealth) return;
 
-    const updateStats = () => {
+    const update = () => {
-      if (video) {
+      if (!video) {
-        const now = Date.now();
+        return;
-        const bytesLoaded = totalBytesLoaded.current;
+      }
+
+      const now = Date.now();
+      const bytesLoaded = totalBytesLoaded.current;
+
+      if (getStats) {
         const timeElapsed = (now - lastTimestamp) / 1000; // seconds
         const bandwidth = (bytesLoaded - lastLoadedBytes) / timeElapsed / 1000; // kBps
 
@@ -774,22 +823,64 @@ function MSEPlayer({
           droppedFrameRate,
         });
       }
+
+      if (!sampleHealth) {
+        return;
+      }
+
+      // a sample taken mid-append would count the bytes without their
+      // media, so it waits one tick for the append to land
+      if (appendingRef.current && !deferred) {
+        deferred = true;
+        return;
+      }
+      deferred = false;
+
+      // the buffered end tracks delivered media, so playback-rate
+      // catch-up does not skew it
+      const bufferedEnd =
+        video.buffered.length > 0
+          ? video.buffered.end(video.buffered.length - 1)
+          : 0;
+      const mediaSeconds = bufferedEnd - lastBufferedEnd;
+      const bytes = bytesLoaded - healthBytes;
+      const wallSeconds = (now - healthTimestamp) / 1000;
+
+      lastBufferedEnd = bufferedEnd;
+      healthBytes = bytesLoaded;
+      healthTimestamp = now;
+
+      // a reconnect starts a new MediaSource timeline
+      if (mediaSeconds < 0) {
+        return;
+      }
+
+      if (
+        document.visibilityState !== "visible" ||
+        (video.paused && bufferedEnd > 0)
+      ) {
+        return;
+      }
+
+      onHealthSampleRef.current?.({ bytes, mediaSeconds, wallSeconds });
     };
 
-    const interval = setInterval(updateStats, 1000); // Update every second
+    const interval = setInterval(update, 1000);
 
     return () => {
       clearInterval(interval);
-      setStats?.({
+      if (getStats) {
-        streamType: "-",
+        setStats?.({
-        bandwidth: 0,
+          streamType: "-",
-        totalFrames: 0,
+          bandwidth: 0,
-        droppedFrames: undefined,
+          totalFrames: 0,
-        decodedFrames: 0,
+          droppedFrames: undefined,
-        droppedFrameRate: 0,
+          decodedFrames: 0,
-      });
+          droppedFrameRate: 0,
+        });
+      }
     };
-  }, [setStats, getStats]);
+  }, [setStats, getStats, sampleHealth]);
 
   return (
     <video

web/src/components/player/PlayerStats.tsx

@@ -5,9 +5,16 @@ import { useTranslation } from "react-i18next";
 type PlayerStatsProps = {
   stats: PlayerStatsType;
   minimal: boolean;
+  streamLabel?: string;
+  streamAuto?: boolean;
 };
 
-export function PlayerStats({ stats, minimal }: PlayerStatsProps) {
+export function PlayerStats({
+  stats,
+  minimal,
+  streamLabel,
+  streamAuto = false,
+}: PlayerStatsProps) {
   const { t } = useTranslation(["components/player", "views/live"]);
   const streamTypeLabel = t(
     `stream.technology.name.${stats.streamType.toLowerCase()}`,
@@ -15,6 +22,16 @@ export function PlayerStats({ stats, minimal }: PlayerStatsProps) {
   );
   const fullStatsContent = (
     <>
+      {streamLabel != undefined && (
+        <p>
+          <span className="text-white/70">{t("stats.stream")}</span>{" "}
+          <span className="text-white">
+            {streamAuto
+              ? t("stats.streamAuto", { stream: streamLabel })
+              : streamLabel}
+          </span>
+        </p>
+      )}
       <p>
         <span className="text-white/70">{t("stats.streamType.title")}</span>{" "}
         <span className="text-white">{streamTypeLabel}</span>

web/src/components/player/WebRTCPlayer.tsx

@@ -1,5 +1,6 @@
 import { baseUrl } from "@/api/baseUrl";
 import {
+  LiveHealthSample,
   LivePlayerError,
   PlayerStatsType,
   TwoWayTalkError,
@@ -24,6 +25,7 @@ type WebRtcPlayerProps = {
   setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
   onPlaying?: () => void;
   onError?: (error: LivePlayerError) => void;
+  onHealthSample?: (sample: LiveHealthSample) => void;
   onMicrophoneError?: (error: TwoWayTalkError) => void;
 };
 
@@ -41,6 +43,7 @@ export default function WebRtcPlayer({
   setFullResolution,
   onPlaying,
   onError,
+  onHealthSample,
   onMicrophoneError,
 }: WebRtcPlayerProps) {
   // metadata
@@ -354,13 +357,20 @@ export default function WebRtcPlayer({
     onPlaying?.();
   };
 
-  // stats
+  // stats and health samples
+
+  const onHealthSampleRef = useRef(onHealthSample);
+  useEffect(() => {
+    onHealthSampleRef.current = onHealthSample;
+  }, [onHealthSample]);
+  const sampleHealth = onHealthSample !== undefined;
 
   useEffect(() => {
-    if (!pcRef.current || !getStats) return;
+    if (!pcRef.current || (!getStats && !sampleHealth)) return;
 
     let lastBytesReceived = 0;
     let lastTimestamp = 0;
+    let lastFreezeSeconds: number | undefined;
 
     const interval = setInterval(async () => {
       if (pcRef.current && videoRef.current && !videoRef.current.paused) {
@@ -369,6 +379,7 @@ export default function WebRtcPlayer({
         let timestamp = 0;
         let framesReceived = 0;
         let framesDecoded = 0;
+        let freezeSeconds: number | undefined;
 
         report.forEach((stat) => {
           if (stat.type === "inbound-rtp" && stat.kind === "video") {
@@ -376,43 +387,72 @@ export default function WebRtcPlayer({
             timestamp = stat.timestamp;
             framesReceived = stat.framesReceived;
             framesDecoded = stat.framesDecoded;
+            freezeSeconds = stat.totalFreezesDuration;
           }
         });
 
         const timeDiff = (timestamp - lastTimestamp) / 1000; // in seconds
-        const bitrate =
-          timeDiff > 0
-            ? (bytesReceived - lastBytesReceived) / timeDiff / 1000
-            : 0; // in kBps
 
-        setStats?.({
+        if (getStats) {
-          streamType: "WebRTC",
+          const bitrate =
-          bandwidth: Math.round(bitrate),
+            timeDiff > 0
-          totalFrames: framesReceived,
+              ? (bytesReceived - lastBytesReceived) / timeDiff / 1000
-          droppedFrames: undefined,
+              : 0; // in kBps
-          decodedFrames: framesDecoded,
+
-          droppedFrameRate: undefined,
+          setStats?.({
-        });
+            streamType: "WebRTC",
+            bandwidth: Math.round(bitrate),
+            totalFrames: framesReceived,
+            droppedFrames: undefined,
+            decodedFrames: framesDecoded,
+            droppedFrameRate: undefined,
+          });
+        }
+
+        if (
+          sampleHealth &&
+          lastTimestamp > 0 &&
+          timeDiff > 0 &&
+          document.visibilityState === "visible"
+        ) {
+          // freeze time does not depend on the camera's frame rate. A
+          // freeze is credited only when it ends, so media goes negative
+          // to carry its full length. Browsers without the field report
+          // full delivery and rely on the stall watchdog
+          const frozen =
+            freezeSeconds !== undefined && lastFreezeSeconds !== undefined
+              ? Math.max(0, freezeSeconds - lastFreezeSeconds)
+              : 0;
+
+          onHealthSampleRef.current?.({
+            bytes: bytesReceived - lastBytesReceived,
+            mediaSeconds: timeDiff - frozen,
+            wallSeconds: timeDiff,
+          });
+        }
 
         lastBytesReceived = bytesReceived;
         lastTimestamp = timestamp;
+        lastFreezeSeconds = freezeSeconds;
       }
     }, 1000);
 
     return () => {
       clearInterval(interval);
-      setStats?.({
+      if (getStats) {
-        streamType: "-",
+        setStats?.({
-        bandwidth: 0,
+          streamType: "-",
-        totalFrames: 0,
+          bandwidth: 0,
-        droppedFrames: undefined,
+          totalFrames: 0,
-        decodedFrames: 0,
+          droppedFrames: undefined,
-        droppedFrameRate: 0,
+          decodedFrames: 0,
-      });
+          droppedFrameRate: 0,
+        });
+      }
     };
     // we need to listen on the value of the ref
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [pcRef, pcRef.current, getStats]);
+  }, [pcRef, pcRef.current, getStats, sampleHealth]);
 
   return (
     <video

web/src/hooks/use-auto-live-stream.ts

@@ -0,0 +1,121 @@
+import { LiveStreamGovernor } from "@/components/player/LiveStreamGovernor";
+import {
+  LiveAutoReason,
+  LiveHealthSample,
+  LivePlayerError,
+} from "@/types/live";
+import { useCallback, useEffect, useMemo, useSyncExternalStore } from "react";
+
+// one governor per camera ladder for the tab session, so reopening a
+// camera resumes where auto settled
+const governors = new Map<string, LiveStreamGovernor>();
+
+function governorFor(cameraName: string, streams: string[]) {
+  const key = `${cameraName}|${streams.join(",")}`;
+  let governor = governors.get(key);
+  if (!governor) {
+    // the extra rung is the jsmpeg floor
+    governor = new LiveStreamGovernor(streams.length + 1);
+    governors.set(key, governor);
+  }
+  return governor;
+}
+
+type UseAutoLiveStreamProps = {
+  cameraName: string;
+  // go2rtc stream names in config order
+  streams: string[];
+  // auto is the stream setting
+  selected: boolean;
+  // mic, debug, force low-bandwidth, the error fallback latch, or unloaded
+  // preferences own the player
+  paused: boolean;
+};
+
+export type AutoLiveStream = {
+  streamName: string | undefined;
+  atFloor: boolean;
+  reason: LiveAutoReason | undefined;
+  onHealthSample: (sample: LiveHealthSample) => void;
+  handleError: (error: LivePlayerError) => boolean;
+  reset: () => void;
+};
+
+export function useAutoLiveStream({
+  cameraName,
+  streams,
+  selected,
+  paused,
+}: UseAutoLiveStreamProps): AutoLiveStream {
+  const governor = useMemo(
+    () => governorFor(cameraName, streams),
+    [cameraName, streams],
+  );
+  const { rung, reason } = useSyncExternalStore(
+    governor.subscribe,
+    governor.getSnapshot,
+  );
+  const active = selected && !paused;
+
+  // evidence from before a pin, a pause, or a remount does not count
+  useEffect(() => {
+    if (active) {
+      governor.resetHistory();
+    }
+  }, [active, governor]);
+
+  useEffect(() => {
+    if (!selected) {
+      return;
+    }
+
+    const saveData =
+      (navigator as Navigator & { connection?: { saveData?: boolean } })
+        .connection?.saveData === true;
+    governor.setHoldLow(saveData);
+  }, [selected, governor]);
+
+  useEffect(() => {
+    if (!active) {
+      return;
+    }
+
+    // a hidden tab delivers no samples, so its time is not clean time
+    const interval = setInterval(() => {
+      if (document.visibilityState === "visible") {
+        governor.tick();
+      } else {
+        governor.resetHistory();
+      }
+    }, 1000);
+
+    return () => clearInterval(interval);
+  }, [active, governor]);
+
+  const onHealthSample = useCallback(
+    (sample: LiveHealthSample) => {
+      if (active) {
+        governor.sample(sample);
+      }
+    },
+    [active, governor],
+  );
+
+  const handleError = useCallback(
+    (error: LivePlayerError) => active && governor.playerError(error),
+    [active, governor],
+  );
+
+  const reset = useCallback(() => governor.reset(), [governor]);
+
+  const floor = streams.length;
+
+  return {
+    streamName: selected ? streams[Math.min(rung, floor - 1)] : undefined,
+    atFloor: selected && rung === floor,
+    reason: selected ? reason : undefined,
+    onHealthSample,
+    handleError,
+    reset,
+  };
+}

web/src/types/live.ts

@@ -31,7 +31,20 @@ export type LiveStreamMetadata = {
   consumers: LiveConsumerMetadata[];
 };
 
-export type LivePlayerError = "stalled" | "startup" | "mse-decode";
+// mse-codec is go2rtc's proof that it has no codec this browser plays;
+// mse-decode also covers transient Safari failures
+export type LivePlayerError =
+  "stalled" | "startup" | "mse-decode" | "mse-codec";
+
+// one second of delivery from the playing transport
+export type LiveHealthSample = {
+  bytes: number;
+  mediaSeconds: number;
+  wallSeconds: number;
+};
+
+// why auto is playing below the top stream
+export type LiveAutoReason = "bandwidth" | "codec" | "saveData" | "floor";
 
 export type TwoWayTalkError = "microphone" | "refused";
 

web/src/views/live/LiveCameraView.tsx

@@ -13,6 +13,8 @@ import CameraFeatureToggle from "@/components/dynamic/CameraFeatureToggle";
 import FilterSwitch from "@/components/filter/FilterSwitch";
 import LivePlayer from "@/components/player/LivePlayer";
 import StreamTechnologySelect from "@/components/player/StreamTechnologySelect";
+import LiveStreamSelect from "@/components/player/LiveStreamSelect";
+import { useAutoLiveStream } from "@/hooks/use-auto-live-stream";
 import { Button } from "@/components/ui/button";
 import { Drawer, DrawerContent, DrawerTrigger } from "@/components/ui/drawer";
 import {
@@ -30,6 +32,7 @@ import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { useWebRTCAvailableForStream } from "@/hooks/use-webrtc-availability";
 import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
 import {
+  LiveAutoReason,
   LivePlayerError,
   TwoWayTalkError,
   LivePlayerMode,
@@ -38,13 +41,7 @@ import {
   WebRTCUnavailableReason,
 } from "@/types/live";
 import { RecordingStartingPoint } from "@/types/record";
-import React, {
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
-  useCallback,
-  useEffect,
-  useMemo,
-  useRef,
-  useState,
-} from "react";
 import {
   isDesktop,
   isFirefox,
@@ -98,14 +95,6 @@ import useSWR from "swr";
 import { cn } from "@/lib/utils";
 import { useSessionPersistence } from "@/hooks/use-session-persistence";
 
-import {
-  Select,
-  SelectContent,
-  SelectGroup,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from "@/components/ui/select";
 import { useUserPersistence } from "@/hooks/use-user-persistence";
 import { Label } from "@/components/ui/label";
 import { Switch } from "@/components/ui/switch";
@@ -157,11 +146,9 @@ export default function LiveCameraView({
 
   // supported features
 
-  const [streamName, setStreamName, streamNameLoaded] =
+  // an absent saved stream means auto
-    useUserPersistence<string>(
+  const [pinnedStream, setPinnedStream, streamNameLoaded, clearPinnedStream] =
-      `${camera.name}-stream`,
+    useUserPersistence<string>(`${camera.name}-stream`);
-      Object.values(camera.live.streams)[0],
-    );
 
   const [
     userPreferredLiveMode,
@@ -180,6 +167,53 @@ export default function LiveCameraView({
   const preferencesLoaded =
     streamNameLoaded && userPreferredLiveModeLoaded && forceLowBandwidthLoaded;
 
+  const [mic, setMic] = useState(false);
+  const [debug, setDebug] = useState(false);
+  // error fallback latches, layered on top in preferredLiveMode
+  const [webRTC, setWebRTC] = useState(false);
+  const [lowBandwidth, setLowBandwidth] = useState(false);
+
+  const liveStreams = useMemo(
+    () => Object.values(camera.live.streams || {}),
+    [camera.live.streams],
+  );
+  const autoAvailable = liveStreams.length > 1;
+  const autoSelected = autoAvailable && pinnedStream == undefined;
+
+  const autoStream = useAutoLiveStream({
+    cameraName: camera.name,
+    streams: liveStreams,
+    selected: autoSelected,
+    paused:
+      !preferencesLoaded ||
+      mic ||
+      debug ||
+      webRTC ||
+      lowBandwidth ||
+      (forceLowBandwidth ?? false),
+  });
+  const {
+    handleError: autoHandleError,
+    reset: resetAutoStream,
+    onHealthSample,
+  } = autoStream;
+
+  // the stream actually playing; everything downstream keys off it
+  const streamName = autoSelected
+    ? autoStream.streamName
+    : (pinnedStream ?? liveStreams[0]);
+
+  const selectStream = useCallback(
+    (stream: string | undefined) => {
+      if (stream == undefined) {
+        clearPinnedStream();
+      } else {
+        setPinnedStream(stream);
+      }
+    },
+    [clearPinnedStream, setPinnedStream],
+  );
+
   const isRestreamed = useMemo(
     () =>
       config &&
@@ -187,18 +221,15 @@ export default function LiveCameraView({
     [config, streamName],
   );
 
-  // validate stored stream name and reset if now invalid
+  // a pin to a stream no longer in config falls back to auto
 
   useEffect(() => {
-    if (!streamNameLoaded) return;
+    if (!streamNameLoaded || pinnedStream == undefined) return;
 
-    const available = Object.values(camera.live.streams || {});
+    if (liveStreams.length > 0 && !liveStreams.includes(pinnedStream)) {
-    if (available.length === 0) return;
+      clearPinnedStream();
-
-    if (streamName != null && !available.includes(streamName)) {
-      setStreamName(available[0]);
     }
-  }, [streamNameLoaded, camera.live.streams, streamName, setStreamName]);
+  }, [streamNameLoaded, liveStreams, pinnedStream, clearPinnedStream]);
 
   const { data: cameraMetadata } = useSWR<LiveStreamMetadata>(
     isRestreamed ? `go2rtc/streams/${streamName}` : null,
@@ -395,7 +426,6 @@ export default function LiveCameraView({
   // playback state
 
   const [audio, setAudio] = useSessionPersistence("liveAudio", false);
-  const [mic, setMic] = useState(false);
 
   // the mic only connects through the WebRTC player, so it can't stay on
   // for a stream that has lost it
@@ -404,9 +434,8 @@ export default function LiveCameraView({
       setMic(false);
     }
   }, [talkAvailable]);
-  const [webRTC, setWebRTC] = useState(false);
+
   const [pip, setPip] = useState(false);
-  const [lowBandwidth, setLowBandwidth] = useState(false);
 
   const [playInBackground, setPlayInBackground] = useUserPersistence<boolean>(
     `${camera.name}-background-play`,
@@ -414,7 +443,6 @@ export default function LiveCameraView({
   );
 
   const [showStats, setShowStats] = useState(false);
-  const [debug, setDebug] = useState(false);
 
   useSearchEffect("debug", (value: string) => {
     if (value === "true") {
@@ -434,7 +462,7 @@ export default function LiveCameraView({
       return "webrtc";
     }
 
-    if (forceLowBandwidth) {
+    if (forceLowBandwidth || autoStream.atFloor) {
       return "jsmpeg";
     }
 
@@ -467,6 +495,7 @@ export default function LiveCameraView({
     isRestreamed,
     resolvedUserMode,
     isWebRTCAvailable,
+    autoStream.atFloor,
   ]);
 
   // A latched error fallback would keep overriding the user's new choice.
@@ -593,6 +622,11 @@ export default function LiveCameraView({
   const handleError = useCallback(
     (e: LivePlayerError) => {
       if (e) {
+        // auto answers congestion and codec failures with a lower stream
+        if (autoHandleError(e)) {
+          return;
+        }
+
         // WebRTC can now be the user's own choice, so the fallback flag no
         // longer implies untried: hopping to the mode that just failed sticks.
         if (preferredLiveMode !== "webrtc" && webRTCUsable) {
@@ -603,9 +637,14 @@ export default function LiveCameraView({
         }
       }
     },
-    [preferredLiveMode, webRTCUsable],
+    [autoHandleError, preferredLiveMode, webRTCUsable],
   );
 
+  const resetStream = useCallback(() => {
+    setLowBandwidth(false);
+    resetAutoStream();
+  }, [resetAutoStream]);
+
   const handleMicrophoneError = useCallback(
     (error: TwoWayTalkError) => {
       setMic(false);
@@ -787,8 +826,11 @@ export default function LiveCameraView({
                 camera.audio_transcription.enabled_in_config
               }
               fullscreen={fullscreen}
-              streamName={streamName ?? ""}
+              pinnedStream={pinnedStream}
-              setStreamName={setStreamName}
+              playingStream={streamName ?? ""}
+              autoAvailable={autoAvailable}
+              autoReason={autoStream.reason}
+              onSelectStream={selectStream}
               userPreferredLiveMode={resolvedUserMode}
               setUserPreferredLiveMode={setUserPreferredLiveMode}
               forceLowBandwidth={forceLowBandwidth ?? false}
@@ -801,7 +843,7 @@ export default function LiveCameraView({
               isRestreamed={isRestreamed ?? false}
               isWebRTCAvailable={isWebRTCAvailable}
               webRTCUnavailableReason={webRTCAvailability.reason}
-              setLowBandwidth={setLowBandwidth}
+              onResetStream={resetStream}
               supportsAudioOutput={supportsAudioOutput}
               supports2WayTalk={supports2WayTalk}
               cameraEnabled={cameraEnabled}
@@ -889,6 +931,8 @@ export default function LiveCameraView({
                   containerRef={containerRef}
                   setFullResolution={setFullResolution}
                   onError={handleError}
+                  onHealthSample={autoSelected ? onHealthSample : undefined}
+                  streamAuto={autoSelected}
                   onMicrophoneError={handleMicrophoneError}
                 />
               </div>
@@ -942,8 +986,11 @@ type FrigateCameraFeaturesProps = {
   autotrackingEnabled: boolean;
   transcriptionEnabled: boolean;
   fullscreen: boolean;
-  streamName: string;
+  pinnedStream: string | undefined;
-  setStreamName?: (value: string | undefined) => void;
+  playingStream: string;
+  autoAvailable: boolean;
+  autoReason?: LiveAutoReason;
+  onSelectStream: (stream: string | undefined) => void;
   userPreferredLiveMode: LivePlayerMode;
   setUserPreferredLiveMode: (value: LivePlayerMode | undefined) => void;
   forceLowBandwidth: boolean;
@@ -956,7 +1003,7 @@ type FrigateCameraFeaturesProps = {
   isRestreamed: boolean;
   isWebRTCAvailable: boolean;
   webRTCUnavailableReason?: WebRTCUnavailableReason;
-  setLowBandwidth: React.Dispatch<React.SetStateAction<boolean>>;
+  onResetStream: () => void;
   supportsAudioOutput: boolean;
   supports2WayTalk: boolean;
   cameraEnabled: boolean;
@@ -970,8 +1017,11 @@ function FrigateCameraFeatures({
   autotrackingEnabled,
   transcriptionEnabled,
   fullscreen,
-  streamName,
+  pinnedStream,
-  setStreamName,
+  playingStream,
+  autoAvailable,
+  autoReason,
+  onSelectStream,
   userPreferredLiveMode,
   setUserPreferredLiveMode,
   forceLowBandwidth,
@@ -984,7 +1034,7 @@ function FrigateCameraFeatures({
   isRestreamed,
   isWebRTCAvailable,
   webRTCUnavailableReason,
-  setLowBandwidth,
+  onResetStream,
   supportsAudioOutput,
   supports2WayTalk,
   cameraEnabled,
@@ -1016,10 +1066,6 @@ function FrigateCameraFeatures({
 
   const isAdmin = useIsAdmin();
 
-  const streamSelectLabel = Object.keys(camera.live.streams).find(
-    (key) => camera.live.streams[key] === streamName,
-  );
-
   // manual event
 
   const recordingEventIdRef = useRef<string | null>(null);
@@ -1363,33 +1409,15 @@ function FrigateCameraFeatures({
                       <Label htmlFor="streaming-method">
                         {t("stream.title")}
                       </Label>
-                      <Select
+                      <LiveStreamSelect
-                        value={streamName}
+                        streams={camera.live.streams}
+                        pinnedStream={pinnedStream}
+                        playingStream={playingStream}
+                        autoAvailable={autoAvailable}
+                        autoReason={autoReason}
+                        onSelect={onSelectStream}
                         disabled={debug || forceLowBandwidth}
-                        onValueChange={(value) => {
+                      />
-                          setStreamName?.(value);
-                        }}
-                      >
-                        <SelectTrigger className="w-full">
-                          <SelectValue>{streamSelectLabel}</SelectValue>
-                        </SelectTrigger>
-
-                        <SelectContent>
-                          <SelectGroup>
-                            {Object.entries(camera.live.streams).map(
-                              ([stream, name]) => (
-                                <SelectItem
-                                  key={stream}
-                                  className="cursor-pointer"
-                                  value={name}
-                                >
-                                  {stream}
-                                </SelectItem>
-                              ),
-                            )}
-                          </SelectGroup>
-                        </SelectContent>
-                      </Select>
 
                       {debug && (
                         <div className="flex flex-row items-center gap-1 text-sm text-muted-foreground">
@@ -1510,7 +1538,7 @@ function FrigateCameraFeatures({
                               aria-label={t("stream.lowBandwidth.resetStream")}
                               variant="outline"
                               size="sm"
-                              onClick={() => setLowBandwidth(false)}
+                              onClick={onResetStream}
                             >
                               <MdOutlineRestartAlt className="size-5 text-primary-variant" />
                               <div className="text-primary-variant">
@@ -1772,33 +1800,15 @@ function FrigateCameraFeatures({
                 Object.values(camera.live.streams).length > 0 && (
                   <div className="mt-1 p-2">
                     <div className="mb-1 text-sm">{t("stream.title")}</div>
-                    <Select
+                    <LiveStreamSelect
-                      value={streamName}
+                      streams={camera.live.streams}
-                      onValueChange={(value) => {
+                      pinnedStream={pinnedStream}
-                        setStreamName?.(value);
+                      playingStream={playingStream}
-                      }}
+                      autoAvailable={autoAvailable}
+                      autoReason={autoReason}
+                      onSelect={onSelectStream}
                       disabled={debug || forceLowBandwidth}
-                    >
+                    />
-                      <SelectTrigger className="w-full">
-                        <SelectValue>{streamSelectLabel}</SelectValue>
-                      </SelectTrigger>
-
-                      <SelectContent>
-                        <SelectGroup>
-                          {Object.entries(camera.live.streams).map(
-                            ([stream, name]) => (
-                              <SelectItem
-                                key={stream}
-                                className="cursor-pointer"
-                                value={name}
-                              >
-                                {stream}
-                              </SelectItem>
-                            ),
-                          )}
-                        </SelectGroup>
-                      </SelectContent>
-                    </Select>
 
                     {debug && (
                       <div className="flex flex-row items-center gap-1 text-sm text-muted-foreground">
@@ -1915,7 +1925,7 @@ function FrigateCameraFeatures({
                             variant="outline"
                             size="sm"
                             disabled={debug}
-                            onClick={() => setLowBandwidth(false)}
+                            onClick={onResetStream}
                           >
                             <MdOutlineRestartAlt className="size-5 text-primary-variant" />
                             <div className="text-primary-variant">

web/src/views/live/LiveDashboardView.tsx

@@ -298,7 +298,10 @@ export default function LiveDashboardView({
     (cameraName: string, error: LivePlayerError) => {
       setPreferredLiveModes((prevModes) => {
         const newModes = { ...prevModes };
-        if (error === "mse-decode" && webRTCUsableStates[cameraName]) {
+        if (
+          (error === "mse-decode" || error === "mse-codec") &&
+          webRTCUsableStates[cameraName]
+        ) {
           newModes[cameraName] = "webrtc";
         } else {
           newModes[cameraName] = "jsmpeg";