Add Auto live view option with transcoding options (#24519)

* implement auto mode for single camera live view * add transcoded live streams and stream ordering for auto mode Cameras can now add lower quality live streams that go2rtc transcodes to H.264 on demand. `live.transcode` takes a source stream and a list of heights and bitrates, and each quality becomes a `{camera}_transcode_{height}p` stream. The config validator adds them to `live.streams` without moving any the user already placed, and drops them when transcoding is disabled or a height changes. `create_config.py` writes them into go2rtc's generated config at startup, and saving the config or deleting a camera syncs them through go2rtc's API, so no restart is needed. They use `#hardware`, so go2rtc picks a hardware encoder and falls back to the CPU when there isn't one. The Live playback settings stream list can be reordered by drag, since its order is the auto ladder. Auto order sorts it by bitrate, measuring native streams through a new admin-only `/go2rtc/streams/{name}/bitrate` endpoint and using the configured bitrate for transcoded ones. A pure reorder wasn't saved before because RJSF, the settings form, and `update_yaml` all ignore map key order. Sections can now mark a map with `orderedMaps`, which sends the whole map with `replace_paths` so `config_set` rewrites it in order. Transcoded streams aren't in `go2rtc.streams`, which only lists yaml streams, so the frontend treated them as not restreamed and fell back to jsmpeg. Every restream check now goes through `isRestreamedStream`. Auto treated any stall with no bytes in the last 2 seconds as a dead camera and handed it to the error fallback, which went straight to jsmpeg. Heavy congestion can stop delivery completely, so congested viewers skipped every lower stream. Auto now declines only when stats show the camera offline, and a stall on a live camera steps down. The stream picker also has a Try highest quality button that sends auto back to the top stream. * fixes
2026-10-06 14:52:47 +03:00 · 2026-09-30 15:10:49 -06:00
parent 1bb61eb808
commit 2754f788cc
46 changed files with 3659 additions and 286 deletions
@@ -0,0 +1,320 @@
+/**
+ * Auto live stream selection in the single-camera view.
+ *
+ * go2rtc is mocked per stream. A failing stream answers MSE codec
+ * negotiation with go2rtc's "codecs not matched" error, the one ladder
+ * trigger Chromium reproduces without real media. A healthy stream
+ * answers negotiation and then sends nothing, so it waits quietly.
+ */
+import type { Locator, Page } from "@playwright/test";
+import { test, expect } from "../fixtures/frigate-test";
+import { LivePage } from "../pages/live.page";
+
+// the mocked profile is admin, so useUserPersistence keys are namespaced
+const STREAM_KEY = "front_door-stream:admin";
+const CODEC_ERROR = "mse: streams: codecs not matched: H265 => H264";
+
+const CONFIG = {
+  go2rtc: {
+    streams: {
+      front_door: ["rtsp://127.0.0.1:8554/front_door"],
+      front_door_sub: ["rtsp://127.0.0.1:8554/front_door_sub"],
+      backyard: ["rtsp://127.0.0.1:8554/backyard"],
+    },
+    webrtc: { candidates: [], ice_servers: [] },
+  },
+  cameras: {
+    front_door: {
+      live: { streams: { Sub: "front_door_sub" } },
+    },
+  },
+};
+
+test.use({ expectedErrors: [/MSE error 'mse-codec'/] });
+
+async function writeIdb(page: Page, entries: Record<string, unknown>) {
+  await page.evaluate(async (data) => {
+    await new Promise<void>((resolve, reject) => {
+      const request = indexedDB.open("keyval-store", 1);
+      request.onupgradeneeded = () =>
+        request.result.createObjectStore("keyval");
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => {
+        const tx = request.result.transaction("keyval", "readwrite");
+        const store = tx.objectStore("keyval");
+        Object.entries(data).forEach(([key, value]) => store.put(value, key));
+        tx.oncomplete = () => resolve();
+        tx.onerror = () => reject(tx.error);
+      };
+    });
+  }, entries);
+}
+
+async function readIdb(page: Page, key: string) {
+  return page.evaluate(async (target) => {
+    return new Promise((resolve, reject) => {
+      const request = indexedDB.open("keyval-store", 1);
+      request.onupgradeneeded = () =>
+        request.result.createObjectStore("keyval");
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => {
+        const tx = request.result.transaction("keyval", "readonly");
+        const get = tx.objectStore("keyval").get(target);
+        get.onsuccess = () => resolve(get.result ?? null);
+        get.onerror = () => reject(get.error);
+      };
+    });
+  }, key);
+}
+
+// records the src of every MSE socket in open order
+async function mockGo2rtc(page: Page, failing: string[] = []) {
+  const opened: string[] = [];
+
+  await page.routeWebSocket("**/live/mse/api/ws**", (ws) => {
+    const src = new URL(ws.url()).searchParams.get("src") ?? "";
+    opened.push(src);
+
+    ws.onMessage((message) => {
+      const msg = JSON.parse(message.toString());
+      if (msg.type !== "mse") {
+        return;
+      }
+
+      ws.send(
+        JSON.stringify(
+          failing.includes(src)
+            ? { type: "error", value: CODEC_ERROR }
+            : { type: "mse", value: 'video/mp4; codecs="avc1.640029"' },
+        ),
+      );
+    });
+  });
+
+  await page.route("**/api/go2rtc/streams/**", (route) =>
+    route.fulfill({
+      json: {
+        producers: [{ medias: ["video, recvonly, H264"] }],
+        consumers: [],
+      },
+    }),
+  );
+
+  return opened;
+}
+
+async function openDesktopSettings(page: Page) {
+  const live = new LivePage(page, true);
+  await expect(live.backButton).toBeVisible({ timeout: 10_000 });
+  await page.locator("button:has(svg)").last().click();
+  const menu = page.locator('[role="menu"], [data-radix-menu-content]').first();
+  await expect(menu).toBeVisible({ timeout: 3_000 });
+  return menu;
+}
+
+function streamTrigger(menu: Locator) {
+  return menu
+    .locator('div:has(> label[for="streaming-method"]) [role="combobox"]')
+    .first();
+}
+
+test.describe("Auto live stream @mobile", () => {
+  test("a codec error on the main stream moves Auto to the sub stream", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, ["front_door"]);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_sub");
+    expect(opened[0]).toBe("front_door");
+  });
+});
+
+test.describe("Transcoded live stream @mobile", () => {
+  test("a pinned transcoded stream plays over MSE", async ({ frigateApp }) => {
+    // go2rtc.streams lists only yaml streams; transcoded ones are generated
+    await frigateApp.installDefaults({
+      config: {
+        ...CONFIG,
+        cameras: {
+          front_door: {
+            live: {
+              streams: {
+                Sub: "front_door_sub",
+                "720p": "front_door_transcode_720p",
+              },
+              transcode: {
+                enabled: true,
+                source: "front_door",
+                qualities: [{ height: 720, bitrate: 1200 }],
+              },
+            },
+          },
+        },
+      },
+    });
+    const opened = await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/");
+    await writeIdb(frigateApp.page, {
+      [STREAM_KEY]: "front_door_transcode_720p",
+    });
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_transcode_720p");
+  });
+});
+
+test.describe("Auto live stream selector", () => {
+  test.beforeEach(({ frigateApp }) => {
+    test.skip(frigateApp.isMobile, "Desktop dropdown only");
+  });
+
+  test("no saved stream opens Auto on the first stream", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect.poll(() => opened[0], { timeout: 10_000 }).toBe("front_door");
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Auto · front_door");
+  });
+
+  test("a codec error shows the codec reason", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, ["front_door"]);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_sub");
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Auto · Sub");
+    await expect(
+      menu.getByText(/because this browser can't play a higher-quality stream/),
+    ).toBeVisible();
+  });
+
+  test("trying the highest quality retries the top stream", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, ["front_door"]);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_sub");
+    const menu = await openDesktopSettings(frigateApp.page);
+    const beforeRetry = opened.length;
+    await menu.getByRole("button", { name: "Try highest quality" }).click();
+
+    await expect
+      .poll(() => opened.slice(beforeRetry), { timeout: 10_000 })
+      .toContain("front_door");
+  });
+
+  test("a pinned main takes the legacy fallback instead of stepping down", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, ["front_door"]);
+
+    await frigateApp.goto("/");
+    await writeIdb(frigateApp.page, { [STREAM_KEY]: "front_door" });
+    await frigateApp.goto("/#front_door");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(
+      menu.getByText(/Live view is in low-bandwidth mode/),
+    ).toBeVisible({ timeout: 10_000 });
+    expect(opened).not.toContain("front_door_sub");
+  });
+
+  test("selecting Auto clears the saved stream", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/");
+    await writeIdb(frigateApp.page, { [STREAM_KEY]: "front_door_sub" });
+    await frigateApp.goto("/#front_door");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Sub");
+    await streamTrigger(menu).click();
+    await frigateApp.page.getByRole("option", { name: /^Auto/ }).click();
+
+    await expect.poll(() => readIdb(frigateApp.page, STREAM_KEY)).toBeNull();
+  });
+
+  test("a stale pin falls back to Auto", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/");
+    await writeIdb(frigateApp.page, { [STREAM_KEY]: "removed_stream" });
+    await frigateApp.goto("/#front_door");
+
+    await expect.poll(() => readIdb(frigateApp.page, STREAM_KEY)).toBeNull();
+    await expect.poll(() => opened[0], { timeout: 10_000 }).toBe("front_door");
+  });
+
+  test("a single-stream camera offers no Auto", async ({ frigateApp }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    await mockGo2rtc(frigateApp.page);
+
+    await frigateApp.goto("/#backyard");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await streamTrigger(menu).click();
+    await expect(
+      frigateApp.page.getByRole("option", { name: "backyard" }),
+    ).toBeVisible();
+    await expect(
+      frigateApp.page.getByRole("option", { name: /^Auto/ }),
+    ).toHaveCount(0);
+  });
+
+  test("Reset stream at the floor returns to the top stream", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.installDefaults({ config: CONFIG });
+    const opened = await mockGo2rtc(frigateApp.page, [
+      "front_door",
+      "front_door_sub",
+    ]);
+
+    await frigateApp.goto("/#front_door");
+
+    await expect
+      .poll(() => opened, { timeout: 10_000 })
+      .toContain("front_door_sub");
+    expect(opened[0]).toBe("front_door");
+
+    const menu = await openDesktopSettings(frigateApp.page);
+    await expect(streamTrigger(menu)).toHaveText("Auto · Low bandwidth");
+    await expect(
+      menu.getByText(/Playing the low-bandwidth feed/),
+    ).toBeVisible();
+
+    const beforeReset = opened.length;
+    await menu.getByRole("button", { name: "Reset stream" }).click();
+
+    // leaving the floor changes the stream and the mode in one render,
+    // which must still open each stream exactly once
+    await expect
+      .poll(() => opened.slice(beforeReset), { timeout: 10_000 })
+      .toEqual(["front_door", "front_door_sub"]);
+  });
+});
@@ -0,0 +1,285 @@
+/**
+ * Live playback stream order and transcoded streams -- MEDIUM tier.
+ *
+ * The stream list order is the Auto ladder, so the form saves it with
+ * replace_paths. Transcoded streams are generated from live.transcode and
+ * appear in the list as soon as the switch flips.
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { test, expect } from "../../fixtures/frigate-test";
+import type { Page } from "@playwright/test";
+import { configFactory } from "../../fixtures/mock-data/config";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CONFIG_SCHEMA = JSON.parse(
+  readFileSync(
+    resolve(__dirname, "../../fixtures/mock-data/config-schema.json"),
+    "utf-8",
+  ),
+);
+
+const GO2RTC_STREAMS = {
+  front_door_main: ["rtsp://user:pass@192.168.0.20:554/main"],
+  front_door_sub: ["rtsp://user:pass@192.168.0.20:554/sub"],
+};
+
+const TRANSCODE = {
+  enabled: false,
+  qualities: [
+    { height: 720, bitrate: 1200 },
+    { height: 480, bitrate: 500 },
+  ],
+};
+
+const SETTINGS_URL = "/settings?page=cameraLivePlayback&camera=front_door";
+
+type RouteOptions = {
+  go2rtcStreams?: Record<string, string[]>;
+  streams?: Record<string, string>;
+  bitrates?: Record<string, number>;
+};
+
+async function installRoutes(page: Page, options: RouteOptions = {}) {
+  const go2rtcStreams = options.go2rtcStreams ?? GO2RTC_STREAMS;
+  const streams = options.streams ?? {
+    Sub: "front_door_sub",
+    Main: "front_door_main",
+  };
+  const live = { streams, transcode: TRANSCODE };
+  const config = configFactory({
+    go2rtc: { streams: go2rtcStreams },
+    cameras: { front_door: { live } },
+  });
+  // configFactory's deepMerge adds override keys onto the base fixture's
+  // streams map instead of replacing it (they share no key names here), so
+  // pin the streams map to exactly what this test wants.
+  config.cameras.front_door.live.streams = streams;
+
+  let lastSaved: Record<string, unknown> | null = null;
+
+  await page.route("**/api/config/schema.json", (route) =>
+    route.fulfill({ json: CONFIG_SCHEMA }),
+  );
+  await page.route("**/api/config", (route) => {
+    if (route.request().method() === "GET") {
+      return route.fulfill({ json: config });
+    }
+    return route.fulfill({ json: { success: true } });
+  });
+  await page.route("**/api/config/raw_paths", (route) =>
+    route.fulfill({
+      json: {
+        go2rtc: { streams: go2rtcStreams },
+        cameras: { front_door: { live } },
+      },
+    }),
+  );
+  await page.route("**/api/config/set", async (route) => {
+    lastSaved = route.request().postDataJSON();
+    await route.fulfill({ json: { success: true, go2rtc_synced: true } });
+  });
+  await page.route("**/api/go2rtc/streams/*/bitrate", (route) => {
+    const name = decodeURIComponent(
+      new URL(route.request().url()).pathname.split("/").slice(-2)[0],
+    );
+    const kbps = options.bitrates?.[name];
+    return kbps === undefined
+      ? route.fulfill({ status: 502, json: { success: false } })
+      : route.fulfill({ json: { success: true, kbps } });
+  });
+
+  return { saved: () => lastSaved };
+}
+
+function streamNames(page: Page) {
+  return page
+    .getByRole("textbox", { name: "Stream name" })
+    .evaluateAll((inputs) =>
+      inputs.map((input) => (input as HTMLInputElement).value),
+    );
+}
+
+async function dragRow(page: Page, from: number, to: number) {
+  const handles = page.getByRole("button", { name: "Drag to reorder" });
+  const start = await handles.nth(from).boundingBox();
+  const end = await handles.nth(to).boundingBox();
+
+  if (!start || !end) throw new Error("drag handle not visible");
+
+  await page.mouse.move(start.x + start.width / 2, start.y + start.height / 2);
+  await page.mouse.down();
+  await page.mouse.move(
+    end.x + end.width / 2,
+    end.y + end.height / 2 + (to > from ? 12 : -12),
+    { steps: 15 },
+  );
+  await page.mouse.up();
+}
+
+test.describe("live stream order and transcoded streams @medium", () => {
+  test.skip(({ frigateApp }) => frigateApp.isMobile, "desktop drag");
+
+  test("dragging a stream saves the new order", async ({ frigateApp }) => {
+    const capture = await installRoutes(frigateApp.page);
+    await frigateApp.goto(SETTINGS_URL);
+
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Sub", "Main"]);
+
+    await dragRow(frigateApp.page, 1, 0);
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Main", "Sub"]);
+
+    await frigateApp.page.getByRole("button", { name: "Save" }).click();
+
+    await expect.poll(() => capture.saved()).not.toBeNull();
+    const saved = capture.saved() as {
+      replace_paths: string[];
+      config_data: {
+        cameras: { front_door: { live: { streams: Record<string, string> } } };
+      };
+    };
+    expect(saved.replace_paths).toEqual(["cameras.front_door.live.streams"]);
+    expect(
+      Object.keys(saved.config_data.cameras.front_door.live.streams),
+    ).toEqual(["Main", "Sub"]);
+  });
+
+  test("save all keeps a reorder made on another page", async ({
+    frigateApp,
+  }) => {
+    const capture = await installRoutes(frigateApp.page);
+    await frigateApp.goto(SETTINGS_URL);
+
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Sub", "Main"]);
+
+    await dragRow(frigateApp.page, 1, 0);
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Main", "Sub"]);
+
+    await frigateApp.page.getByText("Snapshots", { exact: true }).click();
+    await frigateApp.page.getByRole("button", { name: "Save All" }).click();
+
+    await expect.poll(() => capture.saved()).not.toBeNull();
+    const saved = capture.saved() as {
+      replace_paths: string[];
+      config_data: {
+        cameras: { front_door: { live: { streams: Record<string, string> } } };
+      };
+    };
+    expect(saved.replace_paths).toEqual(["cameras.front_door.live.streams"]);
+    expect(
+      Object.keys(saved.config_data.cameras.front_door.live.streams),
+    ).toEqual(["Main", "Sub"]);
+  });
+
+  test("enabling transcoding saves the shown source", async ({
+    frigateApp,
+  }) => {
+    const capture = await installRoutes(frigateApp.page);
+    await frigateApp.goto(SETTINGS_URL);
+
+    await frigateApp.page
+      .getByRole("switch", { name: "Enable transcoded streams" })
+      .click();
+    await frigateApp.page.getByRole("button", { name: "Save" }).click();
+
+    await expect.poll(() => capture.saved()).not.toBeNull();
+    const saved = capture.saved() as {
+      config_data: {
+        cameras: {
+          front_door: { live: { transcode: Record<string, unknown> } };
+        };
+      };
+    };
+    // pinned, so dragging another stream to the top can't change it
+    expect(saved.config_data.cameras.front_door.live.transcode).toMatchObject({
+      enabled: true,
+      source: "front_door_sub",
+    });
+  });
+
+  test("the transcode switch adds and removes transcoded rows", async ({
+    frigateApp,
+  }) => {
+    await installRoutes(frigateApp.page);
+    await frigateApp.goto(SETTINGS_URL);
+
+    const toggle = frigateApp.page.getByRole("switch", {
+      name: "Enable transcoded streams",
+    });
+    const save = frigateApp.page.getByRole("button", { name: "Save" });
+
+    await toggle.click();
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Sub", "Main", "720p", "480p"]);
+    await expect(
+      frigateApp.page.getByText("Transcoded", { exact: true }),
+    ).toHaveCount(2);
+    await expect(save).toBeEnabled();
+
+    await toggle.click();
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Sub", "Main"]);
+    await expect(save).toBeDisabled();
+  });
+
+  test("auto order sorts measured and set bitrates", async ({ frigateApp }) => {
+    await installRoutes(frigateApp.page, {
+      bitrates: { front_door_main: 4000, front_door_sub: 800 },
+    });
+    await frigateApp.goto(SETTINGS_URL);
+
+    await frigateApp.page
+      .getByRole("switch", { name: "Enable transcoded streams" })
+      .click();
+    await frigateApp.page.getByRole("button", { name: "Auto order" }).click();
+
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Main", "720p", "Sub", "480p"]);
+    await expect(frigateApp.page.getByText("4000 kbps")).toBeVisible();
+    await expect(frigateApp.page.getByText("1200 kbps (set)")).toBeVisible();
+  });
+
+  test("an unmeasurable stream sorts last", async ({ frigateApp }) => {
+    await installRoutes(frigateApp.page, {
+      bitrates: { front_door_main: 4000 },
+    });
+    await frigateApp.goto(SETTINGS_URL);
+
+    await frigateApp.page.getByRole("button", { name: "Auto order" }).click();
+
+    await expect
+      .poll(() => streamNames(frigateApp.page))
+      .toEqual(["Main", "Sub"]);
+    await expect(frigateApp.page.getByText("Couldn't measure")).toBeVisible();
+  });
+
+  test("transcoding needs a go2rtc stream", async ({ frigateApp }) => {
+    await installRoutes(frigateApp.page, {
+      go2rtcStreams: {},
+      streams: { front_door: "front_door" },
+    });
+    await frigateApp.goto(SETTINGS_URL);
+
+    await expect(
+      frigateApp.page.getByRole("switch", {
+        name: "Enable transcoded streams",
+      }),
+    ).toBeDisabled();
+    await expect(
+      frigateApp.page.getByText(/need a go2rtc stream to read from/),
+    ).toBeVisible();
+  });
+});