Live streaming tech selection (#24374)

* allow users to select live streaming technology

* fix webrtc being downgraded to mse on load

`useUserPersistence` seeds state with the default and loads asynchronously, so the first render always used `mse` instead of the saved choice, and `useWebRTCGloballyAvailable` reports `checking` until the probe settles and re-enters that state on every consumer mount, so a saved `webrtc` was rewritten to `mse` even after the probe had already passed. On Safari the MSE player then timed out and latched the jsmpeg fallback. A pending probe now counts as available, the player waits on `autoLive` until the stored preferences load, and `handleError` gates on the mode in use since the fallback flag no longer implies webrtc is untried. A rejected IndexedDB read also resolves `loaded` now, so a blocked store can't leave the player waiting forever.

* add support for configurable ICE servers in WebRTC player

* add mic error state, fix dialog overwriting saved choice and dashboard ignoring stream

* tweaks

web/src/utils/cameraUtil.ts

@@ -174,7 +174,7 @@ export function detectCameraAudioFeatures(
 
   const twoWayAudio =
     (!requireSecureContext || window.isSecureContext) &&
-    metadata.producers.find(
+    (metadata.producers ?? []).find(
       (prod) =>
         prod.medias &&
         prod.medias.find((media) => media.includes("audio, sendonly")) !=
@@ -182,7 +182,7 @@ export function detectCameraAudioFeatures(
     ) != undefined;
 
   const audioOutput =
-    metadata.producers.find(
+    (metadata.producers ?? []).find(
       (prod) =>
         prod.medias &&
         prod.medias.find((media) => media.includes("audio, recvonly")) !=
@@ -195,6 +195,62 @@ export function detectCameraAudioFeatures(
   };
 }
 
+const MEDIA_DIRECTIONS = new Set(["sendonly", "recvonly", "sendrecv"]);
+
+// "sendonly" is the two-way-talk backchannel, not playback audio.
+const PLAYBACK_DIRECTIONS = new Set(["recvonly", "sendrecv"]);
+
+// Parses a go2rtc media line: "audio, recvonly, OPUS/48000/2" -> ["OPUS"],
+// stripping the "/clockrate[/channels]" suffix.
+function codecsFromMedia(media: string): string[] {
+  const parts = media.split(",").map((p) => p.trim());
+  return parts
+    .slice(1)
+    .filter((p) => !MEDIA_DIRECTIONS.has(p.toLowerCase()))
+    .map((p) => p.split("/")[0].toUpperCase());
+}
+
+function mediaDirection(media: string): string | undefined {
+  return media
+    .split(",")
+    .map((p) => p.trim().toLowerCase())
+    .find((p) => MEDIA_DIRECTIONS.has(p));
+}
+
+function codecsForKind(
+  metadata: LiveStreamMetadata | null | undefined,
+  kind: "video" | "audio",
+  directions?: Set<string>,
+): string[] {
+  if (!metadata) return [];
+  const codecs = new Set<string>();
+  for (const producer of metadata.producers ?? []) {
+    for (const media of producer.medias ?? []) {
+      if (!media.trim().toLowerCase().startsWith(kind)) continue;
+      if (directions) {
+        const direction = mediaDirection(media);
+        if (!direction || !directions.has(direction)) continue;
+      }
+      codecsFromMedia(media).forEach((c) => codecs.add(c));
+    }
+  }
+  return Array.from(codecs);
+}
+
+export function getStreamVideoCodecs(
+  metadata: LiveStreamMetadata | null | undefined,
+): string[] {
+  return codecsForKind(metadata, "video");
+}
+
+// go2rtc exposes a WebRTC-capable playback codec only when one is configured
+// (e.g. an opus transcode) or the camera streams G.711 natively.
+export function getPlaybackAudioCodecs(
+  metadata: LiveStreamMetadata | null | undefined,
+): string[] {
+  return codecsForKind(metadata, "audio", PLAYBACK_DIRECTIONS);
+}
+
 const REPLAY_CAMERA_PREFIX = "_replay_";
 
 /**

web/src/utils/uiSettingsTransfer.ts

@@ -10,6 +10,7 @@ export type TransferSection = "layouts" | "streaming" | "preferences";
 const cameraStreamingSettingsSchema = z.object({
   streamName: z.string(),
   streamType: z.enum(["no-streaming", "smart", "continuous"]),
+  playerMode: z.enum(["mse", "webrtc", "jsmpeg"]).optional().catch(undefined),
   compatibilityMode: z.boolean(),
   playAudio: z.boolean(),
   volume: z.number(),

web/src/utils/webrtcProbe.ts

@@ -0,0 +1,140 @@
+import { baseUrl } from "@/api/baseUrl";
+
+/**
+ * Performs a single real WebRTC handshake against go2rtc to verify that a
+ * media connection can actually be established (validates candidates, port
+ * 8555 reachability, and STUN/TURN end-to-end). Result is cached per page
+ * session via a module-level promise.
+ */
+
+export type WebRTCProbeResult = {
+  ok: boolean;
+  detail?: string;
+};
+
+let probePromise: Promise<WebRTCProbeResult> | null = null;
+
+function describeError(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+function runProbe(
+  testStream: string,
+  iceServers: RTCIceServer[],
+  timeoutMs: number,
+): Promise<WebRTCProbeResult> {
+  return new Promise<WebRTCProbeResult>((resolve) => {
+    let settled = false;
+    const wsURL = `${baseUrl.replace(/^http/, "ws")}live/webrtc/api/ws?src=${testStream}`;
+
+    const pc = new RTCPeerConnection({
+      bundlePolicy: "max-bundle",
+      iceServers,
+    });
+    let ws: WebSocket | null = null;
+
+    const cleanup = (result: WebRTCProbeResult) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      try {
+        ws?.close();
+      } catch {
+        // ignore
+      }
+      try {
+        pc.close();
+      } catch {
+        // ignore
+      }
+      resolve(result);
+    };
+
+    const fail = (detail: string) => cleanup({ ok: false, detail });
+
+    const timer = setTimeout(
+      () => fail(`no ICE connection within ${timeoutMs}ms`),
+      timeoutMs,
+    );
+
+    pc.oniceconnectionstatechange = () => {
+      const state = pc.iceConnectionState;
+      if (state === "connected" || state === "completed") {
+        cleanup({ ok: true });
+      } else if (state === "failed" || state === "closed") {
+        fail(`ICE connection state: ${state}`);
+      }
+    };
+
+    pc.addTransceiver("video", { direction: "recvonly" });
+
+    try {
+      ws = new WebSocket(wsURL);
+    } catch (err) {
+      fail(`WebSocket to go2rtc could not be opened: ${describeError(err)}`);
+      return;
+    }
+
+    ws.addEventListener("error", () => fail("WebSocket to go2rtc errored"));
+
+    ws.addEventListener("open", () => {
+      pc.addEventListener("icecandidate", (ev) => {
+        if (!ev.candidate || !ws) return;
+        ws.send(
+          JSON.stringify({
+            type: "webrtc/candidate",
+            value: ev.candidate.candidate,
+          }),
+        );
+      });
+
+      pc.createOffer()
+        .then((offer) => pc.setLocalDescription(offer))
+        .then(() => {
+          ws?.send(
+            JSON.stringify({
+              type: "webrtc/offer",
+              value: pc.localDescription?.sdp,
+            }),
+          );
+        })
+        .catch((err) =>
+          fail(`failed to create the local offer: ${describeError(err)}`),
+        );
+    });
+
+    ws.addEventListener("message", (ev) => {
+      let msg: { type: string; value: string };
+      try {
+        msg = JSON.parse((ev as MessageEvent).data);
+      } catch {
+        return;
+      }
+      if (msg.type === "webrtc/candidate") {
+        pc.addIceCandidate({ candidate: msg.value, sdpMid: "0" }).catch((err) =>
+          fail(`remote ICE candidate rejected: ${describeError(err)}`),
+        );
+      } else if (msg.type === "webrtc/answer") {
+        pc.setRemoteDescription({ type: "answer", sdp: msg.value }).catch(
+          (err) => fail(`remote answer rejected: ${describeError(err)}`),
+        );
+      }
+    });
+  });
+}
+
+export function probeWebRTCAvailability(
+  testStream: string,
+  iceServers: RTCIceServer[],
+  timeoutMs: number = 5000,
+): Promise<WebRTCProbeResult> {
+  if (!probePromise) {
+    probePromise = runProbe(testStream, iceServers, timeoutMs);
+  }
+  return probePromise;
+}
+
+/** Clears the cached probe result (e.g. when go2rtc config changes). */
+export function resetWebRTCProbe(): void {
+  probePromise = null;
+}

web/src/utils/webrtcUtil.ts

@@ -0,0 +1,108 @@
+import { FrigateConfig } from "@/types/frigateConfig";
+import { WebRTCUnavailableReason } from "@/types/live";
+
+/**
+ * Helpers for detecting what the current browser can do over WebRTC.
+ * Used to gate the WebRTC live streaming option (e.g. H.265 is only
+ * decodable over WebRTC on Chrome 136+ / Safari 18+ with HEVC hardware).
+ */
+
+const DEFAULT_ICE_SERVERS: RTCIceServer[] = [
+  { urls: "stun:stun.l.google.com:19302" },
+];
+
+type ConfiguredIceServers = FrigateConfig["go2rtc"]["webrtc"]["ice_servers"];
+
+/** ICE servers for browser peer connections, falling back to public STUN. */
+export function webRTCIceServers(
+  configured: ConfiguredIceServers,
+): RTCIceServer[] {
+  if (!configured?.length) {
+    return DEFAULT_ICE_SERVERS;
+  }
+
+  return configured.map((server) => ({
+    urls: server.urls,
+    username: server.username,
+    credential: server.credential,
+  }));
+}
+
+export function browserSupportsWebRTC(): boolean {
+  return typeof window !== "undefined" && "RTCPeerConnection" in window;
+}
+
+/** Codec aliases that refer to the same underlying codec. */
+const CODEC_ALIASES: Record<string, string> = {
+  HEVC: "H265",
+  AVC: "H264",
+};
+
+function normalizeCodec(codec: string): string {
+  const upper = codec.toUpperCase();
+  return CODEC_ALIASES[upper] ?? upper;
+}
+
+/** The set of video codec names the browser can receive over WebRTC. */
+export function browserWebRTCVideoCodecs(): string[] {
+  if (
+    typeof RTCRtpReceiver === "undefined" ||
+    typeof RTCRtpReceiver.getCapabilities !== "function"
+  ) {
+    return [];
+  }
+
+  const caps = RTCRtpReceiver.getCapabilities("video");
+  if (!caps) return [];
+
+  const codecs = new Set<string>();
+  for (const codec of caps.codecs) {
+    // mimeType is like "video/H264"; ignore infra codecs (rtx, red, ulpfec)
+    const name = codec.mimeType.split("/")[1]?.toUpperCase();
+    if (!name || ["RTX", "RED", "ULPFEC", "FLEXFEC-03"].includes(name)) {
+      continue;
+    }
+    codecs.add(normalizeCodec(name));
+  }
+  return Array.from(codecs);
+}
+
+export function browserSupportsWebRTCVideoCodec(codec: string): boolean {
+  return browserWebRTCVideoCodecs().includes(normalizeCodec(codec));
+}
+
+const DOCS_URL =
+  "https://docs.frigate.video/configuration/live/#selecting-a-streaming-technology";
+
+// Availability is consumed by several components at once, so without this the
+// same fact prints once per consumer.
+const loggedMessages = new Set<string>();
+
+/**
+ * Logs why WebRTC is unavailable, supplementing the short message shown inline
+ * in the technology selector with the underlying detail.
+ *
+ * @param scope - Stream name for per-stream reasons, omitted for global ones
+ * @param reason - The reason surfaced in the UI
+ * @param description - The detail the UI message leaves out
+ */
+export function logWebRTCUnavailable(
+  scope: string | undefined,
+  reason: WebRTCUnavailableReason,
+  description: string,
+): void {
+  const message = `${scope ? `${scope} - ` : ""}WebRTC unavailable '${reason}': ${description} See the documentation: ${DOCS_URL}`;
+
+  if (loggedMessages.has(message)) {
+    return;
+  }
+  loggedMessages.add(message);
+
+  // eslint-disable-next-line no-console
+  console.warn(message);
+}
+
+/** Clears the emitted messages (e.g. when go2rtc config changes). */
+export function resetWebRTCUnavailableLog(): void {
+  loggedMessages.clear();
+}