retry failed WebRTC probes and skip offline streams

The connectivity probe only ever tried the first go2rtc stream and cached its result for the whole page session, so a single offline camera at the top of the go2rtc config marked WebRTC unreachable for every camera until a reload, as did any brief network hiccup. The probe now starts with the stream being viewed and moves on to the next one when go2rtc reports that it can't open the stream's source. A failed result is only reused for 30 seconds, and it's retried on the next mount or when the page becomes visible again.

web/src/hooks/use-webrtc-availability.ts

@@ -79,7 +79,6 @@ export function evaluateStreamWebRTCAvailability(args: {
 /** Console detail for a reason decided by the global availability check. */
 function describeGlobalReason(
   reason: WebRTCUnavailableReason,
-  testStream: string | undefined,
   probeDetail: string | undefined,
 ): string {
   switch (reason) {
@@ -88,7 +87,7 @@ function describeGlobalReason(
     case "not-configured":
       return "No candidates or ice_servers are set under go2rtc.webrtc.";
     case "unreachable":
-      return `The connectivity probe against stream '${testStream}' failed: ${probeDetail ?? "unknown cause"}.`;
+      return `The connectivity probe failed against ${probeDetail ?? "an unknown stream"}.`;
     default:
       return "";
   }
@@ -98,9 +97,13 @@ function describeGlobalReason(
 let lastProbeSignature: string | null = null;
 
 /**
- * Once-per-session: browser support, go2rtc config, and a live handshake probe.
+ * Browser support, go2rtc config, and a live handshake probe shared by the
+ * page. A failed probe is retried on a later mount or when the page becomes
+ * visible again once its cached result has expired.
  */
-export function useWebRTCGloballyAvailable(): GlobalAvailability {
+export function useWebRTCGloballyAvailable(
+  preferredStream?: string,
+): GlobalAvailability {
   const { data: config } = useSWR<FrigateConfig>("config");
   const [probe, setProbe] = useState<{
     state: "pending" | "pass" | "fail";
@@ -118,11 +121,19 @@ export function useWebRTCGloballyAvailable(): GlobalAvailability {
     );
   }, [config]);
 
-  // Representative restreamed stream to probe against.
-  const testStream = useMemo(() => {
-    const streams = config?.go2rtc?.streams ?? {};
-    return Object.keys(streams)[0];
-  }, [config]);
+  // the stream being viewed is the one most likely to be online
+  const testStreams = useMemo(() => {
+    const streams = Object.keys(config?.go2rtc?.streams ?? {});
+
+    if (!preferredStream || !streams.includes(preferredStream)) {
+      return streams;
+    }
+
+    return [preferredStream, ...streams.filter((s) => s !== preferredStream)];
+  }, [config, preferredStream]);
+  const testStreamsKey = testStreams.join("\n");
+
+  const [retryToken, setRetryToken] = useState(0);
 
   const iceServers = useMemo(
     () => webRTCIceServers(config?.go2rtc?.webrtc?.ice_servers),
@@ -136,9 +147,9 @@ export function useWebRTCGloballyAvailable(): GlobalAvailability {
       JSON.stringify({
         candidates: config?.go2rtc?.webrtc?.candidates ?? [],
         iceServers,
-        testStream,
+        streams: Object.keys(config?.go2rtc?.streams ?? {}),
       }),
-    [config, iceServers, testStream],
+    [config, iceServers],
   );
 
   useEffect(() => {
@@ -153,11 +164,29 @@ export function useWebRTCGloballyAvailable(): GlobalAvailability {
   }, [probeSignature]);
 
   useEffect(() => {
-    if (!browserOk || !configured || !testStream) {
+    if (probe.state !== "fail") {
+      return;
+    }
+
+    const onVisibilityChange = () => {
+      if (document.visibilityState === "visible") {
+        setRetryToken((token) => token + 1);
+      }
+    };
+
+    document.addEventListener("visibilitychange", onVisibilityChange);
+    return () =>
+      document.removeEventListener("visibilitychange", onVisibilityChange);
+  }, [probe.state]);
+
+  useEffect(() => {
+    const streams = testStreamsKey ? testStreamsKey.split("\n") : [];
+
+    if (!browserOk || !configured || streams.length === 0) {
       return;
     }
     let cancelled = false;
-    probeWebRTCAvailability(testStream, iceServers).then((result) => {
+    probeWebRTCAvailability(streams, iceServers).then((result) => {
       if (!cancelled) {
         setProbe({
           state: result.ok ? "pass" : "fail",
@@ -168,7 +197,7 @@ export function useWebRTCGloballyAvailable(): GlobalAvailability {
     return () => {
       cancelled = true;
     };
-  }, [browserOk, configured, testStream, iceServers]);
+  }, [browserOk, configured, testStreamsKey, iceServers, retryToken]);
 
   const availability = useMemo<GlobalAvailability>(() => {
     if (!browserOk) {
@@ -196,9 +225,9 @@ export function useWebRTCGloballyAvailable(): GlobalAvailability {
     logWebRTCUnavailable(
       undefined,
       reason,
-      describeGlobalReason(reason, testStream, probe.detail),
+      describeGlobalReason(reason, probe.detail),
     );
-  }, [config, availability, testStream, probe.detail]);
+  }, [config, availability, probe.detail]);
 
   return availability;
 }
@@ -208,7 +237,8 @@ export function useWebRTCAvailableForStream(
   metadata: LiveStreamMetadata | null | undefined,
   streamName?: string,
 ): StreamAvailability {
-  const { globallyAvailable, globalReason } = useWebRTCGloballyAvailable();
+  const { globallyAvailable, globalReason } =
+    useWebRTCGloballyAvailable(streamName);
 
   const availability = useMemo(
     () =>

web/src/utils/webrtcProbe.ts

@@ -1,18 +1,29 @@
 import { baseUrl } from "@/api/baseUrl";
 
 /**
- * Performs a single real WebRTC handshake against go2rtc to verify that a
- * media connection can actually be established (validates candidates, port
- * 8555 reachability, and STUN/TURN end-to-end). Result is cached per page
- * session via a module-level promise.
+ * Performs a real WebRTC handshake against go2rtc to verify that a media
+ * connection can actually be established (validates candidates, port 8555
+ * reachability, and STUN/TURN end-to-end). A success is cached for the page
+ * session, a failure only for PROBE_FAILURE_TTL_MS.
  */
 
 export type WebRTCProbeResult = {
   ok: boolean;
   detail?: string;
+  // go2rtc could not open the stream's source, which says nothing about
+  // whether WebRTC itself can connect
+  streamError?: boolean;
 };
 
-let probePromise: Promise<WebRTCProbeResult> | null = null;
+const PROBE_FAILURE_TTL_MS = 30_000;
+
+// streams tried per probe when go2rtc refuses the earlier ones
+const MAX_PROBE_STREAMS = 3;
+
+let cachedProbe: {
+  promise: Promise<WebRTCProbeResult>;
+  failedAt?: number;
+} | null = null;
 
 function describeError(err: unknown): string {
   return err instanceof Error ? err.message : String(err);
@@ -118,23 +129,67 @@ function runProbe(
         pc.setRemoteDescription({ type: "answer", sdp: msg.value }).catch(
           (err) => fail(`remote answer rejected: ${describeError(err)}`),
         );
+      } else if (msg.type === "error") {
+        cleanup({ ok: false, streamError: true, detail: msg.value });
       }
     });
   });
 }
 
+async function runProbes(
+  testStreams: string[],
+  iceServers: RTCIceServer[],
+  timeoutMs: number,
+): Promise<WebRTCProbeResult> {
+  let result: WebRTCProbeResult = { ok: false, detail: "no stream to probe" };
+
+  for (const stream of testStreams.slice(0, MAX_PROBE_STREAMS)) {
+    result = await runProbe(stream, iceServers, timeoutMs);
+
+    if (!result.ok) {
+      result.detail = `stream '${stream}': ${result.detail}`;
+    }
+
+    if (!result.streamError) {
+      break;
+    }
+  }
+
+  return result;
+}
+
+/** Probes the streams in order, moving on only when go2rtc refuses one. */
 export function probeWebRTCAvailability(
-  testStream: string,
+  testStreams: string[],
   iceServers: RTCIceServer[],
   timeoutMs: number = 5000,
 ): Promise<WebRTCProbeResult> {
-  if (!probePromise) {
-    probePromise = runProbe(testStream, iceServers, timeoutMs);
+  if (
+    cachedProbe &&
+    (cachedProbe.failedAt === undefined ||
+      Date.now() - cachedProbe.failedAt < PROBE_FAILURE_TTL_MS)
+  ) {
+    return cachedProbe.promise;
   }
-  return probePromise;
+
+  const entry: NonNullable<typeof cachedProbe> = {
+    promise: runProbes(testStreams, iceServers, timeoutMs)
+      .catch((err): WebRTCProbeResult => ({
+        ok: false,
+        detail: describeError(err),
+      }))
+      .then((result) => {
+        if (!result.ok) {
+          entry.failedAt = Date.now();
+        }
+        return result;
+      }),
+  };
+  cachedProbe = entry;
+  return entry.promise;
 }
 
 /** Clears the cached probe result (e.g. when go2rtc config changes). */
 export function resetWebRTCProbe(): void {
-  probePromise = null;
+  cachedProbe = null;
 }