Add sub stream recording with adaptive quality playback (#24009)

* add sub stream recording with adaptive quality playback

Optionally record a second, lower bitrate stream alongside the main
recording stream via a `record_sub` input role and `record.sub` config block, with its own retention windows.
Recordings rows now carry the stream type plus the media details needed to serve both streams from one manifest: video codec, audio presence, audio codec and rate, and a record-time keyframe index.

Playback resolves coverage across both streams and merges them into a single VOD sequence, falling back to a discontinuity manifest with per-clip init segments when the media signatures differ. The player exposes a quality selector, and an auto governor picks the stream from stall time, bandwidth, codec support, and the save-data hint.

* fix tests and i18n

web/src/components/player/HlsVideoPlayer.tsx

@@ -26,7 +26,7 @@ import { useIsAdmin } from "@/hooks/use-is-admin";
 // Android native hls does not seek correctly
 const USE_NATIVE_HLS = false;
 const HLS_MIME_TYPE = "application/vnd.apple.mpegurl" as const;
-const unsupportedErrorCodes = [
+const unsupportedErrorCodes: number[] = [
   MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED,
   MediaError.MEDIA_ERR_DECODE,
 ];
@@ -58,6 +58,14 @@ type HlsVideoPlayerProps = {
   onSnapshot?: (playTime: number) => Promise<void> | void;
   toggleFullscreen?: () => void;
   onError?: (error: RecordingPlayerError) => void;
+  onStallStart?: () => void;
+  onStallEnd?: () => void;
+  onSeekStart?: () => void;
+  onBandwidthSample?: (estimateBps: number, levelBitrateBps?: number) => void;
+  onFatalNetworkError?: () => boolean;
+  onFatalCodecError?: () => boolean;
+  initialBandwidthEstimate?: number;
+  bufferLength?: number;
   isDetailMode?: boolean;
   camera?: string;
   currentTimeOverride?: number;
@@ -86,6 +94,14 @@ export default function HlsVideoPlayer({
   onSnapshot,
   toggleFullscreen,
   onError,
+  onStallStart,
+  onStallEnd,
+  onSeekStart,
+  onBandwidthSample,
+  onFatalNetworkError,
+  onFatalCodecError,
+  initialBandwidthEstimate,
+  bufferLength,
   isDetailMode = false,
   camera,
   currentTimeOverride,
@@ -101,9 +117,37 @@ export default function HlsVideoPlayer({
   // playback
 
   const hlsRef = useRef<Hls>(undefined);
-  const [useHlsCompat, setUseHlsCompat] = useState(false);
+  // kept in a ref so changing callback identities do not recreate the
+  // Hls instance; the setup effect must only re-run on source changes
+  const qualitySignalsRef = useRef({
+    onStallStart,
+    onStallEnd,
+    onSeekStart,
+    onBandwidthSample,
+    onFatalNetworkError,
+    onFatalCodecError,
+    initialBandwidthEstimate,
+  });
+  // must resolve before the first render: a mount-effect flip would run
+  // the first source effect in native mode, briefly handing iOS a native
+  // HLS src that hls.js then tears away mid-load
+  const [useHlsCompat, setUseHlsCompat] = useState(() => {
+    if (
+      USE_NATIVE_HLS &&
+      document.createElement("video").canPlayType(HLS_MIME_TYPE)
+    ) {
+      return false;
+    }
+    return Hls.isSupported();
+  });
   const [loadedMetadata, setLoadedMetadata] = useState(false);
   const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
+  // native HLS playback has no MSE, so it recovers from pipeline errors
+  // by reloading the source; one attempt per source
+  const nativeRetryRef = useRef(0);
+  // a ref rather than an effect-scoped counter so the element error
+  // handler can hold its toast while a recovery is still possible
+  const mediaRecoveryBudgetRef = useRef(0);
 
   const applyVideoDimensions = useCallback(
     (width: number, height: number) => {
@@ -153,27 +197,38 @@ export default function HlsVideoPlayer({
   }, [videoRef, applyVideoDimensions]);
 
   useEffect(() => {
-    if (!videoRef.current) {
-      return;
-    }
-
-    if (USE_NATIVE_HLS && videoRef.current.canPlayType(HLS_MIME_TYPE)) {
-      return;
-    } else if (Hls.isSupported()) {
-      setUseHlsCompat(true);
-    }
-  }, [videoRef]);
+    qualitySignalsRef.current = {
+      onStallStart,
+      onStallEnd,
+      onSeekStart,
+      onBandwidthSample,
+      onFatalNetworkError,
+      onFatalCodecError,
+      initialBandwidthEstimate,
+    };
+  }, [
+    onStallStart,
+    onStallEnd,
+    onSeekStart,
+    onBandwidthSample,
+    onFatalNetworkError,
+    onFatalCodecError,
+    initialBandwidthEstimate,
+  ]);
 
   useEffect(() => {
     if (!videoRef.current) {
       return;
     }
 
-    setLoadedMetadata(false);
-
+    // loadedMetadata is intentionally NOT reset here: on a source swap
+    // the element already holds a decoded frame, and keeping it visible
+    // bridges the gap while the new source loads
     const currentPlaybackRate = videoRef.current.playbackRate;
 
     if (!useHlsCompat) {
+      nativeRetryRef.current = 0;
+      mediaRecoveryBudgetRef.current = 0;
       videoRef.current.src = currentSource.playlist;
       videoRef.current.load();
       return;
@@ -181,14 +236,68 @@ export default function HlsVideoPlayer({
 
     // Base HLS configuration
     const hlsConfig: Partial<HlsConfig> = {
-      maxBufferLength: 10,
+      maxBufferLength: bufferLength ?? 10,
       maxBufferSize: 20 * 1000 * 1000,
       startPosition: currentSource.startPosition,
     };
 
-    hlsRef.current = new Hls(hlsConfig);
-    hlsRef.current.attachMedia(videoRef.current);
-    hlsRef.current.loadSource(currentSource.playlist);
+    // every quality switch and chunk change recreates the instance, so
+    // seed it to keep measured throughput across source swaps
+    const seedEstimate = qualitySignalsRef.current.initialBandwidthEstimate;
+    if (seedEstimate !== undefined && seedEstimate > 0) {
+      hlsConfig.abrEwmaDefaultEstimate = seedEstimate;
+    }
+
+    const hls = new Hls(hlsConfig);
+    hlsRef.current = hls;
+    let networkRecoveryAttempts = 0;
+    mediaRecoveryBudgetRef.current = 1;
+    hls.on(Hls.Events.ERROR, (_event, data) => {
+      if (data.fatal) {
+        if (data.type === Hls.ErrorTypes.NETWORK_ERROR) {
+          // prefer a quality downswitch; fall back to restarting loading
+          const handled =
+            qualitySignalsRef.current.onFatalNetworkError?.() ?? false;
+          if (!handled && networkRecoveryAttempts < 2) {
+            networkRecoveryAttempts += 1;
+            hls.startLoad();
+          }
+        } else if (data.type === Hls.ErrorTypes.MEDIA_ERROR) {
+          // retrying the same codec cannot succeed, so a codec error
+          // prefers a quality downswitch over recovery
+          const isCodecError =
+            data.details ===
+              Hls.ErrorDetails.BUFFER_INCOMPATIBLE_CODECS_ERROR ||
+            data.details === Hls.ErrorDetails.BUFFER_ADD_CODEC_ERROR;
+          if (isCodecError && qualitySignalsRef.current.onFatalCodecError?.()) {
+            return;
+          }
+          if (!isCodecError && mediaRecoveryBudgetRef.current > 0) {
+            mediaRecoveryBudgetRef.current -= 1;
+            hls.recoverMediaError();
+          }
+        }
+        return;
+      }
+
+      // hls.js reports each stall episode only once, so STALL_RESOLVED
+      // below is what closes it
+      if (data.details === Hls.ErrorDetails.BUFFER_STALLED_ERROR) {
+        qualitySignalsRef.current.onStallStart?.();
+      }
+    });
+    hls.on(Hls.Events.STALL_RESOLVED, () => {
+      qualitySignalsRef.current.onStallEnd?.();
+    });
+    hls.on(Hls.Events.FRAG_LOADED, () => {
+      // manifests are single-variant, so the bitrate is always level 0
+      qualitySignalsRef.current.onBandwidthSample?.(
+        hls.bandwidthEstimate,
+        hls.levels?.[0]?.bitrate || undefined,
+      );
+    });
+    hls.attachMedia(videoRef.current);
+    hls.loadSource(currentSource.playlist);
     videoRef.current.playbackRate = currentPlaybackRate;
 
     return () => {
@@ -199,7 +308,7 @@ export default function HlsVideoPlayer({
         hlsRef.current.destroy();
       }
     };
-  }, [videoRef, hlsRef, useHlsCompat, currentSource]);
+  }, [videoRef, hlsRef, useHlsCompat, currentSource, bufferLength]);
 
   // state handling
 
@@ -481,11 +590,17 @@ export default function HlsVideoPlayer({
                 );
               }
             }}
-            onPlaying={onPlaying}
+            onPlaying={() => {
+              qualitySignalsRef.current.onStallEnd?.();
+              onPlaying?.();
+            }}
             onPause={() => {
               setIsPlaying(false);
               clearTimeout(bufferTimeout);
 
+              // paused time must never count as stall time
+              qualitySignalsRef.current.onStallEnd?.();
+
               if (isMobile && mobileCtrlTimeout) {
                 clearTimeout(mobileCtrlTimeout);
               }
@@ -495,13 +610,18 @@ export default function HlsVideoPlayer({
               // while paused and never resumes it on seek, so a seek
               // into unbuffered media would never complete
               hlsRef.current?.resumeBuffering();
+              qualitySignalsRef.current.onSeekStart?.();
             }}
             onWaiting={() => {
-              if (onError != undefined) {
-                if (videoRef.current?.paused) {
-                  return;
-                }
+              if (videoRef.current?.paused) {
+                return;
+              }
 
+              // the only stall signal under native HLS playback, which
+              // emits no hls.js events
+              qualitySignalsRef.current.onStallStart?.();
+
+              if (onError != undefined) {
                 setBufferTimeout(
                   setTimeout(() => {
                     if (
@@ -557,23 +677,52 @@ export default function HlsVideoPlayer({
               }
             }}
             onError={(e) => {
-              if (
-                !hlsRef.current &&
-                // @ts-expect-error code does exist
-                unsupportedErrorCodes.includes(e.target.error.code) &&
-                videoRef.current
-              ) {
-                setLoadedMetadata(false);
-                setUseHlsCompat(true);
-              } else {
-                toast.error(
-                  // @ts-expect-error code does exist
-                  `Failed to play recordings (error ${e.target.error.code}): ${e.target.error.message}`,
-                  {
-                    position: "top-center",
-                  },
-                );
+              const mediaError = (e.target as HTMLVideoElement).error;
+
+              if (!mediaError) {
+                return;
               }
+
+              // an intentional source swap aborts the in-flight load;
+              // that abort is not an error the user can act on
+              if (mediaError.code === MediaError.MEDIA_ERR_ABORTED) {
+                return;
+              }
+
+              // hold the toast while the fatal handler still has a retry
+              // left; a failed recovery raises a second element error
+              if (hlsRef.current && mediaRecoveryBudgetRef.current > 0) {
+                return;
+              }
+
+              if (!hlsRef.current && videoRef.current) {
+                if (
+                  unsupportedErrorCodes.includes(mediaError.code) &&
+                  Hls.isSupported()
+                ) {
+                  setLoadedMetadata(false);
+                  setUseHlsCompat(true);
+                  return;
+                }
+
+                // native pipeline errors around source swaps are usually
+                // transient, and hls.js is no fallback without MSE
+                if (nativeRetryRef.current < 1) {
+                  nativeRetryRef.current += 1;
+                  videoRef.current.load();
+                  return;
+                }
+              }
+
+              toast.error(
+                t("toast.error.playRecordingsFailed", {
+                  code: mediaError.code,
+                  message: mediaError.message,
+                }),
+                {
+                  position: "top-center",
+                },
+              );
             }}
           />
         </div>