adaptive playback rate and intelligent switching improvements

web/src/components/player/LivePlayer.tsx

@@ -13,7 +13,6 @@ import {
   LivePlayerMode,
   VideoResolutionType,
 } from "@/types/live";
-import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { getIconForLabel } from "@/utils/iconUtil";
 import Chip from "../indicators/Chip";
 import { capitalizeFirstLetter } from "@/utils/stringUtil";
@@ -25,7 +24,7 @@ type LivePlayerProps = {
   containerRef?: React.MutableRefObject<HTMLDivElement | null>;
   className?: string;
   cameraConfig: CameraConfig;
-  preferredLiveMode?: LivePlayerMode;
+  preferredLiveMode: LivePlayerMode;
   showStillWithoutActivity?: boolean;
   windowVisible?: boolean;
   playAudio?: boolean;
@@ -70,8 +69,6 @@ export default function LivePlayer({
 
   // camera live state
 
-  const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode);
-
   const [liveReady, setLiveReady] = useState(false);
 
   const liveReadyRef = useRef(liveReady);
@@ -152,7 +149,7 @@ export default function LivePlayer({
   let player;
   if (!autoLive) {
     player = null;
-  } else if (liveMode == "webrtc") {
+  } else if (preferredLiveMode == "webrtc") {
     player = (
       <WebRtcPlayer
         className={`size-full rounded-lg md:rounded-2xl ${liveReady ? "" : "hidden"}`}
@@ -166,7 +163,7 @@ export default function LivePlayer({
         onError={onError}
       />
     );
-  } else if (liveMode == "mse") {
+  } else if (preferredLiveMode == "mse") {
     if ("MediaSource" in window || "ManagedMediaSource" in window) {
       player = (
         <MSEPlayer
@@ -187,7 +184,7 @@ export default function LivePlayer({
         </div>
       );
     }
-  } else if (liveMode == "jsmpeg") {
+  } else if (preferredLiveMode == "jsmpeg") {
     if (cameraActive || !showStillWithoutActivity || liveReady) {
       player = (
         <JSMpegPlayer
@@ -209,6 +206,10 @@ export default function LivePlayer({
     player = <ActivityIndicator />;
   }
 
+  useEffect(() => {
+    console.log(cameraConfig.name, "switching to", preferredLiveMode);
+  }, [preferredLiveMode]);
+
   return (
     <div
       ref={cameraRef ?? internalContainerRef}

web/src/components/player/MsePlayer.tsx

@@ -216,6 +216,7 @@ function MSEPlayer({
               type: "mse",
               // @ts-expect-error for typing
               value: codecs(MediaSource.isTypeSupported),
+              duration: Number.POSITIVE_INFINITY, // https://stackoverflow.com/questions/74461792/mediasource-api-safari-pauses-video-on-buffer-underflow
             },
             3000,
           ).catch(() => {
@@ -245,6 +246,7 @@ function MSEPlayer({
             {
               type: "mse",
               value: codecs(MediaSource.isTypeSupported),
+              duration: Number.POSITIVE_INFINITY, // https://stackoverflow.com/questions/74461792/mediasource-api-safari-pauses-video-on-buffer-underflow
             },
             3000,
           ).catch(() => {
@@ -268,7 +270,23 @@ function MSEPlayer({
     onmessageRef.current["mse"] = (msg) => {
       if (msg.type !== "mse") return;
 
-      const sb = msRef.current?.addSourceBuffer(msg.value);
+      let sb: SourceBuffer | undefined;
+      try {
+        sb = msRef.current?.addSourceBuffer(msg.value);
+      } catch (e) {
+        // Safari sometimes throws this error
+        if (e instanceof DOMException && e.name === "InvalidStateError") {
+          if (wsRef.current) {
+            onDisconnect();
+          }
+          console.log(camera, "threw InvalidStateError");
+          onError?.("mse-decode");
+          return;
+        } else {
+          throw e; // Re-throw if it's not the error we're handling
+        }
+      }
+
       sb?.addEventListener("updateend", () => {
         if (sb.updating) return;
 
@@ -322,6 +340,24 @@ function MSEPlayer({
     return averageBufferTime * 1.5;
   };
 
+  const calculateAdaptivePlaybackRate = (
+    bufferTime: number,
+    bufferThreshold: number,
+  ) => {
+    const alpha = 0.2; // aggressiveness of playback rate increase
+    const beta = 0.5; // steepness of exponential growth
+
+    // don't adjust playback rate if we're close enough to live
+    if (
+      (bufferTime <= bufferThreshold && bufferThreshold < 3) ||
+      bufferTime < 3
+    ) {
+      return 1;
+    }
+    const rate = 1 + alpha * Math.exp(beta * bufferTime - bufferThreshold);
+    return Math.min(rate, 2);
+  };
+
   useEffect(() => {
     if (!playbackEnabled) {
       return;
@@ -407,13 +443,17 @@ function MSEPlayer({
         onPlaying?.();
         setIsPlaying(true);
         lastJumpTimeRef.current = Date.now();
+        console.log(camera, "loaded mse data");
       }}
       muted={!audioEnabled}
       onPause={handlePause}
       onProgress={() => {
         const bufferTime = getBufferedTime(videoRef.current);
 
-        if (videoRef.current && videoRef.current.playbackRate === 1) {
+        if (
+          videoRef.current &&
+          (videoRef.current.playbackRate === 1 || bufferTime < 3)
+        ) {
           if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) {
             bufferTimes.current.push(bufferTime);
           } else {
@@ -434,6 +474,21 @@ function MSEPlayer({
           onPlaying?.();
         }
 
+        // if we have more than 10 seconds of buffer, something's wrong so error out
+        if (
+          isPlaying &&
+          playbackEnabled &&
+          (bufferThreshold > 10 || bufferTime > 10)
+        ) {
+          onDisconnect();
+          onError?.("stalled");
+        }
+
+        const playbackRate = calculateAdaptivePlaybackRate(
+          bufferTime,
+          bufferThreshold,
+        );
+
         // if we're above our rolling average threshold or have > 3 seconds of
         // buffered data and we're playing, we may have drifted from actual live
         // time, so increase playback rate to compensate
@@ -441,16 +496,25 @@ function MSEPlayer({
           videoRef.current &&
           isPlaying &&
           playbackEnabled &&
-          (bufferTime > bufferThreshold || bufferTime > 3) &&
           Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT
         ) {
-          videoRef.current.playbackRate = 1.1;
-        } else {
-          if (videoRef.current) {
-            videoRef.current.playbackRate = 1;
-          }
+          videoRef.current.playbackRate = playbackRate;
         }
 
+        console.log(
+          camera,
+          "isPlaying?",
+          isPlaying,
+          "playbackEnabled?",
+          playbackEnabled,
+          "bufferTime",
+          bufferTime,
+          "bufferThreshold",
+          bufferThreshold,
+          "playbackRate",
+          playbackRate,
+        );
+
         if (onError != undefined) {
           if (videoRef.current?.paused) {
             return;

web/src/views/live/LiveCameraView.tsx

@@ -227,6 +227,10 @@ export default function LiveCameraView({
       return "webrtc";
     }
 
+    if (!isRestreamed) {
+      return "jsmpeg";
+    }
+
     return "mse";
   }, [lowBandwidth, mic, webRTC, isRestreamed]);