From 2b0c3a7bf6de892c08e15da80bb6729b59c32457 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:27:26 -0500 Subject: [PATCH] adaptive playback rate and intelligent switching improvements --- web/src/components/player/LivePlayer.tsx | 15 ++--- web/src/components/player/MsePlayer.tsx | 80 +++++++++++++++++++++--- web/src/views/live/LiveCameraView.tsx | 4 ++ 3 files changed, 84 insertions(+), 15 deletions(-) diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx index 67057a278..02f73cf4e 100644 --- a/web/src/components/player/LivePlayer.tsx +++ b/web/src/components/player/LivePlayer.tsx @@ -13,7 +13,6 @@ import { LivePlayerMode, VideoResolutionType, } from "@/types/live"; -import useCameraLiveMode from "@/hooks/use-camera-live-mode"; import { getIconForLabel } from "@/utils/iconUtil"; import Chip from "../indicators/Chip"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; @@ -25,7 +24,7 @@ type LivePlayerProps = { containerRef?: React.MutableRefObject; className?: string; cameraConfig: CameraConfig; - preferredLiveMode?: LivePlayerMode; + preferredLiveMode: LivePlayerMode; showStillWithoutActivity?: boolean; windowVisible?: boolean; playAudio?: boolean; @@ -70,8 +69,6 @@ export default function LivePlayer({ // camera live state - const liveMode = useCameraLiveMode(cameraConfig, preferredLiveMode); - const [liveReady, setLiveReady] = useState(false); const liveReadyRef = useRef(liveReady); @@ -152,7 +149,7 @@ export default function LivePlayer({ let player; if (!autoLive) { player = null; - } else if (liveMode == "webrtc") { + } else if (preferredLiveMode == "webrtc") { player = ( ); - } else if (liveMode == "mse") { + } else if (preferredLiveMode == "mse") { if ("MediaSource" in window || "ManagedMediaSource" in window) { player = ( ); } - } else if (liveMode == "jsmpeg") { + } else if (preferredLiveMode == "jsmpeg") { if (cameraActive || !showStillWithoutActivity || liveReady) { player = ( ; } + useEffect(() => { + console.log(cameraConfig.name, "switching to", preferredLiveMode); + }, [preferredLiveMode]); + return (
{ @@ -245,6 +246,7 @@ function MSEPlayer({ { type: "mse", value: codecs(MediaSource.isTypeSupported), + duration: Number.POSITIVE_INFINITY, // https://stackoverflow.com/questions/74461792/mediasource-api-safari-pauses-video-on-buffer-underflow }, 3000, ).catch(() => { @@ -268,7 +270,23 @@ function MSEPlayer({ onmessageRef.current["mse"] = (msg) => { if (msg.type !== "mse") return; - const sb = msRef.current?.addSourceBuffer(msg.value); + let sb: SourceBuffer | undefined; + try { + sb = msRef.current?.addSourceBuffer(msg.value); + } catch (e) { + // Safari sometimes throws this error + if (e instanceof DOMException && e.name === "InvalidStateError") { + if (wsRef.current) { + onDisconnect(); + } + console.log(camera, "threw InvalidStateError"); + onError?.("mse-decode"); + return; + } else { + throw e; // Re-throw if it's not the error we're handling + } + } + sb?.addEventListener("updateend", () => { if (sb.updating) return; @@ -322,6 +340,24 @@ function MSEPlayer({ return averageBufferTime * 1.5; }; + const calculateAdaptivePlaybackRate = ( + bufferTime: number, + bufferThreshold: number, + ) => { + const alpha = 0.2; // aggressiveness of playback rate increase + const beta = 0.5; // steepness of exponential growth + + // don't adjust playback rate if we're close enough to live + if ( + (bufferTime <= bufferThreshold && bufferThreshold < 3) || + bufferTime < 3 + ) { + return 1; + } + const rate = 1 + alpha * Math.exp(beta * bufferTime - bufferThreshold); + return Math.min(rate, 2); + }; + useEffect(() => { if (!playbackEnabled) { return; @@ -407,13 +443,17 @@ function MSEPlayer({ onPlaying?.(); setIsPlaying(true); lastJumpTimeRef.current = Date.now(); + console.log(camera, "loaded mse data"); }} muted={!audioEnabled} onPause={handlePause} onProgress={() => { const bufferTime = getBufferedTime(videoRef.current); - if (videoRef.current && videoRef.current.playbackRate === 1) { + if ( + videoRef.current && + (videoRef.current.playbackRate === 1 || bufferTime < 3) + ) { if (bufferTimes.current.length < MAX_BUFFER_ENTRIES) { bufferTimes.current.push(bufferTime); } else { @@ -434,6 +474,21 @@ function MSEPlayer({ onPlaying?.(); } + // if we have more than 10 seconds of buffer, something's wrong so error out + if ( + isPlaying && + playbackEnabled && + (bufferThreshold > 10 || bufferTime > 10) + ) { + onDisconnect(); + onError?.("stalled"); + } + + const playbackRate = calculateAdaptivePlaybackRate( + bufferTime, + bufferThreshold, + ); + // if we're above our rolling average threshold or have > 3 seconds of // buffered data and we're playing, we may have drifted from actual live // time, so increase playback rate to compensate @@ -441,16 +496,25 @@ function MSEPlayer({ videoRef.current && isPlaying && playbackEnabled && - (bufferTime > bufferThreshold || bufferTime > 3) && Date.now() - lastJumpTimeRef.current > BUFFERING_COOLDOWN_TIMEOUT ) { - videoRef.current.playbackRate = 1.1; - } else { - if (videoRef.current) { - videoRef.current.playbackRate = 1; - } + videoRef.current.playbackRate = playbackRate; } + console.log( + camera, + "isPlaying?", + isPlaying, + "playbackEnabled?", + playbackEnabled, + "bufferTime", + bufferTime, + "bufferThreshold", + bufferThreshold, + "playbackRate", + playbackRate, + ); + if (onError != undefined) { if (videoRef.current?.paused) { return; diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index be264c9f5..9dab7d916 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -227,6 +227,10 @@ export default function LiveCameraView({ return "webrtc"; } + if (!isRestreamed) { + return "jsmpeg"; + } + return "mse"; }, [lowBandwidth, mic, webRTC, isRestreamed]);