From 758df09da38061e24d49a3ea26f32e334085cb69 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 31 May 2024 07:52:42 -0600 Subject: [PATCH] Handle error when live view stalls (#11665) * Handle error when live view stalls * Manually calculate buffer timeout * Formatting --- web/src/components/player/LivePlayer.tsx | 9 +++++- web/src/components/player/MsePlayer.tsx | 34 ++++++++++++++++++++-- web/src/components/player/WebRTCPlayer.tsx | 32 ++++++++++++++++++++ web/src/types/live.ts | 2 ++ web/src/views/live/LiveCameraView.tsx | 10 ++++++- 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx index af1d47f9c..6c223d651 100644 --- a/web/src/components/player/LivePlayer.tsx +++ b/web/src/components/player/LivePlayer.tsx @@ -8,7 +8,11 @@ import JSMpegPlayer from "./JSMpegPlayer"; import { MdCircle } from "react-icons/md"; import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; import { useCameraActivity } from "@/hooks/use-camera-activity"; -import { LivePlayerMode, VideoResolutionType } from "@/types/live"; +import { + LivePlayerError, + LivePlayerMode, + VideoResolutionType, +} from "@/types/live"; import useCameraLiveMode from "@/hooks/use-camera-live-mode"; import { getIconForLabel } from "@/utils/iconUtil"; import Chip from "../indicators/Chip"; @@ -30,6 +34,7 @@ type LivePlayerProps = { autoLive?: boolean; onClick?: () => void; setFullResolution?: React.Dispatch>; + onError?: (error: LivePlayerError) => void; }; export default function LivePlayer({ @@ -47,6 +52,7 @@ export default function LivePlayer({ autoLive = true, onClick, setFullResolution, + onError, }: LivePlayerProps) { // camera activity @@ -145,6 +151,7 @@ export default function LivePlayer({ onPlaying={() => setLiveReady(true)} pip={pip} setFullResolution={setFullResolution} + onError={onError} /> ); } else { diff --git a/web/src/components/player/MsePlayer.tsx b/web/src/components/player/MsePlayer.tsx index 841b824db..d240f0a83 100644 --- a/web/src/components/player/MsePlayer.tsx +++ b/web/src/components/player/MsePlayer.tsx @@ -1,5 +1,5 @@ import { baseUrl } from "@/api/baseUrl"; -import { VideoResolutionType } from "@/types/live"; +import { LivePlayerError, VideoResolutionType } from "@/types/live"; import { SetStateAction, useCallback, @@ -17,6 +17,7 @@ type MSEPlayerProps = { pip?: boolean; onPlaying?: () => void; setFullResolution?: React.Dispatch>; + onError?: (error: LivePlayerError) => void; }; function MSEPlayer({ @@ -27,6 +28,7 @@ function MSEPlayer({ pip = false, onPlaying, setFullResolution, + onError, }: MSEPlayerProps) { const RECONNECT_TIMEOUT: number = 30000; @@ -45,6 +47,7 @@ function MSEPlayer({ const [wsState, setWsState] = useState(WebSocket.CLOSED); const [connectTS, setConnectTS] = useState(0); + const [bufferTimeout, setBufferTimeout] = useState(); const videoRef = useRef(null); const wsRef = useRef(null); @@ -308,7 +311,34 @@ function MSEPlayer({ onPlaying?.(); }} muted={!audioEnabled} - onError={() => { + onProgress={ + onError != undefined + ? () => { + if (videoRef.current?.paused) { + return; + } + + if (bufferTimeout) { + clearTimeout(bufferTimeout); + setBufferTimeout(undefined); + } + + setBufferTimeout( + setTimeout(() => { + onError("stalled"); + }, 3000), + ); + } + : undefined + } + onError={(e) => { + if ( + // @ts-expect-error code does exist + e.target.error.code == MediaError.MEDIA_ERR_NETWORK + ) { + onError?.("startup"); + } + if (wsRef.current) { wsRef.current.close(); wsRef.current = null; diff --git a/web/src/components/player/WebRTCPlayer.tsx b/web/src/components/player/WebRTCPlayer.tsx index 8f966254b..8f64e6cae 100644 --- a/web/src/components/player/WebRTCPlayer.tsx +++ b/web/src/components/player/WebRTCPlayer.tsx @@ -1,4 +1,5 @@ import { baseUrl } from "@/api/baseUrl"; +import { LivePlayerError } from "@/types/live"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; type WebRtcPlayerProps = { @@ -10,6 +11,7 @@ type WebRtcPlayerProps = { iOSCompatFullScreen?: boolean; // ios doesn't support fullscreen divs so we must support the video element pip?: boolean; onPlaying?: () => void; + onError?: (error: LivePlayerError) => void; }; export default function WebRtcPlayer({ @@ -21,6 +23,7 @@ export default function WebRtcPlayer({ iOSCompatFullScreen = false, pip = false, onPlaying, + onError, }: WebRtcPlayerProps) { // metadata @@ -32,6 +35,7 @@ export default function WebRtcPlayer({ const pcRef = useRef(); const videoRef = useRef(null); + const [bufferTimeout, setBufferTimeout] = useState(); const PeerConnection = useCallback( async (media: string) => { @@ -198,11 +202,39 @@ export default function WebRtcPlayer({ playsInline muted={!audioEnabled} onLoadedData={onPlaying} + onProgress={ + onError != undefined + ? () => { + if (videoRef.current?.paused) { + return; + } + + if (bufferTimeout) { + clearTimeout(bufferTimeout); + setBufferTimeout(undefined); + } + + setBufferTimeout( + setTimeout(() => { + onError("stalled"); + }, 3000), + ); + } + : undefined + } onClick={ iOSCompatFullScreen ? () => setiOSCompatControls(!iOSCompatControls) : undefined } + onError={(e) => { + if ( + // @ts-expect-error code does exist + e.target.error.code == MediaError.MEDIA_ERR_NETWORK + ) { + onError?.("startup"); + } + }} /> ); } diff --git a/web/src/types/live.ts b/web/src/types/live.ts index 6b34c0e5b..7002304f5 100644 --- a/web/src/types/live.ts +++ b/web/src/types/live.ts @@ -30,3 +30,5 @@ export type LiveStreamMetadata = { producers: LiveProducerMetadata[]; consumers: LiveConsumerMetadata[]; }; + +export type LivePlayerError = "stalled" | "startup"; diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 9e897cbd6..a298e8462 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -191,6 +191,7 @@ export default function LiveCameraView({ const [audio, setAudio] = useState(false); const [mic, setMic] = useState(false); const [pip, setPip] = useState(false); + const [lowBandwidth, setLowBandwidth] = useState(false); const [fullResolution, setFullResolution] = useState({ width: 0, @@ -202,8 +203,14 @@ export default function LiveCameraView({ return "webrtc"; } + if (lowBandwidth) { + return "jsmpeg"; + } + return "mse"; - }, [mic]); + }, [lowBandwidth, mic]); + + // layout state const windowAspectRatio = useMemo(() => { return windowWidth / windowHeight; @@ -419,6 +426,7 @@ export default function LiveCameraView({ pip={pip} setFullResolution={setFullResolution} containerRef={containerRef} + onError={() => setLowBandwidth(true)} /> {camera.onvif.host != "" && (