From 714be8f414a267858642d9c979caffd2e036c0eb Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 31 May 2024 06:18:28 -0600 Subject: [PATCH] Handle error when live view stalls --- web/src/components/player/LivePlayer.tsx | 5 ++++- web/src/components/player/MsePlayer.tsx | 18 +++++++++++++++++- web/src/components/player/WebRTCPlayer.tsx | 18 +++++++++++++++++- web/src/types/live.ts | 2 ++ web/src/views/live/LiveCameraView.tsx | 10 +++++++++- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/web/src/components/player/LivePlayer.tsx b/web/src/components/player/LivePlayer.tsx index af1d47f9c..420ceb06d 100644 --- a/web/src/components/player/LivePlayer.tsx +++ b/web/src/components/player/LivePlayer.tsx @@ -8,7 +8,7 @@ import JSMpegPlayer from "./JSMpegPlayer"; import { MdCircle } from "react-icons/md"; import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; import { useCameraActivity } from "@/hooks/use-camera-activity"; -import { LivePlayerMode, VideoResolutionType } from "@/types/live"; +import { LivePlayerError, LivePlayerMode, VideoResolutionType } from "@/types/live"; import useCameraLiveMode from "@/hooks/use-camera-live-mode"; import { getIconForLabel } from "@/utils/iconUtil"; import Chip from "../indicators/Chip"; @@ -30,6 +30,7 @@ type LivePlayerProps = { autoLive?: boolean; onClick?: () => void; setFullResolution?: React.Dispatch>; + onError?: (error: LivePlayerError) => void; }; export default function LivePlayer({ @@ -47,6 +48,7 @@ export default function LivePlayer({ autoLive = true, onClick, setFullResolution, + onError, }: LivePlayerProps) { // camera activity @@ -145,6 +147,7 @@ export default function LivePlayer({ onPlaying={() => setLiveReady(true)} pip={pip} setFullResolution={setFullResolution} + onError={onError} /> ); } else { diff --git a/web/src/components/player/MsePlayer.tsx b/web/src/components/player/MsePlayer.tsx index 841b824db..38568434d 100644 --- a/web/src/components/player/MsePlayer.tsx +++ b/web/src/components/player/MsePlayer.tsx @@ -1,5 +1,5 @@ import { baseUrl } from "@/api/baseUrl"; -import { VideoResolutionType } from "@/types/live"; +import { LivePlayerError, VideoResolutionType } from "@/types/live"; import { SetStateAction, useCallback, @@ -17,6 +17,7 @@ type MSEPlayerProps = { pip?: boolean; onPlaying?: () => void; setFullResolution?: React.Dispatch>; + onError?: (error: LivePlayerError) => void; }; function MSEPlayer({ @@ -27,6 +28,7 @@ function MSEPlayer({ pip = false, onPlaying, setFullResolution, + onError, }: MSEPlayerProps) { const RECONNECT_TIMEOUT: number = 30000; @@ -45,6 +47,7 @@ function MSEPlayer({ const [wsState, setWsState] = useState(WebSocket.CLOSED); const [connectTS, setConnectTS] = useState(0); + const [receivedData, setReceivedData] = useState(false); const videoRef = useRef(null); const wsRef = useRef(null); @@ -100,6 +103,7 @@ function MSEPlayer({ const onConnect = useCallback(() => { if (!videoRef.current?.isConnected || !wsURL || wsRef.current) return false; + setReceivedData(false); setWsState(WebSocket.CONNECTING); setConnectTS(Date.now()); @@ -306,9 +310,21 @@ function MSEPlayer({ onLoadedData={() => { handleLoadedMetadata?.(); onPlaying?.(); + setReceivedData(true); }} muted={!audioEnabled} + onStalled={() => { + if (receivedData) { + onError?.("stalled"); + } else { + onError?.("startup"); + } + }} onError={() => { + if (!receivedData) { + onError?.("startup"); + } + if (wsRef.current) { wsRef.current.close(); wsRef.current = null; diff --git a/web/src/components/player/WebRTCPlayer.tsx b/web/src/components/player/WebRTCPlayer.tsx index 8f966254b..0311c8c44 100644 --- a/web/src/components/player/WebRTCPlayer.tsx +++ b/web/src/components/player/WebRTCPlayer.tsx @@ -1,4 +1,5 @@ import { baseUrl } from "@/api/baseUrl"; +import { LivePlayerError } from "@/types/live"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; type WebRtcPlayerProps = { @@ -10,6 +11,7 @@ type WebRtcPlayerProps = { iOSCompatFullScreen?: boolean; // ios doesn't support fullscreen divs so we must support the video element pip?: boolean; onPlaying?: () => void; + onError?: (error: LivePlayerError) => void; }; export default function WebRtcPlayer({ @@ -21,6 +23,7 @@ export default function WebRtcPlayer({ iOSCompatFullScreen = false, pip = false, onPlaying, + onError, }: WebRtcPlayerProps) { // metadata @@ -32,6 +35,7 @@ export default function WebRtcPlayer({ const pcRef = useRef(); const videoRef = useRef(null); + const [receivedData, setReceivedData] = useState(false); const PeerConnection = useCallback( async (media: string) => { @@ -164,6 +168,8 @@ export default function WebRtcPlayer({ pcRef.current.close(); pcRef.current = undefined; } + + setReceivedData(false); }; }, [ camera, @@ -197,7 +203,17 @@ export default function WebRtcPlayer({ autoPlay playsInline muted={!audioEnabled} - onLoadedData={onPlaying} + onLoadedData={() => { + setReceivedData(true); + onPlaying?.(); + }} + onStalled={() => { + if (receivedData) { + onError?.("stalled"); + } else { + onError?.("startup"); + } + }} onClick={ iOSCompatFullScreen ? () => setiOSCompatControls(!iOSCompatControls) diff --git a/web/src/types/live.ts b/web/src/types/live.ts index 6b34c0e5b..7002304f5 100644 --- a/web/src/types/live.ts +++ b/web/src/types/live.ts @@ -30,3 +30,5 @@ export type LiveStreamMetadata = { producers: LiveProducerMetadata[]; consumers: LiveConsumerMetadata[]; }; + +export type LivePlayerError = "stalled" | "startup"; diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index f6d85a565..2e932ec58 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -190,6 +190,7 @@ export default function LiveCameraView({ const [audio, setAudio] = useState(false); const [mic, setMic] = useState(false); const [pip, setPip] = useState(false); + const [lowBandwidth, setLowBandwidth] = useState(false); const [fullResolution, setFullResolution] = useState({ width: 0, @@ -201,8 +202,14 @@ export default function LiveCameraView({ return "webrtc"; } + if (lowBandwidth) { + return "jsmpeg"; + } + return "mse"; - }, [mic]); + }, [lowBandwidth, mic]); + + // layout state const windowAspectRatio = useMemo(() => { return windowWidth / windowHeight; @@ -406,6 +413,7 @@ export default function LiveCameraView({ pip={pip} setFullResolution={setFullResolution} containerRef={containerRef} + onError={() => setLowBandwidth(true)} /> {camera.onvif.host != "" && (