Handle error when live view stalls

web/src/components/player/LivePlayer.tsx

@@ -8,7 +8,7 @@ import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle } from "react-icons/md";
 import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip";
 import { useCameraActivity } from "@/hooks/use-camera-activity";
-import { LivePlayerMode, VideoResolutionType } from "@/types/live";
+import { LivePlayerError, LivePlayerMode, VideoResolutionType } from "@/types/live";
 import useCameraLiveMode from "@/hooks/use-camera-live-mode";
 import { getIconForLabel } from "@/utils/iconUtil";
 import Chip from "../indicators/Chip";
@@ -30,6 +30,7 @@ type LivePlayerProps = {
   autoLive?: boolean;
   onClick?: () => void;
   setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
+  onError?: (error: LivePlayerError) => void;
 };
 
 export default function LivePlayer({
@@ -47,6 +48,7 @@ export default function LivePlayer({
   autoLive = true,
   onClick,
   setFullResolution,
+  onError,
 }: LivePlayerProps) {
   // camera activity
 
@@ -145,6 +147,7 @@ export default function LivePlayer({
           onPlaying={() => setLiveReady(true)}
           pip={pip}
           setFullResolution={setFullResolution}
+          onError={onError}
         />
       );
     } else {

web/src/components/player/MsePlayer.tsx

@@ -1,5 +1,5 @@
 import { baseUrl } from "@/api/baseUrl";
-import { VideoResolutionType } from "@/types/live";
+import { LivePlayerError, VideoResolutionType } from "@/types/live";
 import {
   SetStateAction,
   useCallback,
@@ -17,6 +17,7 @@ type MSEPlayerProps = {
   pip?: boolean;
   onPlaying?: () => void;
   setFullResolution?: React.Dispatch<SetStateAction<VideoResolutionType>>;
+  onError?: (error: LivePlayerError) => void;
 };
 
 function MSEPlayer({
@@ -27,6 +28,7 @@ function MSEPlayer({
   pip = false,
   onPlaying,
   setFullResolution,
+  onError,
 }: MSEPlayerProps) {
   const RECONNECT_TIMEOUT: number = 30000;
 
@@ -45,6 +47,7 @@ function MSEPlayer({
 
   const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
   const [connectTS, setConnectTS] = useState<number>(0);
+  const [receivedData, setReceivedData] = useState(false);
 
   const videoRef = useRef<HTMLVideoElement>(null);
   const wsRef = useRef<WebSocket | null>(null);
@@ -100,6 +103,7 @@ function MSEPlayer({
   const onConnect = useCallback(() => {
     if (!videoRef.current?.isConnected || !wsURL || wsRef.current) return false;
 
+    setReceivedData(false);
     setWsState(WebSocket.CONNECTING);
 
     setConnectTS(Date.now());
@@ -306,9 +310,21 @@ function MSEPlayer({
       onLoadedData={() => {
         handleLoadedMetadata?.();
         onPlaying?.();
+        setReceivedData(true);
       }}
       muted={!audioEnabled}
+      onStalled={() => {
+        if (receivedData) {
+          onError?.("stalled");
+        } else {
+          onError?.("startup");
+        }
+      }}
       onError={() => {
+        if (!receivedData) {
+          onError?.("startup");
+        }
+
         if (wsRef.current) {
           wsRef.current.close();
           wsRef.current = null;

web/src/components/player/WebRTCPlayer.tsx

@@ -1,4 +1,5 @@
 import { baseUrl } from "@/api/baseUrl";
+import { LivePlayerError } from "@/types/live";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 
 type WebRtcPlayerProps = {
@@ -10,6 +11,7 @@ type WebRtcPlayerProps = {
   iOSCompatFullScreen?: boolean; // ios doesn't support fullscreen divs so we must support the video element
   pip?: boolean;
   onPlaying?: () => void;
+  onError?: (error: LivePlayerError) => void;
 };
 
 export default function WebRtcPlayer({
@@ -21,6 +23,7 @@ export default function WebRtcPlayer({
   iOSCompatFullScreen = false,
   pip = false,
   onPlaying,
+  onError,
 }: WebRtcPlayerProps) {
   // metadata
 
@@ -32,6 +35,7 @@ export default function WebRtcPlayer({
 
   const pcRef = useRef<RTCPeerConnection | undefined>();
   const videoRef = useRef<HTMLVideoElement | null>(null);
+  const [receivedData, setReceivedData] = useState(false);
 
   const PeerConnection = useCallback(
     async (media: string) => {
@@ -164,6 +168,8 @@ export default function WebRtcPlayer({
         pcRef.current.close();
         pcRef.current = undefined;
       }
+
+      setReceivedData(false);
     };
   }, [
     camera,
@@ -197,7 +203,17 @@ export default function WebRtcPlayer({
       autoPlay
       playsInline
       muted={!audioEnabled}
-      onLoadedData={onPlaying}
+      onLoadedData={() => {
+        setReceivedData(true);
+        onPlaying?.();
+      }}
+      onStalled={() => {
+        if (receivedData) {
+          onError?.("stalled");
+        } else {
+          onError?.("startup");
+        }
+      }}
       onClick={
         iOSCompatFullScreen
           ? () => setiOSCompatControls(!iOSCompatControls)

web/src/types/live.ts

@@ -30,3 +30,5 @@ export type LiveStreamMetadata = {
   producers: LiveProducerMetadata[];
   consumers: LiveConsumerMetadata[];
 };
+
+export type LivePlayerError = "stalled" | "startup";

web/src/views/live/LiveCameraView.tsx

@@ -190,6 +190,7 @@ export default function LiveCameraView({
   const [audio, setAudio] = useState(false);
   const [mic, setMic] = useState(false);
   const [pip, setPip] = useState(false);
+  const [lowBandwidth, setLowBandwidth] = useState(false);
 
   const [fullResolution, setFullResolution] = useState<VideoResolutionType>({
     width: 0,
@@ -201,8 +202,14 @@ export default function LiveCameraView({
       return "webrtc";
     }
 
+    if (lowBandwidth) {
+      return "jsmpeg";
+    }
+
     return "mse";
-  }, [mic]);
+  }, [lowBandwidth, mic]);
+
+  // layout state
 
   const windowAspectRatio = useMemo(() => {
     return windowWidth / windowHeight;
@@ -406,6 +413,7 @@ export default function LiveCameraView({
                 pip={pip}
                 setFullResolution={setFullResolution}
                 containerRef={containerRef}
+                onError={() => setLowBandwidth(true)}
               />
             </div>
             {camera.onvif.host != "" && (