Merge branch 'dev' into yaml_loader

.devcontainer/devcontainer.json

@@ -10,9 +10,9 @@
   "features": {
     "ghcr.io/devcontainers/features/common-utils:1": {}
   },
-  "forwardPorts": [8080, 5000, 5001, 5173, 8554, 8555],
+  "forwardPorts": [8971, 5000, 5001, 5173, 8554, 8555],
   "portsAttributes": {
-    "8080": {
+    "8971": {
       "label": "External NGINX",
       "onAutoForward": "silent"
     },

docker/main/Dockerfile

@@ -78,7 +78,7 @@ COPY --from=ov-converter /models/ssdlite_mobilenet_v2.bin openvino-model/
 RUN wget -q https://github.com/openvinotoolkit/open_model_zoo/raw/master/data/dataset_classes/coco_91cl_bkgr.txt -O openvino-model/coco_91cl_bkgr.txt && \
     sed -i 's/truck/car/g' openvino-model/coco_91cl_bkgr.txt
 # Get Audio Model and labels
-RUN wget -qO cpu_audio_model.tflite https://tfhub.dev/google/lite-model/yamnet/classification/tflite/1?lite-format=tflite
+RUN wget -qO cpu_audio_model.tflite https://www.kaggle.com/api/v1/models/google/yamnet/tfLite/classification-tflite/1/download
 COPY audio-labelmap.txt .
 
 

docker/main/rootfs/etc/s6-overlay/s6-rc.d/certsync/run

@@ -34,7 +34,7 @@ do
             ;;
     esac
 
-    liveprint=`echo | openssl s_client -showcerts -connect 127.0.0.1:8080 2>&1 | openssl x509 -fingerprint 2>&1 | grep -i fingerprint  || echo 'failed'`
+    liveprint=`echo | openssl s_client -showcerts -connect 127.0.0.1:8971 2>&1 | openssl x509 -fingerprint 2>&1 | grep -i fingerprint  || echo 'failed'`
 
     case "$liveprint" in
         *Fingerprint*)

docker/main/rootfs/usr/local/nginx/conf/nginx.conf

@@ -59,9 +59,6 @@ http {
     include go2rtc_upstream.conf;
 
     server {
-        # intended for internal traffic, not protected by auth
-        listen [::]:5000 ipv6only=off;
-
         include listen.conf;
 
         # vod settings

docker/main/rootfs/usr/local/nginx/templates/listen.gotmpl

@@ -1,9 +1,12 @@
 {{ if not .enabled }}
 # intended for external traffic, protected by auth
-listen [::]:8080 ipv6only=off;
+listen 8971;
 {{ else }}
 # intended for external traffic, protected by auth
-listen [::]:8080 ipv6only=off ssl;
+listen 8971 ssl;
+
+# intended for internal traffic, not protected by auth
+listen 5000;
 
 ssl_certificate /etc/letsencrypt/live/frigate/fullchain.pem;
 ssl_certificate_key /etc/letsencrypt/live/frigate/privkey.pem;

docs/docs/configuration/advanced.md

@@ -106,7 +106,53 @@ Some labels have special handling and modifications can disable functionality.
 
 :::
 
-## Custom ffmpeg build
+## Network Configuration
+
+Changes to Frigate's internal network configuration can be made by bind mounting nginx.conf into the container. For example:
+
+```yaml
+services:
+  frigate:
+    container_name: frigate
+    ...
+    volumes:
+      ...
+      - /path/to/your/nginx.conf:/usr/local/nginx/conf/nginx.conf
+```
+
+### Enabling IPv6
+
+IPv6 is disabled by default, to enable IPv6 listen.gotmpl needs to be bind mounted with IPv6 enabled. For example:
+
+```
+{{ if not .enabled }}
+# intended for external traffic, protected by auth
+listen 8971;
+{{ else }}
+# intended for external traffic, protected by auth
+listen 8971 ssl;
+
+# intended for internal traffic, not protected by auth
+listen 5000;
+```
+
+becomes
+
+```
+{{ if not .enabled }}
+# intended for external traffic, protected by auth
+listen [::]:8971 ipv6only=off;
+{{ else }}
+# intended for external traffic, protected by auth
+listen [::]:8971 ipv6only=off ssl;
+
+# intended for internal traffic, not protected by auth
+listen [::]:5000 ipv6only=off;
+```
+
+## Custom Dependencies
+
+### Custom ffmpeg build
 
 Included with Frigate is a build of ffmpeg that works for the vast majority of users. However, there exists some hardware setups which have incompatibilities with the included build. In this case, a docker volume mapping can be used to overwrite the included ffmpeg build with an ffmpeg build that works for your specific hardware setup.
 
@@ -118,7 +164,7 @@ To do this:
 
 NOTE: The folder that is mapped from the host needs to be the folder that contains `/bin`. So if the full structure is `/home/appdata/frigate/custom-ffmpeg/bin/ffmpeg` then `/home/appdata/frigate/custom-ffmpeg` needs to be mapped to `/usr/lib/btbn-ffmpeg`.
 
-## Custom go2rtc version
+### Custom go2rtc version
 
 Frigate currently includes go2rtc v1.9.4, there may be certain cases where you want to run a different version of go2rtc.
 

docs/docs/configuration/authentication.md

@@ -13,7 +13,7 @@ The following ports are available to access the Frigate web UI.
 
 | Port   | Description                                                                                                                                                                                                  |
 | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `8080` | Authenticated UI and API. Reverse proxies should use this port.                                                                                                                                              |
+| `8971` | Authenticated UI and API. Reverse proxies should use this port.                                                                                                                                              |
 | `5000` | Internal unauthenticated UI and API access. Access to this port should be limited. Intended to be used within the docker network for services that integrate with Frigate and do not support authentication. |
 
 ## Onboarding

docs/docs/configuration/live.md

@@ -7,13 +7,15 @@ Frigate intelligently displays your camera streams on the Live view dashboard. Y
 
 ## Live View technologies
 
-Frigate intelligently uses three different streaming technologies to display your camera streams. The highest quality and fluency of the Live view requires the bundled `go2rtc` to be configured as shown in the [step by step guide](/guides/configuring_go2rtc).
+Frigate intelligently uses three different streaming technologies to display your camera streams on the dashboard and the single camera view, switching between available modes based on network bandwidth, player errors, or required features like two-way talk. The highest quality and fluency of the Live view requires the bundled `go2rtc` to be configured as shown in the [step by step guide](/guides/configuring_go2rtc).
 
-| Source | Latency | Frame Rate                            | Resolution     | Audio                        | Requires go2rtc | Other Limitations                                |
+The jsmpeg live view will use more browser and client GPU resources. Using go2rtc is highly recommended and will provide a superior experience.
-| ------ | ------- | ------------------------------------- | -------------- | ---------------------------- | --------------- | ------------------------------------------------ |
+
-| jsmpeg | low     | same as `detect -> fps`, capped at 10 | same as detect | no                           | no              | none                                             |
+| Source | Latency | Frame Rate                            | Resolution | Audio                        | Requires go2rtc | Other Limitations                                                                    |
-| mse    | low     | native                                | native         | yes (depends on audio codec) | yes             | iPhone requires iOS 17.1+, Firefox is h.264 only |
+| ------ | ------- | ------------------------------------- | ---------- | ---------------------------- | --------------- | ------------------------------------------------------------------------------------ |
-| webrtc | lowest  | native                                | native         | yes (depends on audio codec) | yes             | requires extra config, doesn't support h.265     |
+| jsmpeg | low     | same as `detect -> fps`, capped at 10 | 720p       | no                           | no              | resolution is configurable, but go2rtc is recommended if you want higher resolutions |
+| mse    | low     | native                                | native     | yes (depends on audio codec) | yes             | iPhone requires iOS 17.1+, Firefox is h.264 only                                     |
+| webrtc | lowest  | native                                | native     | yes (depends on audio codec) | yes             | requires extra config, doesn't support h.265                                         |
 
 ### Audio Support
 

docs/docs/configuration/reference.md

@@ -65,7 +65,7 @@ database:
 
 # Optional: TLS configuration
 tls:
-  # Optional: Enable TLS for port 8080 (default: shown below)
+  # Optional: Enable TLS for port 8971 (default: shown below)
   enabled: True
 
 # Optional: Proxy configuration

docs/docs/configuration/tls.md

@@ -5,7 +5,7 @@ title: TLS
 
 # TLS
 
-Frigate's integrated NGINX server supports TLS certificates. By default Frigate will generate a self signed certificate that will be used for port 8080. Frigate is designed to make it easy to use whatever tool you prefer to manage certificates.
+Frigate's integrated NGINX server supports TLS certificates. By default Frigate will generate a self signed certificate that will be used for port 8971. Frigate is designed to make it easy to use whatever tool you prefer to manage certificates.
 
 Frigate is often running behind a reverse proxy that manages TLS certificates for multiple services. You will likely need to set your reverse proxy to allow self signed certificates or you can disable TLS in Frigate's config. However, if you are running on a dedicated device that's separate from your proxy or if you expose Frigate directly to the internet, you may want to configure TLS with valid certificates.
 
@@ -44,13 +44,13 @@ frigate:
 
 Frigate automatically compares the fingerprint of the certificate at `/etc/letsencrypt/live/frigate/fullchain.pem` against the fingerprint of the TLS cert in NGINX every minute. If these differ, the NGINX config is reloaded to pick up the updated certificate.
 
-If you issue Frigate valid certificates you will likely want to configure it to run on port 443 so you can access it without a port number like `https://your-frigate-domain.com` by mapping 8080 to 443.
+If you issue Frigate valid certificates you will likely want to configure it to run on port 443 so you can access it without a port number like `https://your-frigate-domain.com` by mapping 8971 to 443.
 
 ```yaml
 frigate:
   ...
   ports:
-    - "443:8080"
+    - "443:8971"
   ...
 ```
 

docs/docs/frigate/installation.md

@@ -34,7 +34,7 @@ The following ports are used by Frigate and can be mapped via docker as required
 
 | Port   | Description                                                                                                                                                                |
 | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `8080` | Authenticated UI and API access without TLS. Reverse proxies should use this port.                                                                                         |
+| `8971` | Authenticated UI and API access without TLS. Reverse proxies should use this port.                                                                                         |
 | `5000` | Internal unauthenticated UI and API access. Access to this port should be limited. Intended to be used within the docker network for services that integrate with Frigate. |
 | `8554` | RTSP restreaming. By default, these streams are unauthenticated. Authentication can be configured in go2rtc section of config.                                             |
 | `8555` | WebRTC connections for low latency live views.                                                                                                                             |
@@ -171,7 +171,7 @@ services:
         tmpfs:
           size: 1000000000
     ports:
-      - "8080:8080"
+      - "8971:8971"
       # - "5000:5000" # Internal unauthenticated access. Expose carefully.
       - "8554:8554" # RTSP feeds
       - "8555:8555/tcp" # WebRTC over tcp
@@ -194,7 +194,7 @@ docker run -d \
   -v /path/to/your/config:/config \
   -v /etc/localtime:/etc/localtime:ro \
   -e FRIGATE_RTSP_PASSWORD='password' \
-  -p 8080:8080 \
+  -p 8971:8971 \
   -p 8554:8554 \
   -p 8555:8555/tcp \
   -p 8555:8555/udp \
@@ -370,7 +370,7 @@ docker run \
   --network=bridge \
   --privileged \
   --workdir=/opt/frigate \
-  -p 8080:8080 \
+  -p 8971:8971 \
   -p 8554:8554 \
   -p 8555:8555 \
   -p 8555:8555/udp \

docs/docs/guides/getting_started.md

@@ -117,7 +117,7 @@ services:
         tmpfs:
           size: 1000000000
     ports:
-      - "8080:8080"
+      - "8971:8971"
       - "8554:8554" # RTSP feeds
 ```
 
@@ -137,7 +137,7 @@ cameras:
             - detect
 ```
 
-Now you should be able to start Frigate by running `docker compose up -d` from within the folder containing `docker-compose.yml`. On startup, an admin user and password will be created and outputted in the logs. You can see this by running `docker logs frigate`. Frigate should now be accessible at `https://server_ip:8080` where you can login with the `admin` user and finish the configuration using the built-in configuration editor.
+Now you should be able to start Frigate by running `docker compose up -d` from within the folder containing `docker-compose.yml`. On startup, an admin user and password will be created and outputted in the logs. You can see this by running `docker logs frigate`. Frigate should now be accessible at `https://server_ip:8971` where you can login with the `admin` user and finish the configuration using the built-in configuration editor.
 
 ## Configuring Frigate
 

docs/docs/guides/reverse_proxy.md

@@ -38,20 +38,20 @@ Here we access Frigate via https://cctv.mydomain.co.uk
     ServerName cctv.mydomain.co.uk
 
     ProxyPreserveHost On
-    ProxyPass "/"  "http://frigatepi.local:8080/"
+    ProxyPass "/"  "http://frigatepi.local:8971/"
-    ProxyPassReverse "/"  "http://frigatepi.local:8080/"
+    ProxyPassReverse "/"  "http://frigatepi.local:8971/"
 
-    ProxyPass /ws ws://frigatepi.local:8080/ws
+    ProxyPass /ws ws://frigatepi.local:8971/ws
-    ProxyPassReverse /ws ws://frigatepi.local:8080/ws
+    ProxyPassReverse /ws ws://frigatepi.local:8971/ws
 
-    ProxyPass /live/ ws://frigatepi.local:8080/live/
+    ProxyPass /live/ ws://frigatepi.local:8971/live/
-    ProxyPassReverse /live/ ws://frigatepi.local:8080/live/
+    ProxyPassReverse /live/ ws://frigatepi.local:8971/live/
 
     RewriteEngine on
     RewriteCond %{HTTP:Upgrade} =websocket [NC]
-    RewriteRule /(.*)  ws://frigatepi.local:8080/$1 [P,L]
+    RewriteRule /(.*)  ws://frigatepi.local:8971/$1 [P,L]
     RewriteCond %{HTTP:Upgrade} !=websocket [NC]
-    RewriteRule /(.*)  http://frigatepi.local:8080/$1 [P,L]
+    RewriteRule /(.*)  http://frigatepi.local:8971/$1 [P,L]
 </VirtualHost>
 ```
 
@@ -101,7 +101,7 @@ This is set in `$server` and `$port` this should match your ports you have expos
 server {
   set $forward_scheme http;
   set $server         "192.168.100.2"; # FRIGATE SERVER LOCATION
-  set $port           8080;
+  set $port           8971;
 
   listen 80;
   listen 443 ssl http2;

frigate/api/media.py

@@ -637,7 +637,7 @@ def vod_event(id):
         # If the recordings are not found and the event started more than 5 minutes ago, set has_clip to false
         if (
             event.start_time < datetime.now().timestamp() - 300
-            and type(vod_response) == tuple
+            and type(vod_response) is tuple
             and len(vod_response) == 2
             and vod_response[1] == 404
         ):

frigate/config.py

@@ -116,7 +116,7 @@ class UIConfig(FrigateBaseModel):
 
 
 class TlsConfig(FrigateBaseModel):
-    enabled: bool = Field(default=True, title="Enable TLS for port 8080")
+    enabled: bool = Field(default=True, title="Enable TLS for port 8971")
 
 
 class HeaderMappingConfig(FrigateBaseModel):

web/src/components/camera/CameraImage.tsx

@@ -26,13 +26,12 @@ export default function CameraImage({
 
   const { name } = config ? config.cameras[camera] : "";
   const enabled = config ? config.cameras[camera].enabled : "True";
-  const [isPortraitImage, setIsPortraitImage] = useState(false);
 
   const [{ width: containerWidth, height: containerHeight }] =
     useResizeObserver(containerRef);
 
   const requestHeight = useMemo(() => {
-    if (!config || containerHeight == 0) {
+    if (!config || containerHeight == 0 || !hasLoaded) {
       return 360;
     }
 
@@ -40,7 +39,14 @@ export default function CameraImage({
       config.cameras[camera].detect.height,
       Math.round(containerHeight * (isDesktop ? 1.1 : 1.25)),
     );
-  }, [config, camera, containerHeight]);
+  }, [config, camera, containerHeight, hasLoaded]);
+
+  const isPortraitImage = useMemo(() => {
+    if (imgRef.current && containerWidth && containerHeight && hasLoaded) {
+      const { naturalHeight, naturalWidth } = imgRef.current;
+      return naturalWidth / naturalHeight < containerWidth / containerHeight;
+    }
+  }, [containerWidth, containerHeight, hasLoaded]);
 
   useEffect(() => {
     if (!config || !imgRef.current) {
@@ -61,13 +67,6 @@ export default function CameraImage({
           onLoad={() => {
             setHasLoaded(true);
 
-            if (imgRef.current) {
-              const { naturalHeight, naturalWidth } = imgRef.current;
-              setIsPortraitImage(
-                naturalWidth / naturalHeight < containerWidth / containerHeight,
-              );
-            }
-
             if (onload) {
               onload();
             }

web/src/components/player/BirdseyeLivePlayer.tsx

@@ -12,7 +12,7 @@ type LivePlayerProps = {
   birdseyeConfig: BirdseyeConfig;
   liveMode: LivePlayerMode;
   onClick?: () => void;
-  containerRef?: React.MutableRefObject<HTMLDivElement | null>;
+  containerRef: React.MutableRefObject<HTMLDivElement | null>;
 };
 
 export default function BirdseyeLivePlayer({
@@ -54,6 +54,7 @@ export default function BirdseyeLivePlayer({
         width={birdseyeConfig.width}
         height={birdseyeConfig.height}
         containerRef={containerRef}
+        playbackEnabled={true}
       />
     );
   } else {
@@ -62,6 +63,7 @@ export default function BirdseyeLivePlayer({
 
   return (
     <div
+      ref={containerRef}
       className={cn(
         "relative flex w-full cursor-pointer justify-center",
         className,

web/src/components/player/JSMpegPlayer.tsx

@@ -3,14 +3,15 @@ import { useResizeObserver } from "@/hooks/resize-observer";
 import { cn } from "@/lib/utils";
 // @ts-expect-error we know this doesn't have types
 import JSMpeg from "@cycjimmy/jsmpeg-player";
-import React, { useEffect, useMemo, useRef, useId, useState } from "react";
+import React, { useEffect, useMemo, useRef, useState } from "react";
 
 type JSMpegPlayerProps = {
   className?: string;
   camera: string;
   width: number;
   height: number;
-  containerRef?: React.MutableRefObject<HTMLDivElement | null>;
+  containerRef: React.MutableRefObject<HTMLDivElement | null>;
+  playbackEnabled: boolean;
   onPlaying?: () => void;
 };
 
@@ -20,18 +21,21 @@ export default function JSMpegPlayer({
   height,
   className,
   containerRef,
+  playbackEnabled,
   onPlaying,
 }: JSMpegPlayerProps) {
   const url = `${baseUrl.replace(/^http/, "ws")}live/jsmpeg/${camera}`;
-  const playerRef = useRef<HTMLDivElement | null>(null);
+  const videoRef = useRef<HTMLDivElement>(null);
-  const videoRef = useRef(null);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
   const internalContainerRef = useRef<HTMLDivElement | null>(null);
   const onPlayingRef = useRef(onPlaying);
   const [showCanvas, setShowCanvas] = useState(false);
 
   const selectedContainerRef = useMemo(
-    () => containerRef ?? internalContainerRef,
+    () => (containerRef.current ? containerRef : internalContainerRef),
-    [containerRef, internalContainerRef],
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [containerRef, containerRef.current, internalContainerRef],
   );
 
   const [{ width: containerWidth, height: containerHeight }] =
@@ -83,39 +87,64 @@ export default function JSMpegPlayer({
     }
   }, [scaledHeight, aspectRatio]);
 
-  const uniqueId = useId();
-
   useEffect(() => {
     onPlayingRef.current = onPlaying;
   }, [onPlaying]);
 
   useEffect(() => {
-    if (!playerRef.current || videoRef.current) {
+    if (!selectedContainerRef?.current || !url) {
       return;
     }
 
-    videoRef.current = new JSMpeg.VideoElement(
+    const videoWrapper = videoRef.current;
-      playerRef.current,
+    const canvas = canvasRef.current;
-      url,
+    let hasData = false;
-      { canvas: `#${CSS.escape(uniqueId)}` },
+    let videoElement: JSMpeg.VideoElement | null = null;
-      {
+
-        protocols: [],
+    if (videoWrapper && playbackEnabled) {
-        audio: false,
+      // Delayed init to avoid issues with react strict mode
-        videoBufferSize: 1024 * 1024 * 4,
+      const initPlayer = setTimeout(() => {
-        onPlay: () => {
+        videoElement = new JSMpeg.VideoElement(
-          setShowCanvas(true);
+          videoWrapper,
-          onPlayingRef.current?.();
+          url,
-        },
+          { canvas: canvas },
-      },
+          {
-    );
+            protocols: [],
-  }, [url, uniqueId]);
+            audio: false,
+            videoBufferSize: 1024 * 1024 * 4,
+            onVideoDecode: () => {
+              if (!hasData) {
+                hasData = true;
+                setShowCanvas(true);
+                onPlayingRef.current?.();
+              }
+            },
+          },
+        );
+      }, 0);
+
+      return () => {
+        clearTimeout(initPlayer);
+        if (videoElement) {
+          try {
+            // this causes issues in react strict mode
+            // https://stackoverflow.com/questions/76822128/issue-with-cycjimmy-jsmpeg-player-in-react-18-cannot-read-properties-of-null-o
+            videoElement.destroy();
+            // eslint-disable-next-line no-empty
+          } catch (e) {}
+        }
+      };
+    }
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [playbackEnabled, url]);
 
   return (
-    <div className={className}>
+    <div className={cn(className, !containerRef.current && "size-full")}>
-      <div className="size-full" ref={internalContainerRef}>
+      <div className="internal-jsmpeg-container" ref={internalContainerRef}>
-        <div ref={playerRef} className={cn("jsmpeg", !showCanvas && "hidden")}>
+        <div ref={videoRef} className={cn("jsmpeg", !showCanvas && "hidden")}>
           <canvas
-            id={uniqueId}
+            ref={canvasRef}
             style={{
               width: scaledWidth ?? width,
               height: scaledHeight ?? height,

web/src/components/player/LivePlayer.tsx

@@ -2,7 +2,7 @@ import WebRtcPlayer from "./WebRTCPlayer";
 import { CameraConfig } from "@/types/frigateConfig";
 import AutoUpdatingCameraImage from "../camera/AutoUpdatingCameraImage";
 import ActivityIndicator from "../indicators/activity-indicator";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import MSEPlayer from "./MsePlayer";
 import JSMpegPlayer from "./JSMpegPlayer";
 import { MdCircle } from "react-icons/md";
@@ -55,6 +55,7 @@ export default function LivePlayer({
   setFullResolution,
   onError,
 }: LivePlayerProps) {
+  const internalContainerRef = useRef<HTMLDivElement | null>(null);
   // camera activity
 
   const { activeMotion, activeTracking, objects, offline } =
@@ -73,20 +74,12 @@ export default function LivePlayer({
 
   const [liveReady, setLiveReady] = useState(false);
   useEffect(() => {
-    if (!autoLive) {
+    if (!autoLive || !liveReady) {
-      return;
-    }
-
-    if (!liveReady) {
-      if (cameraActive && liveMode == "jsmpeg") {
-        setLiveReady(true);
-      }
-
       return;
     }
 
     if (!cameraActive) {
-      setTimeout(() => setLiveReady(false), 500);
+      setLiveReady(false);
     }
     // live mode won't change
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -181,7 +174,8 @@ export default function LivePlayer({
           camera={cameraConfig.live.stream_name}
           width={cameraConfig.detect.width}
           height={cameraConfig.detect.height}
-          containerRef={containerRef}
+          playbackEnabled={cameraActive || !showStillWithoutActivity}
+          containerRef={containerRef ?? internalContainerRef}
           onPlaying={playerIsPlaying}
         />
       );
@@ -194,7 +188,7 @@ export default function LivePlayer({
 
   return (
     <div
-      ref={cameraRef}
+      ref={cameraRef ?? internalContainerRef}
       data-camera={cameraConfig.name}
       className={cn(
         "relative flex w-full cursor-pointer justify-center outline",

web/src/components/player/MsePlayer.tsx

@@ -31,7 +31,7 @@ function MSEPlayer({
   setFullResolution,
   onError,
 }: MSEPlayerProps) {
-  const RECONNECT_TIMEOUT: number = 30000;
+  const RECONNECT_TIMEOUT: number = 10000;
 
   const CODECS: string[] = [
     "avc1.640029", // H.264 high 4.1 (Chromecast 1st and 2nd Gen)
@@ -45,10 +45,12 @@ function MSEPlayer({
   ];
 
   const visibilityCheck: boolean = !pip;
+  const [safariPlaying, setSafariPlaying] = useState(false);
 
   const [wsState, setWsState] = useState<number>(WebSocket.CLOSED);
   const [connectTS, setConnectTS] = useState<number>(0);
   const [bufferTimeout, setBufferTimeout] = useState<NodeJS.Timeout>();
+  const [errorCount, setErrorCount] = useState<number>(0);
 
   const videoRef = useRef<HTMLVideoElement>(null);
   const wsRef = useRef<WebSocket | null>(null);
@@ -117,12 +119,21 @@ function MSEPlayer({
   }, [wsURL]);
 
   const onDisconnect = useCallback(() => {
-    if (wsRef.current && wsState == WebSocket.OPEN) {
+    if (bufferTimeout) {
+      clearTimeout(bufferTimeout);
+      setBufferTimeout(undefined);
+    }
+
+    if ((isSafari || isIOS) && safariPlaying) {
+      setSafariPlaying(false);
+    }
+
+    if (wsRef.current && wsState != WebSocket.CLOSED) {
       setWsState(WebSocket.CLOSED);
       wsRef.current.close();
       wsRef.current = null;
     }
-  }, [wsState]);
+  }, [wsState, bufferTimeout, safariPlaying]);
 
   const onOpen = () => {
     setWsState(WebSocket.OPEN);
@@ -162,6 +173,26 @@ function MSEPlayer({
     reconnect();
   };
 
+  const sendWithTimeout = (value: object, timeout: number) => {
+    return new Promise<void>((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        reject(new Error("Timeout waiting for response"));
+      }, timeout);
+
+      send(value);
+
+      // Override the onmessageRef handler for mse type to resolve the promise on response
+      const originalHandler = onmessageRef.current["mse"];
+      onmessageRef.current["mse"] = (msg) => {
+        if (msg.type === "mse") {
+          clearTimeout(timeoutId);
+          if (originalHandler) originalHandler(msg);
+          resolve();
+        }
+      };
+    });
+  };
+
   const onMse = () => {
     if ("ManagedMediaSource" in window) {
       const MediaSource = window.ManagedMediaSource;
@@ -169,10 +200,22 @@ function MSEPlayer({
       msRef.current?.addEventListener(
         "sourceopen",
         () => {
-          send({
+          sendWithTimeout(
-            type: "mse",
+            {
-            // @ts-expect-error for typing
+              type: "mse",
-            value: codecs(MediaSource.isTypeSupported),
+              // @ts-expect-error for typing
+              value: codecs(MediaSource.isTypeSupported),
+            },
+            3000,
+          ).catch(() => {
+            if (wsRef.current) {
+              onDisconnect();
+            }
+            if (isIOS || isSafari) {
+              onError?.("mse-decode");
+            } else {
+              onError?.("startup");
+            }
           });
         },
         { once: true },
@@ -187,9 +230,21 @@ function MSEPlayer({
         "sourceopen",
         () => {
           URL.revokeObjectURL(videoRef.current?.src || "");
-          send({
+          sendWithTimeout(
-            type: "mse",
+            {
-            value: codecs(MediaSource.isTypeSupported),
+              type: "mse",
+              value: codecs(MediaSource.isTypeSupported),
+            },
+            3000,
+          ).catch(() => {
+            if (wsRef.current) {
+              onDisconnect();
+            }
+            if (isIOS || isSafari) {
+              onError?.("mse-decode");
+            } else {
+              onError?.("startup");
+            }
           });
         },
         { once: true },
@@ -260,10 +315,6 @@ function MSEPlayer({
 
     return () => {
       onDisconnect();
-      if (bufferTimeout) {
-        clearTimeout(bufferTimeout);
-        setBufferTimeout(undefined);
-      }
     };
     // we know that these deps are correct
     // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -305,6 +356,23 @@ function MSEPlayer({
     videoRef.current.requestPictureInPicture();
   }, [pip, videoRef]);
 
+  // ensure we disconnect for slower connections
+
+  useEffect(() => {
+    if (wsState === WebSocket.OPEN && !playbackEnabled) {
+      if (bufferTimeout) {
+        clearTimeout(bufferTimeout);
+        setBufferTimeout(undefined);
+      }
+
+      setTimeout(() => {
+        if (!playbackEnabled) onDisconnect();
+      }, 10000);
+    }
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [playbackEnabled]);
+
   return (
     <video
       ref={videoRef}
@@ -317,7 +385,8 @@ function MSEPlayer({
       }}
       muted={!audioEnabled}
       onProgress={() => {
-        if (isSafari || isIOS) {
+        if ((isSafari || isIOS) && !safariPlaying) {
+          setSafariPlaying(true);
           onPlaying?.();
         }
         if (onError != undefined) {
@@ -334,8 +403,10 @@ function MSEPlayer({
             setTimeout(() => {
               if (
                 document.visibilityState === "visible" &&
-                wsRef.current != null
+                wsRef.current != null &&
+                videoRef.current
               ) {
+                onDisconnect();
                 onError("stalled");
               }
             }, 3000),
@@ -347,6 +418,9 @@ function MSEPlayer({
           // @ts-expect-error code does exist
           e.target.error.code == MediaError.MEDIA_ERR_NETWORK
         ) {
+          if (wsRef.current) {
+            onDisconnect();
+          }
           onError?.("startup");
         }
 
@@ -355,15 +429,22 @@ function MSEPlayer({
           e.target.error.code == MediaError.MEDIA_ERR_DECODE &&
           (isSafari || isIOS)
         ) {
+          if (wsRef.current) {
+            onDisconnect();
+          }
           onError?.("mse-decode");
-          clearTimeout(bufferTimeout);
-          setBufferTimeout(undefined);
         }
 
+        setErrorCount((prevCount) => prevCount + 1);
+
         if (wsRef.current) {
-          wsRef.current.close();
+          onDisconnect();
-          wsRef.current = null;
+          if (errorCount >= 3) {
-          reconnect(5000);
+            // too many mse errors, try jsmpeg
+            onError?.("startup");
+          } else {
+            reconnect(5000);
+          }
         }
       }}
     />

web/src/pages/UIPlayground.tsx

@@ -130,6 +130,7 @@ const generateRandomEvent = (): ReviewSegment => {
 function UIPlayground() {
   const { data: config } = useSWR<FrigateConfig>("config");
   const contentRef = useRef<HTMLDivElement>(null);
+  const containerRef = useRef<HTMLDivElement>(null);
   const reviewTimelineRef = useRef<HTMLDivElement>(null);
   const [mockEvents, setMockEvents] = useState<ReviewSegment[]>([]);
   const [mockMotionData, setMockMotionData] = useState<MotionData[]>([]);
@@ -344,11 +345,12 @@ function UIPlayground() {
                   Zoom In
                 </Button>
               </p>
-              <div className="">
+              <div ref={containerRef} className="">
                 {birdseyeConfig && (
                   <BirdseyeLivePlayer
                     birdseyeConfig={birdseyeConfig}
                     liveMode={birdseyeConfig.restream ? "mse" : "jsmpeg"}
+                    containerRef={containerRef}
                   />
                 )}
               </div>

web/src/views/live/DraggableGridLayout.tsx

@@ -20,13 +20,13 @@ import {
 } from "react-grid-layout";
 import "react-grid-layout/css/styles.css";
 import "react-resizable/css/styles.css";
-import { LivePlayerMode } from "@/types/live";
+import { LivePlayerError, LivePlayerMode } from "@/types/live";
 import { ASPECT_VERTICAL_LAYOUT, ASPECT_WIDE_LAYOUT } from "@/types/record";
 import { Skeleton } from "@/components/ui/skeleton";
 import { useResizeObserver } from "@/hooks/resize-observer";
 import { isEqual } from "lodash";
 import useSWR from "swr";
-import { isDesktop, isMobile, isSafari } from "react-device-detect";
+import { isDesktop, isMobile } from "react-device-detect";
 import BirdseyeLivePlayer from "@/components/player/BirdseyeLivePlayer";
 import LivePlayer from "@/components/player/LivePlayer";
 import { IoClose } from "react-icons/io5";
@@ -73,6 +73,31 @@ export default function DraggableGridLayout({
   const { data: config } = useSWR<FrigateConfig>("config");
   const birdseyeConfig = useMemo(() => config?.birdseye, [config]);
 
+  // preferred live modes per camera
+
+  const [preferredLiveModes, setPreferredLiveModes] = useState<{
+    [key: string]: LivePlayerMode;
+  }>({});
+
+  useEffect(() => {
+    if (!cameras) return;
+    const newPreferredLiveModes = cameras.reduce(
+      (acc, camera) => {
+        const isRestreamed =
+          config &&
+          Object.keys(config.go2rtc.streams || {}).includes(
+            camera.live.stream_name,
+          );
+
+        acc[camera.name] = isRestreamed ? "mse" : "jsmpeg";
+        return acc;
+      },
+      {} as { [key: string]: LivePlayerMode },
+    );
+
+    setPreferredLiveModes(newPreferredLiveModes);
+  }, [cameras, config]);
+
   const ResponsiveGridLayout = useMemo(() => WidthProvider(Responsive), []);
 
   const [gridLayout, setGridLayout, isGridLayoutLoaded] = usePersistence<
@@ -429,10 +454,21 @@ export default function DraggableGridLayout({
                     windowVisible && visibleCameras.includes(camera.name)
                   }
                   cameraConfig={camera}
-                  preferredLiveMode={isSafari ? "webrtc" : "mse"}
+                  preferredLiveMode={preferredLiveModes[camera.name] ?? "mse"}
                   onClick={() => {
                     !isEditMode && onSelectCamera(camera.name);
                   }}
+                  onError={(e) => {
+                    setPreferredLiveModes((prevModes) => {
+                      const newModes = { ...prevModes };
+                      if (e === "mse-decode") {
+                        newModes[camera.name] = "webrtc";
+                      } else {
+                        newModes[camera.name] = "jsmpeg";
+                      }
+                      return newModes;
+                    });
+                  }}
                 >
                   {isEditMode && showCircles && <CornerCircles />}
                 </LivePlayerGridItem>
@@ -590,6 +626,7 @@ type LivePlayerGridItemProps = {
   cameraConfig: CameraConfig;
   preferredLiveMode: LivePlayerMode;
   onClick: () => void;
+  onError: (e: LivePlayerError) => void;
 };
 
 const LivePlayerGridItem = React.forwardRef<
@@ -609,6 +646,7 @@ const LivePlayerGridItem = React.forwardRef<
       cameraConfig,
       preferredLiveMode,
       onClick,
+      onError,
       ...props
     },
     ref,
@@ -629,6 +667,7 @@ const LivePlayerGridItem = React.forwardRef<
           cameraConfig={cameraConfig}
           preferredLiveMode={preferredLiveMode}
           onClick={onClick}
+          onError={onError}
           containerRef={ref as React.RefObject<HTMLDivElement>}
         />
         {children}

web/src/views/live/LiveDashboardView.tsx

@@ -16,7 +16,6 @@ import {
   isDesktop,
   isMobile,
   isMobileOnly,
-  isSafari,
   isTablet,
 } from "react-device-detect";
 import useSWR from "swr";
@@ -24,6 +23,7 @@ import DraggableGridLayout from "./DraggableGridLayout";
 import { IoClose } from "react-icons/io5";
 import { LuLayoutDashboard } from "react-icons/lu";
 import { cn } from "@/lib/utils";
+import { LivePlayerMode } from "@/types/live";
 
 type LiveDashboardViewProps = {
   cameras: CameraConfig[];
@@ -99,6 +99,29 @@ export default function LiveDashboardView({
   // camera live views
 
   const [autoLiveView] = usePersistence("autoLiveView", true);
+  const [preferredLiveModes, setPreferredLiveModes] = useState<{
+    [key: string]: LivePlayerMode;
+  }>({});
+
+  useEffect(() => {
+    if (!cameras) return;
+    const newPreferredLiveModes = cameras.reduce(
+      (acc, camera) => {
+        const isRestreamed =
+          config &&
+          Object.keys(config.go2rtc.streams || {}).includes(
+            camera.live.stream_name,
+          );
+
+        acc[camera.name] = isRestreamed ? "mse" : "jsmpeg";
+        return acc;
+      },
+      {} as { [key: string]: LivePlayerMode },
+    );
+
+    setPreferredLiveModes(newPreferredLiveModes);
+  }, [cameras, config]);
+
   const [windowVisible, setWindowVisible] = useState(true);
   const visibilityListener = useCallback(() => {
     setWindowVisible(document.visibilityState == "visible");
@@ -289,9 +312,20 @@ export default function LiveDashboardView({
                   windowVisible && visibleCameras.includes(camera.name)
                 }
                 cameraConfig={camera}
-                preferredLiveMode={isSafari ? "webrtc" : "mse"}
+                preferredLiveMode={preferredLiveModes[camera.name] ?? "mse"}
                 autoLive={autoLiveView}
                 onClick={() => onSelectCamera(camera.name)}
+                onError={(e) => {
+                  setPreferredLiveModes((prevModes) => {
+                    const newModes = { ...prevModes };
+                    if (e === "mse-decode") {
+                      newModes[camera.name] = "webrtc";
+                    } else {
+                      newModes[camera.name] = "jsmpeg";
+                    }
+                    return newModes;
+                  });
+                }}
               />
             );
           })}