Live view improvements (#24967)

* hold the last frame on screen while switching live streams

Switching streams in the single camera view blanked the player until the new stream played, because the old player unmounts in the same render and the hidden new player gave the card no height. The outgoing video's last frame is now drawn to a canvas before the remount and covers the gap until the new stream plays, and the card takes the camera's aspect ratio so it keeps its size while the video is hidden.

* show live streams as a one-tap grid in the mobile drawer

Switching streams on mobile meant scrolling past the camera toggles, opening a dropdown, and picking a stream. The drawer now opens with the stream section, and streams are a wrapping grid of toggles sized to their names, so a switch is two taps. A single stream shows as plain text. The audio and two-way talk lines show a spinner while the stream's go2rtc metadata loads instead of reporting audio as unavailable, which showed on the first switch to a transcoded stream.

* don't capture the switch frame on tiles that show the camera still

web/public/locales/en/views/live.json

@@ -173,6 +173,9 @@
       "label": "Play in background",
       "tips": "Enable this option to continue streaming when the player is hidden."
     },
+    "metadata": {
+      "loading": "Fetching stream metadata…"
+    },
     "mode": "Streaming Technology",
     "technology": {
       "description": "Choose your preferred streaming technology. Frigate may still fall back to low bandwidth mode on playback or network errors.",

web/src/components/player/LivePlayer.tsx

@@ -248,10 +248,21 @@ export default function LivePlayer({
   // remounts them in the same render and the new player is hidden until it
   // plays. jsmpeg plays the camera, not the stream, and keeps playing
   const [renderedStream, setRenderedStream] = useState(streamName);
+  const [switchFrame, setSwitchFrame] = useState<HTMLCanvasElement>();
   if (renderedStream !== streamName) {
     setRenderedStream(streamName);
     if (preferredLiveMode !== "jsmpeg") {
       setLiveReady(false);
+
+      // the old video is still in the DOM until this render commits, so its
+      // last frame can cover the gap until the new stream plays. Tiles that
+      // show the camera still already cover it
+      if (!showStillWithoutActivity) {
+        const frame = captureVideoFrame(overlayRef.current);
+        if (frame) {
+          setSwitchFrame(frame);
+        }
+      }
     }
   }
 
@@ -263,6 +274,7 @@ export default function LivePlayer({
 
   const playerIsPlaying = useCallback(() => {
     setLiveReady(true);
+    setSwitchFrame(undefined);
   }, []);
 
   // enabled states
@@ -416,6 +428,12 @@ export default function LivePlayer({
           </div>
         )}
       {player}
+      {switchFrame && !liveReady && (
+        <div
+          className="absolute inset-0"
+          ref={(node) => node?.replaceChildren(switchFrame)}
+        />
+      )}
       {cameraEnabled &&
         !offline &&
         (!showStillWithoutActivity || isReEnabling) &&
@@ -582,3 +600,19 @@ export default function LivePlayer({
     </div>
   );
 }
+
+function captureVideoFrame(
+  container: HTMLElement | null,
+): HTMLCanvasElement | undefined {
+  const video = container?.querySelector("video");
+  if (!video || video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
+    return undefined;
+  }
+
+  const canvas = document.createElement("canvas");
+  canvas.width = video.videoWidth;
+  canvas.height = video.videoHeight;
+  canvas.className = "size-full object-contain";
+  canvas.getContext("2d")?.drawImage(video, 0, 0);
+  return canvas;
+}

web/src/components/player/LiveStreamSelect.tsx

@@ -12,11 +12,15 @@ import { useTranslation } from "react-i18next";
 import { LuInfo } from "react-icons/lu";
 import { MdOutlineRestartAlt } from "react-icons/md";
 import { Button } from "@/components/ui/button";
+import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
 
 // Radix Select needs a non-empty value. Auto itself is stored as the
 // absence of a pinned stream
 const AUTO_VALUE = "__frigate_auto__";
 
+const GRID_ITEM =
+  "h-auto min-h-10 min-w-[calc((100%-1rem)/3)] grow whitespace-normal bg-secondary [overflow-wrap:anywhere] px-2 py-2 text-secondary-foreground hover:bg-secondary hover:text-secondary-foreground data-[state=on]:bg-selected data-[state=on]:text-selected-foreground data-[state=on]:hover:bg-selected data-[state=on]:hover:text-selected-foreground";
+
 type LiveStreamSelectProps = {
   // friendly name to go2rtc stream name, in ladder order
   streams: Record<string, string>;
@@ -29,6 +33,7 @@ type LiveStreamSelectProps = {
   // sends auto back to the top of the ladder
   onRetry: () => void;
   disabled: boolean;
+  variant?: "select" | "grid";
 };
 
 export default function LiveStreamSelect({
@@ -40,6 +45,7 @@ export default function LiveStreamSelect({
   onSelect,
   onRetry,
   disabled,
+  variant = "select",
 }: LiveStreamSelectProps) {
   const { t } = useTranslation(["views/live"]);
 
@@ -52,48 +58,78 @@ export default function LiveStreamSelect({
   const auto = autoAvailable && pinnedStream == undefined;
   const value = auto ? AUTO_VALUE : (pinnedStream ?? playingStream);
 
+  const autoLabel = auto
+    ? t("stream.auto.selected", {
+        stream:
+          autoReason === "floor"
+            ? t("stream.auto.floorLabel")
+            : labelFor(playingStream),
+      })
+    : t("stream.auto.label");
+
   return (
     <>
-      <Select
-        value={value}
-        disabled={disabled}
-        onValueChange={(next) =>
-          onSelect(next === AUTO_VALUE ? undefined : next)
-        }
-      >
-        <SelectTrigger className="w-full">
-          <SelectValue>
-            {auto
-              ? t("stream.auto.selected", {
-                  stream:
-                    autoReason === "floor"
-                      ? t("stream.auto.floorLabel")
-                      : labelFor(playingStream),
-                })
-              : labelFor(value)}
-          </SelectValue>
-        </SelectTrigger>
+      {variant === "grid" &&
+      !autoAvailable &&
+      Object.keys(streams).length <= 1 ? (
+        <div className="text-sm">{labelFor(playingStream)}</div>
+      ) : variant === "grid" ? (
+        <ToggleGroup
+          type="single"
+          className="flex flex-wrap items-stretch gap-2"
+          value={value}
+          disabled={disabled}
+          onValueChange={(next) => {
+            // radix sends an empty value when the active item is tapped again
+            if (next) {
+              onSelect(next === AUTO_VALUE ? undefined : next);
+            }
+          }}
+        >
+          {autoAvailable && (
+            <ToggleGroupItem value={AUTO_VALUE} className={GRID_ITEM}>
+              {autoLabel}
+            </ToggleGroupItem>
+          )}
+          {Object.entries(streams).map(([label, name]) => (
+            <ToggleGroupItem key={label} value={name} className={GRID_ITEM}>
+              {label}
+            </ToggleGroupItem>
+          ))}
+        </ToggleGroup>
+      ) : (
+        <Select
+          value={value}
+          disabled={disabled}
+          onValueChange={(next) =>
+            onSelect(next === AUTO_VALUE ? undefined : next)
+          }
+        >
+          <SelectTrigger className="w-full">
+            <SelectValue>{auto ? autoLabel : labelFor(value)}</SelectValue>
+          </SelectTrigger>
 
-        <SelectContent>
-          <SelectGroup>
-            {autoAvailable && (
-              <SelectItem className="cursor-pointer" value={AUTO_VALUE}>
-                <span className="flex flex-col gap-0.5 whitespace-normal">
-                  <span>{t("stream.auto.label")}</span>
-                  <span className="text-xs text-muted-foreground">
-                    {t("stream.auto.tips")}
+          <SelectContent>
+            <SelectGroup>
+              {autoAvailable && (
+                <SelectItem className="cursor-pointer" value={AUTO_VALUE}>
+                  <span className="flex flex-col gap-0.5 whitespace-normal">
+                    <span>{t("stream.auto.label")}</span>
+                    <span className="text-xs text-muted-foreground">
+                      {t("stream.auto.tips")}
+                    </span>
                   </span>
-                </span>
-              </SelectItem>
-            )}
-            {Object.entries(streams).map(([label, name]) => (
-              <SelectItem key={label} className="cursor-pointer" value={name}>
-                {label}
-              </SelectItem>
-            ))}
-          </SelectGroup>
-        </SelectContent>
-      </Select>
+                </SelectItem>
+              )}
+              {Object.entries(streams).map(([label, name]) => (
+                <SelectItem key={label} className="cursor-pointer" value={name}>
+                  {label}
+                </SelectItem>
+              ))}
+            </SelectGroup>
+          </SelectContent>
+        </Select>
+      )}
 
       {auto && autoReason && !disabled && (
         <div className="flex flex-col items-center gap-3">

web/src/views/live/LiveCameraView.tsx

@@ -230,15 +230,16 @@ export default function LiveCameraView({
     }
   }, [streamNameLoaded, liveStreams, pinnedStream, clearPinnedStream]);
 
-  const { data: cameraMetadata } = useSWR<LiveStreamMetadata>(
-    isRestreamed ? `go2rtc/streams/${streamName}` : null,
-    {
-      revalidateOnFocus: false,
-      revalidateOnReconnect: false,
-      revalidateIfStale: false,
-      dedupingInterval: 60000,
-    },
-  );
+  const { data: cameraMetadata, isLoading: cameraMetadataLoading } =
+    useSWR<LiveStreamMetadata>(
+      isRestreamed ? `go2rtc/streams/${streamName}` : null,
+      {
+        revalidateOnFocus: false,
+        revalidateOnReconnect: false,
+        revalidateIfStale: false,
+        dedupingInterval: 60000,
+      },
+    );
 
   const webRTCAvailability = useWebRTCAvailableForStream(
     cameraMetadata,
@@ -847,6 +848,7 @@ export default function LiveCameraView({
               onResetStream={resetStream}
               supportsAudioOutput={supportsAudioOutput}
               supports2WayTalk={supports2WayTalk}
+              streamMetadataLoading={cameraMetadataLoading}
               cameraEnabled={cameraEnabled}
               debug={debug}
               setDebug={setDebug}
@@ -873,9 +875,12 @@ export default function LiveCameraView({
                   growClassName,
                 )}
                 ref={clickOverlayRef}
-                style={{
-                  aspectRatio: constrainedAspectRatio,
-                }}
+                style={
+                  {
+                    aspectRatio: constrainedAspectRatio,
+                    "--camera-aspect": cameraAspectRatio,
+                  } as React.CSSProperties
+                }
               >
                 {clickOverlay && overlaySize.width > 0 && (
                   <div
@@ -914,7 +919,10 @@ export default function LiveCameraView({
                 )}
                 <LivePlayer
                   key={camera.name}
-                  className={fullscreen ? "rounded-none" : ""}
+                  className={cn(
+                    "aspect-[var(--camera-aspect)]",
+                    fullscreen && "rounded-none",
+                  )}
                   windowVisible
                   showStillWithoutActivity={false}
                   alwaysShowCameraName={false}
@@ -1007,6 +1015,7 @@ type FrigateCameraFeaturesProps = {
   onResetStream: () => void;
   supportsAudioOutput: boolean;
   supports2WayTalk: boolean;
+  streamMetadataLoading: boolean;
   cameraEnabled: boolean;
   debug: boolean;
   setDebug: (debug: boolean) => void;
@@ -1038,6 +1047,7 @@ function FrigateCameraFeatures({
   onResetStream,
   supportsAudioOutput,
   supports2WayTalk,
+  streamMetadataLoading,
   cameraEnabled,
   debug,
   setDebug,
@@ -1434,7 +1444,12 @@ function FrigateCameraFeatures({
                         !debug &&
                         isRestreamed && (
                           <div className="flex flex-row items-center gap-1 text-sm text-muted-foreground">
-                            {supportsAudioOutput ? (
+                            {streamMetadataLoading ? (
+                              <>
+                                <ActivityIndicator className="size-4" />
+                                <div>{t("stream.metadata.loading")}</div>
+                              </>
+                            ) : supportsAudioOutput ? (
                               <>
                                 <LuCheck className="size-4 text-success" />
                                 <div>{t("stream.audio.available")}</div>
@@ -1693,72 +1708,7 @@ function FrigateCameraFeatures({
       <DrawerContent className="max-h-[75dvh] overflow-hidden rounded-2xl">
         <div className="scrollbar-container mt-2 flex h-auto flex-col gap-2 overflow-y-auto px-2 py-4">
           <>
-            {isAdmin && (
-              <>
-                <FilterSwitch
-                  label={t("cameraSettings.camera")}
-                  isChecked={enabledState == "ON"}
-                  onCheckedChange={() =>
-                    sendEnabled(enabledState == "ON" ? "OFF" : "ON")
-                  }
-                />
-                <FilterSwitch
-                  label={t("cameraSettings.objectDetection")}
-                  isChecked={detectState == "ON"}
-                  onCheckedChange={() =>
-                    sendDetect(detectState == "ON" ? "OFF" : "ON")
-                  }
-                />
-                {recordingEnabled && (
-                  <FilterSwitch
-                    label={t("cameraSettings.recording")}
-                    isChecked={recordState == "ON"}
-                    onCheckedChange={() =>
-                      sendRecord(recordState == "ON" ? "OFF" : "ON")
-                    }
-                  />
-                )}
-                <FilterSwitch
-                  label={t("cameraSettings.snapshots")}
-                  isChecked={snapshotState == "ON"}
-                  onCheckedChange={() =>
-                    sendSnapshot(snapshotState == "ON" ? "OFF" : "ON")
-                  }
-                />
-                {audioDetectEnabled && (
-                  <FilterSwitch
-                    label={t("cameraSettings.audioDetection")}
-                    isChecked={audioState == "ON"}
-                    onCheckedChange={() =>
-                      sendAudio(audioState == "ON" ? "OFF" : "ON")
-                    }
-                  />
-                )}
-                {audioDetectEnabled && transcriptionEnabled && (
-                  <FilterSwitch
-                    label={t("cameraSettings.transcription")}
-                    disabled={audioState == "OFF"}
-                    isChecked={transcriptionState == "ON"}
-                    onCheckedChange={() =>
-                      sendTranscription(
-                        transcriptionState == "ON" ? "OFF" : "ON",
-                      )
-                    }
-                  />
-                )}
-                {autotrackingEnabled && (
-                  <FilterSwitch
-                    label={t("cameraSettings.autotracking")}
-                    isChecked={autotrackingState == "ON"}
-                    onCheckedChange={() =>
-                      sendAutotracking(autotrackingState == "ON" ? "OFF" : "ON")
-                    }
-                  />
-                )}
-              </>
-            )}
-
-            <div className="mt-3 flex flex-col gap-5">
+            <div className="mb-3 flex flex-col gap-5">
               {!isRestreamed && (
                 <div className="flex flex-col gap-2 p-2">
                   <Label>{t("stream.title")}</Label>
@@ -1811,6 +1761,7 @@ function FrigateCameraFeatures({
                       onSelect={onSelectStream}
                       onRetry={onResetStream}
                       disabled={debug || forceLowBandwidth}
+                      variant="grid"
                     />
 
                     {debug && (
@@ -1826,7 +1777,12 @@ function FrigateCameraFeatures({
                       !debug &&
                       isRestreamed && (
                         <div className="mt-1 flex flex-row items-center gap-1 text-sm text-muted-foreground">
-                          {supportsAudioOutput ? (
+                          {streamMetadataLoading ? (
+                            <>
+                              <ActivityIndicator className="size-4" size={14} />
+                              <div>{t("stream.metadata.loading")}</div>
+                            </>
+                          ) : supportsAudioOutput ? (
                             <>
                               <LuCheck className="size-4 text-success" />
                               <div>{t("stream.audio.available")}</div>
@@ -1955,6 +1911,73 @@ function FrigateCameraFeatures({
                     </p>
                   </div>
                 )}
+            </div>
+            {isAdmin && (
+              <>
+                <FilterSwitch
+                  label={t("cameraSettings.camera")}
+                  isChecked={enabledState == "ON"}
+                  onCheckedChange={() =>
+                    sendEnabled(enabledState == "ON" ? "OFF" : "ON")
+                  }
+                />
+                <FilterSwitch
+                  label={t("cameraSettings.objectDetection")}
+                  isChecked={detectState == "ON"}
+                  onCheckedChange={() =>
+                    sendDetect(detectState == "ON" ? "OFF" : "ON")
+                  }
+                />
+                {recordingEnabled && (
+                  <FilterSwitch
+                    label={t("cameraSettings.recording")}
+                    isChecked={recordState == "ON"}
+                    onCheckedChange={() =>
+                      sendRecord(recordState == "ON" ? "OFF" : "ON")
+                    }
+                  />
+                )}
+                <FilterSwitch
+                  label={t("cameraSettings.snapshots")}
+                  isChecked={snapshotState == "ON"}
+                  onCheckedChange={() =>
+                    sendSnapshot(snapshotState == "ON" ? "OFF" : "ON")
+                  }
+                />
+                {audioDetectEnabled && (
+                  <FilterSwitch
+                    label={t("cameraSettings.audioDetection")}
+                    isChecked={audioState == "ON"}
+                    onCheckedChange={() =>
+                      sendAudio(audioState == "ON" ? "OFF" : "ON")
+                    }
+                  />
+                )}
+                {audioDetectEnabled && transcriptionEnabled && (
+                  <FilterSwitch
+                    label={t("cameraSettings.transcription")}
+                    disabled={audioState == "OFF"}
+                    isChecked={transcriptionState == "ON"}
+                    onCheckedChange={() =>
+                      sendTranscription(
+                        transcriptionState == "ON" ? "OFF" : "ON",
+                      )
+                    }
+                  />
+                )}
+                {autotrackingEnabled && (
+                  <FilterSwitch
+                    label={t("cameraSettings.autotracking")}
+                    isChecked={autotrackingState == "ON"}
+                    onCheckedChange={() =>
+                      sendAutotracking(autotrackingState == "ON" ? "OFF" : "ON")
+                    }
+                  />
+                )}
+              </>
+            )}
+
+            <div className="mt-3 flex flex-col gap-5">
               {isRestreamed &&
                 Object.values(camera.live.streams).length > 0 && (
                   <div className="flex flex-col gap-2">