Miscellaneous Fixes (#21024)

* fix wording in reference config

* spacing tweaks

* make live view settings drawer scrollable

* clarify audio transcription docs

* change audio transcription icon to activity indicator when transcription is in progress

the backend doesn't implement any kind of queueing for speech event transcription

* tracking details tweaks

- Add attribute box overlay and area
- Add score
- Throttle swr revalidation during video component rerendering

* add mse codecs to console debug on errors

* add camera name

web/src/components/overlay/ObjectTrackOverlay.tsx

@@ -42,6 +42,7 @@ type ObjectData = {
   pathPoints: PathPoint[];
   currentZones: string[];
   currentBox?: number[];
+  currentAttributeBox?: number[];
 };
 
 export default function ObjectTrackOverlay({
@@ -105,6 +106,12 @@ export default function ObjectTrackOverlay({
     selectedObjectIds.length > 0
       ? ["event_ids", { ids: selectedObjectIds.join(",") }]
       : null,
+    null,
+    {
+      revalidateOnFocus: false,
+      revalidateOnReconnect: false,
+      dedupingInterval: 30000,
+    },
   );
 
   // Fetch timeline data for each object ID using fixed number of hooks
@@ -112,7 +119,12 @@ export default function ObjectTrackOverlay({
     selectedObjectIds.length > 0
       ? `timeline?source_id=${selectedObjectIds.join(",")}&limit=1000`
       : null,
-    { revalidateOnFocus: false },
+    null,
+    {
+      revalidateOnFocus: false,
+      revalidateOnReconnect: false,
+      dedupingInterval: 30000,
+    },
   );
 
   const getZonesFriendlyNames = (zones: string[], config: FrigateConfig) => {
@@ -270,6 +282,7 @@ export default function ObjectTrackOverlay({
           );
 
         const currentBox = nearbyTimelineEvent?.data?.box;
+        const currentAttributeBox = nearbyTimelineEvent?.data?.attribute_box;
 
         return {
           objectId,
@@ -278,6 +291,7 @@ export default function ObjectTrackOverlay({
           pathPoints: combinedPoints,
           currentZones,
           currentBox,
+          currentAttributeBox,
         };
       })
       .filter((obj: ObjectData) => obj.pathPoints.length > 0); // Only include objects with path data
@@ -482,6 +496,20 @@ export default function ObjectTrackOverlay({
                 />
               </g>
             )}
+            {objData.currentAttributeBox && showBoundingBoxes && (
+              <g>
+                <rect
+                  x={objData.currentAttributeBox[0] * videoWidth}
+                  y={objData.currentAttributeBox[1] * videoHeight}
+                  width={objData.currentAttributeBox[2] * videoWidth}
+                  height={objData.currentAttributeBox[3] * videoHeight}
+                  fill="none"
+                  stroke={objData.color}
+                  strokeWidth={boxStroke}
+                  opacity="0.9"
+                />
+              </g>
+            )}
           </g>
         );
       })}

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -92,6 +92,7 @@ import { DialogPortal } from "@radix-ui/react-dialog";
 import { useDetailStream } from "@/context/detail-stream-context";
 import { PiSlidersHorizontalBold } from "react-icons/pi";
 import { HiSparkles } from "react-icons/hi";
+import { useAudioTranscriptionProcessState } from "@/api/ws";
 
 const SEARCH_TABS = ["snapshot", "tracking_details"] as const;
 export type SearchTab = (typeof SEARCH_TABS)[number];
@@ -1076,6 +1077,11 @@ function ObjectDetailsTab({
       });
   }, [search, t]);
 
+  // audio transcription processing state
+
+  const { payload: audioTranscriptionProcessState } =
+    useAudioTranscriptionProcessState();
+
   // frigate+ submission
 
   type SubmissionState = "reviewing" | "uploading" | "submitted";
@@ -1431,10 +1437,20 @@ function ObjectDetailsTab({
                   <TooltipTrigger asChild>
                     <button
                       aria-label={t("itemMenu.audioTranscription.label")}
-                      className="text-primary/40 hover:text-primary/80"
+                      className={cn(
+                        "text-primary/40",
+                        audioTranscriptionProcessState === "processing"
+                          ? "cursor-not-allowed"
+                          : "hover:text-primary/80",
+                      )}
                       onClick={onTranscribe}
+                      disabled={audioTranscriptionProcessState === "processing"}
                     >
-                      <FaMicrophone className="size-4" />
+                      {audioTranscriptionProcessState === "processing" ? (
+                        <ActivityIndicator className="size-4" />
+                      ) : (
+                        <FaMicrophone className="size-4" />
+                      )}
                     </button>
                   </TooltipTrigger>
                   <TooltipContent>

web/src/components/overlay/detail/TrackingDetails.tsx

@@ -75,12 +75,15 @@ export function TrackingDetails({
     setIsVideoLoading(true);
   }, [event.id]);
 
-  const { data: eventSequence } = useSWR<TrackingDetailsSequence[]>([
-    "timeline",
+  const { data: eventSequence } = useSWR<TrackingDetailsSequence[]>(
+    ["timeline", { source_id: event.id }],
+    null,
     {
-      source_id: event.id,
+      revalidateOnFocus: false,
+      revalidateOnReconnect: false,
+      dedupingInterval: 30000,
     },
-  ]);
+  );
 
   const { data: config } = useSWR<FrigateConfig>("config");
 
@@ -104,6 +107,12 @@ export function TrackingDetails({
           },
         ]
       : null,
+    null,
+    {
+      revalidateOnFocus: false,
+      revalidateOnReconnect: false,
+      dedupingInterval: 30000,
+    },
   );
 
   // Convert a timeline timestamp to actual video player time, accounting for
@@ -714,53 +723,6 @@ export function TrackingDetails({
                     )}
                     <div className="space-y-2">
                       {eventSequence.map((item, idx) => {
-                        const isActive =
-                          Math.abs(
-                            (effectiveTime ?? 0) - (item.timestamp ?? 0),
-                          ) <= 0.5;
-                        const formattedEventTimestamp = config
-                          ? formatUnixTimestampToDateTime(item.timestamp ?? 0, {
-                              timezone: config.ui.timezone,
-                              date_format:
-                                config.ui.time_format == "24hour"
-                                  ? t(
-                                      "time.formattedTimestampHourMinuteSecond.24hour",
-                                      { ns: "common" },
-                                    )
-                                  : t(
-                                      "time.formattedTimestampHourMinuteSecond.12hour",
-                                      { ns: "common" },
-                                    ),
-                              time_style: "medium",
-                              date_style: "medium",
-                            })
-                          : "";
-
-                        const ratio =
-                          Array.isArray(item.data.box) &&
-                          item.data.box.length >= 4
-                            ? (
-                                aspectRatio *
-                                (item.data.box[2] / item.data.box[3])
-                              ).toFixed(2)
-                            : "N/A";
-                        const areaPx =
-                          Array.isArray(item.data.box) &&
-                          item.data.box.length >= 4
-                            ? Math.round(
-                                (config.cameras[event.camera]?.detect?.width ??
-                                  0) *
-                                  (config.cameras[event.camera]?.detect
-                                    ?.height ?? 0) *
-                                  (item.data.box[2] * item.data.box[3]),
-                              )
-                            : undefined;
-                        const areaPct =
-                          Array.isArray(item.data.box) &&
-                          item.data.box.length >= 4
-                            ? (item.data.box[2] * item.data.box[3]).toFixed(4)
-                            : undefined;
-
                         return (
                           <div
                             key={`${item.timestamp}-${item.source_id ?? ""}-${idx}`}
@@ -770,11 +732,7 @@ export function TrackingDetails({
                           >
                             <LifecycleIconRow
                               item={item}
-                              isActive={isActive}
-                              formattedEventTimestamp={formattedEventTimestamp}
-                              ratio={ratio}
-                              areaPx={areaPx}
-                              areaPct={areaPct}
+                              event={event}
                               onClick={() => handleLifecycleClick(item)}
                               setSelectedZone={setSelectedZone}
                               getZoneColor={getZoneColor}
@@ -798,11 +756,7 @@ export function TrackingDetails({
 
 type LifecycleIconRowProps = {
   item: TrackingDetailsSequence;
-  isActive?: boolean;
-  formattedEventTimestamp: string;
-  ratio: string;
-  areaPx?: number;
-  areaPct?: string;
+  event: Event;
   onClick: () => void;
   setSelectedZone: (z: string) => void;
   getZoneColor: (zoneName: string) => number[] | undefined;
@@ -812,11 +766,7 @@ type LifecycleIconRowProps = {
 
 function LifecycleIconRow({
   item,
-  isActive,
-  formattedEventTimestamp,
-  ratio,
-  areaPx,
-  areaPct,
+  event,
   onClick,
   setSelectedZone,
   getZoneColor,
@@ -826,9 +776,101 @@ function LifecycleIconRow({
   const { t } = useTranslation(["views/explore", "components/player"]);
   const { data: config } = useSWR<FrigateConfig>("config");
   const [isOpen, setIsOpen] = useState(false);
-
   const navigate = useNavigate();
 
+  const aspectRatio = useMemo(() => {
+    if (!config) {
+      return 16 / 9;
+    }
+
+    return (
+      config.cameras[event.camera].detect.width /
+      config.cameras[event.camera].detect.height
+    );
+  }, [config, event]);
+
+  const isActive = useMemo(
+    () => Math.abs((effectiveTime ?? 0) - (item.timestamp ?? 0)) <= 0.5,
+    [effectiveTime, item.timestamp],
+  );
+
+  const formattedEventTimestamp = useMemo(
+    () =>
+      config
+        ? formatUnixTimestampToDateTime(item.timestamp ?? 0, {
+            timezone: config.ui.timezone,
+            date_format:
+              config.ui.time_format == "24hour"
+                ? t("time.formattedTimestampHourMinuteSecond.24hour", {
+                    ns: "common",
+                  })
+                : t("time.formattedTimestampHourMinuteSecond.12hour", {
+                    ns: "common",
+                  }),
+            time_style: "medium",
+            date_style: "medium",
+          })
+        : "",
+    [config, item.timestamp, t],
+  );
+
+  const ratio = useMemo(
+    () =>
+      Array.isArray(item.data.box) && item.data.box.length >= 4
+        ? (aspectRatio * (item.data.box[2] / item.data.box[3])).toFixed(2)
+        : "N/A",
+    [aspectRatio, item.data.box],
+  );
+
+  const areaPx = useMemo(
+    () =>
+      Array.isArray(item.data.box) && item.data.box.length >= 4
+        ? Math.round(
+            (config?.cameras[event.camera]?.detect?.width ?? 0) *
+              (config?.cameras[event.camera]?.detect?.height ?? 0) *
+              (item.data.box[2] * item.data.box[3]),
+          )
+        : undefined,
+    [config, event.camera, item.data.box],
+  );
+
+  const attributeAreaPx = useMemo(
+    () =>
+      Array.isArray(item.data.attribute_box) &&
+      item.data.attribute_box.length >= 4
+        ? Math.round(
+            (config?.cameras[event.camera]?.detect?.width ?? 0) *
+              (config?.cameras[event.camera]?.detect?.height ?? 0) *
+              (item.data.attribute_box[2] * item.data.attribute_box[3]),
+          )
+        : undefined,
+    [config, event.camera, item.data.attribute_box],
+  );
+
+  const attributeAreaPct = useMemo(
+    () =>
+      Array.isArray(item.data.attribute_box) &&
+      item.data.attribute_box.length >= 4
+        ? (item.data.attribute_box[2] * item.data.attribute_box[3]).toFixed(4)
+        : undefined,
+    [item.data.attribute_box],
+  );
+
+  const areaPct = useMemo(
+    () =>
+      Array.isArray(item.data.box) && item.data.box.length >= 4
+        ? (item.data.box[2] * item.data.box[3]).toFixed(4)
+        : undefined,
+    [item.data.box],
+  );
+
+  const score = useMemo(() => {
+    if (item.data.score !== undefined) {
+      return (item.data.score * 100).toFixed(0) + "%";
+    }
+    return "N/A";
+  }, [item.data.score]);
+
   return (
     <div
       role="button"
@@ -856,16 +898,28 @@ function LifecycleIconRow({
             <div className="text-md flex items-start break-words text-left">
               {getLifecycleItemDescription(item)}
             </div>
-            <div className="mt-1 flex flex-wrap items-center gap-2 text-xs text-secondary-foreground md:gap-5">
-              <div className="flex items-center gap-1">
+            <div className="my-2 ml-2 flex flex-col flex-wrap items-start gap-1.5 text-xs text-secondary-foreground">
+              <div className="flex items-center gap-1.5">
+                <span className="text-primary-variant">
+                  {t("trackingDetails.lifecycleItemDesc.header.score")}
+                </span>
+                <span className="font-medium text-primary">{score}</span>
+              </div>
+              <div className="flex items-center gap-1.5">
                 <span className="text-primary-variant">
                   {t("trackingDetails.lifecycleItemDesc.header.ratio")}
                 </span>
                 <span className="font-medium text-primary">{ratio}</span>
               </div>
-              <div className="flex items-center gap-1">
+              <div className="flex items-center gap-1.5">
                 <span className="text-primary-variant">
-                  {t("trackingDetails.lifecycleItemDesc.header.area")}
+                  {t("trackingDetails.lifecycleItemDesc.header.area")}{" "}
+                  {attributeAreaPx !== undefined &&
+                    attributeAreaPct !== undefined && (
+                      <span className="text-primary-variant">
+                        ({getTranslatedLabel(item.data.label)})
+                      </span>
+                    )}
                 </span>
                 {areaPx !== undefined && areaPct !== undefined ? (
                   <span className="font-medium text-primary">
@@ -876,9 +930,25 @@ function LifecycleIconRow({
                   <span>N/A</span>
                 )}
               </div>
+              {attributeAreaPx !== undefined &&
+                attributeAreaPct !== undefined && (
+                  <div className="flex items-center gap-1.5">
+                    <span className="text-primary-variant">
+                      {t("trackingDetails.lifecycleItemDesc.header.area")} (
+                      {getTranslatedLabel(item.data.attribute)})
+                    </span>
+                    <span className="font-medium text-primary">
+                      {t("information.pixels", {
+                        ns: "common",
+                        area: attributeAreaPx,
+                      })}{" "}
+                      · {attributeAreaPct}%
+                    </span>
+                  </div>
+                )}
 
               {item.data?.zones && item.data.zones.length > 0 && (
-                <div className="flex flex-wrap items-center gap-2">
+                <div className="mt-1 flex flex-wrap items-center gap-2">
                   {item.data.zones.map((zone, zidx) => {
                     const color = getZoneColor(zone)?.join(",") ?? "0,0,0";
                     return (

web/src/components/player/MsePlayer.tsx

@@ -82,6 +82,7 @@ function MSEPlayer({
     [key: string]: (msg: { value: string; type: string }) => void;
   }>({});
   const msRef = useRef<MediaSource | null>(null);
+  const mseCodecRef = useRef<string | null>(null);
 
   const wsURL = useMemo(() => {
     return `${baseUrl.replace(/^http/, "ws")}live/mse/api/ws?src=${camera}`;
@@ -93,6 +94,10 @@ function MSEPlayer({
       console.error(
         `${camera} - MSE error '${error}': ${description} See the documentation: https://docs.frigate.video/configuration/live/#live-player-error-messages`,
       );
+      if (mseCodecRef.current) {
+        // eslint-disable-next-line no-console
+        console.error(`${camera} - MSE codec in use: ${mseCodecRef.current}`);
+      }
       onError?.(error);
     },
     [camera, onError],
@@ -299,6 +304,9 @@ function MSEPlayer({
     onmessageRef.current["mse"] = (msg) => {
       if (msg.type !== "mse") return;
 
+      // Store the codec value for error logging
+      mseCodecRef.current = msg.value;
+
       let sb: SourceBuffer | undefined;
       try {
         sb = msRef.current?.addSourceBuffer(msg.value);