From 53dc069c6d4ce420e93a70f1f5fa436b6f9b2d74 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Mon, 26 May 2025 07:24:18 -0500
Subject: [PATCH] frontend live transcription

---
 web/src/types/frigateConfig.ts        |  9 +++
 web/src/views/live/LiveCameraView.tsx | 82 +++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)

diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts
index 355d4cb72..cf2bf1476 100644
--- a/web/src/types/frigateConfig.ts
+++ b/web/src/types/frigateConfig.ts
@@ -41,6 +41,11 @@ export interface CameraConfig {
     min_volume: number;
     num_threads: number;
   };
+  audio_transcription: {
+    enabled: boolean;
+    enabled_in_config: boolean;
+    live_enabled: boolean;
+  };
   best_image_timeout: number;
   birdseye: {
     enabled: boolean;
@@ -296,6 +301,10 @@ export interface FrigateConfig {
     num_threads: number;
   };
 
+  audio_transcription: {
+    enabled: boolean;
+  };
+
   birdseye: BirdseyeConfig;
 
   cameras: {
diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx
index b57f0c8a7..0ab908fce 100644
--- a/web/src/views/live/LiveCameraView.tsx
+++ b/web/src/views/live/LiveCameraView.tsx
@@ -1,11 +1,13 @@
 import {
   useAudioState,
+  useAudioTranscriptionState,
   useAutotrackingState,
   useDetectState,
   useEnabledState,
   usePtzCommand,
   useRecordingsState,
   useSnapshotsState,
+  useTrackedObjectUpdate,
 } from "@/api/ws";
 import CameraFeatureToggle from "@/components/dynamic/CameraFeatureToggle";
 import FilterSwitch from "@/components/filter/FilterSwitch";
@@ -90,6 +92,8 @@ import {
   LuX,
 } from "react-icons/lu";
 import {
+  MdClosedCaption,
+  MdClosedCaptionDisabled,
   MdNoPhotography,
   MdOutlineRestartAlt,
   MdPersonOff,
@@ -196,6 +200,33 @@ export default function LiveCameraView({
   const { payload: enabledState } = useEnabledState(camera.name);
   const cameraEnabled = enabledState === "ON";
 
+  // for audio transcriptions
+
+  const { payload: audioTranscriptionState, send: sendTranscription } =
+    useAudioTranscriptionState(camera.name);
+  const { payload: wsUpdate } = useTrackedObjectUpdate();
+  const transcriptionRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (
+      wsUpdate &&
+      wsUpdate.type == "transcription" &&
+      wsUpdate.camera == camera.name
+    ) {
+      if (transcriptionRef.current) {
+        transcriptionRef.current.scrollTop =
+          transcriptionRef.current.scrollHeight;
+      }
+    }
+  }, [wsUpdate, camera.name]);
+
+  useEffect(() => {
+    return () => {
+      // disable transcriptions when unmounting
+      if (audioTranscriptionState == "ON") sendTranscription("OFF");
+    };
+  }, [audioTranscriptionState, sendTranscription]);
+
   // click overlay for ptzs
 
   const [clickOverlay, setClickOverlay] = useState(false);
@@ -566,6 +597,9 @@ export default function LiveCameraView({
                 autotrackingEnabled={
                   camera.onvif.autotracking.enabled_in_config
                 }
+                transcriptionEnabled={
+                  camera.audio_transcription.enabled_in_config
+                }
                 fullscreen={fullscreen}
                 streamName={streamName ?? ""}
                 setStreamName={setStreamName}
@@ -625,6 +659,19 @@ export default function LiveCameraView({
               />
             </div>
           </TransformComponent>
+          {camera?.audio?.enabled_in_config &&
+            audioTranscriptionState == "ON" &&
+            wsUpdate &&
+            wsUpdate.type === "transcription" &&
+            wsUpdate.camera === camera.name &&
+            wsUpdate.text !== "" && (
+              <div
+                ref={transcriptionRef}
+                className="text-md scrollbar-container absolute bottom-4 left-1/2 max-h-[15vh] w-[75%] -translate-x-1/2 overflow-y-auto rounded-lg bg-black/70 p-2 text-white md:w-[50%]"
+              >
+                {wsUpdate.text}
+              </div>
+            )}
         </div>
       </div>
       {camera.onvif.host != "" && (
@@ -983,6 +1030,7 @@ type FrigateCameraFeaturesProps = {
   recordingEnabled: boolean;
   audioDetectEnabled: boolean;
   autotrackingEnabled: boolean;
+  transcriptionEnabled: boolean;
   fullscreen: boolean;
   streamName: string;
   setStreamName?: (value: string | undefined) => void;
@@ -1002,6 +1050,7 @@ function FrigateCameraFeatures({
   recordingEnabled,
   audioDetectEnabled,
   autotrackingEnabled,
+  transcriptionEnabled,
   fullscreen,
   streamName,
   setStreamName,
@@ -1033,6 +1082,8 @@ function FrigateCameraFeatures({
   const { payload: audioState, send: sendAudio } = useAudioState(camera.name);
   const { payload: autotrackingState, send: sendAutotracking } =
     useAutotrackingState(camera.name);
+  const { payload: transcriptionState, send: sendTranscription } =
+    useAudioTranscriptionState(camera.name);
 
   // roles
 
@@ -1196,6 +1247,27 @@ function FrigateCameraFeatures({
                 disabled={!cameraEnabled}
               />
             )}
+            {audioDetectEnabled && transcriptionEnabled && (
+              <CameraFeatureToggle
+                className="p-2 md:p-0"
+                variant={fullscreen ? "overlay" : "primary"}
+                Icon={
+                  transcriptionState == "ON"
+                    ? MdClosedCaption
+                    : MdClosedCaptionDisabled
+                }
+                isActive={transcriptionState == "ON"}
+                title={
+                  transcriptionState == "ON"
+                    ? t("transcription.disable")
+                    : t("transcription.enable")
+                }
+                onClick={() =>
+                  sendTranscription(transcriptionState == "ON" ? "OFF" : "ON")
+                }
+                disabled={!cameraEnabled || audioState == "OFF"}
+              />
+            )}
             {autotrackingEnabled && (
               <CameraFeatureToggle
                 className="p-2 md:p-0"
@@ -1558,6 +1630,16 @@ function FrigateCameraFeatures({
                   }
                 />
               )}
+              {audioDetectEnabled && transcriptionEnabled && (
+                <FilterSwitch
+                  label={t("cameraSettings.transcription")}
+                  disabled={audioState == "OFF"}
+                  isChecked={transcriptionState == "ON"}
+                  onCheckedChange={() =>
+                    sendTranscription(transcriptionState == "ON" ? "OFF" : "ON")
+                  }
+                />
+              )}
               {autotrackingEnabled && (
                 <FilterSwitch
                   label={t("cameraSettings.autotracking")}