Miscellaneous Fixes (#21024)

* fix wording in reference config

* spacing tweaks

* make live view settings drawer scrollable

* clarify audio transcription docs

* change audio transcription icon to activity indicator when transcription is in progress

the backend doesn't implement any kind of queueing for speech event transcription

* tracking details tweaks

- Add attribute box overlay and area
- Add score
- Throttle swr revalidation during video component rerendering

* add mse codecs to console debug on errors

* add camera name

frigate/api/classification.py

@@ -542,6 +542,7 @@ def transcribe_audio(request: Request, body: AudioTranscriptionBody):
             status_code=409,  # 409 Conflict
         )
     else:
+        logger.debug(f"Failed to transcribe audio, response: {response}")
         return JSONResponse(
             content={
                 "success": False,

frigate/comms/dispatcher.py

@@ -23,6 +23,7 @@ from frigate.const import (
     NOTIFICATION_TEST,
     REQUEST_REGION_GRID,
     UPDATE_AUDIO_ACTIVITY,
+    UPDATE_AUDIO_TRANSCRIPTION_STATE,
     UPDATE_BIRDSEYE_LAYOUT,
     UPDATE_CAMERA_ACTIVITY,
     UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
@@ -61,6 +62,7 @@ class Dispatcher:
         self.model_state: dict[str, ModelStatusTypesEnum] = {}
         self.embeddings_reindex: dict[str, Any] = {}
         self.birdseye_layout: dict[str, Any] = {}
+        self.audio_transcription_state: str = "idle"
         self._camera_settings_handlers: dict[str, Callable] = {
             "audio": self._on_audio_command,
             "audio_transcription": self._on_audio_transcription_command,
@@ -178,6 +180,19 @@ class Dispatcher:
         def handle_model_state() -> None:
             self.publish("model_state", json.dumps(self.model_state.copy()))
 
+        def handle_update_audio_transcription_state() -> None:
+            if payload:
+                self.audio_transcription_state = payload
+                self.publish(
+                    "audio_transcription_state",
+                    json.dumps(self.audio_transcription_state),
+                )
+
+        def handle_audio_transcription_state() -> None:
+            self.publish(
+                "audio_transcription_state", json.dumps(self.audio_transcription_state)
+            )
+
         def handle_update_embeddings_reindex_progress() -> None:
             self.embeddings_reindex = payload
             self.publish(
@@ -264,10 +279,12 @@ class Dispatcher:
             UPDATE_MODEL_STATE: handle_update_model_state,
             UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress,
             UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout,
+            UPDATE_AUDIO_TRANSCRIPTION_STATE: handle_update_audio_transcription_state,
             NOTIFICATION_TEST: handle_notification_test,
             "restart": handle_restart,
             "embeddingsReindexProgress": handle_embeddings_reindex_progress,
             "modelState": handle_model_state,
+            "audioTranscriptionState": handle_audio_transcription_state,
             "birdseyeLayout": handle_birdseye_layout,
             "onConnect": handle_on_connect,
         }

frigate/const.py

@@ -113,6 +113,7 @@ CLEAR_ONGOING_REVIEW_SEGMENTS = "clear_ongoing_review_segments"
 UPDATE_CAMERA_ACTIVITY = "update_camera_activity"
 UPDATE_AUDIO_ACTIVITY = "update_audio_activity"
 EXPIRE_AUDIO_ACTIVITY = "expire_audio_activity"
+UPDATE_AUDIO_TRANSCRIPTION_STATE = "update_audio_transcription_state"
 UPDATE_EVENT_DESCRIPTION = "update_event_description"
 UPDATE_REVIEW_DESCRIPTION = "update_review_description"
 UPDATE_MODEL_STATE = "update_model_state"

frigate/data_processing/post/audio_transcription.py

@@ -13,6 +13,7 @@ from frigate.config import FrigateConfig
 from frigate.const import (
     CACHE_DIR,
     MODEL_CACHE_DIR,
+    UPDATE_AUDIO_TRANSCRIPTION_STATE,
     UPDATE_EVENT_DESCRIPTION,
 )
 from frigate.data_processing.types import PostProcessDataEnum
@@ -190,6 +191,8 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
                 self.transcription_running = False
                 self.transcription_thread = None
 
+            self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "idle")
+
     def handle_request(self, topic: str, request_data: dict[str, any]) -> str | None:
         if topic == "transcribe_audio":
             event = request_data["event"]
@@ -203,6 +206,8 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
 
                 # Mark as running and start the thread
                 self.transcription_running = True
+                self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "processing")
+
                 self.transcription_thread = threading.Thread(
                     target=self._transcription_wrapper, args=(event,), daemon=True
                 )

frigate/timeline.py

@@ -109,6 +109,7 @@ class TimelineProcessor(threading.Thread):
                     event_data["region"],
                 ),
                 "attribute": "",
+                "score": event_data["score"],
             },
         }