close onvif sessions on shutdown

`OnvifController.close()` only stopped its event loop, so the aiohttp sessions each `ONVIFCamera` holds and the `_poll_config_updates` task were left to be garbage collected during interpreter shutdown, when their warnings can no longer be logged. Every restart ended with a run of `Unclosed client session` and `Task was destroyed but it is pending!` logging errors, which only became visible once restart started exiting the process itself under non-root. `close()` now closes each camera's client and cancels the tasks on the loop before stopping it.

docs/docs/configuration/advanced/reference.md

@@ -800,7 +800,7 @@ lpr:
 # to Google or OpenAI's LLMs to generate descriptions. GenAI features can be configured at
 # the camera level to enhance privacy for indoor cameras.
 # NOTE: genai is a map of named providers. Each key is a name you choose for the provider,
-#       and each role (chat, descriptions, embeddings) may be assigned to exactly one provider.
+#       and each role (chat, descriptions, embeddings, transcribe) may be assigned to exactly one provider.
 genai:
   # Required: name of the provider (chosen by you, used to reference it elsewhere)
   my_provider:
@@ -813,11 +813,13 @@ genai:
     # Required: The model to use with the provider.
     model: gemini-1.5-flash
     # Optional: Roles this provider handles (default: shown below)
-    # Each role (chat, descriptions, embeddings) must be assigned to exactly one provider.
+    # Each role (chat, descriptions, embeddings, transcribe) must be assigned to exactly
+    # one provider.
     roles:
       - chat
       - descriptions
       - embeddings
+      - transcribe
     # Optional additional args to pass to the GenAI Provider (default: None)
     provider_options:
       keep_alive: -1
@@ -830,13 +832,19 @@ genai:
 audio_transcription:
   # Optional: Enable live and speech event audio transcription (default: shown below)
   enabled: False
+  # Optional: The transcription backend (default: shown below)
+  # Either 'whisper' for Frigate's built-in local models, or the name of a genai
+  # provider that has 'transcribe' in its roles. device and model_size are ignored
+  # when a genai provider is named.
+  model: whisper
   # Optional: The device to run the models on for live transcription. (default: shown below)
   device: CPU
   # Optional: Set the model size used for live transcription. (default: shown below)
   model_size: small
   # Optional: Set the language used for transcription translation. (default: shown below)
-  # List of language codes: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
-  language: en
+  # Use 'auto' to let the model detect the language, or a language code from
+  # https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
+  language: auto
 
 # Optional: Configuration for classification models
 classification:

docs/docs/configuration/audio_detectors.md

@@ -204,7 +204,7 @@ Frequently-heard labels like `speech` can generate a lot of events, and each eve
 
 ### Audio Transcription
 
-Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`. The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
+Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`, and can alternatively offload transcription to a [GenAI provider](#genai-provider). The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
 
 :::info
 
@@ -224,6 +224,7 @@ To enable transcription, configure it globally and optionally disable for specif
 **Global:** Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
 
 - Set **Enable audio transcription** to on
+- Set **Audio transcription model or GenAI provider name** to `whisper` for Frigate's built-in local models, or to the name of a GenAI provider
 - Set **Transcription device** to the desired device
 - Set **Model size** to the desired size
 
@@ -235,6 +236,7 @@ To enable transcription, configure it globally and optionally disable for specif
 ```yaml
 audio_transcription:
   enabled: True
+  model: whisper
   device: ...
   model_size: ...
 ```
@@ -263,20 +265,88 @@ The optional config parameters that can be set at the global level include:
 - **`enabled`**: Enable or disable the audio transcription feature.
   - Default: `False`
   - It is recommended to only configure the features at the global level, and enable it at the individual camera level.
+- **`model`**: The transcription backend.
+  - Default: `whisper`
+  - `whisper` uses Frigate's built-in local models, described by `device` and `model_size` below.
+  - Any other value must name a key in your `genai` config whose entry has `transcribe` in its `roles`. See [GenAI Provider](#genai-provider).
 - **`device`**: Device to use to run transcription and translation models.
   - Default: `CPU`
   - This can be `CPU` or `GPU`. The `sherpa-onnx` models are lightweight and run on the CPU only. The `whisper` models can run on GPU but are only supported on CUDA hardware.
+  - Ignored when `model` names a GenAI provider.
 - **`model_size`**: The size of the model used for live transcription.
   - Default: `small`
   - This can be `small` or `large`. The `small` setting uses `sherpa-onnx` models that are fast, lightweight, and always run on the CPU but are not as accurate as the `whisper` model.
-  - This config option applies to **live transcription only**. Recorded `speech` events will always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
-- **`language`**: Defines the language used by `whisper` to translate `speech` audio events (and live audio only if using the `large` model).
-  - Default: `en`
-  - You must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
+  - This config option applies to **live transcription only**. With `model: whisper`, recorded `speech` events always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
+  - Ignored when `model` names a GenAI provider.
+- **`language`**: Defines the language used to transcribe and translate `speech` audio events (and live audio only if using the `large` model or a GenAI provider).
+  - Default: `auto`
+  - `auto` lets the model detect the language itself, which most models do well. Set an explicit language only if detection is picking the wrong one.
+  - Otherwise you must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
   - Transcriptions for `speech` events are translated.
   - Live audio is translated only if you are using the `large` model. The `small` `sherpa-onnx` model is English-only.
 
-The only field that is valid at the camera level is `enabled`.
+The only field that is valid at the camera level is `enabled`. In particular `model` is global only: the transcription backend is a process-wide resource shared by every camera.
+
+#### GenAI Provider
+
+Frigate can send audio to a GenAI provider for transcription when that provider has the `transcribe` role. This is useful if you already run a GenAI provider, or if you do not have the CPU/GPU headroom for a local whisper model. Supported providers are **OpenAI**, **Azure OpenAI**, **Gemini**, and **llama.cpp** with an audio-capable model (a dedicated ASR model such as Qwen3-ASR, or a general multimodal model that accepts audio). Ollama is not supported as it has no audio input.
+
+To use a GenAI provider for audio transcription:
+
+1. Configure a GenAI provider with `transcribe` in its `roles`.
+2. Set the audio transcription model to that GenAI config key (e.g. `whisper_cloud`).
+
+<ConfigTabs>
+<TabItem value="ui">
+
+Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
+
+| Field                                                | Description                                                                                              |
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
+| **Audio transcription model or GenAI provider name** | Set to the GenAI config key (e.g. `whisper_cloud`) to use a configured GenAI provider for transcription |
+
+The GenAI provider must also be configured with the `transcribe` role under <NavPath path="Settings > Enrichments > Generative AI" />.
+
+</TabItem>
+<TabItem value="yaml">
+
+```yaml
+genai:
+  whisper_cloud:
+    provider: openai
+    api_key: your-api-key
+    model: gpt-transcribe
+    roles:
+      - transcribe
+
+audio_transcription:
+  enabled: True
+  model: whisper_cloud
+  language: en
+```
+
+</TabItem>
+</ConfigTabs>
+
+:::warning
+
+**Give `transcribe` its own `genai` entry.** A `genai` entry has a single `model` string that is shared by every role it holds, so `roles: [descriptions, transcribe]` would send the same model name to both the chat endpoint and the transcription endpoint. Transcription models and chat models are almost never the same model, so define a dedicated entry as shown above.
+
+:::
+
+:::warning
+
+**Live transcription against a metered provider is billed continuously.** In live mode Frigate uploads an overlapping ~2 second window of audio roughly once per second, per camera, for as long as audio stays above that camera's `audio.min_volume`. Windows below that threshold are never uploaded, which is what keeps a quiet camera near zero requests, but a camera pointed at a busy street will keep sending.
+
+Three things keep this opt-in: `transcribe` is not one of the default roles, live transcription is off by default, and the volume gate suppresses silence. Transcription of recorded `speech` events is unaffected - it remains a manual, one-request-per-event action.
+
+:::
+
+`device` and `model_size` have no effect on this path and no local model is ever downloaded.
+
+`language` defaults to `auto`, which sends no language hint and lets the model detect it. Most audio models detect language well, so leave it on `auto` unless detection is picking the wrong one.
+
+When set explicitly, it is sent as the transcription endpoint's native `language` parameter for OpenAI, Azure, and llama.cpp, and as part of the prompt for Gemini. This matters for dedicated ASR models such as Qwen3-ASR: they read the prompt as contextual biasing rather than as an instruction, so a language named in the prompt is ignored, while the endpoint parameter is honored.
 
 #### Live transcription
 
@@ -292,6 +362,8 @@ Results can be error-prone due to a number of factors, including:
 
 For speech sources close to the camera with minimal background noise, use the `small` model.
 
+A [GenAI provider](#genai-provider) is generally the most accurate option for live transcription, at the cost of a network round trip per window. That round trip has to stay under about a second to keep up with the audio; if it does not, Frigate drops the oldest buffered audio rather than letting the backlog grow.
+
 If you have CUDA hardware, you can experiment with the `large` `whisper` model on GPU. Performance is not quite as fast as the `sherpa-onnx` `small` model, but live transcription is far more accurate. Using the `large` model with CPU will likely be too slow for real-time transcription.
 
 #### Transcription and translation of `speech` audio events
@@ -308,7 +380,7 @@ Only one `speech` event may be transcribed at a time. Frigate does not automatic
 
 :::
 
-Recorded `speech` events will always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient.
+With `model: whisper`, recorded `speech` events always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient. With a [GenAI provider](#genai-provider), the recorded clip is sent to the provider instead and no local model is used.
 
 #### FAQ
 

docs/docs/configuration/genai/config.md

@@ -43,7 +43,7 @@ genai:
 
 The examples on this page all use `my_provider`, but the name is arbitrary and is only used to reference the provider elsewhere in the config (for example, `semantic_search.model`).
 
-Each provider handles one or more **roles**: `chat`, `descriptions`, and `embeddings`. A provider handles all three by default, and each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
+Each provider handles one or more **roles**: `chat`, `descriptions`, `embeddings`, and `transcribe`. A provider handles the first three by default; `transcribe` must always be listed explicitly, and is not available on Ollama, which has no audio input. Each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
 
 If the provider you choose requires an API key, you may either directly paste it in your configuration, or store it in an environment variable prefixed with `FRIGATE_`.
 
@@ -63,11 +63,11 @@ Running Generative AI models on CPU is not recommended, as high inference times
 
 You must use a vision-capable model with Frigate. The following models are recommended for local deployment of the `descriptions` and `chat` roles:
 
-| Model               | Notes                                                                                                                                                                |
-| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `qwen3-vl`          | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object.                                              |
-| `qwen3.6`/`qwen3.8` | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
-| `gemma4`            | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action.                                     |
+| Model               | Review [frame mode](/configuration/genai/genai_review#frame-mode) | Notes                                                                                                                                                                |
+| ------------------- | --------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `qwen3-vl`          | `frames`                                                              | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object. Follows a sequence of frames on its own.     |
+| `qwen3.6`/`qwen3.8` | `frames`                                                              | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
+| `gemma4`            | `annotated_frames`                                                    | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action. Loses track of activity that repeats or reverses, so it benefits from annotated frames. |
 
 #### Embedding models
 
@@ -77,6 +77,17 @@ The `embeddings` role needs a different kind of model. Text queries are matched
 | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `qwen3-vl-embedding` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Must be served by llama.cpp started with `--embeddings` and `--mmproj`. |
 
+#### Transcription models
+
+The `transcribe` role needs a model that accepts audio input. A text-only or vision-only model cannot serve this role. The following are recommended for local deployment of the `transcribe` role:
+
+| Model       | Notes                                                                                                                                                                                                      |
+| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `qwen3-asr` | Dedicated speech recognition model covering 30 languages, and the better choice for transcription quality. It only transcribes, so it cannot be shared with the `descriptions` or `chat` roles.            |
+| `gemma4`    | General multimodal model that accepts audio as well as images, so one served model can cover `transcribe` alongside the other roles. Transcript quality is below `qwen3-asr`, particularly on noisy audio. |
+
+Both must be served by llama.cpp started with the matching audio `--mmproj`. llama.cpp only reports audio support when an audio projector is loaded. Without it Frigate sees the model as text-only and the `transcribe` role is unavailable in the UI. Frigate transcribes through the server's `/v1/audio/transcriptions` route, which llama.cpp serves for any audio-capable model.
+
 :::info
 
 Each model is available in multiple parameter sizes (3b, 4b, 8b, etc.). Larger sizes are more capable of complex tasks and understanding of situations, but requires more memory and computational resources. It is recommended to try multiple models and experiment to see which performs best.

docs/docs/configuration/genai/review_summaries.md

@@ -192,6 +192,43 @@ review:
 </TabItem>
 </ConfigTabs>
 
+### Frame Mode
+
+Review items are sent to the model as a sequence of still frames. Some models follow that sequence well on their own; others lose track of activity that repeats or reverses, and describe a single trip when the subject actually made several. The `frame_mode` option controls how those frames are presented.
+
+- `frames` (default): the prompt followed by the frames, exactly as earlier versions of Frigate sent them.
+- `annotated_frames`: each frame is preceded by its frame number and elapsed time, along with notes describing what the object tracker recorded at that moment, such as an object being first detected, starting to move, turning around, stopping, or no longer being detected.
+
+The notes come from tracking data rather than from the images, so they describe activity the model may not have picked up on its own. In testing with a person carrying three waste bins to the curb one at a time, `gemma4` described a single trip on every attempt with `frames`, and consistently described multiple trips with `annotated_frames`. Models that already handle these sequences well, such as the `qwen3-vl` family, gain little and should stay on `frames`.
+
+Annotated mode also caps the number of frames, since the notes already establish the order of events and extra near-duplicate frames tend to crowd out the middle of a clip. Longer review items are sampled more sparsely as a result, and typically use fewer tokens than `frames` mode for the same item.
+
+:::note
+
+Annotated mode needs tracking data for the review item. If none is available, Frigate falls back to sending plain frames for that item.
+
+:::
+
+<ConfigTabs>
+<TabItem value="ui">
+
+Navigate to <NavPath path="Settings > Global configuration > Review" />.
+
+- Set **GenAI config > Frame mode** to the desired mode (e.g., `annotated_frames`)
+
+</TabItem>
+<TabItem value="yaml">
+
+```yaml {4}
+review:
+  genai:
+    enabled: true
+    frame_mode: annotated_frames
+```
+
+</TabItem>
+</ConfigTabs>
+
 ### Response Style
 
 Different models respond to the built-in prompt with very different writing styles: some produce natural narration while others sound short and mechanical. The `response_style` option selects a writing style preset that rewords the prompt's instructions for the user-facing fields (the title, short summary, and scene description). Presets replace those instructions rather than adding extra ones, so the model never receives competing style directions.

frigate/api/media.py

@@ -258,15 +258,15 @@ async def latest_frame(
 
                 frame = request.app.camera_error_image
 
-        height = int(params.height or str(frame.shape[0]))
-        width = int(height * frame.shape[1] / frame.shape[0])
-
         if frame is None:
             return JSONResponse(
                 content={"success": False, "message": "Unable to get valid frame"},
                 status_code=500,
             )
 
+        height = int(params.height or str(frame.shape[0]))
+        width = int(height * frame.shape[1] / frame.shape[0])
+
         if height < 1 or width < 1:
             return JSONResponse(
                 content="Invalid height / width requested :: {} / {}".format(
@@ -886,9 +886,8 @@ async def vod_event(
     # If the recordings are not found and the event started more than 5 minutes ago, set has_clip to false
     if (
         event.start_time < datetime.now().timestamp() - 300
-        and type(vod_response) is tuple
-        and len(vod_response) == 2
-        and vod_response[1] == 404
+        and isinstance(vod_response, JSONResponse)
+        and vod_response.status_code == 404
     ):
         Event.update(has_clip=False).where(Event.id == event_id).execute()
 
@@ -956,64 +955,80 @@ async def event_snapshot(
     event_complete = False
     jpg_bytes = None
     frame_time = 0
+
     try:
         event = Event.get(Event.id == event_id, Event.end_time != None)
-        event_complete = True
         await require_camera_access(event.camera, request=request)
+    except DoesNotExist:
+        event = None
+
+    if event is not None:
+        event_complete = True
+
         if not event.has_snapshot:
             return JSONResponse(
                 content={"success": False, "message": "Snapshot not available"},
                 status_code=404,
             )
-        snapshot_settings = _resolve_snapshot_settings(
-            request.app.frigate_config.cameras[event.camera].snapshots, params
-        )
-        jpg_bytes, frame_time = get_event_snapshot_bytes(
-            event,
-            ext="jpg",
-            timestamp=snapshot_settings["timestamp"],
-            bounding_box=snapshot_settings["bounding_box"],
-            crop=snapshot_settings["crop"],
-            height=snapshot_settings["height"],
-            quality=snapshot_settings["quality"],
-            timestamp_style=request.app.frigate_config.cameras[
-                event.camera
-            ].timestamp_style,
-            colormap=request.app.frigate_config.model_for_camera(event.camera).colormap,
-        )
-    except DoesNotExist:
-        # see if the object is currently being tracked
+
         try:
-            camera_states: list[CameraState] = (
-                request.app.detected_frames_processor.get_camera_states()
+            snapshot_settings = _resolve_snapshot_settings(
+                request.app.frigate_config.cameras[event.camera].snapshots, params
+            )
+            jpg_bytes, frame_time = get_event_snapshot_bytes(
+                event,
+                ext="jpg",
+                timestamp=snapshot_settings["timestamp"],
+                bounding_box=snapshot_settings["bounding_box"],
+                crop=snapshot_settings["crop"],
+                height=snapshot_settings["height"],
+                quality=snapshot_settings["quality"],
+                timestamp_style=request.app.frigate_config.cameras[
+                    event.camera
+                ].timestamp_style,
+                colormap=request.app.frigate_config.model_for_camera(
+                    event.camera
+                ).colormap,
             )
-            for camera_state in camera_states:
-                if event_id in camera_state.tracked_objects:
-                    tracked_obj = camera_state.tracked_objects.get(event_id)
-                    if tracked_obj is not None:
-                        snapshot_settings = _resolve_snapshot_settings(
-                            camera_state.camera_config.snapshots, params
-                        )
-                        jpg_bytes, frame_time = tracked_obj.get_img_bytes(
-                            ext="jpg",
-                            timestamp=snapshot_settings["timestamp"],
-                            bounding_box=snapshot_settings["bounding_box"],
-                            crop=snapshot_settings["crop"],
-                            height=snapshot_settings["height"],
-                            quality=snapshot_settings["quality"],
-                        )
-                        await require_camera_access(camera_state.name, request=request)
         except Exception:
             return JSONResponse(
-                content={"success": False, "message": "Ongoing event not found"},
+                content={"success": False, "message": "Unknown error occurred"},
                 status_code=404,
             )
-    except Exception:
-        return JSONResponse(
-            content={"success": False, "message": "Unknown error occurred"},
-            status_code=404,
+    else:
+        # see if the object is currently being tracked
+        camera_states: list[CameraState] = (
+            request.app.detected_frames_processor.get_camera_states()
         )
 
+        for camera_state in camera_states:
+            tracked_obj = camera_state.tracked_objects.get(event_id)
+
+            if tracked_obj is None:
+                continue
+
+            await require_camera_access(camera_state.name, request=request)
+
+            try:
+                snapshot_settings = _resolve_snapshot_settings(
+                    camera_state.camera_config.snapshots, params
+                )
+                jpg_bytes, frame_time = tracked_obj.get_img_bytes(
+                    ext="jpg",
+                    timestamp=snapshot_settings["timestamp"],
+                    bounding_box=snapshot_settings["bounding_box"],
+                    crop=snapshot_settings["crop"],
+                    height=snapshot_settings["height"],
+                    quality=snapshot_settings["quality"],
+                )
+            except Exception:
+                return JSONResponse(
+                    content={"success": False, "message": "Ongoing event not found"},
+                    status_code=404,
+                )
+
+            break
+
     if jpg_bytes is None:
         return JSONResponse(
             content={"success": False, "message": "Live frame not available"},
@@ -1062,19 +1077,25 @@ async def event_thumbnail(
 
     if not thumbnail_bytes:
         # see if the object is currently being tracked
-        try:
-            camera_states = request.app.detected_frames_processor.get_camera_states()
-            for camera_state in camera_states:
-                if event_id in camera_state.tracked_objects:
-                    tracked_obj = camera_state.tracked_objects.get(event_id)
-                    if tracked_obj is not None:
-                        await require_camera_access(camera_state.name, request=request)
-                        thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
-        except Exception:
-            return JSONResponse(
-                content={"success": False, "message": "Event not found"},
-                status_code=404,
-            )
+        camera_states = request.app.detected_frames_processor.get_camera_states()
+
+        for camera_state in camera_states:
+            tracked_obj = camera_state.tracked_objects.get(event_id)
+
+            if tracked_obj is None:
+                continue
+
+            await require_camera_access(camera_state.name, request=request)
+
+            try:
+                thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
+            except Exception:
+                return JSONResponse(
+                    content={"success": False, "message": "Event not found"},
+                    status_code=404,
+                )
+
+            break
 
     if not thumbnail_bytes:
         return JSONResponse(

frigate/config/camera/genai.py

@@ -1,7 +1,7 @@
 from enum import Enum
-from typing import Any
+from typing import Any, Self
 
-from pydantic import Field
+from pydantic import Field, model_validator
 
 from ..base import FrigateBaseModel
 from ..env import EnvString
@@ -21,6 +21,17 @@ class GenAIRoleEnum(str, Enum):
     chat = "chat"
     descriptions = "descriptions"
     embeddings = "embeddings"
+    transcribe = "transcribe"
+
+
+# Providers that can accept audio input for the transcribe role. Ollama has no
+# audio input support, so claiming the role there would fail at request time.
+TRANSCRIBE_CAPABLE_PROVIDERS = {
+    GenAIProviderEnum.openai,
+    GenAIProviderEnum.azure_openai,
+    GenAIProviderEnum.gemini,
+    GenAIProviderEnum.llamacpp,
+}
 
 
 class GenAIConfig(FrigateBaseModel):
@@ -52,7 +63,7 @@ class GenAIConfig(FrigateBaseModel):
             GenAIRoleEnum.chat,
         ],
         title="Roles",
-        description="GenAI roles (chat, descriptions, embeddings); one provider per role.",
+        description="GenAI roles (chat, descriptions, embeddings, transcribe); one provider per role. Only chat, descriptions, and embeddings are granted by default; transcribe must be listed explicitly.",
     )
     provider_options: dict[str, Any] = Field(
         default={},
@@ -66,3 +77,17 @@ class GenAIConfig(FrigateBaseModel):
         description="Runtime options passed to the provider for each inference call.",
         json_schema_extra={"additionalProperties": {}},
     )
+
+    @model_validator(mode="after")
+    def validate_transcribe_provider(self) -> Self:
+        """Reject the transcribe role on providers that cannot accept audio input."""
+        if (
+            GenAIRoleEnum.transcribe in self.roles
+            and self.provider not in TRANSCRIBE_CAPABLE_PROVIDERS
+        ):
+            raise ValueError(
+                f"GenAI provider '{self.provider.value}' does not support audio input "
+                "and cannot be given the 'transcribe' role."
+            )
+
+        return self

frigate/config/camera/review.py

@@ -9,6 +9,7 @@ __all__ = [
     "DetectionsConfig",
     "AlertsConfig",
     "ImageSourceEnum",
+    "ReviewFrameModeEnum",
     "ReviewResponseStyleEnum",
 ]
 
@@ -20,6 +21,13 @@ class ImageSourceEnum(str, Enum):
     recordings = "recordings"
 
 
+class ReviewFrameModeEnum(str, Enum):
+    """How review frames are presented to the GenAI provider."""
+
+    frames = "frames"
+    annotated_frames = "annotated_frames"
+
+
 class ReviewResponseStyleEnum(str, Enum):
     """Writing style presets for GenAI review descriptions."""
 
@@ -153,6 +161,11 @@ class GenAIReviewConfig(FrigateBaseModel):
         description="Preferred language to request from the GenAI provider for generated responses.",
         default=None,
     )
+    frame_mode: ReviewFrameModeEnum = Field(
+        default=ReviewFrameModeEnum.frames,
+        title="Frame mode",
+        description="How frames are presented to the model. 'frames' sends the prompt followed by the frames, which suits models that track a sequence well on their own. 'annotated_frames' labels each frame and interleaves notes derived from object tracking, which helps models that lose track of activity that repeats or reverses.",
+    )
     response_style: ReviewResponseStyleEnum = Field(
         default=ReviewResponseStyleEnum.default,
         title="Response style",

frigate/config/classification.py

@@ -5,6 +5,7 @@ from pydantic import ConfigDict, Field, field_validator
 from .base import FrigateBaseModel
 
 __all__ = [
+    "AudioTranscriptionModelEnum",
     "CameraFaceRecognitionConfig",
     "CameraLicensePlateRecognitionConfig",
     "CameraAudioTranscriptionConfig",
@@ -20,6 +21,10 @@ class SemanticSearchModelEnum(str, Enum):
     jinav2 = "jinav2"
 
 
+class AudioTranscriptionModelEnum(str, Enum):
+    whisper = "whisper"
+
+
 class EnrichmentsDeviceEnum(str, Enum):
     GPU = "GPU"
     CPU = "CPU"
@@ -53,10 +58,35 @@ class AudioTranscriptionConfig(FrigateBaseModel):
         description="Enable or disable automatic audio transcription for all cameras; can be overridden per-camera.",
     )
     language: str = Field(
-        default="en",
+        default="auto",
         title="Transcription language",
-        description="Language code used for transcription/translation (for example 'en' for English). See https://whisper-api.com/docs/languages/ for supported language codes.",
+        description="Language code used for transcription/translation (for example 'en' for English), or 'auto' to let the model detect it. See https://whisper-api.com/docs/languages/ for supported language codes.",
     )
+    model: AudioTranscriptionModelEnum | str | None = Field(
+        default=AudioTranscriptionModelEnum.whisper,
+        title="Audio transcription model or GenAI provider name",
+        description="The transcription backend: 'whisper' for Frigate's built-in local models, or the name of a GenAI provider with the transcribe role.",
+    )
+
+    @field_validator("model", mode="before")
+    @classmethod
+    def coerce_model_enum(cls, v):
+        # An absent value ("model:" with nothing after it, or an explicit null)
+        # means unspecified, so fall back to the built-in backend. Left as None
+        # it would pass the GenAI-provider validation, which only inspects
+        # strings, and then be treated as a provider name that resolves to no
+        # client, turning transcription into a silent no-op.
+        if v is None or (isinstance(v, str) and not v.strip()):
+            return AudioTranscriptionModelEnum.whisper
+
+        if isinstance(v, str):
+            try:
+                return AudioTranscriptionModelEnum(v)
+            except ValueError:
+                return v
+
+        return v
+
     device: EnrichmentsDeviceEnum = Field(
         default=EnrichmentsDeviceEnum.CPU,
         title="Transcription device",

frigate/config/config.py

@@ -56,6 +56,7 @@ from .camera.timestamp import TimestampStyleConfig
 from .camera_group import CameraGroupConfig
 from .classification import (
     AudioTranscriptionConfig,
+    AudioTranscriptionModelEnum,
     ClassificationConfig,
     FaceRecognitionConfig,
     LicensePlateRecognitionConfig,
@@ -884,7 +885,7 @@ class FrigateConfig(FrigateBaseModel):
         # set notifications state
         self.notifications.enabled_in_config = self.notifications.enabled
 
-        # validate genai: each role (chat, descriptions, embeddings) at most once
+        # validate genai: each role (chat, descriptions, embeddings, transcribe) at most once
         role_to_name: dict[GenAIRoleEnum, str] = {}
         for name, genai_cfg in self.genai.items():
             for role in genai_cfg.roles:
@@ -1245,6 +1246,35 @@ class FrigateConfig(FrigateBaseModel):
         for model in self.models:
             model.create_colormap(colored_labels)
 
+        # validate audio_transcription.model when it is a GenAI provider name.
+        # this runs here rather than beside the semantic_search check because the
+        # global->camera merge above is what resolves camera-level enablement.
+        transcription_active = self.audio_transcription.enabled or any(
+            camera.audio_transcription.enabled for camera in self.cameras.values()
+        )
+
+        if (
+            transcription_active
+            and isinstance(self.audio_transcription.model, str)
+            and not isinstance(
+                self.audio_transcription.model, AudioTranscriptionModelEnum
+            )
+        ):
+            if self.audio_transcription.model not in self.genai:
+                raise ValueError(
+                    f"audio_transcription.model '{self.audio_transcription.model}' is not a "
+                    "valid GenAI config key. Must match a key in genai config."
+                )
+
+            if (
+                GenAIRoleEnum.transcribe
+                not in self.genai[self.audio_transcription.model].roles
+            ):
+                raise ValueError(
+                    f"GenAI provider '{self.audio_transcription.model}' must have "
+                    "'transcribe' in its roles for audio transcription."
+                )
+
         # Check audio transcription and audio detection requirements
         if self.audio_transcription.enabled:
             # If audio transcription is enabled globally, at least one camera must have audio detection enabled

frigate/data_processing/post/audio_transcription.py

@@ -10,6 +10,7 @@ from peewee import DoesNotExist
 
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
+from frigate.config.classification import AudioTranscriptionModelEnum
 from frigate.const import (
     CACHE_DIR,
     MODEL_CACHE_DIR,
@@ -18,8 +19,13 @@ from frigate.const import (
 )
 from frigate.data_processing.types import PostProcessDataEnum
 from frigate.embeddings.embeddings import Embeddings
+from frigate.genai.manager import GenAIClientManager
 from frigate.types import TrackedObjectUpdateTypesEnum
-from frigate.util.audio import get_audio_from_recording
+from frigate.util.audio import (
+    clean_transcript,
+    get_audio_from_recording,
+    resolve_language,
+)
 
 from ..types import DataProcessorMetrics
 from .api import PostProcessorApi
@@ -34,15 +40,25 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
         requestor: InterProcessRequestor,
         embeddings: Embeddings,
         metrics: DataProcessorMetrics,
+        genai_manager: GenAIClientManager | None = None,
     ):
         super().__init__(config, metrics, None)
         self.config = config
         self.requestor = requestor
         self.embeddings = embeddings
+        self.genai_manager = genai_manager
         self.recognizer = None
         self.transcription_lock = threading.Lock()
         self.transcription_thread: threading.Thread | None = None
         self.transcription_running = False
+        self._use_genai = not isinstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+
+        if self._use_genai:
+            # never build the local recognizer on the GenAI path; WhisperModel
+            # downloads several hundred MB on first use
+            return
 
         # faster-whisper handles model downloading automatically
         self.model_path = os.path.join(MODEL_CACHE_DIR, "whisper")
@@ -147,6 +163,31 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
             logger.error(f"Error in audio transcription post-processing: {e}")
 
     def __transcribe_audio(self, audio_data: bytes) -> str | None:
+        """Transcribe WAV audio data with the configured backend."""
+        if self._use_genai:
+            return self.__transcribe_audio_genai(audio_data)
+
+        return self.__transcribe_audio_whisper(audio_data)
+
+    def __transcribe_audio_genai(self, audio_data: bytes) -> str | None:
+        """Hand the WAV bytes to the GenAI provider holding the transcribe role."""
+        client = self.genai_manager.transcribe_client if self.genai_manager else None
+
+        if not client:
+            logger.error(
+                "audio_transcription.model is '%s' (GenAI provider) but no transcribe "
+                "client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
+                self.config.audio_transcription.model,
+            )
+            return None
+
+        text = client.transcribe(
+            audio_data,
+            language=resolve_language(self.config.audio_transcription.language),
+        )
+        return clean_transcript(text) or None
+
+    def __transcribe_audio_whisper(self, audio_data: bytes) -> str | None:
         """Transcribe WAV audio data using faster-whisper."""
         if not self.recognizer:
             logger.debug("Recognizer not initialized")
@@ -160,7 +201,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
 
             segments, info = self.recognizer.transcribe(
                 temp_wav,
-                language=self.config.audio_transcription.language,
+                language=resolve_language(self.config.audio_transcription.language),
                 beam_size=5,
             )
 

frigate/data_processing/post/review_annotations.py

@@ -0,0 +1,394 @@
+"""Frame annotations derived from object tracking data.
+
+Builds short notes describing what changed during a review item, keyed to the
+frames sampled from it. Everything here comes from tracked object data already
+in the database (each event's `path_data` trajectory and the timeline's
+stationary/active changes), so the notes can be stated to the model as fact
+rather than as something it must perceive.
+"""
+
+import logging
+import math
+from collections.abc import Sequence
+from typing import Any
+
+from frigate.models import Event, Timeline
+
+logger = logging.getLogger(__name__)
+
+# Movement smaller than this (normalized frame units) between two path points
+# is treated as the object holding still rather than travelling.
+STILL_THRESHOLD = 0.02
+
+# A heading change beyond this (dot product against the leg's own heading)
+# counts as the object turning back rather than curving.
+REVERSAL_DOT = -0.3
+
+# A run of travel shorter than this (normalized frame units) is treated as
+# milling about rather than going somewhere. Without it, a subject pacing in
+# one spot produces a burst of contradictory "turns around" notes on a single
+# frame.
+MIN_LEG_DISTANCE = 0.08
+
+# Movement that begins within this many seconds of detection is folded into
+# the detection note, so each arrival reads as one event instead of several.
+DETECT_MOVE_MERGE_SECONDS = 2.0
+
+STATE_CHANGE_PHRASES = {
+    "stationary": "has stopped moving",
+    "active": "starts moving again",
+}
+
+Point = tuple[float, float, float]
+Leg = tuple[int, int]
+
+
+def describe_position(x: float, y: float) -> str:
+    """Name a normalized frame position in plain terms."""
+    horizontal = "left" if x < 0.34 else ("right" if x > 0.66 else "center")
+    vertical = "top" if y < 0.34 else ("bottom" if y > 0.66 else "middle")
+
+    if horizontal == "center" and vertical == "middle":
+        return "the middle of the frame"
+
+    if horizontal == "center":
+        return f"the {vertical} of the frame"
+
+    if vertical == "middle":
+        return f"the {horizontal} of the frame"
+
+    return f"the {vertical} {horizontal} of the frame"
+
+
+def describe_heading(dx: float, dy: float) -> str:
+    """Name a direction of travel in frame terms.
+
+    y grows downward in normalized coordinates, so a falling y reads as moving
+    toward the top of the frame.
+    """
+    parts = []
+
+    if abs(dy) > abs(dx) * 0.4:
+        parts.append("down" if dy > 0 else "up")
+
+    if abs(dx) > abs(dy) * 0.4:
+        parts.append("right" if dx > 0 else "left")
+
+    return " and ".join(parts) if parts else "in place"
+
+
+def event_name(event: dict[str, Any]) -> str:
+    """Name an object for the notes, e.g. 'a person' or 'waste bin "Compost"'.
+
+    Objects are never numbered or given track identifiers. Frigate opens a new
+    tracked object whenever a subject is re-detected, so the tracking data
+    cannot say whether two entries are the same subject, and the notes stay
+    ambiguous rather than implying either answer.
+    """
+    label = str(event["label"]).replace("_", " ").replace("-verified", "")
+    sub_label = event.get("sub_label")
+
+    if sub_label:
+        return f'{label} "{sub_label}"'
+
+    article = "an" if label[:1].lower() in "aeiou" else "a"
+    return f"{article} {label}"
+
+
+def path_legs(points: list[Point]) -> list[Leg]:
+    """Split a trajectory into runs of travel in a consistent direction.
+
+    A leg ends when the subject starts moving back against the direction that
+    leg established, and only once the leg has covered MIN_LEG_DISTANCE, so
+    jitter around a standing subject does not register as a turn.
+
+    Returns (start, end) index pairs into `points`.
+    """
+    legs: list[Leg] = []
+    start = 0
+
+    for i in range(1, len(points)):
+        lx = points[i][0] - points[start][0]
+        ly = points[i][1] - points[start][1]
+        leg_distance = (lx * lx + ly * ly) ** 0.5
+
+        if leg_distance < MIN_LEG_DISTANCE:
+            continue
+
+        sx = points[i][0] - points[i - 1][0]
+        sy = points[i][1] - points[i - 1][1]
+        step = (sx * sx + sy * sy) ** 0.5
+
+        if step < STILL_THRESHOLD:
+            continue
+
+        dot = (lx / leg_distance) * (sx / step) + (ly / leg_distance) * (sy / step)
+
+        if dot < REVERSAL_DOT:
+            legs.append((start, i - 1))
+            start = i - 1
+
+    if start < len(points) - 1:
+        legs.append((start, len(points) - 1))
+
+    return [
+        (a, b)
+        for a, b in legs
+        if ((points[b][0] - points[a][0]) ** 2 + (points[b][1] - points[a][1]) ** 2)
+        ** 0.5
+        >= MIN_LEG_DISTANCE
+    ]
+
+
+def path_points(path_data: list[Any]) -> list[Point]:
+    """Flatten path_data into (x, y, timestamp) tuples, or [] if malformed."""
+    try:
+        return [(p[0][0], p[0][1], p[1]) for p in path_data or []]
+    except (IndexError, TypeError):
+        logger.debug("Malformed path_data, skipping trajectory notes")
+        return []
+
+
+def leg_start_time(points: list[Point], leg: Leg) -> float:
+    """When a leg's movement actually began.
+
+    path_data always keeps an object's first two samples, so a leg can open
+    with points recorded long before the object moved. The first sample that
+    has left the leg's origin is the earliest evidence of movement.
+    """
+    a, b = leg
+    x0, y0, t0 = points[a]
+
+    for x, y, t in points[a + 1 : b + 1]:
+        if math.hypot(x - x0, y - y0) >= STILL_THRESHOLD:
+            return t
+
+    return t0
+
+
+def leg_heading(points: list[Point], leg: Leg) -> str:
+    a, b = leg
+    return describe_heading(points[b][0] - points[a][0], points[b][1] - points[a][1])
+
+
+def leg_phrase(points: list[Point], leg: Leg, first: bool) -> str:
+    """Describe the start of a leg, e.g. 'turns around at ... and heads left'."""
+    heading = leg_heading(points, leg)
+    place = describe_position(points[leg[0]][0], points[leg[0]][1])
+
+    if first:
+        return f"starts moving {heading} from {place}"
+
+    return f"turns around at {place} and heads {heading}"
+
+
+def path_moments(path_data: list[Any]) -> list[tuple[float, str]]:
+    """Key moments in one trajectory as (timestamp, phrase).
+
+    Emits one note per leg of travel. Where the last leg ends is left out:
+    path_data only records significant movement, so its final point cannot
+    distinguish an object coming to rest from one leaving the frame.
+    """
+    points = path_points(path_data)
+
+    if len(points) < 2:
+        return []
+
+    return [
+        (leg_start_time(points, leg), leg_phrase(points, leg, index == 0))
+        for index, leg in enumerate(path_legs(points))
+    ]
+
+
+def build_timeline(
+    events: list[dict[str, Any]],
+    span_end: float,
+    state_changes: Sequence[dict[str, Any]] = (),
+) -> list[tuple[float, str]]:
+    """All annotated moments across every event, in time order.
+
+    Only changes are noted, since those are what sparse frames miss; an
+    object's state at the end of the clip is visible in the last frame.
+    `span_end` is the timestamp of the last sampled frame, and moments past it
+    describe nothing the model can see. A track ending means the object
+    stopped being detected, which may or may not mean it left the frame.
+
+    `state_changes` are timeline rows (timestamp, source_id, class_type); the
+    stationary and active ones become "has stopped moving" / "starts moving
+    again". Frigate only marks an object stationary after it has been still
+    for a while, which the past-tense wording reflects.
+
+    Each object keeps its own notes. Folding an object into the note of the
+    person moving it ("alongside ...") was tried and made models lose track of
+    where the object went.
+    """
+    changes_by_event: dict[str, list[tuple[float, str]]] = {}
+
+    for change in state_changes:
+        phrase = STATE_CHANGE_PHRASES.get(change["class_type"])
+
+        if phrase:
+            changes_by_event.setdefault(change["source_id"], []).append(
+                (change["timestamp"], phrase)
+            )
+
+    timeline: list[tuple[float, str]] = []
+
+    for event in sorted(events, key=lambda e: e["start_time"]):
+        # Frame extraction can come up short at the end of a clip, leaving
+        # objects that only appear after the last frame we actually have.
+        if event["start_time"] > span_end:
+            continue
+
+        points = path_points(event.get("path_data") or [])
+        legs = path_legs(points) if len(points) >= 2 else []
+        name = event_name(event)
+        detected_at = event["start_time"]
+        where = describe_position(points[0][0], points[0][1]) if points else "the frame"
+        merges = bool(legs) and leg_start_time(points, legs[0]) - detected_at <= (
+            DETECT_MOVE_MERGE_SECONDS
+        )
+        remaining = list(enumerate(legs))
+        moments: list[tuple[float, str]] = []
+
+        if merges:
+            heading = leg_heading(points, legs[0])
+            timeline.append(
+                (detected_at, f"{name} first detected at {where}, moving {heading}")
+            )
+            remaining = remaining[1:]
+        else:
+            timeline.append((detected_at, f"{name} first detected at {where}"))
+
+        for index, leg in remaining:
+            moments.append(
+                (
+                    leg_start_time(points, leg),
+                    leg_phrase(points, leg, index == 0),
+                )
+            )
+
+        moments.extend(
+            (timestamp, phrase)
+            for timestamp, phrase in changes_by_event.get(event["id"], [])
+            if timestamp >= detected_at
+        )
+
+        for timestamp, phrase in moments:
+            if timestamp <= span_end:
+                timeline.append((timestamp, f"{name} {phrase}"))
+
+        if event["end_time"] and event["end_time"] <= span_end:
+            timeline.append((event["end_time"], f"{name} is no longer detected"))
+
+    return sorted(timeline, key=lambda m: m[0])
+
+
+def annotations_by_frame(
+    timeline: list[tuple[float, str]], frame_times: list[float]
+) -> dict[int, list[str]]:
+    """Bucket timeline moments onto the frame that follows each one.
+
+    A moment is attached to the first frame at or after it happened, so the
+    note always precedes the image in which the change becomes visible.
+    """
+    buckets: dict[int, list[str]] = {}
+
+    if not frame_times:
+        return buckets
+
+    for timestamp, phrase in timeline:
+        index = next(
+            (i for i, ft in enumerate(frame_times) if ft >= timestamp),
+            len(frame_times) - 1,
+        )
+        buckets.setdefault(index, []).append(phrase)
+
+    return buckets
+
+
+def get_tracked_events(detection_ids: list[str]) -> list[dict[str, Any]]:
+    """Load the tracked objects behind a review item's detections."""
+    if not detection_ids:
+        return []
+
+    rows = list(
+        Event.select(
+            Event.id,
+            Event.label,
+            Event.sub_label,
+            Event.start_time,
+            Event.end_time,
+            Event.data,
+        )
+        .where(Event.id << detection_ids)
+        .dicts()
+        .iterator()
+    )
+
+    return [
+        {
+            "id": row["id"],
+            "label": row["label"],
+            "sub_label": row["sub_label"],
+            "start_time": row["start_time"],
+            "end_time": row["end_time"],
+            "path_data": (row["data"] or {}).get("path_data") or [],
+        }
+        for row in rows
+        if row["start_time"] is not None
+    ]
+
+
+def get_state_changes(detection_ids: list[str]) -> list[dict[str, Any]]:
+    """Stationary/active changes the timeline recorded for these objects."""
+    if not detection_ids:
+        return []
+
+    return list(
+        Timeline.select(Timeline.timestamp, Timeline.source_id, Timeline.class_type)
+        .where(
+            (Timeline.source_id << detection_ids)
+            & (Timeline.class_type << list(STATE_CHANGE_PHRASES))
+        )
+        .dicts()
+        .iterator()
+    )
+
+
+def build_frame_captions(
+    detection_ids: list[str],
+    frame_times: list[float],
+) -> list[str]:
+    """A caption for each sampled frame, in frame order.
+
+    Every frame gets its index and elapsed time so the model can tell them
+    apart; frames where something changed also carry the tracker notes for
+    that moment. Returns an empty list when there is nothing to say, which
+    callers treat as a reason to fall back to sending plain frames.
+    """
+    if not frame_times:
+        return []
+
+    events = get_tracked_events(detection_ids)
+
+    if not events:
+        logger.debug("No tracked events found for review item, skipping annotations")
+        return []
+
+    timeline = build_timeline(events, frame_times[-1], get_state_changes(detection_ids))
+    buckets = annotations_by_frame(timeline, frame_times)
+
+    if not buckets:
+        return []
+
+    total = len(frame_times)
+    origin = frame_times[0]
+    captions: list[str] = []
+
+    for index, timestamp in enumerate(frame_times):
+        lines = [f"Frame {index + 1} of {total} (+{timestamp - origin:.1f}s):"]
+        lines.extend(f"[tracker] {note}" for note in buckets.get(index, []))
+        captions.append("\n".join(lines))
+
+    return captions

frigate/data_processing/post/review_descriptions.py

@@ -19,7 +19,11 @@ from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
 from frigate.config.camera import CameraConfig
-from frigate.config.camera.review import GenAIReviewConfig, ImageSourceEnum
+from frigate.config.camera.review import (
+    GenAIReviewConfig,
+    ImageSourceEnum,
+    ReviewFrameModeEnum,
+)
 from frigate.const import (
     ATTRIBUTE_LABEL_DISPLAY_MAP,
     CACHE_DIR,
@@ -36,6 +40,7 @@ from frigate.util.image import get_image_from_recording
 
 from ..post.api import PostProcessorApi
 from ..types import DataProcessorMetrics
+from .review_annotations import build_frame_captions
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +48,7 @@ RECORDING_BUFFER_EXTENSION_PERCENT = 0.10
 MIN_RECORDING_DURATION = 10
 MAX_IMAGE_TOKENS = 24000
 MAX_FRAMES_PER_SECOND = 1
+MAX_ANNOTATED_FRAMES = 28
 
 
 class ReviewDescriptionProcessor(PostProcessorApi):
@@ -67,6 +73,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         duration: float,
         image_source: ImageSourceEnum = ImageSourceEnum.preview,
         height: int = 480,
+        frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
     ) -> int:
         """Calculate optimal number of frames based on event duration, context size,
         image source, and resolution.
@@ -80,6 +87,8 @@ class ReviewDescriptionProcessor(PostProcessorApi):
           - MAX_FRAMES_PER_SECOND x duration, to avoid drowning short events in
             near-duplicate frames where the model latches onto the redundant middle
             and skips the start/end action
+          - MAX_ANNOTATED_FRAMES in annotated mode, where the tracking notes
+            already carry the sequence
         """
         client = self.genai_manager.description_client
 
@@ -125,6 +134,10 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         max_frames_by_tokens = int(image_token_budget / tokens_per_image)
         max_frames_by_duration = int(duration * MAX_FRAMES_PER_SECOND)
         max_frames = min(max_frames_by_tokens, max_frames_by_duration)
+
+        if frame_mode == ReviewFrameModeEnum.annotated_frames:
+            max_frames = min(max_frames, MAX_ANNOTATED_FRAMES)
+
         return max(max_frames, 3)
 
     def process_data(
@@ -166,6 +179,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 return
 
             image_source = camera_config.review.genai.image_source
+            frame_mode = camera_config.review.genai.frame_mode
 
             if image_source == ImageSourceEnum.recordings:
                 buffer_extension = get_recording_buffer_extension(
@@ -174,40 +188,43 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 final_data["start_time"] -= buffer_extension
                 final_data["end_time"] += buffer_extension
 
-                thumbs = self.get_recording_frames(
+                frames = self.get_recording_frames(
                     camera,
                     final_data["start_time"],
                     final_data["end_time"],
                     height=480,  # Use 480p for good balance between quality and token usage
+                    frame_mode=frame_mode,
                 )
 
-                if not thumbs:
+                if not frames:
                     # Fallback to preview frames if no recordings available
                     logger.warning(
                         f"No recording frames found for {camera}, falling back to preview frames"
                     )
-                    thumbs = self.get_preview_frames_as_bytes(
+                    frames = self.get_preview_frames_as_bytes(
                         camera,
                         final_data["start_time"],
                         final_data["end_time"],
                         final_data["thumb_path"],
                         id,
                         camera_config.review.genai.debug_save_thumbnails,
+                        frame_mode,
                     )
                 elif camera_config.review.genai.debug_save_thumbnails:
-                    self.save_debug_recording_frames(id, thumbs)
+                    self.save_debug_recording_frames(id, frames)
             else:
                 # Use preview frames
-                thumbs = self.get_preview_frames_as_bytes(
+                frames = self.get_preview_frames_as_bytes(
                     camera,
                     final_data["start_time"],
                     final_data["end_time"],
                     final_data["thumb_path"],
                     id,
                     camera_config.review.genai.debug_save_thumbnails,
+                    frame_mode,
                 )
 
-            self.start_analysis(camera_config, final_data, thumbs)
+            self.start_analysis(camera_config, final_data, frames)
 
     def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
         if topic == EmbeddingsRequestEnum.regenerate_review_description.value:
@@ -386,31 +403,50 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         buffer_extension = get_recording_buffer_extension(
             final_data["end_time"] - final_data["start_time"]
         )
-        thumbs = self.get_recording_frames(
+        frames = self.get_recording_frames(
             str(review.camera),
             final_data["start_time"] - buffer_extension,
             final_data["end_time"] + buffer_extension,
             height=480,
+            frame_mode=camera_config.review.genai.frame_mode,
         )
 
-        if not thumbs:
+        if not frames:
             logger.error(
                 "No recording frames are available for review item %s", review_id
             )
             return
 
         if camera_config.review.genai.debug_save_thumbnails:
-            self.save_debug_recording_frames(review_id, thumbs)
+            self.save_debug_recording_frames(review_id, frames)
 
-        self.start_analysis(camera_config, final_data, thumbs)
+        self.start_analysis(camera_config, final_data, frames)
 
     def start_analysis(
         self,
         camera_config: CameraConfig,
         final_data: dict[str, Any],
-        thumbs: list[bytes],
+        frames: list[tuple[bytes, float]],
     ) -> None:
         """Kick off description generation for a review item in the background."""
+        thumbs = [frame for frame, _ in frames]
+        captions: list[str] = []
+
+        if (
+            camera_config.review.genai.frame_mode
+            == ReviewFrameModeEnum.annotated_frames
+        ):
+            captions = build_frame_captions(
+                final_data["data"].get("detections") or [],
+                [timestamp for _, timestamp in frames],
+            )
+
+            if not captions:
+                logger.debug(
+                    "No tracking annotations for review item %s, sending plain frames",
+                    final_data["id"],
+                )
+
         self.review_desc_dps.update()
         threading.Thread(
             target=run_analysis,
@@ -421,19 +457,22 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 camera_config,
                 final_data,
                 thumbs,
+                captions,
                 camera_config.review.genai,
                 sorted(self.config.all_labels),
                 self.config.all_attributes,
             ),
         ).start()
 
-    def save_debug_recording_frames(self, review_id: str, thumbs: list[bytes]) -> None:
+    def save_debug_recording_frames(
+        self, review_id: str, frames: list[tuple[bytes, float]]
+    ) -> None:
         """Write the recording frames sent to the provider out for debugging."""
         Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
             parents=True, exist_ok=True
         )
 
-        for idx, frame_bytes in enumerate(thumbs):
+        for idx, (frame_bytes, _) in enumerate(frames):
             with open(
                 os.path.join(CLIPS_DIR, f"genai-requests/{review_id}/{idx}.jpg"),
                 "wb",
@@ -445,7 +484,9 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         camera: str,
         start_time: float,
         end_time: float,
-    ) -> list[str]:
+        frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
+    ) -> list[tuple[str, float]]:
+        """Preview frame paths paired with the time each one was captured."""
         preview_dir = os.path.join(CACHE_DIR, "preview_frames")
         file_start = f"preview_{camera}-"
         start_file = f"{file_start}{start_time}.webp"
@@ -477,18 +518,29 @@ class ReviewDescriptionProcessor(PostProcessorApi):
 
         frame_count = len(all_frames)
         desired_frame_count = self.calculate_frame_count(
-            camera, duration=end_time - start_time
+            camera,
+            duration=end_time - start_time,
+            frame_mode=frame_mode,
         )
 
+        def with_timestamp(path: str) -> tuple[str, float]:
+            # Preview frames are named preview_<camera>-<timestamp>.webp
+            stem = os.path.basename(path).removesuffix(".webp")
+
+            try:
+                return (path, float(stem.removeprefix(file_start)))
+            except ValueError:
+                return (path, start_time)
+
         if frame_count <= desired_frame_count:
-            return all_frames
+            return [with_timestamp(f) for f in all_frames]
 
         selected_frames = []
         step_size = (frame_count - 1) / (desired_frame_count - 1)
 
         for i in range(desired_frame_count):
             index = round(i * step_size)
-            selected_frames.append(all_frames[index])
+            selected_frames.append(with_timestamp(all_frames[index]))
 
         return selected_frames
 
@@ -498,11 +550,12 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         start_time: float,
         end_time: float,
         height: int = 480,
-    ) -> list[bytes]:
-        """Get frames from recordings at specified timestamps."""
+        frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
+    ) -> list[tuple[bytes, float]]:
+        """Get frames from recordings paired with the time each was captured."""
         duration = end_time - start_time
         desired_frame_count = self.calculate_frame_count(
-            camera, duration, ImageSourceEnum.recordings, height
+            camera, duration, ImageSourceEnum.recordings, height, frame_mode
         )
 
         # Calculate evenly spaced timestamps throughout the duration
@@ -540,7 +593,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             except DoesNotExist:
                 return None
 
-        frames = []
+        frames: list[tuple[bytes, float]] = []
 
         for timestamp in timestamps:
             try:
@@ -553,7 +606,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                     image_data = extract_frame_from_recording(rounded_timestamp)
 
                 if image_data:
-                    frames.append(image_data)
+                    frames.append((image_data, timestamp))
                 else:
                     logger.warning(
                         f"No recording found for {camera} at timestamp {timestamp}"
@@ -574,7 +627,8 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         thumb_path_fallback: str,
         review_id: str,
         save_debug: bool,
-    ) -> list[bytes]:
+        frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
+    ) -> list[tuple[bytes, float]]:
         """Get preview frames and convert them to JPEG bytes.
 
         Args:
@@ -586,14 +640,14 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             save_debug: Whether to save debug thumbnails
 
         Returns:
-            List of JPEG image bytes
+            List of (JPEG image bytes, capture timestamp) pairs
         """
-        frame_paths = self.get_cache_frames(camera, start_time, end_time)
+        frame_paths = self.get_cache_frames(camera, start_time, end_time, frame_mode)
         if not frame_paths:
-            frame_paths = [thumb_path_fallback]
+            frame_paths = [(thumb_path_fallback, start_time)]
 
-        thumbs = []
-        for idx, thumb_path in enumerate(frame_paths):
+        thumbs: list[tuple[bytes, float]] = []
+        for idx, (thumb_path, timestamp) in enumerate(frame_paths):
             thumb_data = cv2.imread(thumb_path)
 
             if thumb_data is None:
@@ -606,7 +660,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 ".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100]
             )
             if ret:
-                thumbs.append(jpg.tobytes())
+                thumbs.append((jpg.tobytes(), timestamp))
 
             if save_debug:
                 Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
@@ -641,6 +695,7 @@ def run_analysis(
     camera_config: CameraConfig,
     final_data: dict[str, Any],
     thumbs: list[bytes],
+    frame_captions: list[str],
     genai_config: GenAIReviewConfig,
     labelmap_objects: list[str],
     attribute_labels: list[str],
@@ -697,6 +752,7 @@ def run_analysis(
         genai_config.debug_save_thumbnails,
         genai_config.activity_context_prompt,
         genai_config.response_style,
+        frame_captions,
     )
     review_inference_speed.update(datetime.datetime.now().timestamp() - start)
 

frigate/data_processing/real_time/audio_transcription.py

@@ -1,16 +1,19 @@
 """Handle processing audio for speech transcription using sherpa-onnx with FFmpeg pipe."""
 
+import collections
 import logging
 import os
 import queue
 import threading
-from typing import Any
+import time
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import CameraConfig, FrigateConfig
-from frigate.const import MODEL_CACHE_DIR
+from frigate.config.classification import AudioTranscriptionModelEnum
+from frigate.const import AUDIO_DURATION, MODEL_CACHE_DIR
 from frigate.data_processing.common.audio_transcription.model import (
     AudioTranscriptionModelRunner,
 )
@@ -18,12 +21,38 @@ from frigate.data_processing.real_time.whisper_online import (
     FasterWhisperASR,
     OnlineASRProcessor,
 )
+from frigate.util.audio import (
+    clean_transcript,
+    pcm16_to_wav,
+    resolve_language,
+    stitch_transcripts,
+)
 
 from ..types import DataProcessorMetrics
 from .api import RealTimeProcessorApi
 
+if TYPE_CHECKING:
+    # importing frigate.genai eagerly would pull the provider SDKs into the
+    # audio process even when transcription runs on a local model
+    from frigate.genai.manager import GenAIClientManager
+
 logger = logging.getLogger(__name__)
 
+# Number of ~0.975s audio detector chunks per GenAI request. The window advances
+# one chunk at a time, so two chunks means a 50% overlap: every word lands whole
+# in at least one window, which whisper-family models need to avoid hallucinating
+# on a clipped clip. The cadence is fixed by the audio detector's frame size, so
+# this is a constant rather than a config knob.
+GENAI_WINDOW_CHUNKS = 2
+
+# Bound the queue at ~30s of audio so a slow or hung provider cannot grow it
+# without limit. The producer is the ffmpeg read thread and must never block.
+AUDIO_QUEUE_MAXSIZE = int(30 / AUDIO_DURATION)
+
+# A backed-up queue drops a chunk per cycle, so warning on each one would spam
+# the log once a second per camera for as long as the provider stays slow.
+AUDIO_DROP_WARN_INTERVAL = 10.0
+
 
 class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
     def __init__(
@@ -31,9 +60,10 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
         config: FrigateConfig,
         camera_config: CameraConfig,
         requestor: InterProcessRequestor,
-        model_runner: AudioTranscriptionModelRunner,
+        model_runner: AudioTranscriptionModelRunner | None,
         metrics: DataProcessorMetrics,
         stop_event: threading.Event,
+        genai_manager: "GenAIClientManager | None" = None,
     ):
         super().__init__(config, metrics)
         self.config = config
@@ -42,11 +72,31 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
         self.stream: Any = None
         self.whisper_model: FasterWhisperASR | None = None
         self.model_runner = model_runner
+        self.genai_manager = genai_manager
         self.transcription_segments: list[str] = []
-        self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue()
+        self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue(
+            maxsize=AUDIO_QUEUE_MAXSIZE
+        )
         self.stop_event = stop_event
+        self._use_genai = not isinstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+        # sliding window of raw int16 chunks; the deque's maxlen is what evicts
+        # the oldest chunk and so produces the overlap
+        self._genai_window: collections.deque[np.ndarray] = collections.deque(
+            maxlen=GENAI_WINDOW_CHUNKS
+        )
+        self._genai_committed = ""
+        # set by the producer when it discards a chunk, so the consumer knows the
+        # audio it is about to receive is not contiguous with what it buffered
+        self._audio_dropped = threading.Event()
+        self._last_drop_warning = 0.0
 
     def __build_recognizer(self) -> None:
+        if self._use_genai:
+            # nothing local to load; never import sherpa or FasterWhisperASR
+            return
+
         try:
             if self.config.audio_transcription.model_size == "large":
                 # Whisper models need to be per-process and can only run one stream at a time
@@ -64,7 +114,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
                 self.stream = OnlineASRProcessor(
                     asr=self.whisper_model,
                 )
-            else:
+            elif self.model_runner is not None:
                 logger.debug(f"Loading sherpa stream for {self.camera_config.name}")
                 self.stream = self.model_runner.model.create_stream()
             logger.debug(
@@ -76,6 +126,15 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             )
 
     def __process_audio_stream(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
+        # must precede both the model_runner guard (model_runner is None on this
+        # path) and the float32 normalization below (GenAI wants untouched int16)
+        if self._use_genai:
+            return self.__process_audio_genai(audio_data)
+
+        if self.model_runner is None:
+            logger.debug("Audio transcription (live) model runner not initialized")
+            return None
+
         if (
             self.model_runner.model is None
             and self.config.audio_transcription.model_size == "small"
@@ -140,6 +199,82 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             logger.error(f"Error processing audio stream: {e}")
             return None
 
+    def __process_audio_genai(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
+        """Transcribe a sliding overlapped window through the GenAI provider."""
+        client = self.genai_manager.transcribe_client if self.genai_manager else None
+
+        if not client:
+            logger.error(
+                "audio_transcription.model is '%s' (GenAI provider) but no transcribe "
+                "client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
+                self.config.audio_transcription.model,
+            )
+            return None
+
+        if self._audio_dropped.is_set():
+            self._audio_dropped.clear()
+
+            # Chunks were discarded between what is buffered and this one, so
+            # concatenating them would splice non-adjacent audio into one window
+            # and destroy the overlap the stitcher depends on.
+            self._genai_window.clear()
+
+            if self._genai_committed:
+                # the transcript has a gap in it; close the utterance out rather
+                # than stitching across missing speech
+                return self.__end_genai_utterance()
+
+        self._genai_window.append(audio_data)
+
+        if len(self._genai_window) < GENAI_WINDOW_CHUNKS:
+            # wait for a full window so the first request is never a clipped clip
+            return None
+
+        window = np.concatenate(list(self._genai_window))
+
+        # Silence gate, using the same threshold audio detection uses. Gate the
+        # whole window rather than individual chunks; this is the primary cost
+        # and privacy brake and is what keeps a quiet camera near zero requests.
+        window_as_float = window.astype(np.float32)
+        rms = float(np.sqrt(np.mean(np.absolute(np.square(window_as_float)))))
+
+        if rms < self.camera_config.audio.min_volume:
+            logger.debug(
+                f"Window RMS {rms:.1f} below min_volume, skipping transcription"
+            )
+            return self.__end_genai_utterance()
+
+        text = client.transcribe(
+            pcm16_to_wav(window),
+            language=resolve_language(self.config.audio_transcription.language),
+        )
+
+        # cleaning has to come first: a silent window often comes back as the
+        # model's preamble alone, which is silence, not a word to commit
+        cleaned = clean_transcript(text)
+
+        if not cleaned:
+            return self.__end_genai_utterance()
+
+        self._genai_committed = stitch_transcripts(self._genai_committed, cleaned)
+
+        # no VAD on this path, so mirror the whisper branch's heuristic endpoint
+        is_endpoint = (
+            self._genai_committed.endswith((".", "!", "?"))
+            and len(self._genai_committed) > 300
+        )
+
+        logger.debug(f"GenAI transcription: '{self._genai_committed}'")
+
+        return self._genai_committed, is_endpoint
+
+    def __end_genai_utterance(self) -> tuple[str, bool] | None:
+        """Close out the current utterance when a window carries no speech."""
+        if not self._genai_committed:
+            return None
+
+        return self._genai_committed, True
+
     def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
         pass
 
@@ -148,8 +283,38 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             logger.debug("No audio data provided for transcription")
             return None
 
-        # enqueue audio data for processing in the thread
-        self.audio_queue.put((obj_data, audio))
+        # enqueue audio data for processing in the thread. never block: the
+        # producer is the ffmpeg read thread that audio detection depends on,
+        # so on a backlog drop the oldest chunk instead.
+        try:
+            self.audio_queue.put_nowait((obj_data, audio))
+        except queue.Full:
+            try:
+                self.audio_queue.get_nowait()
+                self.audio_queue.task_done()
+            except queue.Empty:
+                pass
+
+            # the stream now has a hole in it, which the consumer has to know
+            # about before it splices the next chunk onto what it already holds
+            self._audio_dropped.set()
+
+            now = time.monotonic()
+
+            if now - self._last_drop_warning >= AUDIO_DROP_WARN_INTERVAL:
+                self._last_drop_warning = now
+                logger.warning(
+                    "Audio transcription queue for %s is full, dropping audio. The "
+                    "provider is not keeping up with the %.2fs chunk rate",
+                    self.camera_config.name,
+                    AUDIO_DURATION,
+                )
+
+            try:
+                self.audio_queue.put_nowait((obj_data, audio))
+            except queue.Full:
+                pass
+
         return None
 
     def run(self) -> None:
@@ -205,6 +370,14 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
                 break
 
     def reset(self) -> None:
+        if self._use_genai:
+            self._genai_committed = ""
+            # stale audio carried across an utterance boundary would be
+            # re-transcribed into the next one
+            self._genai_window.clear()
+            logger.debug("Stream reset")
+            return
+
         if self.config.audio_transcription.model_size == "large":
             # get final output from whisper
             output = self.stream.finish()
@@ -218,7 +391,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             # reset whisper
             self.stream.init()
             self.transcription_segments = []
-        else:
+        elif self.model_runner is not None:
             # reset sherpa
             self.model_runner.model.reset(self.stream)
 
@@ -226,6 +399,24 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
 
     def check_unload_model(self) -> None:
         # regularly called in the loop in audio maintainer
+        if self._use_genai:
+            # no model to unload, but this is the hook that fires when
+            # live_enabled flips off. guard on emptiness: called ~1x/s per camera.
+            if self._genai_committed or self._genai_window:
+                logger.debug(
+                    f"Clearing GenAI transcription state for {self.camera_config.name}"
+                )
+                self.clear_audio_queue()
+                self._genai_committed = ""
+                self._genai_window.clear()
+
+                self.requestor.send_data(
+                    f"{self.camera_config.name}/audio/transcription",
+                    "",
+                )
+
+            return
+
         if (
             self.config.audio_transcription.model_size == "large"
             and self.whisper_model is not None
@@ -270,6 +461,10 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
         self, topic: str, request_data: dict[str, Any]
     ) -> dict[str, Any] | None:
         if topic == "clear_audio_recognizer":
+            if self._use_genai:
+                self.reset()
+                return {"message": "Audio transcription state cleared", "success": True}
+
             self.stream = None
             self.__build_recognizer()
             return {"message": "Audio recognizer cleared and rebuilt", "success": True}

frigate/embeddings/maintainer.py

@@ -68,7 +68,7 @@ from frigate.events.types import (
     RegenerateDescriptionEnum,
 )
 from frigate.genai import GenAIClientManager
-from frigate.models import Event, Recordings, ReviewSegment, Trigger
+from frigate.models import Event, Recordings, ReviewSegment, Timeline, Trigger
 from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
 from frigate.util.file import get_event_thumbnail_bytes
@@ -139,7 +139,7 @@ class EmbeddingMaintainer(threading.Thread):
             ),
             load_vec_extension=True,
         )
-        models = [Event, Recordings, ReviewSegment, Trigger]
+        models = [Event, Recordings, ReviewSegment, Timeline, Trigger]
         db.bind(models)
 
         self.genai_manager = GenAIClientManager(config)
@@ -251,7 +251,11 @@ class EmbeddingMaintainer(threading.Thread):
         ):
             self.post_processors.append(
                 AudioTranscriptionPostProcessor(
-                    self.config, self.requestor, self.embeddings, metrics
+                    self.config,
+                    self.requestor,
+                    self.embeddings,
+                    metrics,
+                    self.genai_manager,
                 )
             )
 

frigate/events/audio.py

@@ -19,6 +19,7 @@ from frigate.config.camera.updater import (
     CameraConfigUpdateEnum,
     CameraConfigUpdateSubscriber,
 )
+from frigate.config.classification import AudioTranscriptionModelEnum
 from frigate.const import (
     AUDIO_DURATION,
     AUDIO_FORMAT,
@@ -112,18 +113,31 @@ class AudioProcessor(FrigateProcess):
 
         threading.current_thread().name = "process:audio_manager"
 
+        self.transcription_model_runner: AudioTranscriptionModelRunner | None = None
+        self.genai_manager: Any = None
+
         if any(
             c.enabled_in_config and c.audio_transcription.enabled
             for c in self.config.cameras.values()
         ):
-            self.transcription_model_runner: AudioTranscriptionModelRunner | None = (
-                AudioTranscriptionModelRunner(
+            if isinstance(
+                self.config.audio_transcription.model, AudioTranscriptionModelEnum
+            ):
+                # AudioTranscriptionModelRunner.__init__ unconditionally fetches
+                # sherpa-onnx or whisper weights, so only build it on the local path
+                self.transcription_model_runner = AudioTranscriptionModelRunner(
                     self.config.audio_transcription.device or "AUTO",
                     self.config.audio_transcription.model_size,
                 )
-            )
-        else:
-            self.transcription_model_runner = None
+            else:
+                # imported here rather than at module scope: frigate.genai pulls in
+                # numpy, the provider SDKs, frigate.models, and the prompt builders,
+                # and this process runs at PROCESS_PRIORITY_HIGH. built after the
+                # fork because SDK clients hold sockets and TLS state that must not
+                # cross it; clients themselves stay lazy behind the role property.
+                from frigate.genai.manager import GenAIClientManager
+
+                self.genai_manager = GenAIClientManager(self.config)
 
         config_subscriber = CameraConfigUpdateSubscriber(
             self.config,
@@ -151,6 +165,7 @@ class AudioProcessor(FrigateProcess):
                 self.camera_metrics,
                 self.transcription_model_runner,
                 self.stop_event,  # type: ignore[arg-type]
+                self.genai_manager,
             )
             self.audio_threads[name] = thread
             thread.start()
@@ -200,6 +215,7 @@ class AudioEventMaintainer(threading.Thread):
         camera_metrics: DictProxy,
         audio_transcription_model_runner: AudioTranscriptionModelRunner | None,
         stop_event: threading.Event,
+        genai_manager: Any = None,
     ) -> None:
         super().__init__(name=f"{camera.name}_audio_event_processor")
 
@@ -222,6 +238,7 @@ class AudioEventMaintainer(threading.Thread):
         self.logpipe = LogPipe(f"ffmpeg.{self.camera_config.name}.audio")
         self.audio_listener: subprocess.Popen[Any] | None = None
         self.audio_transcription_model_runner = audio_transcription_model_runner
+        self.genai_manager = genai_manager
         self.transcription_processor = None
         self.transcription_thread = None
 
@@ -238,9 +255,9 @@ class AudioEventMaintainer(threading.Thread):
         )
         self.detection_publisher = DetectionPublisher(DetectionTypeEnum.audio.value)
 
-        if (
-            self.camera_config.audio_transcription.enabled
-            and self.audio_transcription_model_runner is not None
+        if self.camera_config.audio_transcription.enabled and (
+            self.audio_transcription_model_runner is not None
+            or self.genai_manager is not None
         ):
             # init the transcription processor for this camera
             self.transcription_processor = AudioTranscriptionRealTimeProcessor(
@@ -250,6 +267,7 @@ class AudioEventMaintainer(threading.Thread):
                 model_runner=self.audio_transcription_model_runner,
                 metrics=self.camera_metrics[self.camera_config.name],
                 stop_event=self.stop_event,
+                genai_manager=self.genai_manager,
             )
 
             self.transcription_thread = threading.Thread(

frigate/genai/__init__.py

@@ -105,8 +105,21 @@ class GenAIClient:
         debug_save: bool,
         activity_context_prompt: str,
         response_style: str = "default",
+        frame_captions: list[str] | None = None,
     ) -> ReviewMetadata | None:
-        """Generate a description for the review item activity."""
+        """Generate a description for the review item activity.
+
+        `frame_captions` holds one caption per thumbnail for the annotated
+        frame mode; each is sent directly before its frame.
+        """
+        if frame_captions and len(frame_captions) != len(thumbnails):
+            logger.warning(
+                "Got %d frame captions for %d thumbnails, sending plain frames",
+                len(frame_captions),
+                len(thumbnails),
+            )
+            frame_captions = None
+
         context_prompt = build_review_description_prompt(
             review_data,
             thumbnails,
@@ -114,6 +127,7 @@ class GenAIClient:
             preferred_language,
             activity_context_prompt,
             response_style,
+            frame_captions,
         )
 
         logger.debug(
@@ -129,9 +143,30 @@ class GenAIClient:
             ) as f:
                 f.write(context_prompt)
 
+            if frame_captions:
+                # One file per frame, numbered to match the image it precedes
+                # (0.txt goes with 0.jpg), so the debug folder replays without
+                # having to re-derive the mapping.
+                for index, caption in enumerate(frame_captions):
+                    with open(
+                        os.path.join(
+                            CLIPS_DIR,
+                            "genai-requests",
+                            review_data["id"],
+                            f"{index}.txt",
+                        ),
+                        "w",
+                    ) as f:
+                        f.write(caption)
+
         response_format = build_review_description_response_format(concerns)
 
-        response = self._send(context_prompt, thumbnails, response_format)
+        response = self._send(
+            context_prompt,
+            thumbnails,
+            response_format,
+            image_captions=frame_captions,
+        )
 
         if debug_save and response:
             with open(
@@ -269,6 +304,7 @@ class GenAIClient:
         images: list[bytes],
         response_format: dict | None = None,
         enable_thinking: bool = False,
+        image_captions: list[str] | None = None,
     ) -> str | None:
         """Submit a request to the provider.
 
@@ -276,6 +312,10 @@ class GenAIClient:
         ``supports_toggleable_thinking``. Description-style callers leave it
         at the default (off) since synthesis tasks don't benefit from
         reasoning traces.
+
+        ``image_captions`` carries one caption per image, to be placed
+        immediately before its image so the model can tell the frames apart.
+        Providers build their request order with ``interleave_images``.
         """
         return None
 
@@ -298,6 +338,11 @@ class GenAIClient:
         """Whether the configured model can generate embeddings via embed()."""
         return False
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Whether the configured model can transcribe audio via transcribe()."""
+        return False
+
     def list_models(self) -> list[str]:
         """Return the list of model names available from this provider.
 
@@ -305,6 +350,21 @@ class GenAIClient:
         """
         return []
 
+    def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
+        """Return capability flags for each model the provider serves.
+
+        Only providers whose backend advertises capabilities per model can
+        populate this; llama.cpp reports input modalities for every model it
+        serves, so one request describes them all. An empty mapping means "no
+        per-model information available", and callers fall back to this
+        client's own capability properties, which describe only the configured
+        model. A model absent from a non-empty mapping means the same thing.
+
+        Returns:
+            Model name (including aliases) to its capability flags
+        """
+        return {}
+
     def get_context_size(self) -> int:
         """Get the context window size for this provider in tokens."""
         return 4096
@@ -336,6 +396,33 @@ class GenAIClient:
         )
         return []
 
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe speech audio to text.
+
+        Audio is passed as a self-describing blob rather than raw samples so
+        every provider receives a container it can declare, and WAV framing
+        lives in one place instead of in each plugin.
+
+        Args:
+            audio: The encoded audio payload (WAV bytes by default)
+            language: Optional ISO language hint for the provider
+            mime_type: Media type of ``audio``
+
+        Returns:
+            The transcript, or None when the provider cannot produce one
+        """
+        logger.warning(
+            "%s does not support transcription. "
+            "This method should be overridden by the provider implementation.",
+            self.__class__.__name__,
+        )
+        return None
+
     def chat_with_tools(
         self,
         messages: list[dict[str, Any]],

frigate/genai/manager.py

@@ -110,6 +110,12 @@ class GenAIClientManager:
         name = self._role_map.get(GenAIRoleEnum.embeddings)
         return self._get_client(name) if name else None
 
+    @property
+    def transcribe_client(self) -> "GenAIClient | None":
+        """Client configured for the transcribe role."""
+        name = self._role_map.get(GenAIRoleEnum.transcribe)
+        return self._get_client(name) if name else None
+
     def role_info(self) -> dict[str, dict[str, Any]]:
         """Return the model selected for each configured role and its context size.
 
@@ -144,5 +150,11 @@ class GenAIClientManager:
                 "roles": [r.value for r in genai_cfg.roles],
                 "supports_toggleable_thinking": client.supports_toggleable_thinking,
                 "supports_embeddings": client.supports_embeddings,
+                "supports_transcription": client.supports_transcription,
+                # Capabilities of the configured model are above; this maps every
+                # model the provider serves to its own, so the UI can react to a
+                # model selected but not yet saved. Empty when the provider
+                # cannot report capabilities without loading a model.
+                "model_capabilities": client.list_model_capabilities(),
             }
         return result

frigate/genai/plugins/azure-openai.py

@@ -13,6 +13,19 @@ overrides what is genuinely Azure-specific:
 - Context size: Azure does not expose a per-model ``max_model_len`` field
   reliably, so we keep the historical 128K default rather than the
   model-name heuristic used by OpenAI.
+
+Transcription is inherited too: :class:`openai.AzureOpenAI` exposes the same
+``audio.transcriptions.create``. Two Azure-specific caveats apply when using
+the ``transcribe`` role:
+
+- ``model`` must be the Azure *deployment* name, not the underlying model name.
+- The ``api-version`` parsed from ``base_url`` must be 2024-06-01 or later;
+  earlier versions have no transcriptions route and the 404 surfaces only as a
+  generic provider error.
+- Because ``model`` is a deployment name, the inherited check that picks
+  ``languages`` over ``language`` for gpt-transcribe cannot fire unless the
+  deployment happens to be named after the model. Name the deployment
+  ``gpt-transcribe`` to get the right field, or leave the language on ``auto``.
 """
 
 import logging

frigate/genai/plugins/gemini.py

@@ -13,9 +13,14 @@ from google.genai.types import FunctionCallingConfigMode
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
+from frigate.genai.utils import interleave_images
 
 logger = logging.getLogger(__name__)
 
+# Gemini requests carrying inline data are capped at ~20 MB total; stay well
+# under it so the request fails as a log line rather than a 400.
+GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024
+
 
 def _decode_thought_signature(value: Any) -> bytes | None:
     """Decode a base64-encoded thought_signature carried across conversation turns."""
@@ -118,11 +123,16 @@ class GeminiClient(GenAIClient):
         images: list[bytes],
         response_format: dict | None = None,
         enable_thinking: bool = False,
+        image_captions: list[str] | None = None,
     ) -> str | None:
         """Submit a request to Gemini."""
-        contents = [prompt] + [
-            types.Part.from_bytes(data=img, mime_type="image/jpeg") for img in images
+        contents: list[Any] = [
+            part
+            if isinstance(part, str)
+            else types.Part.from_bytes(data=part, mime_type="image/jpeg")
+            for part in interleave_images(prompt, images, image_captions)
         ]
+
         try:
             # Merge runtime_options into generation_config if provided
             generation_config_dict: dict[str, Any] = {"candidate_count": 1}
@@ -136,7 +146,7 @@ class GeminiClient(GenAIClient):
 
             response = self.provider.models.generate_content(
                 model=self.genai_config.model,
-                contents=contents,  # type: ignore[arg-type]
+                contents=contents,
                 config=types.GenerateContentConfig(
                     **generation_config_dict,
                 ),
@@ -157,6 +167,58 @@ class GeminiClient(GenAIClient):
             return None
         return description
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Gemini models accept inline audio parts."""
+        return True
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio by sending it as an inline part alongside a prompt."""
+        if len(audio) > GEMINI_MAX_INLINE_BYTES:
+            logger.warning(
+                "Audio payload of %d bytes exceeds the Gemini inline limit; skipping transcription",
+                len(audio),
+            )
+            return None
+
+        prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
+
+        if language:
+            prompt += f" The speech is in language '{language}'."
+
+        try:
+            contents: list[Any] = [
+                prompt,
+                types.Part.from_bytes(data=audio, mime_type=mime_type),
+            ]
+            response = self.provider.models.generate_content(
+                model=self.genai_config.model,
+                contents=contents,
+                config=types.GenerateContentConfig(candidate_count=1),
+            )
+        except errors.APIError as e:
+            logger.warning("Gemini returned an error: %s", str(e))
+            return None
+        except Exception as e:
+            logger.warning("An unexpected error occurred with Gemini: %s", str(e))
+            return None
+
+        try:
+            if response.text is None:
+                return None
+
+            transcript = response.text.strip()
+        except (ValueError, AttributeError):
+            # No transcript was generated
+            return None
+
+        return transcript or None
+
     def list_models(self) -> list[str]:
         """Return available model names from Gemini."""
         try:

frigate/genai/plugins/llama_cpp.py

@@ -14,7 +14,7 @@ from PIL import Image
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
-from frigate.genai.utils import parse_tool_calls_from_message
+from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
 
 logger = logging.getLogger(__name__)
 
@@ -333,6 +333,7 @@ class LlamaCppClient(GenAIClient):
         images: list[bytes],
         response_format: dict | None = None,
         enable_thinking: bool = False,
+        image_captions: list[str] | None = None,
     ) -> str | None:
         """Submit a request to llama.cpp server."""
         if self.provider is None:
@@ -342,18 +343,17 @@ class LlamaCppClient(GenAIClient):
             return None
 
         try:
-            content = [
-                {
-                    "type": "text",
-                    "text": prompt,
-                }
-            ]
-            for image in images:
-                encoded_image = base64.b64encode(image).decode("utf-8")
+            content: list[dict[str, Any]] = []
+            for part in interleave_images(prompt, images, image_captions):
+                if isinstance(part, str):
+                    content.append({"type": "text", "text": part})
+                    continue
+
+                encoded_image = base64.b64encode(part).decode("utf-8")
                 content.append(
                     {
                         "type": "image_url",
-                        "image_url": {  # type: ignore[dict-item]
+                        "image_url": {
                             "url": f"data:image/jpeg;base64,{encoded_image}",
                         },
                     }
@@ -408,6 +408,126 @@ class LlamaCppClient(GenAIClient):
         """Whether the loaded model supports audio input."""
         return self._supports_audio
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Audio-capable models can transcribe through chat completions."""
+        return self._supports_audio
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio through the OpenAI-compatible transcriptions route.
+
+        llama.cpp serves /v1/audio/transcriptions for any audio-capable model,
+        not only a separately loaded whisper (ggml-org/llama.cpp#21863), so it
+        covers exactly the models supports_transcription detects. It takes the
+        language as a native multipart field, which is the only thing dedicated
+        ASR models honor: they read the chat prompt as contextual biasing, so
+        asking one there to use a language does nothing.
+
+        Falls back to chat completions when the server predates that route.
+        """
+        if self.provider is None:
+            logger.warning(
+                "llama.cpp provider has not been initialized, audio will not be transcribed. Check your llama.cpp configuration."
+            )
+            return None
+
+        if not self._supports_audio:
+            logger.warning(
+                "llama.cpp model '%s' does not accept audio input",
+                self.genai_config.model,
+            )
+            return None
+
+        try:
+            data = {"model": self.genai_config.model, "response_format": "json"}
+
+            if language:
+                data["language"] = language
+
+            response = self._post(
+                f"{self.provider}/v1/audio/transcriptions",
+                files={"file": ("audio.wav", audio, mime_type)},
+                data=data,
+                timeout=self.timeout,
+            )
+
+            if response.status_code == 404:
+                logger.debug(
+                    "llama.cpp server has no /v1/audio/transcriptions route, using chat completions"
+                )
+                return self._transcribe_via_chat(audio, language)
+
+            response.raise_for_status()
+            result = response.json()
+            text = result.get("text") if isinstance(result, dict) else None
+
+            return str(text).strip() or None if text else None
+        except Exception as e:
+            logger.warning("llama.cpp returned an error: %s", str(e))
+            return None
+
+    def _transcribe_via_chat(self, audio: bytes, language: str | None) -> str | None:
+        """Transcribe through /v1/chat/completions, for servers without the
+        transcriptions route.
+
+        The _media_marker / multimodal_data convention is an /embeddings-only
+        protocol, so no marker-refresh retry is needed here.
+        """
+        prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
+
+        if language:
+            prompt += f" The speech is in language '{language}'."
+
+        try:
+            encoded_audio = base64.b64encode(audio).decode("utf-8")
+            payload: dict[str, Any] = {
+                "model": self.genai_config.model,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "input_audio",
+                                "input_audio": {
+                                    "data": encoded_audio,
+                                    "format": "wav",
+                                },
+                            },
+                        ],
+                    },
+                ],
+                **self.provider_options,
+            }
+
+            response = self._post(
+                f"{self.provider}/v1/chat/completions",
+                json=payload,
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            if (
+                result is not None
+                and "choices" in result
+                and len(result["choices"]) > 0
+            ):
+                choice = result["choices"][0]
+
+                if "message" in choice and choice["message"].get("content"):
+                    return str(choice["message"]["content"].strip()) or None
+
+            return None
+        except Exception as e:
+            logger.warning("llama.cpp returned an error: %s", str(e))
+            return None
+
     @property
     def supports_tools(self) -> bool:
         """Whether the loaded model supports tool/function calling."""
@@ -417,28 +537,73 @@ class LlamaCppClient(GenAIClient):
     def supports_toggleable_thinking(self) -> bool:
         return self._supports_reasoning
 
-    def list_models(self) -> list[str]:
-        """Return available model IDs from the llama.cpp server."""
+    def _fetch_models_data(self) -> list[dict[str, Any]]:
+        """Return the raw /v1/models entries, or an empty list if unreachable."""
         base_url = self.provider or (
             self.genai_config.base_url.rstrip("/")
             if self.genai_config.base_url
             else None
         )
+
         if base_url is None:
             return []
+
         try:
             response = self._get(f"{base_url}/v1/models", timeout=10)
             response.raise_for_status()
-            models = []
-            for m in response.json().get("data", []):
-                models.append(m.get("id", "unknown"))
-                for alias in m.get("aliases", []):
-                    models.append(alias)
-            return sorted(models)
+            data = response.json().get("data", [])
         except Exception as e:
             logger.warning("Failed to list llama.cpp models: %s", e)
             return []
 
+        return data if isinstance(data, list) else []
+
+    def list_models(self) -> list[str]:
+        """Return available model IDs from the llama.cpp server."""
+        models: set[str] = set()
+
+        # llama-server lists the id among the aliases when --alias is set
+        for m in self._fetch_models_data():
+            models.add(m.get("id", "unknown"))
+            models.update(m.get("aliases", []))
+
+        return sorted(models)
+
+    def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
+        """Report input modalities for every model the server serves.
+
+        Since ggml-org/llama.cpp#22952 each /v1/models entry carries
+        architecture.input_modalities, so a single request describes every
+        model rather than just the configured one. That is what lets the UI
+        answer "can the model I just picked transcribe" before the config is
+        saved and a client for it exists.
+
+        Models whose entry predates that field are omitted rather than reported
+        as incapable, so an older server falls back to the /props probe instead
+        of silently losing capabilities it actually has.
+        """
+        capabilities: dict[str, dict[str, bool]] = {}
+
+        for model in self._fetch_models_data():
+            architecture = model.get("architecture") or {}
+            modalities = architecture.get("input_modalities")
+
+            if not isinstance(modalities, list) or not modalities:
+                continue
+
+            flags = {
+                "supports_vision": "image" in modalities,
+                "supports_transcription": "audio" in modalities,
+            }
+
+            names = [model.get("id"), *(model.get("aliases") or [])]
+
+            for name in names:
+                if isinstance(name, str) and name:
+                    capabilities[name] = flags
+
+        return capabilities
+
     def get_context_size(self) -> int:
         """Get the context window size for llama.cpp.
 

frigate/genai/plugins/ollama.py

@@ -14,7 +14,7 @@ from ollama import ResponseError
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
-from frigate.genai.utils import parse_tool_calls_from_message
+from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
 
 logger = logging.getLogger(__name__)
 
@@ -50,6 +50,28 @@ def _extract_ollama_stats(response: Any) -> dict[str, Any] | None:
     return stats or None
 
 
+# Ollama replaces each occurrence of this marker in a message, in order, with
+# the next image from the message's images list. Without markers it puts every
+# image before the text.
+IMAGE_PLACEHOLDER = "[img]"
+
+
+def _flatten_parts(parts: list[str | bytes]) -> tuple[str, list[bytes] | None]:
+    """Collapse ordered text and image parts into Ollama's (content, images)
+    shape, marking where each image goes so the order survives."""
+    text: list[str] = []
+    images: list[bytes] = []
+
+    for part in parts:
+        if isinstance(part, bytes):
+            text.append(IMAGE_PLACEHOLDER)
+            images.append(part)
+        elif part:
+            text.append(part)
+
+    return "\n".join(text), (images or None)
+
+
 def _normalize_multimodal_content(
     content: Any,
 ) -> tuple[str | None, list[bytes] | None]:
@@ -58,13 +80,13 @@ def _normalize_multimodal_content(
     The chat API constructs user messages with content as a list of
     ``{"type": "text"}`` and ``{"type": "image_url"}`` parts when a tool
     returns a live frame. Ollama's SDK requires content to be a string and
-    images to be passed in a separate field, so we extract each.
+    images to be passed in a separate field, so images are pulled out and
+    their positions marked with placeholders.
     """
     if not isinstance(content, list):
         return content, None
 
-    text_parts: list[str] = []
-    images: list[bytes] = []
+    parts: list[str | bytes] = []
     for part in content:
         if not isinstance(part, dict):
             continue
@@ -72,17 +94,20 @@ def _normalize_multimodal_content(
         if part_type == "text":
             text = part.get("text")
             if text:
-                text_parts.append(str(text))
+                parts.append(str(text))
         elif part_type == "image_url":
             url = (part.get("image_url") or {}).get("url", "")
             if isinstance(url, str) and url.startswith("data:"):
                 try:
                     encoded = url.split(",", 1)[1]
-                    images.append(base64.b64decode(encoded, validate=True))
+                    parts.append(base64.b64decode(encoded, validate=True))
                 except (ValueError, IndexError, binascii.Error) as e:
                     logger.debug("Failed to decode multimodal image url: %s", e)
 
-    return ("\n".join(text_parts) if text_parts else None), (images or None)
+    if not parts:
+        return None, None
+
+    return _flatten_parts(parts)
 
 
 @register_genai_provider(GenAIProviderEnum.ollama)
@@ -196,58 +221,46 @@ class OllamaClient(GenAIClient):
         images: list[bytes],
         response_format: dict | None = None,
         enable_thinking: bool = False,
+        image_captions: list[str] | None = None,
     ) -> str | None:
-        """Submit a request to Ollama"""
+        """Submit a request to Ollama through the chat API, the same path the
+        tool-calling chat uses, with image placeholders keeping any captions
+        next to their frames."""
         if self.provider is None:
             logger.warning(
                 "Ollama provider has not been initialized, a description will not be generated. Check your Ollama configuration."
             )
             return None
+
+        content, message_images = _flatten_parts(
+            interleave_images(prompt, images, image_captions)
+        )
+        message: dict[str, Any] = {"role": "user", "content": content}
+
+        if message_images:
+            message["images"] = message_images
+
+        request_params = self._build_request_params(
+            [message], None, None, enable_thinking=enable_thinking
+        )
+
+        if response_format and response_format.get("type") == "json_schema":
+            schema = response_format.get("json_schema", {}).get("schema")
+            if schema:
+                request_params["format"] = self._clean_schema_for_ollama(schema)
+
+        logger.debug(
+            "Ollama chat request: model=%s, prompt_len=%s, image_count=%s, "
+            "has_format=%s, think=%s",
+            self.genai_config.model,
+            len(prompt),
+            len(images),
+            "format" in request_params,
+            request_params.get("think"),
+        )
+
         try:
-            ollama_options = {
-                **self.provider_options,
-                **self.genai_config.runtime_options,
-            }
-            if response_format and response_format.get("type") == "json_schema":
-                schema = response_format.get("json_schema", {}).get("schema")
-                if schema:
-                    ollama_options["format"] = self._clean_schema_for_ollama(schema)
-            if self.supports_toggleable_thinking:
-                ollama_options["think"] = enable_thinking
-            logger.debug(
-                "Ollama generate request: model=%s, prompt_len=%s, image_count=%s, "
-                "has_format=%s, options=%s",
-                self.genai_config.model,
-                len(prompt),
-                len(images) if images else 0,
-                "format" in ollama_options,
-                {k: v for k, v in ollama_options.items() if k != "format"},
-            )
-            result = self.provider.generate(
-                self.genai_config.model,
-                prompt,
-                images=images if images else None,
-                **ollama_options,
-            )
-            logger.debug(
-                "Ollama generate response: done=%s, done_reason=%s, eval_count=%s, "
-                "prompt_eval_count=%s, response_len=%s",
-                result.get("done"),
-                result.get("done_reason"),
-                result.get("eval_count"),
-                result.get("prompt_eval_count"),
-                len(result.get("response", "") or ""),
-            )
-            response_text = str(result["response"]).strip()
-            if not response_text:
-                logger.warning(
-                    "Ollama returned a blank response for model %s (done_reason=%s, "
-                    "eval_count=%s). Check model output, ensure thinking is disabled.",
-                    self.genai_config.model,
-                    result.get("done_reason"),
-                    result.get("eval_count"),
-                )
-            return response_text
+            response = self.provider.chat(**request_params)
         except (
             TimeoutException,
             ResponseError,
@@ -257,6 +270,27 @@ class OllamaClient(GenAIClient):
             logger.warning("Ollama returned an error: %s", str(e))
             return None
 
+        logger.debug(
+            "Ollama chat response: done=%s, done_reason=%s, eval_count=%s, "
+            "prompt_eval_count=%s",
+            response.get("done"),
+            response.get("done_reason"),
+            response.get("eval_count"),
+            response.get("prompt_eval_count"),
+        )
+        response_text = self._message_from_response(response)["content"] or ""
+
+        if not response_text:
+            logger.warning(
+                "Ollama returned a blank response for model %s (done_reason=%s, "
+                "eval_count=%s). Check model output, ensure thinking is disabled.",
+                self.genai_config.model,
+                response.get("done_reason"),
+                response.get("eval_count"),
+            )
+
+        return response_text
+
     def list_models(self) -> list[str]:
         """Return available model names from the Ollama server."""
         client = self.provider
@@ -306,6 +340,8 @@ class OllamaClient(GenAIClient):
             }
             if images:
                 msg_dict["images"] = images
+            elif msg.get("images"):
+                msg_dict["images"] = msg["images"]
             if msg.get("tool_call_id"):
                 msg_dict["tool_call_id"] = msg["tool_call_id"]
             if msg.get("name"):

frigate/genai/plugins/openai.py

@@ -11,9 +11,16 @@ from openai import OpenAI
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
+from frigate.genai.utils import interleave_images
 
 logger = logging.getLogger(__name__)
 
+# gpt-transcribe replaced the singular `language` field with a `languages` array
+# and rejects a request that sends both. Older transcription models
+# (gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1) still take the singular
+# form. https://developers.openai.com/api/docs/guides/speech-to-text
+_LANGUAGES_ARRAY_MODEL_PREFIX = "gpt-transcribe"
+
 
 def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None:
     """Build a stats dict from an OpenAI-compatible usage object."""
@@ -63,21 +70,21 @@ class OpenAIClient(GenAIClient):
         images: list[bytes],
         response_format: dict | None = None,
         enable_thinking: bool = False,
+        image_captions: list[str] | None = None,
     ) -> str | None:
         """Submit a request to OpenAI."""
-        encoded_images = [base64.b64encode(image).decode("utf-8") for image in images]
-        messages_content: list[dict] = [
-            {
-                "type": "text",
-                "text": prompt,
-            }
-        ]
-        for image in encoded_images:
+        messages_content: list[dict] = []
+        for part in interleave_images(prompt, images, image_captions):
+            if isinstance(part, str):
+                messages_content.append({"type": "text", "text": part})
+                continue
+
+            encoded = base64.b64encode(part).decode("utf-8")
             messages_content.append(
                 {
                     "type": "image_url",
                     "image_url": {
-                        "url": f"data:image/jpeg;base64,{image}",
+                        "url": f"data:image/jpeg;base64,{encoded}",
                         "detail": "low",
                     },
                 }
@@ -133,6 +140,51 @@ class OpenAIClient(GenAIClient):
             logger.warning("OpenAI returned an error: %s", str(e))
             return None
 
+    @property
+    def supports_transcription(self) -> bool:
+        """OpenAI exposes /v1/audio/transcriptions for its speech models."""
+        return True
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio via the OpenAI audio transcriptions endpoint."""
+        try:
+            # runtime_options are chat-completion parameters; the transcriptions
+            # endpoint rejects unknown fields, so they are deliberately not splatted
+            # in here the way _send() does.
+            request_params: dict[str, Any] = {
+                "model": self.genai_config.model,
+                "file": ("audio.wav", audio, mime_type),
+                "response_format": "text",
+                "timeout": self.timeout,
+            }
+
+            if language:
+                if (
+                    self.genai_config.model.strip()
+                    .lower()
+                    .startswith(_LANGUAGES_ARRAY_MODEL_PREFIX)
+                ):
+                    # not a typed parameter on the SDK method, so it has to ride
+                    # along in extra_body
+                    request_params["extra_body"] = {"languages": [language]}
+                else:
+                    request_params["language"] = language
+
+            result = self.provider.audio.transcriptions.create(**request_params)
+        except (TimeoutException, Exception) as e:
+            logger.warning("OpenAI returned an error: %s", str(e))
+            return None
+
+        # response_format="text" yields a bare string, but some compatible
+        # servers still return the object form
+        text = result if isinstance(result, str) else getattr(result, "text", None)
+        return text.strip() if text else None
+
     def list_models(self) -> list[str]:
         """Return available model IDs from the OpenAI-compatible API."""
         try:

frigate/genai/prompts.py

@@ -59,6 +59,13 @@ def get_review_field_guidelines(response_style: str = "default") -> dict[str, st
     }
 
 
+# Explains the per-frame labels and tracker notes used by the annotated frame
+# mode. Neither the notes nor this guidance say whether repeated detections are
+# the same subject, since the tracking data cannot tell.
+FRAME_ANNOTATION_GUIDANCE = """- Each image below is immediately preceded by a text label giving its frame number and how many seconds into the sequence it was captured. Use these labels to track the order of events and the time between them.
+- Some images below are preceded by notes from the camera's object tracker recording what changed at that point: an object being first detected, starting to move, reversing direction, stopping, or no longer being detected. These notes come from tracking data rather than from the images, and they are reliable. Use them to establish how many distinct activities occur and in what order, and describe every one of them."""
+
+
 def build_review_description_prompt(
     review_data: dict[str, Any],
     thumbnails: list[bytes],
@@ -66,8 +73,13 @@ def build_review_description_prompt(
     preferred_language: str | None,
     activity_context_prompt: str,
     response_style: str = "default",
+    frame_captions: list[str] | None = None,
 ) -> str:
-    """Build the prompt for review activity description generation."""
+    """Build the prompt for review activity description generation.
+
+    When `frame_captions` is set, each caption is sent directly before its
+    image, so the prompt explains that layout.
+    """
 
     def get_concern_prompt() -> str:
         if concerns:
@@ -93,6 +105,7 @@ def build_review_description_prompt(
             return "\n- (No objects detected)"
 
     fields = get_review_field_guidelines(response_style)
+    frame_guidance = f"\n{FRAME_ANNOTATION_GUIDANCE}" if frame_captions else ""
 
     return f"""
 Your task is to analyze a sequence of images taken in chronological order from a security camera.
@@ -130,7 +143,7 @@ Respond with a JSON object matching the provided schema. Field-specific guidance
 ## Sequence Details
 
 - Camera: {review_data["camera"]}
-- Total frames: {len(thumbnails)} (Frame 1 = earliest, Frame {len(thumbnails)} = latest)
+- Total frames: {len(thumbnails)} (Frame 1 = earliest, Frame {len(thumbnails)} = latest){frame_guidance}
 - Activity started at {review_data["start"]} and lasted {review_data["duration"]} seconds
 - Zones involved: {", ".join(review_data["zones"]) if review_data["zones"] else "None"}
 

frigate/genai/utils.py

@@ -7,6 +7,25 @@ from typing import Any
 logger = logging.getLogger(__name__)
 
 
+def interleave_images(
+    prompt: str, images: list[bytes], captions: list[str] | None = None
+) -> list[str | bytes]:
+    """The prompt, then each image preceded by its caption when one is given.
+
+    Providers map the text and image parts onto their own request format, so
+    every provider sends the same order.
+    """
+    parts: list[str | bytes] = [prompt]
+
+    for index, image in enumerate(images):
+        if captions and index < len(captions):
+            parts.append(captions[index])
+
+        parts.append(image)
+
+    return parts
+
+
 def parse_tool_calls_from_message(
     message: dict[str, Any],
 ) -> list[dict[str, Any]] | None:

frigate/ptz/onvif.py

@@ -1120,6 +1120,18 @@ class OnvifController:
                     f"Camera {camera_name} is still in ONVIF 'MOVING' status."
                 )
 
+    async def _shutdown(self) -> None:
+        """Close the camera sessions and cancel the tasks running on the loop."""
+        for cam_name in list(self.cams):
+            await self._close_camera(cam_name)
+
+        tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
+
+        for task in tasks:
+            task.cancel()
+
+        await asyncio.gather(*tasks, return_exceptions=True)
+
     def close(self) -> None:
         """Gracefully shut down the ONVIF controller."""
         if not hasattr(self, "loop") or self.loop.is_closed():
@@ -1127,6 +1139,16 @@ class OnvifController:
             return
 
         logger.info("Exiting ONVIF controller...")
+
+        # anything left open here is garbage collected during interpreter
+        # shutdown, where its warnings can no longer be logged cleanly
+        try:
+            asyncio.run_coroutine_threadsafe(self._shutdown(), self.loop).result(
+                timeout=5
+            )
+        except TimeoutError:
+            logger.debug("Timed out closing ONVIF sessions")
+
         self.config_subscriber.stop()
 
         def stop_and_cleanup():

frigate/test/test_audio_transcription_genai.py

@@ -0,0 +1,286 @@
+"""Config validation and the historical path for the audio_transcription GenAI backend."""
+
+import unittest
+from copy import deepcopy
+from unittest.mock import MagicMock, patch
+
+from pydantic import ValidationError
+
+from frigate.config import FrigateConfig
+from frigate.config.camera.genai import GenAIConfig, GenAIRoleEnum
+from frigate.config.classification import AudioTranscriptionModelEnum
+from frigate.const import UPDATE_EVENT_DESCRIPTION
+from frigate.data_processing.post.audio_transcription import (
+    AudioTranscriptionPostProcessor,
+)
+from frigate.data_processing.types import PostProcessDataEnum
+
+
+class TestAudioTranscriptionGenAIConfig(unittest.TestCase):
+    def setUp(self):
+        self.base = {
+            "mqtt": {"host": "mqtt"},
+            "cameras": {
+                "back": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {
+                                "path": "rtsp://10.0.0.1:554/video",
+                                "roles": ["detect", "audio"],
+                            }
+                        ]
+                    },
+                    "detect": {"height": 1080, "width": 1920, "fps": 5},
+                    "audio": {"enabled": True},
+                }
+            },
+        }
+
+    def _config(self, **overrides) -> dict:
+        config = deepcopy(self.base)
+        config.update(deepcopy(overrides))
+        return config
+
+    def _provider(self, roles: list[str]) -> dict:
+        return {
+            "whisper_cloud": {
+                "provider": "openai",
+                "model": "gpt-4o-transcribe",
+                "api_key": "k",
+                "roles": roles,
+            }
+        }
+
+    def test_default_model_is_whisper_enum(self):
+        config = FrigateConfig(**self._config())
+        self.assertEqual(
+            config.audio_transcription.model, AudioTranscriptionModelEnum.whisper
+        )
+
+    def test_whisper_string_coerces_to_enum(self):
+        config = FrigateConfig(
+            **self._config(audio_transcription={"enabled": True, "model": "whisper"})
+        )
+        self.assertIsInstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+
+    def test_provider_name_stays_a_string(self):
+        config = FrigateConfig(
+            **self._config(
+                genai=self._provider(["transcribe"]),
+                audio_transcription={"enabled": True, "model": "whisper_cloud"},
+            )
+        )
+        self.assertNotIsInstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+        self.assertEqual(config.audio_transcription.model, "whisper_cloud")
+
+    def test_unspecified_model_falls_back_to_whisper(self):
+        """An empty value must not read as a GenAI provider that resolves to no client."""
+        for value in (None, "", "   "):
+            with self.subTest(repr(value)):
+                config = FrigateConfig(
+                    **self._config(
+                        audio_transcription={"enabled": True, "model": value}
+                    )
+                )
+                self.assertIs(
+                    config.audio_transcription.model,
+                    AudioTranscriptionModelEnum.whisper,
+                )
+
+    def test_missing_genai_key_raises(self):
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(
+                **self._config(
+                    audio_transcription={"enabled": True, "model": "nope"},
+                )
+            )
+        self.assertIn("is not a valid GenAI config key", str(ctx.exception))
+
+    def test_provider_without_role_raises(self):
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(
+                **self._config(
+                    genai=self._provider(["descriptions"]),
+                    audio_transcription={"enabled": True, "model": "whisper_cloud"},
+                )
+            )
+        self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
+
+    def test_global_off_camera_on_still_validates(self):
+        """Global-off/camera-on is a supported deployment and must not skip the check."""
+        config = self._config(
+            genai=self._provider(["descriptions"]),
+            audio_transcription={"enabled": False, "model": "whisper_cloud"},
+        )
+        config["cameras"]["back"]["audio_transcription"] = {"enabled": True}
+
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(**config)
+
+        self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
+
+    def test_disabled_transcription_skips_validation(self):
+        config = FrigateConfig(
+            **self._config(audio_transcription={"enabled": False, "model": "nope"})
+        )
+        self.assertEqual(config.audio_transcription.model, "nope")
+
+    def test_camera_level_model_is_rejected(self):
+        config = self._config()
+        config["cameras"]["back"]["audio_transcription"] = {
+            "enabled": True,
+            "model": "whisper",
+        }
+
+        with self.assertRaises(ValidationError):
+            FrigateConfig(**config)
+
+    def test_default_roles_do_not_include_transcribe(self):
+        """Backward compatibility: existing providers must not silently claim it."""
+        genai = GenAIConfig(provider="openai", model="gpt-4o")
+        self.assertNotIn(GenAIRoleEnum.transcribe, genai.roles)
+
+    def test_two_providers_claiming_transcribe_raises(self):
+        genai = self._provider(["transcribe"])
+        genai["other"] = {
+            "provider": "gemini",
+            "model": "gemini-2.0-flash",
+            "api_key": "k",
+            "roles": ["transcribe"],
+        }
+
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(
+                **self._config(
+                    genai=genai,
+                    audio_transcription={"enabled": True, "model": "whisper_cloud"},
+                )
+            )
+
+        self.assertIn("each role must have", str(ctx.exception))
+
+    def test_transcribe_rejected_on_provider_without_audio_input(self):
+        with self.assertRaises(ValidationError) as ctx:
+            GenAIConfig(provider="ollama", model="llava", roles=["transcribe"])
+
+        self.assertIn("does not support audio input", str(ctx.exception))
+
+
+class TestAudioTranscriptionPostProcessorGenAI(unittest.TestCase):
+    """The recorded-speech path must reach the provider and skip the local model."""
+
+    def setUp(self):
+        self.config = FrigateConfig(
+            **{
+                "mqtt": {"host": "mqtt"},
+                "genai": {
+                    "whisper_cloud": {
+                        "provider": "openai",
+                        "model": "gpt-4o-transcribe",
+                        "api_key": "k",
+                        "roles": ["transcribe"],
+                    }
+                },
+                "audio_transcription": {
+                    "enabled": True,
+                    "model": "whisper_cloud",
+                    "language": "en",
+                },
+                "cameras": {
+                    "back": {
+                        "ffmpeg": {
+                            "inputs": [
+                                {
+                                    "path": "rtsp://10.0.0.1:554/video",
+                                    "roles": ["detect", "audio"],
+                                }
+                            ]
+                        },
+                        "detect": {"height": 1080, "width": 1920, "fps": 5},
+                        "audio": {"enabled": True},
+                    }
+                },
+            }
+        )
+
+        self.client = MagicMock()
+        self.client.transcribe.return_value = "recorded speech"
+        self.manager = MagicMock()
+        self.manager.transcribe_client = self.client
+        self.requestor = MagicMock()
+
+        self.processor = AudioTranscriptionPostProcessor(
+            self.config,
+            self.requestor,
+            MagicMock(),
+            MagicMock(),
+            self.manager,
+        )
+
+    def _process(self):
+        self.processor.process_data(
+            {
+                "event_id": "1234.5-abc",
+                "camera": "back",
+                "event": {
+                    "id": "1234.5-abc",
+                    "camera": "back",
+                    "start_time": 100.0,
+                    "end_time": 110.0,
+                    "data": {},
+                },
+            },
+            PostProcessDataEnum.tracked_object,
+        )
+
+    def test_local_recognizer_is_never_built(self):
+        self.assertTrue(self.processor._use_genai)
+        self.assertIsNone(self.processor.recognizer)
+
+    def test_audio_bytes_and_language_reach_the_client(self):
+        with patch(
+            "frigate.data_processing.post.audio_transcription.get_audio_from_recording",
+            return_value=b"RIFF....WAVE",
+        ):
+            self._process()
+
+        self.client.transcribe.assert_called_once()
+        self.assertEqual(self.client.transcribe.call_args.args[0], b"RIFF....WAVE")
+        self.assertEqual(self.client.transcribe.call_args.kwargs["language"], "en")
+
+    def test_transcript_is_published_as_the_description(self):
+        with patch(
+            "frigate.data_processing.post.audio_transcription.get_audio_from_recording",
+            return_value=b"RIFF....WAVE",
+        ):
+            self._process()
+
+        topics = [call.args[0] for call in self.requestor.send_data.call_args_list]
+        self.assertIn(UPDATE_EVENT_DESCRIPTION, topics)
+
+        payload = next(
+            call.args[1]
+            for call in self.requestor.send_data.call_args_list
+            if call.args[0] == UPDATE_EVENT_DESCRIPTION
+        )
+        self.assertEqual(payload["description"], "recorded speech")
+        self.assertEqual(payload["id"], "1234.5-abc")
+
+    def test_missing_client_publishes_nothing(self):
+        self.manager.transcribe_client = None
+
+        with patch(
+            "frigate.data_processing.post.audio_transcription.get_audio_from_recording",
+            return_value=b"RIFF....WAVE",
+        ):
+            self._process()
+
+        self.requestor.send_data.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/test/test_audio_transcription_live_buffer.py

@@ -0,0 +1,288 @@
+"""Live GenAI transcription: sliding overlapped windows and their lifecycle."""
+
+import io
+import threading
+import unittest
+import wave
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+
+from frigate.config import FrigateConfig
+from frigate.const import AUDIO_DURATION, AUDIO_SAMPLE_RATE
+from frigate.data_processing.real_time.audio_transcription import (
+    GENAI_WINDOW_CHUNKS,
+    AudioTranscriptionRealTimeProcessor,
+)
+
+CHUNK_SAMPLES = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE))
+
+
+def _chunk(amplitude: int) -> np.ndarray:
+    """One audio-detector-sized chunk of int16 samples at a constant amplitude."""
+    return np.full(CHUNK_SAMPLES, amplitude, dtype=np.int16)
+
+
+class TestLiveGenAITranscription(unittest.TestCase):
+    def setUp(self):
+        self.config = FrigateConfig(
+            **{
+                "mqtt": {"host": "mqtt"},
+                "genai": {
+                    "whisper_cloud": {
+                        "provider": "openai",
+                        "model": "gpt-4o-transcribe",
+                        "api_key": "k",
+                        "roles": ["transcribe"],
+                    }
+                },
+                "audio_transcription": {
+                    "enabled": True,
+                    "model": "whisper_cloud",
+                    "language": "en",
+                },
+                "cameras": {
+                    "back": {
+                        "ffmpeg": {
+                            "inputs": [
+                                {
+                                    "path": "rtsp://10.0.0.1:554/video",
+                                    "roles": ["detect", "audio"],
+                                }
+                            ]
+                        },
+                        "detect": {"height": 1080, "width": 1920, "fps": 5},
+                        "audio": {"enabled": True},
+                    }
+                },
+            }
+        )
+
+        self.client = MagicMock()
+        self.client.transcribe.return_value = "hello"
+        self.manager = MagicMock()
+        self.manager.transcribe_client = self.client
+        self.requestor = MagicMock()
+
+        self.processor = AudioTranscriptionRealTimeProcessor(
+            config=self.config,
+            camera_config=self.config.cameras["back"],
+            requestor=self.requestor,
+            model_runner=None,
+            metrics=MagicMock(),
+            stop_event=threading.Event(),
+            genai_manager=self.manager,
+        )
+
+    def _feed(self, chunk: np.ndarray):
+        return (
+            self.processor._AudioTranscriptionRealTimeProcessor__process_audio_stream(
+                chunk
+            )
+        )
+
+    def _sent_wav(self, call_index: int) -> wave.Wave_read:
+        payload = self.client.transcribe.call_args_list[call_index].args[0]
+        return wave.open(io.BytesIO(payload), "rb")
+
+    def test_uses_genai_path(self):
+        self.assertTrue(self.processor._use_genai)
+
+    def test_first_chunk_does_not_transcribe(self):
+        self.assertIsNone(self._feed(_chunk(4000)))
+        self.client.transcribe.assert_not_called()
+
+    def test_full_window_transcribes_two_chunks(self):
+        self._feed(_chunk(4000))
+        result = self._feed(_chunk(4000))
+
+        self.assertEqual(result, ("hello", False))
+        self.client.transcribe.assert_called_once()
+        self.assertEqual(
+            self.client.transcribe.call_args.kwargs["language"],
+            "en",
+        )
+
+        with self._sent_wav(0) as wav:
+            self.assertEqual(wav.getnchannels(), 1)
+            self.assertEqual(wav.getsampwidth(), 2)
+            self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
+            self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
+
+    def test_window_slides_with_overlap(self):
+        """The third chunk's window is chunks 2+3, not 3 alone and not 1+2+3."""
+        self.client.transcribe.side_effect = ["one two", "two three"]
+
+        self._feed(_chunk(1000))
+        self._feed(_chunk(2000))
+        result = self._feed(_chunk(3000))
+
+        self.assertEqual(self.client.transcribe.call_count, 2)
+
+        with self._sent_wav(1) as wav:
+            self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
+            samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        self.assertEqual(samples[0], 2000)
+        self.assertEqual(samples[-1], 3000)
+
+        # the shared "two" appears once
+        self.assertEqual(result, ("one two three", False))
+
+    def test_silent_window_is_not_uploaded(self):
+        self._feed(_chunk(0))
+        self.assertIsNone(self._feed(_chunk(0)))
+        self.client.transcribe.assert_not_called()
+
+    def test_silent_window_ends_a_pending_utterance(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        # the gate covers the whole window, so it takes GENAI_WINDOW_CHUNKS
+        # silent chunks to push the last speech out of it
+        self._feed(_chunk(0))
+
+        self.assertEqual(self._feed(_chunk(0)), ("hello", True))
+
+    def test_empty_transcript_ends_a_pending_utterance(self):
+        self.client.transcribe.side_effect = ["hello", ""]
+
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
+
+    def test_empty_transcript_with_nothing_pending_returns_none(self):
+        self.client.transcribe.return_value = ""
+
+        self._feed(_chunk(4000))
+        self.assertIsNone(self._feed(_chunk(4000)))
+
+    def test_reset_clears_committed_text_and_window(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        self.processor.reset()
+
+        self.assertEqual(self.processor._genai_committed, "")
+        self.assertEqual(len(self.processor._genai_window), 0)
+
+        # a fresh window is required again before the next request
+        self.client.transcribe.reset_mock()
+        self._feed(_chunk(4000))
+        self.client.transcribe.assert_not_called()
+
+    def test_check_unload_model_clears_once_then_is_idempotent(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        self.processor.check_unload_model()
+
+        self.requestor.send_data.assert_called_once_with("back/audio/transcription", "")
+        self.assertEqual(self.processor._genai_committed, "")
+        self.assertEqual(len(self.processor._genai_window), 0)
+
+        self.processor.check_unload_model()
+        self.requestor.send_data.assert_called_once()
+
+    def test_build_recognizer_never_loads_a_local_model(self):
+        with patch(
+            "frigate.data_processing.real_time.audio_transcription.FasterWhisperASR"
+        ) as whisper:
+            self.processor._AudioTranscriptionRealTimeProcessor__build_recognizer()
+
+        whisper.assert_not_called()
+        self.assertIsNone(self.processor.stream)
+
+    def test_clear_audio_recognizer_request_only_resets(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        with patch.object(
+            self.processor,
+            "_AudioTranscriptionRealTimeProcessor__build_recognizer",
+        ) as build:
+            result = self.processor.handle_request("clear_audio_recognizer", {})
+
+        build.assert_not_called()
+        self.assertTrue(result["success"])
+        self.assertEqual(self.processor._genai_committed, "")
+
+    def test_missing_client_logs_and_returns_none(self):
+        self.manager.transcribe_client = None
+
+        self.assertIsNone(self._feed(_chunk(4000)))
+        self.assertIsNone(self._feed(_chunk(4000)))
+
+    def test_dropped_audio_discards_the_buffered_window(self):
+        """A gap in the stream must not be spliced into a single window.
+
+        Dropping a queued chunk leaves the next one non-adjacent to what is
+        buffered, so concatenating them would hand the provider audio with a
+        hole in it and break the 50% overlap the stitcher relies on.
+        """
+        self._feed(_chunk(4000))
+        self.assertEqual(len(self.processor._genai_window), 1)
+
+        # the producer discards a chunk while the consumer is blocked
+        self.processor._audio_dropped.set()
+
+        self._feed(_chunk(5000))
+
+        # the buffered chunk was discarded, so this one starts a fresh window
+        self.assertEqual(len(self.processor._genai_window), 1)
+        self.client.transcribe.assert_not_called()
+
+        # and the window that does go out holds only contiguous audio
+        self._feed(_chunk(5000))
+        self.client.transcribe.assert_called_once()
+
+        with self._sent_wav(0) as wav:
+            samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
+        self.assertTrue((samples == 5000).all(), "window spliced across the gap")
+
+    def test_dropped_audio_ends_a_pending_utterance(self):
+        """Committed text cannot be stitched across missing speech."""
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+        self.assertEqual(self.processor._genai_committed, "hello")
+
+        self.processor._audio_dropped.set()
+
+        self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
+        self.assertEqual(len(self.processor._genai_window), 0)
+
+    def test_drop_flag_is_consumed_once(self):
+        self.processor._audio_dropped.set()
+        self._feed(_chunk(4000))
+
+        self.assertFalse(self.processor._audio_dropped.is_set())
+
+    def test_full_queue_flags_a_drop(self):
+        for i in range(self.processor.audio_queue.maxsize + 1):
+            self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
+
+        self.assertTrue(self.processor._audio_dropped.is_set())
+
+    def test_queue_is_bounded_and_drops_oldest(self):
+        maxsize = self.processor.audio_queue.maxsize
+        self.assertGreater(maxsize, 0)
+
+        for i in range(maxsize + 5):
+            self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
+
+        self.assertEqual(self.processor.audio_queue.qsize(), maxsize)
+
+        # the newest chunk survived, the oldest did not
+        remaining = []
+        while not self.processor.audio_queue.empty():
+            remaining.append(self.processor.audio_queue.get_nowait()[1][0])
+
+        self.assertEqual(remaining[-1], maxsize + 5)
+        self.assertNotIn(1, remaining)
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/test/test_audio_util.py

@@ -0,0 +1,176 @@
+"""Tests for the WAV helpers and transcript stitcher in frigate.util.audio."""
+
+import io
+import struct
+import unittest
+import wave
+
+import numpy as np
+
+from frigate.const import AUDIO_SAMPLE_RATE
+from frigate.util.audio import fix_wav_header, pcm16_to_wav, stitch_transcripts
+
+
+def _wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
+    buffer = io.BytesIO()
+
+    with wave.open(buffer, "wb") as out:
+        out.setnchannels(1)
+        out.setsampwidth(2)
+        out.setframerate(sample_rate)
+        out.writeframes(samples.tobytes())
+
+    return buffer.getvalue()
+
+
+class TestPcm16ToWav(unittest.TestCase):
+    def test_round_trips_through_wave(self):
+        samples = np.arange(-1000, 1000, dtype=np.int16)
+
+        with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
+            self.assertEqual(wav.getnchannels(), 1)
+            self.assertEqual(wav.getsampwidth(), 2)
+            self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
+            self.assertEqual(wav.getnframes(), samples.size)
+            decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        np.testing.assert_array_equal(decoded, samples)
+
+    def test_casts_non_int16_input(self):
+        samples = np.array([0.0, 100.0, -100.0], dtype=np.float32)
+
+        with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
+            decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        np.testing.assert_array_equal(decoded, np.array([0, 100, -100], np.int16))
+
+    def test_honors_sample_rate(self):
+        with wave.open(
+            io.BytesIO(pcm16_to_wav(np.zeros(4, np.int16), 8000)), "rb"
+        ) as w:
+            self.assertEqual(w.getframerate(), 8000)
+
+
+class TestFixWavHeader(unittest.TestCase):
+    def test_rewrites_placeholder_sizes(self):
+        samples = np.arange(64, dtype=np.int16)
+        data = bytearray(_wav(samples))
+
+        # ffmpeg piping to non-seekable stdout leaves both sizes unpatched
+        struct.pack_into("<I", data, 4, 0xFFFFFFFF)
+        data_offset = data.index(b"data")
+        struct.pack_into("<I", data, data_offset + 4, 0xFFFFFFFF)
+
+        fixed = fix_wav_header(bytes(data))
+
+        self.assertEqual(struct.unpack_from("<I", fixed, 4)[0], len(fixed) - 8)
+        self.assertEqual(
+            struct.unpack_from("<I", fixed, data_offset + 4)[0],
+            len(fixed) - (data_offset + 8),
+        )
+
+        with wave.open(io.BytesIO(fixed), "rb") as wav:
+            self.assertEqual(wav.getnframes(), samples.size)
+
+    def test_leaves_a_well_formed_header_alone(self):
+        data = _wav(np.arange(32, dtype=np.int16))
+        self.assertEqual(fix_wav_header(data), data)
+
+    def test_non_riff_payload_passes_through(self):
+        self.assertEqual(fix_wav_header(b"not a wav"), b"not a wav")
+        self.assertEqual(fix_wav_header(b""), b"")
+
+
+class TestStitchTranscripts(unittest.TestCase):
+    def test_table(self):
+        cases = [
+            # (committed, incoming, expected, description)
+            (
+                "the quick brown",
+                "brown fox jumps",
+                "the quick brown fox jumps",
+                "one word",
+            ),
+            (
+                "and then the quick brown",
+                "the quick brown fox",
+                "and then the quick brown fox",
+                "multi word",
+            ),
+            (
+                "hello there",
+                "general kenobi",
+                "hello there general kenobi",
+                "no overlap",
+            ),
+            ("the quick brown fox", "brown fox", "the quick brown fox", "contained"),
+            ("", "first words", "first words", "empty committed"),
+            ("already here", "", "already here", "empty incoming"),
+            (
+                "  spaced   out  ",
+                "out again",
+                "spaced out again",
+                "whitespace normalized",
+            ),
+        ]
+
+        for committed, incoming, expected, description in cases:
+            with self.subTest(description):
+                self.assertEqual(stitch_transcripts(committed, incoming), expected)
+
+    def test_overlap_found_mid_window(self):
+        """The shared run is rarely at the start of the new window.
+
+        The provider re-transcribes the overlapping audio independently and
+        often renders its first word differently, so anchoring the match to the
+        start of the incoming window duplicates the whole phrase.
+        """
+        self.assertEqual(
+            stitch_transcripts(
+                "this is just gonna be a fun time", "It's gonna be a fun time."
+            ),
+            "this is just gonna be a fun time",
+        )
+
+    def test_overlap_longer_than_five_words(self):
+        """The cap is bounded by window duration, not by the old 5-word n-gram."""
+        self.assertEqual(
+            stitch_transcripts(
+                "well anyway one two three four five six",
+                "one two three four five six seven",
+            ),
+            "well anyway one two three four five six seven",
+        )
+
+    def test_repeated_phrase_keeps_its_second_utterance(self):
+        """Preferring the earliest match is what protects a real repeat."""
+        self.assertEqual(
+            stitch_transcripts("a b c fun time", "fun time fun time"),
+            "a b c fun time fun time",
+        )
+
+    def test_window_wholly_repeating_the_tail_is_dropped(self):
+        """The accepted trade-off: an entirely redundant window adds nothing."""
+        self.assertEqual(stitch_transcripts("go go go", "go go go"), "go go go")
+
+    def test_revises_a_mistranscribed_tail(self):
+        """A wrong last word would otherwise block every alignment.
+
+        Those words came from the newest audio, which the next window re-covers,
+        so replacing them is better than duplicating the phrase behind them.
+        """
+        self.assertEqual(
+            stitch_transcripts("Yeah. this is Jessica.", "This is just gonna be fun."),
+            "Yeah. this is just gonna be fun.",
+        )
+
+    def test_revision_needs_more_than_one_shared_word(self):
+        """A revision deletes published text, so it takes real evidence."""
+        self.assertEqual(
+            stitch_transcripts("the cat sat on a mat", "a dog barked"),
+            "the cat sat on a mat a dog barked",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/test/test_dst_transitions.py

@@ -0,0 +1,54 @@
+"""Tests for get_dst_transitions."""
+
+import datetime
+import unittest
+
+from frigate.util.time import get_dst_transitions
+
+
+class TestDstTransitions(unittest.TestCase):
+    def test_dst_transition_splits_periods_at_the_transition(self):
+        start = datetime.datetime(2026, 3, 7, 12, tzinfo=datetime.UTC).timestamp()
+        end = start + 2 * 86400
+        spring = datetime.datetime(2026, 3, 8, 7, tzinfo=datetime.UTC).timestamp()
+        self.assertEqual(
+            get_dst_transitions("America/New_York", start, end),
+            [(start, spring, -18000), (spring, end, -14400)],
+        )
+
+    def test_dst_transition_is_not_reported_a_day_late(self):
+        # local midnight on the day of the change used to report the
+        # transition a full day after it actually happened
+        start = datetime.datetime(2024, 3, 10, 5, tzinfo=datetime.UTC).timestamp()
+        end = start + 3 * 86400
+        spring = datetime.datetime(2024, 3, 10, 7, tzinfo=datetime.UTC).timestamp()
+        self.assertEqual(
+            get_dst_transitions("America/New_York", start, end),
+            [(start, spring, -18000), (spring, end, -14400)],
+        )
+
+    def test_dst_transition_after_the_last_daily_probe_is_found(self):
+        start = datetime.datetime(2024, 11, 2, 12, tzinfo=datetime.UTC).timestamp()
+        end = datetime.datetime(2024, 11, 3, 10, tzinfo=datetime.UTC).timestamp()
+        fall = datetime.datetime(2024, 11, 3, 6, tzinfo=datetime.UTC).timestamp()
+        self.assertEqual(
+            get_dst_transitions("America/New_York", start, end),
+            [(start, fall, -14400), (fall, end, -18000)],
+        )
+
+    def test_no_transition_returns_a_single_period(self):
+        start = datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC).timestamp()
+        end = start + 5 * 86400
+        self.assertEqual(
+            get_dst_transitions("America/New_York", start, end),
+            [(start, end, -14400)],
+        )
+
+    def test_invalid_zone_retains_utc_fallback(self):
+        self.assertEqual(
+            get_dst_transitions("Invalid/Timezone", 100, 200), [(100, 200, 0)]
+        )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

frigate/test/test_genai_providers.py

@@ -360,9 +360,70 @@ class TestOllamaProvider(unittest.TestCase):
         from frigate.genai.plugins.ollama import _normalize_multimodal_content
 
         text, images = _normalize_multimodal_content(MULTIMODAL_MESSAGES[-1]["content"])
-        self.assertIn("live image", text)
-        self.assertEqual(len(images), 1)
-        self.assertEqual(images[0], b"\xff\xd8\xff\xd9")
+        self.assertEqual(
+            text, "Here is the current live image from camera 'front'.\n[img]"
+        )
+        self.assertEqual(images, [b"\xff\xd8\xff\xd9"])
+
+    def test_normalize_keeps_text_and_image_order(self):
+        from frigate.genai.plugins.ollama import _normalize_multimodal_content
+
+        text, images = _normalize_multimodal_content(
+            [
+                {"type": "text", "text": "intro"},
+                {"type": "text", "text": "Frame 1"},
+                {"type": "image_url", "image_url": {"url": _IMAGE_DATA_URI}},
+                {"type": "text", "text": "Frame 2"},
+                {"type": "image_url", "image_url": {"url": _IMAGE_DATA_URI}},
+            ]
+        )
+        self.assertEqual(text, "intro\nFrame 1\n[img]\nFrame 2\n[img]")
+        self.assertEqual(len(images), 2)
+
+    def test_send_uses_chat_with_captions_before_each_image(self):
+        client = self._client()
+        client.provider = MagicMock()
+        client.provider.chat.return_value = {
+            "message": {"content": '{"ok": true}'},
+            "done": True,
+            "done_reason": "stop",
+        }
+        client._supports_thinking_cache = False
+
+        result = client._send(
+            "prompt",
+            [b"a", b"b"],
+            {"type": "json_schema", "json_schema": {"schema": {"type": "object"}}},
+            image_captions=["Frame 1 of 2", "Frame 2 of 2"],
+        )
+
+        self.assertEqual(result, '{"ok": true}')
+        client.provider.generate.assert_not_called()
+        params = client.provider.chat.call_args.kwargs
+        self.assertEqual(
+            params["messages"],
+            [
+                {
+                    "role": "user",
+                    "content": "prompt\nFrame 1 of 2\n[img]\nFrame 2 of 2\n[img]",
+                    "images": [b"a", b"b"],
+                }
+            ],
+        )
+        self.assertEqual(params["format"], {"type": "object"})
+        self.assertNotIn("think", params)
+
+    def test_send_without_captions_puts_images_after_prompt(self):
+        client = self._client()
+        client.provider = MagicMock()
+        client.provider.chat.return_value = {"message": {"content": "ok"}, "done": True}
+        client._supports_thinking_cache = False
+
+        client._send("prompt", [b"a"])
+
+        message = client.provider.chat.call_args.kwargs["messages"][0]
+        self.assertEqual(message["content"], "prompt\n[img]")
+        self.assertEqual(message["images"], [b"a"])
 
 
 # ---------------------------------------------------------------------------
@@ -519,6 +580,281 @@ class TestLlamaCppProvider(unittest.TestCase):
         client = self._validated_client(4096, {"context_size": 32768})
         self.assertEqual(client.get_context_size(), 32768)
 
+    def test_list_models_dedupes_alias_matching_id(self):
+        client = self._client()
+        models_data = [
+            {"id": "qwen3-asr", "aliases": ["qwen3-asr"]},
+            {"id": "gemma", "aliases": ["gemma", "g4"]},
+        ]
+        with patch.object(client, "_fetch_models_data", return_value=models_data):
+            self.assertEqual(client.list_models(), ["g4", "gemma", "qwen3-asr"])
+
+
+# ---------------------------------------------------------------------------
+# transcribe role
+# ---------------------------------------------------------------------------
+WAV_BYTES = b"RIFF$\x00\x00\x00WAVEfmt "
+
+
+class TestOpenAITranscribe(unittest.TestCase):
+    def _client(self):
+        return _make_client(
+            "openai",
+            model="gpt-4o-transcribe",
+            api_key="k",
+            base_url="http://localhost:9999/v1",
+            runtime_options={"temperature": 0.7},
+        )
+
+    def test_supports_transcription(self):
+        self.assertTrue(self._client().supports_transcription)
+
+    def test_passes_file_tuple_and_language(self):
+        client = self._client()
+        create = MagicMock(return_value="  hello there  ")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "hello there")
+
+        kwargs = create.call_args.kwargs
+        self.assertEqual(kwargs["model"], "gpt-4o-transcribe")
+        self.assertEqual(kwargs["file"], ("audio.wav", WAV_BYTES, "audio/wav"))
+        self.assertEqual(kwargs["language"], "en")
+        self.assertEqual(kwargs["response_format"], "text")
+
+    def test_does_not_forward_runtime_options(self):
+        """runtime_options are chat parameters; /audio/transcriptions rejects them."""
+        client = self._client()
+        create = MagicMock(return_value="hi")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        client.transcribe(WAV_BYTES)
+
+        self.assertNotIn("temperature", create.call_args.kwargs)
+
+    def test_gpt_transcribe_uses_languages_array(self):
+        """gpt-transcribe replaced `language` with a `languages` array."""
+        client = _make_client("openai", model="gpt-transcribe", api_key="k")
+        create = MagicMock(return_value="hi")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        client.transcribe(WAV_BYTES, language="en")
+
+        kwargs = create.call_args.kwargs
+        self.assertEqual(kwargs["extra_body"], {"languages": ["en"]})
+        # sending both fields is rejected by the API
+        self.assertNotIn("language", kwargs)
+
+    def test_older_models_use_singular_language(self):
+        for model in ("gpt-4o-transcribe", "whisper-1"):
+            with self.subTest(model):
+                client = _make_client("openai", model=model, api_key="k")
+                create = MagicMock(return_value="hi")
+                client.provider = SimpleNamespace(
+                    audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+                )
+
+                client.transcribe(WAV_BYTES, language="en")
+
+                kwargs = create.call_args.kwargs
+                self.assertEqual(kwargs["language"], "en")
+                self.assertNotIn("extra_body", kwargs)
+
+    def test_object_response_form(self):
+        client = self._client()
+        create = MagicMock(return_value=SimpleNamespace(text="hi"))
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES), "hi")
+
+    def test_error_returns_none(self):
+        client = self._client()
+        create = MagicMock(side_effect=RuntimeError("boom"))
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertIsNone(client.transcribe(WAV_BYTES))
+
+
+class TestAzureOpenAITranscribe(unittest.TestCase):
+    def _client(self):
+        return _make_client(
+            "azure_openai",
+            model="my-deployment",
+            api_key="k",
+            base_url="https://example.openai.azure.com/?api-version=2024-06-01",
+        )
+
+    def test_routes_through_azure_client(self):
+        from openai import AzureOpenAI
+
+        client = self._client()
+        self.assertIsInstance(client.provider, AzureOpenAI)
+        self.assertTrue(client.supports_transcription)
+
+    def test_transcribe_inherited(self):
+        client = self._client()
+        create = MagicMock(return_value="azure text")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES, language="fr"), "azure text")
+        self.assertEqual(create.call_args.kwargs["model"], "my-deployment")
+
+
+class TestGeminiTranscribe(unittest.TestCase):
+    def _client(self):
+        return _make_client("gemini", model="gemini-2.0-flash", api_key="k")
+
+    def test_supports_transcription(self):
+        self.assertTrue(self._client().supports_transcription)
+
+    def test_sends_audio_part(self):
+        client = self._client()
+        generate = MagicMock(return_value=SimpleNamespace(text=" spoken words "))
+        client.provider = SimpleNamespace(
+            models=SimpleNamespace(generate_content=generate)
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "spoken words")
+
+        contents = generate.call_args.kwargs["contents"]
+        audio_parts = [
+            p for p in contents if getattr(p, "inline_data", None) is not None
+        ]
+        self.assertEqual(len(audio_parts), 1)
+        self.assertEqual(audio_parts[0].inline_data.mime_type, "audio/wav")
+        self.assertEqual(audio_parts[0].inline_data.data, WAV_BYTES)
+
+    def test_oversized_payload_is_skipped(self):
+        from frigate.genai.plugins.gemini import GEMINI_MAX_INLINE_BYTES
+
+        client = self._client()
+        generate = MagicMock()
+        client.provider = SimpleNamespace(
+            models=SimpleNamespace(generate_content=generate)
+        )
+
+        self.assertIsNone(client.transcribe(b"\x00" * (GEMINI_MAX_INLINE_BYTES + 1)))
+        generate.assert_not_called()
+
+
+class TestLlamaCppTranscribe(unittest.TestCase):
+    def _client(self, supports_audio: bool):
+        cfg = GenAIConfig(
+            provider="llamacpp",
+            model="m",
+            base_url="http://localhost:9999",
+        )
+        info = {
+            "context_size": 4096,
+            "supports_vision": False,
+            "supports_audio": supports_audio,
+            "supports_tools": False,
+            "supports_reasoning": False,
+            "media_marker": "<__media__>",
+        }
+        cls = PROVIDERS[GenAIProviderEnum.llamacpp]
+        with patch.object(cls, "_get_model_info", return_value=info):
+            return cls(cfg, timeout=5)
+
+    def test_supports_transcription_tracks_supports_audio(self):
+        self.assertTrue(self._client(True).supports_transcription)
+        self.assertFalse(self._client(False).supports_transcription)
+
+    @staticmethod
+    def _transcriptions_response(text: str = " transcript "):
+        response = MagicMock()
+        response.status_code = 200
+        response.json.return_value = {"text": text}
+        return response
+
+    @staticmethod
+    def _chat_response(content: str = " fallback transcript "):
+        response = MagicMock()
+        response.status_code = 200
+        response.json.return_value = {"choices": [{"message": {"content": content}}]}
+        return response
+
+    def test_posts_multipart_to_transcriptions(self):
+        client = self._client(True)
+
+        with patch.object(
+            client, "_post", return_value=self._transcriptions_response()
+        ) as post:
+            self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "transcript")
+
+        self.assertTrue(post.call_args.args[0].endswith("/v1/audio/transcriptions"))
+        self.assertEqual(
+            post.call_args.kwargs["files"]["file"],
+            ("audio.wav", WAV_BYTES, "audio/wav"),
+        )
+        self.assertEqual(post.call_args.kwargs["data"]["language"], "en")
+
+    def test_omits_language_when_not_set(self):
+        """An unset language is what lets the model detect one itself."""
+        client = self._client(True)
+
+        with patch.object(
+            client, "_post", return_value=self._transcriptions_response()
+        ) as post:
+            client.transcribe(WAV_BYTES)
+
+        self.assertNotIn("language", post.call_args.kwargs["data"])
+
+    def test_falls_back_to_chat_completions_on_404(self):
+        """Servers predating llama.cpp#21863 have no transcriptions route."""
+        client = self._client(True)
+        missing = MagicMock()
+        missing.status_code = 404
+
+        with patch.object(
+            client, "_post", side_effect=[missing, self._chat_response()]
+        ) as post:
+            self.assertEqual(
+                client.transcribe(WAV_BYTES, language="en"), "fallback transcript"
+            )
+
+        urls = [call.args[0] for call in post.call_args_list]
+        self.assertTrue(urls[0].endswith("/v1/audio/transcriptions"))
+        self.assertTrue(urls[1].endswith("/v1/chat/completions"))
+
+        payload = post.call_args_list[1].kwargs["json"]
+        content = payload["messages"][0]["content"]
+        audio_parts = [p for p in content if p["type"] == "input_audio"]
+        self.assertEqual(len(audio_parts), 1)
+        self.assertEqual(audio_parts[0]["input_audio"]["format"], "wav")
+        self.assertEqual(
+            base64.b64decode(audio_parts[0]["input_audio"]["data"]), WAV_BYTES
+        )
+
+    def test_audio_unsupported_returns_none(self):
+        client = self._client(False)
+
+        with patch.object(client, "_post") as post:
+            self.assertIsNone(client.transcribe(WAV_BYTES))
+
+        post.assert_not_called()
+
+
+class TestBaseClientTranscribe(unittest.TestCase):
+    """Providers that don't implement the role must be inert, not broken."""
+
+    def test_ollama_reports_and_returns_nothing(self):
+        client = _make_client("ollama", model="llava", base_url="http://localhost:9999")
+        self.assertFalse(client.supports_transcription)
+        self.assertIsNone(client.transcribe(WAV_BYTES, language="en"))
+
 
 if __name__ == "__main__":
     unittest.main()

frigate/test/test_ptz_onvif.py

@@ -15,6 +15,8 @@ Also covers the inverse direction: the ptz movement timestamps must not be writt
 for a camera that has autotracking off, because nothing clears them back out.
 """
 
+import asyncio
+import threading
 import unittest
 from unittest.mock import AsyncMock, MagicMock
 
@@ -234,5 +236,40 @@ class TestManualRelativeMoveMetrics(unittest.IsolatedAsyncioTestCase):
         )
 
 
+class TestOnvifClose(unittest.TestCase):
+    """close() must release everything on the loop, since whatever it leaves is
+    garbage collected during interpreter shutdown, where the resulting warnings
+    fail to log and fill the shutdown output with logging errors."""
+
+    def setUp(self) -> None:
+        self.controller = _make_controller(autotracking_enabled=False)
+        self.onvif = self.controller.cams[CAMERA]["onvif"]
+        self.onvif.close = AsyncMock()
+        self.controller.config_subscriber = MagicMock()
+        self.controller.loop = asyncio.new_event_loop()
+        self.controller.loop_thread = threading.Thread(
+            target=self.controller._run_event_loop, daemon=True
+        )
+        self.controller.loop_thread.start()
+        self.addCleanup(self.controller.loop.close)
+
+    def test_close_closes_camera_sessions(self) -> None:
+        self.controller.close()
+
+        self.onvif.close.assert_awaited_once()
+
+    def test_close_cancels_tasks_left_on_the_loop(self) -> None:
+        async def forever() -> None:
+            while True:
+                await asyncio.sleep(1)
+
+        poll = asyncio.run_coroutine_threadsafe(forever(), self.controller.loop)
+
+        self.controller.close()
+
+        self.assertTrue(poll.cancelled())
+        self.assertFalse(self.controller.loop_thread.is_alive())
+
+
 if __name__ == "__main__":
     unittest.main()

frigate/test/test_restart_frigate.py

@@ -0,0 +1,55 @@
+"""Tests for restarting frigate under s6."""
+
+import signal
+import unittest
+from unittest.mock import MagicMock, patch
+
+import psutil
+
+from frigate.util.services import restart_frigate
+
+
+class TestRestartFrigate(unittest.TestCase):
+    def _s6_process(self) -> MagicMock:
+        proc = MagicMock()
+        proc.name.return_value = "s6-svscan"
+        return proc
+
+    @patch("frigate.util.services.os.kill")
+    @patch("frigate.util.services.psutil.Process")
+    def test_terminates_s6_when_permitted(self, mock_process, mock_kill):
+        proc = self._s6_process()
+        mock_process.return_value = proc
+
+        restart_frigate()
+
+        proc.terminate.assert_called_once()
+        mock_kill.assert_not_called()
+
+    @patch("frigate.util.services.os.getpid", return_value=99)
+    @patch("frigate.util.services.os.kill")
+    @patch("frigate.util.services.psutil.Process")
+    def test_exits_self_when_s6_signal_is_denied(
+        self, mock_process, mock_kill, _mock_getpid
+    ):
+        """Running unprivileged, frigate cannot signal root's s6-svscan."""
+        proc = self._s6_process()
+        proc.terminate.side_effect = psutil.AccessDenied(pid=1, name="s6-svscan")
+        mock_process.return_value = proc
+
+        restart_frigate()
+
+        mock_kill.assert_called_once_with(99, signal.SIGINT)
+
+    @patch("frigate.util.services.os.getpid", return_value=99)
+    @patch("frigate.util.services.os.kill")
+    @patch("frigate.util.services.psutil.Process")
+    def test_exits_self_without_s6(self, mock_process, mock_kill, _mock_getpid):
+        proc = MagicMock()
+        proc.name.return_value = "init"
+        mock_process.return_value = proc
+
+        restart_frigate()
+
+        proc.terminate.assert_not_called()
+        mock_kill.assert_called_once_with(99, signal.SIGINT)

frigate/test/test_review_annotations.py

@@ -0,0 +1,316 @@
+"""Tests for tracker-derived review frame annotations."""
+
+import unittest
+
+from frigate.data_processing.post.review_annotations import (
+    annotations_by_frame,
+    build_timeline,
+    describe_heading,
+    describe_position,
+    event_name,
+    path_legs,
+    path_moments,
+)
+
+
+def straight_path(
+    start: tuple[float, float],
+    end: tuple[float, float],
+    steps: int,
+    t0: float,
+) -> list:
+    """A path_data-shaped trajectory travelling in a straight line."""
+    return [
+        [
+            [
+                start[0] + (end[0] - start[0]) * i / (steps - 1),
+                start[1] + (end[1] - start[1]) * i / (steps - 1),
+            ],
+            t0 + i,
+        ]
+        for i in range(steps)
+    ]
+
+
+class TestDescribers(unittest.TestCase):
+    def test_describes_frame_corners(self):
+        self.assertEqual(describe_position(0.9, 0.9), "the bottom right of the frame")
+        self.assertEqual(describe_position(0.1, 0.1), "the top left of the frame")
+        self.assertEqual(describe_position(0.5, 0.5), "the middle of the frame")
+
+    def test_heading_treats_falling_y_as_up(self):
+        self.assertEqual(describe_heading(0.0, -0.5), "up")
+        self.assertEqual(describe_heading(0.0, 0.5), "down")
+        self.assertEqual(describe_heading(-0.5, 0.0), "left")
+
+    def test_heading_combines_axes(self):
+        self.assertEqual(describe_heading(-0.5, -0.5), "up and left")
+
+
+class TestPathLegs(unittest.TestCase):
+    def test_straight_travel_is_one_leg(self):
+        points = [(0.9 - 0.05 * i, 0.6, float(i)) for i in range(10)]
+        self.assertEqual(len(path_legs(points)), 1)
+
+    def test_out_and_back_is_two_legs(self):
+        out = [(0.9 - 0.05 * i, 0.6, float(i)) for i in range(8)]
+        back = [(0.55 + 0.05 * i, 0.6, 8.0 + i) for i in range(8)]
+        self.assertEqual(len(path_legs(out + back)), 2)
+
+    def test_jitter_in_place_produces_no_legs(self):
+        # A subject standing still wobbles by a couple of percent; without the
+        # minimum leg distance this became a burst of contradictory turns.
+        points = [(0.5 + 0.01 * (i % 2), 0.5, float(i)) for i in range(20)]
+        self.assertEqual(path_legs(points), [])
+
+    def test_single_point_path_has_no_moments(self):
+        self.assertEqual(path_moments([[[0.5, 0.5], 1.0]]), [])
+
+    def test_malformed_path_is_ignored(self):
+        self.assertEqual(path_moments([[0.5, 1.0], [0.6, 2.0]]), [])
+
+
+class TestEventNames(unittest.TestCase):
+    def test_unnamed_objects_use_an_indefinite_article(self):
+        self.assertEqual(event_name({"label": "person"}), "a person")
+        self.assertEqual(event_name({"label": "animal"}), "an animal")
+
+    def test_sub_labeled_objects_use_their_name(self):
+        event = {"label": "waste_bin", "sub_label": "Compost"}
+        self.assertEqual(event_name(event), 'waste bin "Compost"')
+
+    def test_repeated_objects_are_never_numbered(self):
+        # Whether these are the same subject is unknown, so the notes must not
+        # imply either answer.
+        events = [
+            {
+                "id": f"1789481994.68448{i}-abcdef",
+                "label": "person",
+                "sub_label": None,
+                "start_time": float(i * 10),
+                "end_time": float(i * 10 + 50),
+                "zones": [],
+                "path_data": straight_path((0.9, 0.6), (0.4, 0.3), 10, i * 10.0),
+            }
+            for i in range(3)
+        ]
+        phrases = [p for _, p in build_timeline(events, span_end=100.0)]
+        self.assertTrue(all(p.startswith("a person ") for p in phrases))
+        self.assertFalse(any("#" in p for p in phrases))
+
+
+class TestTimeline(unittest.TestCase):
+    def setUp(self):
+        self.event = {
+            "id": "1789481994.684479-lpyc2z",
+            "label": "person",
+            "sub_label": None,
+            "start_time": 0.0,
+            "end_time": 20.0,
+            "zones": ["front_yard"],
+            "path_data": straight_path((0.9, 0.6), (0.4, 0.3), 10, 1.0),
+        }
+
+    def test_timeline_is_ordered_and_bounded(self):
+        timeline = build_timeline([self.event], span_end=30.0)
+        times = [t for t, _ in timeline]
+        self.assertEqual(times, sorted(times))
+        self.assertTrue(all(t <= 30.0 for t in times))
+
+    def test_track_identifiers_never_appear(self):
+        timeline = build_timeline([self.event], span_end=30.0)
+        joined = " ".join(phrase for _, phrase in timeline)
+        self.assertNotIn("track", joined.lower())
+        self.assertNotIn(self.event["id"], joined)
+
+    def test_object_still_tracked_at_end_gets_no_closing_note(self):
+        # Its state at the end is visible in the last frame, so nothing is said.
+        timeline = build_timeline([self.event], span_end=15.0)
+        self.assertEqual(
+            [p for _, p in timeline],
+            ["a person first detected at the right of the frame, moving up and left"],
+        )
+
+    def test_object_ending_inside_the_clip_is_no_longer_detected(self):
+        timeline = build_timeline([self.event], span_end=40.0)
+        phrases = [p for _, p in timeline]
+        self.assertTrue(any("no longer detected" in p for p in phrases))
+
+    def test_moments_past_the_last_frame_are_dropped(self):
+        # The subject keeps moving after the final sampled frame; those notes
+        # describe nothing the model can see.
+        late = dict(self.event)
+        out = straight_path((0.9, 0.6), (0.4, 0.6), 8, 100.0)
+        back = straight_path((0.4, 0.6), (0.9, 0.6), 8, 108.0)
+        late["path_data"] = out + back
+        late["start_time"] = 100.0
+        timeline = build_timeline([late], span_end=105.0)
+        self.assertTrue(any("moving left" in p for _, p in timeline))
+        self.assertFalse(any("turns around" in p for _, p in timeline))
+
+
+def track(event_id, label, start, path, sub_label=None, end=None):
+    return {
+        "id": event_id,
+        "label": label,
+        "sub_label": sub_label,
+        "start_time": start,
+        "end_time": end if end is not None else start + 500.0,
+        "zones": [],
+        "path_data": path,
+    }
+
+
+class TestArrival(unittest.TestCase):
+    def test_objects_arriving_together_keep_separate_notes(self):
+        person = track(
+            "1789481994.684479-lpyc2z",
+            "person",
+            0.0,
+            straight_path((0.9, 0.63), (0.58, 0.34), 10, 0.1),
+        )
+        bin_ = track(
+            "1789481995.063395-vlzd7q",
+            "waste_bin",
+            0.3,
+            straight_path((0.91, 0.64), (0.6, 0.33), 10, 0.4),
+            sub_label="Compost",
+        )
+        phrases = [p for _, p in build_timeline([person, bin_], 100.0)]
+        self.assertEqual(
+            phrases,
+            [
+                "a person first detected at the right of the frame, moving up and left",
+                'waste bin "Compost" first detected at the right of the frame, '
+                "moving up and left",
+            ],
+        )
+
+    def test_object_detected_well_before_it_moves_gets_separate_notes(self):
+        # path_data always keeps the first two samples, so a bin sitting in
+        # the yard opens its leg long before it is picked up.
+        path = [[[0.91, 0.64], 0.4]] + straight_path(
+            (0.91, 0.64), (0.6, 0.33), 10, 20.4
+        )
+        bin_ = track(
+            "1789481995.063395-vlzd7q", "waste_bin", 0.3, path, sub_label="Compost"
+        )
+        timeline = build_timeline([bin_], 100.0)
+        self.assertEqual(
+            timeline[0],
+            (0.3, 'waste bin "Compost" first detected at the right of the frame'),
+        )
+        self.assertAlmostEqual(timeline[1][0], 21.4)
+        self.assertEqual(
+            timeline[1][1],
+            'waste bin "Compost" starts moving up and left from the right of the frame',
+        )
+
+
+class TestStateChanges(unittest.TestCase):
+    def test_stationary_and_active_rows_become_notes(self):
+        bin_ = track(
+            "1789481995.063395-vlzd7q",
+            "waste_bin",
+            0.3,
+            straight_path((0.91, 0.64), (0.6, 0.33), 10, 0.4),
+            sub_label="Compost",
+        )
+        changes = [
+            {"timestamp": 20.0, "source_id": bin_["id"], "class_type": "stationary"},
+            {"timestamp": 30.0, "source_id": bin_["id"], "class_type": "active"},
+            {"timestamp": 35.0, "source_id": bin_["id"], "class_type": "entered_zone"},
+            {
+                "timestamp": 40.0,
+                "source_id": "someone-else",
+                "class_type": "stationary",
+            },
+        ]
+        timeline = build_timeline([bin_], 100.0, state_changes=changes)
+        self.assertEqual(
+            timeline[1:],
+            [
+                (20.0, 'waste bin "Compost" has stopped moving'),
+                (30.0, 'waste bin "Compost" starts moving again'),
+            ],
+        )
+
+    def test_state_changes_after_the_last_frame_are_dropped(self):
+        bin_ = track(
+            "1789481995.063395-vlzd7q",
+            "waste_bin",
+            0.3,
+            straight_path((0.91, 0.64), (0.6, 0.33), 10, 0.4),
+            sub_label="Compost",
+        )
+        changes = [
+            {"timestamp": 200.0, "source_id": bin_["id"], "class_type": "stationary"}
+        ]
+        timeline = build_timeline([bin_], 100.0, state_changes=changes)
+        self.assertFalse(any("stopped" in p for _, p in timeline))
+
+
+class TestNoAssumedState(unittest.TestCase):
+    def test_no_note_for_where_movement_ends(self):
+        # Ending a leftward walk still in the right third was read as a turn
+        # back to the right, and "stops moving" was read as standing still.
+        moments = path_moments(straight_path((0.95, 0.6), (0.7, 0.4), 6, 0.0))
+        self.assertEqual(
+            [p for _, p in moments],
+            ["starts moving up and left from the right of the frame"],
+        )
+
+    def test_notes_never_claim_an_object_is_stationary(self):
+        # A track still open at the last frame says nothing about motion.
+        event = {
+            "id": "1789482056.695307-3uhf47",
+            "label": "person",
+            "sub_label": None,
+            "start_time": 0.0,
+            "end_time": 500.0,
+            "zones": ["front_yard"],
+            "path_data": straight_path((0.9, 0.6), (0.4, 0.3), 10, 1.0),
+        }
+        joined = " ".join(p for _, p in build_timeline([event], span_end=20.0))
+        self.assertNotIn("stationary", joined)
+        self.assertNotIn("stops", joined)
+        self.assertNotIn("leaves", joined)
+        self.assertNotIn("still", joined)
+
+
+class TestLateEvents(unittest.TestCase):
+    def test_objects_first_detected_after_the_last_frame_are_skipped(self):
+        # Without this the arrival lands on the final frame, which was
+        # captured before the object appeared.
+        late = track(
+            "1789481999.000000-latear",
+            "person",
+            50.0,
+            straight_path((0.9, 0.6), (0.4, 0.3), 10, 50.1),
+        )
+        self.assertEqual(build_timeline([late], span_end=40.0), [])
+        self.assertEqual(
+            annotations_by_frame(build_timeline([late], 40.0), [0.0, 40.0]), {}
+        )
+
+
+class TestFrameBucketing(unittest.TestCase):
+    def test_moment_attaches_to_the_following_frame(self):
+        frame_times = [0.0, 10.0, 20.0, 30.0]
+        buckets = annotations_by_frame([(12.0, "something happened")], frame_times)
+        self.assertEqual(buckets, {2: ["something happened"]})
+
+    def test_moment_on_a_frame_boundary_uses_that_frame(self):
+        buckets = annotations_by_frame([(10.0, "x")], [0.0, 10.0, 20.0])
+        self.assertEqual(buckets, {1: ["x"]})
+
+    def test_moment_after_the_last_frame_falls_on_the_last_frame(self):
+        buckets = annotations_by_frame([(99.0, "x")], [0.0, 10.0])
+        self.assertEqual(buckets, {1: ["x"]})
+
+    def test_no_frames_yields_no_buckets(self):
+        self.assertEqual(annotations_by_frame([(1.0, "x")], []), {})
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/util/audio.py

@@ -1,16 +1,61 @@
 """Utilities for creating and manipulating audio."""
 
+import io
 import logging
 import os
+import re
+import string
+import struct
 import subprocess as sp
+import wave
 
+import numpy as np
 from pathvalidate import sanitize_filename
 
-from frigate.const import CACHE_DIR, STREAM_TYPE_MAIN, STREAM_TYPE_SUB
+from frigate.const import (
+    AUDIO_SAMPLE_RATE,
+    CACHE_DIR,
+    STREAM_TYPE_MAIN,
+    STREAM_TYPE_SUB,
+)
 from frigate.models import Recordings
 
 logger = logging.getLogger(__name__)
 
+# Ceiling on the run of words the stitcher will treat as an overlap between two
+# consecutive windows. This is an audio-duration bound, not a linguistic one: a
+# window holds GENAI_WINDOW_CHUNKS * AUDIO_DURATION seconds of speech, so at a
+# fast talker's pace it tops out around this many words, and a whole window can
+# legitimately be redundant. The vendored whisper_streaming HypothesisBuffer
+# caps at 5, but there the n-gram is only a tie-break on top of word-level
+# timestamps; here it is the entire alignment, so 5 truncates real overlaps.
+# Sentinel meaning "let the model work out the language". The vendored
+# whisper_streaming code already uses this spelling, so it is the established
+# convention for the audio_transcription.language field.
+AUTO_LANGUAGE = "auto"
+
+MAX_STITCH_NGRAM = 16
+
+# How many trailing committed words the stitcher may discard to find an
+# alignment. Those words came from the newest audio, which the next window
+# re-covers, so when the provider got one of them wrong it blocks every
+# alignment and the whole phrase duplicates. Set to 0 to make committed text
+# strictly append-only.
+MAX_STITCH_REVISE = 3
+
+# A revision deletes text that was already published, so it has to clear a
+# higher bar than a plain append: a single coincidentally shared word is not
+# enough evidence to throw committed words away.
+MIN_STITCH_REVISE_RUN = 2
+
+# ASR models often wrap their output in control markup. Qwen3-ASR, for example,
+# answers "language English<asr_text>Yeah, that works." A structural opening tag
+# marks where the transcript starts, so anything before the last one is metadata.
+# Closing tags (</x>) and pipe-delimited special tokens (<|endoftext|>) are
+# excluded: those mark where the text ends, so text before them must be kept.
+_OPENING_TAG = re.compile(r"<(?![/|])[^<>]*>")
+_ANY_TAG = re.compile(r"<[^<>]*>")
+
 
 def _get_recordings_for_range(
     camera_name: str, start_ts: float, end_ts: float, stream_type: str
@@ -117,7 +162,9 @@ def get_audio_from_recording(
             logger.debug(
                 f"Successfully extracted audio for {camera_name} from {start_ts} to {end_ts}"
             )
-            return process.stdout
+            # ffmpeg writes to a pipe, so it cannot seek back to patch the chunk
+            # sizes it reserved; repair them before any strict consumer sees them
+            return fix_wav_header(process.stdout)
         else:
             logger.error(f"Failed to extract audio: {process.stderr.decode()}")
             return None
@@ -129,3 +176,217 @@ def get_audio_from_recording(
             os.unlink(file_path)
         except OSError:
             pass
+
+
+def fix_wav_header(data: bytes) -> bytes:
+    """Recompute the RIFF and data chunk sizes in a WAV header.
+
+    ffmpeg writing to a non-seekable pipe cannot go back and patch the sizes it
+    reserved, so it leaves 0xFFFFFFFF placeholders. PyAV-based demuxers ignore
+    them, but strict validators may reject the file or read zero frames.
+
+    Args:
+        data: The complete WAV payload
+
+    Returns:
+        The payload with both sizes corrected, or unchanged if it is not a
+        parseable RIFF/WAVE stream
+    """
+    if len(data) < 12 or data[0:4] != b"RIFF" or data[8:12] != b"WAVE":
+        return data
+
+    out = bytearray(data)
+
+    # RIFF size covers everything after the 8-byte RIFF header
+    struct.pack_into("<I", out, 4, len(out) - 8)
+
+    # walk the chunk list to find "data"; every chunk is padded to even length
+    pos = 12
+
+    while pos + 8 <= len(out):
+        chunk_id = bytes(out[pos : pos + 4])
+        (chunk_size,) = struct.unpack_from("<I", out, pos + 4)
+
+        if chunk_id == b"data":
+            struct.pack_into("<I", out, pos + 4, len(out) - (pos + 8))
+            return bytes(out)
+
+        if chunk_size == 0xFFFFFFFF:
+            # an unpatched size before the data chunk leaves nothing to walk
+            break
+
+        pos += 8 + chunk_size + (chunk_size % 2)
+
+    return bytes(out)
+
+
+def pcm16_to_wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
+    """Wrap mono int16 PCM samples in a WAV container.
+
+    Args:
+        samples: The audio samples; converted to int16 if they are not already
+        sample_rate: Sample rate to declare in the header
+
+    Returns:
+        WAV bytes suitable for upload to a GenAI provider
+    """
+    if samples.dtype != np.int16:
+        samples = samples.astype(np.int16)
+
+    buffer = io.BytesIO()
+
+    with wave.open(buffer, "wb") as wav:
+        wav.setnchannels(1)
+        wav.setsampwidth(2)
+        wav.setframerate(sample_rate)
+        wav.writeframes(samples.tobytes())
+
+    return buffer.getvalue()
+
+
+def stitch_transcripts(committed: str, incoming: str) -> str:
+    """Append *incoming* to *committed*, dropping the speech they share.
+
+    Consecutive overlapped transcription windows re-transcribe the same audio at
+    their seam, so the tail of one and the newest one name the same words. Find
+    the longest run that is a suffix of *committed* and occurs anywhere in
+    *incoming*, then keep only what follows that run.
+
+    Searching all of *incoming* rather than just its start is what makes this
+    work in practice. The provider re-transcribes the shared audio independently
+    and often gets its first word or two different ("just gonna" one window,
+    "It's gonna" the next), which leaves the real overlap sitting in the middle
+    of *incoming*. A prefix-anchored match sees no overlap at all there and
+    duplicates the entire phrase.
+
+    Text-level rather than timestamp-level because only some providers return
+    word timings, and this has to work across all of them.
+
+    Args:
+        committed: The transcript accumulated so far
+        incoming: The newest window's transcript
+
+    Returns:
+        The combined transcript
+    """
+    incoming_words = incoming.split()
+
+    if not incoming_words:
+        return committed
+
+    committed_words = committed.split()
+
+    if not committed_words:
+        return " ".join(incoming_words)
+
+    committed_keys = [_overlap_key(word) for word in committed_words]
+    incoming_keys = [_overlap_key(word) for word in incoming_words]
+
+    length, consumed = _find_overlap(committed_keys, incoming_keys)
+
+    if length:
+        return " ".join(committed_words + incoming_words[consumed:])
+
+    # Nothing aligns. Retry against a shortened committed tail: a single word the
+    # provider got wrong at the end of the previous window otherwise blocks every
+    # alignment, and the entire re-transcribed phrase duplicates behind it.
+    best: tuple[int, int, int] | None = None
+
+    for drop in range(1, min(MAX_STITCH_REVISE, len(committed_keys) - 1) + 1):
+        length, consumed = _find_overlap(committed_keys[:-drop], incoming_keys)
+
+        if length < MIN_STITCH_REVISE_RUN:
+            continue
+
+        # longest run wins; ties go to the smallest revision
+        if best is None or length > best[0]:
+            best = (length, drop, consumed)
+
+    if best is None:
+        return " ".join(committed_words + incoming_words)
+
+    _, drop, consumed = best
+
+    return " ".join(committed_words[:-drop] + incoming_words[consumed:])
+
+
+def _find_overlap(
+    committed_keys: list[str], incoming_keys: list[str]
+) -> tuple[int, int]:
+    """Locate the speech *incoming* shares with the end of *committed*.
+
+    Returns the length of the longest run that is a suffix of *committed_keys*
+    and occurs anywhere in *incoming_keys*, along with the index just past that
+    run in *incoming_keys*. Returns ``(0, 0)`` when nothing matches.
+
+    Prefers the longest run so a real overlap is not cut short, and within one
+    length the earliest position, so a phrase genuinely spoken twice keeps its
+    second utterance.
+    """
+    max_run = min(MAX_STITCH_NGRAM, len(committed_keys), len(incoming_keys))
+
+    for length in range(max_run, 0, -1):
+        tail = committed_keys[-length:]
+
+        for start in range(len(incoming_keys) - length + 1):
+            if incoming_keys[start : start + length] == tail:
+                return length, start + length
+
+    return 0, 0
+
+
+def clean_transcript(text: str | None) -> str:
+    """Strip provider control markup and any preamble from a raw transcript.
+
+    A window with no speech often still comes back as the preamble alone
+    ("language English<asr_text>"), which must reduce to an empty string so
+    callers treat it as silence rather than committing it as spoken words.
+
+    Args:
+        text: The provider's raw response
+
+    Returns:
+        The transcript with markup removed and whitespace collapsed
+    """
+    if not text:
+        return ""
+
+    # everything up to and including the last opening tag is metadata
+    openings = list(_OPENING_TAG.finditer(text))
+
+    if openings:
+        text = text[openings[-1].end() :]
+
+    # drop closing tags and special tokens wherever they landed
+    text = _ANY_TAG.sub(" ", text)
+
+    return " ".join(text.split())
+
+
+def _overlap_key(word: str) -> str:
+    """Comparison key for overlap matching.
+
+    Providers re-transcribe the shared audio at a window seam independently, so
+    the same word routinely comes back capitalized differently or with different
+    edge punctuation ("work." vs "Work"). Those differences must not defeat the
+    match, but the original spelling is what gets kept in the output.
+    """
+    key = word.strip(string.punctuation).casefold()
+
+    # a token that is nothing but punctuation would otherwise match any other
+    return key or word
+
+
+def resolve_language(language: str | None) -> str | None:
+    """Turn a configured language into an explicit code, or None for auto-detect.
+
+    Args:
+        language: The configured value, possibly AUTO_LANGUAGE
+
+    Returns:
+        An ISO language code, or None when the backend should detect it
+    """
+    if not language or language == AUTO_LANGUAGE:
+        return None
+
+    return language

frigate/util/config.py

@@ -829,6 +829,57 @@ def rename_hailo_detector(
     return new_config
 
 
+def _camera_enables_transcription(camera: dict[str, Any]) -> bool:
+    """Whether a camera or one of its profiles turns audio transcription on."""
+    sections = [camera.get("audio_transcription")]
+    profiles = camera.get("profiles")
+
+    if isinstance(profiles, dict):
+        for profile in profiles.values():
+            if isinstance(profile, dict):
+                sections.append(profile.get("audio_transcription"))
+
+    return any(
+        isinstance(section, dict) and section.get("enabled") for section in sections
+    )
+
+
+def _migrate_transcription_language(config: dict[str, Any]) -> None:
+    """Pin English for configs written before the language default became auto.
+
+    audio_transcription.language used to default to "en", so a config that
+    turned transcription on without naming a language was transcribing English.
+    The default is now "auto" (let the model detect), which is better for new
+    users but would silently change behavior for existing ones, so write the old
+    value explicitly for anyone actually using the feature.
+    """
+    transcription = config.get("audio_transcription")
+
+    if isinstance(transcription, dict) and "language" in transcription:
+        # named a language already, so nothing was relying on the default
+        return
+
+    enabled = isinstance(transcription, dict) and bool(transcription.get("enabled"))
+
+    if not enabled:
+        enabled = any(
+            _camera_enables_transcription(camera)
+            for camera in config.get("cameras", {}).values()
+            if isinstance(camera, dict)
+        )
+
+    if not enabled:
+        return
+
+    if not isinstance(transcription, dict):
+        # a camera enabled it without a global section, which still picked up
+        # the global default
+        transcription = {}
+        config["audio_transcription"] = transcription
+
+    transcription["language"] = "en"
+
+
 def migrate_019_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
     """Handle migrating Frigate config to 0.19-0."""
     new_config = rename_hailo_detector(config)
@@ -845,6 +896,8 @@ def migrate_019_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
 
         new_config["cameras"][name] = camera_config
 
+    _migrate_transcription_language(new_config)
+
     new_config["version"] = "0.19-0"
     return new_config
 

frigate/util/services.py

@@ -35,12 +35,19 @@ logger = logging.getLogger(__name__)
 
 def restart_frigate():
     proc = psutil.Process(1)
+
     # if this is running via s6, sigterm pid 1
     if proc.name() == "s6-svscan":
-        proc.terminate()
+        try:
+            proc.terminate()
+            return
+        except psutil.AccessDenied:
+            # frigate runs unprivileged, so it cannot signal root's s6-svscan.
+            # exiting this process instead runs frigate/finish, which halts s6
+            logger.debug("Not permitted to signal s6-svscan, exiting instead")
+
     # otherwise, just try and exit frigate
-    else:
-        os.kill(os.getpid(), signal.SIGINT)
+    os.kill(os.getpid(), signal.SIGINT)
 
 
 def print_stack(sig, frame):

frigate/util/time.py

@@ -2,6 +2,7 @@
 
 import datetime
 import logging
+import math
 from zoneinfo import ZoneInfoNotFoundError
 
 import pytz
@@ -43,9 +44,33 @@ def is_current_hour(timestamp: int) -> bool:
     return timestamp < start_of_next_hour
 
 
+def _utc_offset(tz: datetime.tzinfo, timestamp: float) -> float:
+    dt = datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)
+    return dt.astimezone(tz).utcoffset().total_seconds()
+
+
+def _find_transition(
+    tz: datetime.tzinfo, lo: float, hi: float, lo_offset: float
+) -> float:
+    """Bisect (lo, hi] to the second where the UTC offset first differs from lo_offset."""
+    # whole seconds, so the midpoint always advances (a fractional bound can
+    # otherwise leave the midpoint sitting on lo) and lands on the transition
+    low = math.floor(lo)
+    high = math.ceil(hi)
+
+    while high - low > 1:
+        mid = (low + high) // 2
+        if _utc_offset(tz, mid) == lo_offset:
+            low = mid
+        else:
+            high = mid
+
+    return float(high)
+
+
 def get_dst_transitions(
     tz_name: str, start_time: float, end_time: float
-) -> list[tuple[float, float]]:
+) -> list[tuple[float, float, float]]:
     """
     Find DST transition points and return time periods with consistent offsets.
 
@@ -66,28 +91,24 @@ def get_dst_transitions(
 
     periods = []
     current = start_time
-
-    # Get initial offset
-    dt = datetime.datetime.utcfromtimestamp(current).replace(tzinfo=pytz.UTC)
-    local_dt = dt.astimezone(tz)
-    prev_offset = local_dt.utcoffset().total_seconds()
     period_start = start_time
+    prev_offset = _utc_offset(tz, current)
 
-    # Check each day for offset changes
-    while current <= end_time:
-        dt = datetime.datetime.utcfromtimestamp(current).replace(tzinfo=pytz.UTC)
-        local_dt = dt.astimezone(tz)
-        current_offset = local_dt.utcoffset().total_seconds()
+    # Probe at most a day ahead, capped at end_time so a transition after the
+    # last full day is still seen instead of silently kept in the last period.
+    while current < end_time:
+        next_probe = min(current + 86400, end_time)
+        next_offset = _utc_offset(tz, next_probe)
 
-        if current_offset != prev_offset:
-            # Found a transition - close previous period
-            periods.append((period_start, current, prev_offset))
-            period_start = current
-            prev_offset = current_offset
+        if next_offset != prev_offset:
+            transition = _find_transition(tz, current, next_probe, prev_offset)
+            periods.append((period_start, transition, prev_offset))
+            period_start = transition
+            prev_offset = _utc_offset(tz, transition)
+            current = transition
+        else:
+            current = next_probe
 
-        current += 86400  # Check daily
-
-    # Add final period
     periods.append((period_start, end_time, prev_offset))
 
     return periods

web/e2e/specs/settings/genai.spec.ts

@@ -0,0 +1,157 @@
+/**
+ * Generative AI provider settings tests -- MEDIUM tier.
+ *
+ * A model name belongs to its provider, so switching provider clears the model
+ * field. The roles widget strips a role only for a model or provider picked in
+ * the form, never when capability data arrives for the saved entry, which would
+ * dirty the section on load and silently drop the role on the next save.
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { test, expect } from "../../fixtures/frigate-test";
+import type { Page } from "@playwright/test";
+import { configFactory } from "../../fixtures/mock-data/config";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CONFIG_SCHEMA = JSON.parse(
+  readFileSync(
+    resolve(__dirname, "../../fixtures/mock-data/config-schema.json"),
+    "utf-8",
+  ),
+);
+
+const ENTRY = "audio";
+const SETTINGS_URL = "/settings?page=integrationGenerativeAi";
+const UNSAVED = "You have unsaved changes";
+const MODEL_PLACEHOLDER = "Select or enter a model…";
+
+type Entry = {
+  provider: string;
+  model: string;
+  base_url?: string;
+  roles: string[];
+};
+
+type ProviderInfo = {
+  models: string[];
+  supports_transcription: boolean;
+  model_capabilities?: Record<string, { supports_transcription?: boolean }>;
+};
+
+async function installRoutes(page: Page, entry: Entry, info: ProviderInfo) {
+  const config = configFactory({ genai: { [ENTRY]: entry } });
+
+  await page.route("**/api/config/schema.json", (route) =>
+    route.fulfill({ json: CONFIG_SCHEMA }),
+  );
+  await page.route("**/api/config", (route) => {
+    if (route.request().method() === "GET") {
+      return route.fulfill({ json: config });
+    }
+    return route.fulfill({ json: { success: true } });
+  });
+  await page.route("**/api/config/raw_paths", (route) =>
+    route.fulfill({ json: { genai: { [ENTRY]: entry } } }),
+  );
+  await page.route("**/api/genai/models", (route) =>
+    route.fulfill({
+      json: {
+        [ENTRY]: {
+          roles: entry.roles,
+          supports_toggleable_thinking: false,
+          supports_embeddings: true,
+          model_capabilities: {},
+          ...info,
+        },
+      },
+    }),
+  );
+}
+
+function roleSwitch(page: Page, role: string) {
+  return page.locator(`#root_${ENTRY}_roles-${role}`);
+}
+
+test.describe("genai provider settings @medium", () => {
+  test("a saved role the provider cannot confirm stays and is not dirty", async ({
+    frigateApp,
+  }) => {
+    // The server does not serve the saved model, so the backend reports every
+    // capability as false for the entry.
+    await installRoutes(
+      frigateApp.page,
+      {
+        provider: "llamacpp",
+        model: "stale-model",
+        base_url: "http://llama:8080",
+        roles: ["transcribe"],
+      },
+      { models: ["qwen3-asr"], supports_transcription: false },
+    );
+    await frigateApp.goto(SETTINGS_URL);
+
+    await expect(roleSwitch(frigateApp.page, "transcribe")).toBeVisible();
+    await expect(roleSwitch(frigateApp.page, "transcribe")).toBeChecked();
+
+    // Give any stripping effect time to fire, then confirm the section stayed
+    // clean.
+    await frigateApp.page.waitForTimeout(1000);
+    await expect(frigateApp.page.getByText(UNSAVED)).toBeHidden();
+    await expect(roleSwitch(frigateApp.page, "transcribe")).toBeChecked();
+  });
+
+  test("switching provider clears the model", async ({ frigateApp }) => {
+    await installRoutes(
+      frigateApp.page,
+      {
+        provider: "openai",
+        model: "gpt-4o",
+        roles: ["descriptions"],
+      },
+      { models: ["gpt-4o"], supports_transcription: true },
+    );
+    await frigateApp.goto(SETTINGS_URL);
+
+    const model = frigateApp.page.locator(`#root_${ENTRY}_model`);
+    await expect(model).toHaveText("gpt-4o");
+
+    await frigateApp.page.locator(`#root_${ENTRY}_provider`).click();
+    await frigateApp.page.getByRole("option", { name: "llamacpp" }).click();
+
+    await expect(model).toHaveText(MODEL_PLACEHOLDER);
+    await expect(frigateApp.page.getByText(UNSAVED)).toBeVisible();
+  });
+
+  test("picking a model that cannot transcribe strips the role", async ({
+    frigateApp,
+  }) => {
+    await installRoutes(
+      frigateApp.page,
+      {
+        provider: "llamacpp",
+        model: "qwen3-asr",
+        base_url: "http://llama:8080",
+        roles: ["transcribe"],
+      },
+      {
+        models: ["qwen3-asr", "text-only"],
+        supports_transcription: true,
+        model_capabilities: {
+          "qwen3-asr": { supports_transcription: true },
+          "text-only": { supports_transcription: false },
+        },
+      },
+    );
+    await frigateApp.goto(SETTINGS_URL);
+
+    await expect(roleSwitch(frigateApp.page, "transcribe")).toBeChecked();
+
+    await frigateApp.page.locator(`#root_${ENTRY}_model`).click();
+    await frigateApp.page.getByRole("option", { name: "text-only" }).click();
+
+    await expect(roleSwitch(frigateApp.page, "transcribe")).toBeHidden();
+    await expect(frigateApp.page.getByText(UNSAVED)).toBeVisible();
+  });
+});

web/e2e/specs/settings/runtime-overrides.spec.ts

@@ -0,0 +1,117 @@
+/**
+ * Runtime override tests -- MEDIUM tier.
+ *
+ * Live view, MQTT, and Home Assistant toggles change a camera's running config
+ * without touching yaml, and the change persists across restarts. The settings
+ * form edits yaml, so it keeps showing the saved value. Without a marker on the
+ * field and runtime-aware wording on dependent warnings, the two read as a
+ * contradiction: "audio detection is not enabled" next to a switch that is on.
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { test, expect } from "../../fixtures/frigate-test";
+import type { Page } from "@playwright/test";
+import { configFactory } from "../../fixtures/mock-data/config";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CONFIG_SCHEMA = JSON.parse(
+  readFileSync(
+    resolve(__dirname, "../../fixtures/mock-data/config-schema.json"),
+    "utf-8",
+  ),
+);
+
+const CAMERA = "front_door";
+const TRANSCRIPTION_URL = `/settings?page=cameraAudioTranscription&camera=${CAMERA}`;
+const AUDIO_URL = `/settings?page=cameraAudioEvents&camera=${CAMERA}`;
+
+const CONFIG_DISABLED = /Audio detection is not enabled for this camera/;
+const RUNTIME_DISABLED =
+  /Audio detection is enabled in your config, but it is currently turned off/;
+
+type AudioState = { enabled: boolean; enabled_in_config: boolean };
+
+async function installRoutes(page: Page, audio: AudioState) {
+  const config = configFactory({
+    cameras: {
+      [CAMERA]: {
+        audio,
+        // audio detection only runs on a stream carrying the audio role
+        ffmpeg: {
+          inputs: [
+            {
+              path: "rtsp://user:pass@host/front",
+              roles: ["record", "detect", "audio"],
+            },
+          ],
+        },
+        audio_transcription: { enabled: true, enabled_in_config: true },
+      },
+    },
+  });
+
+  await page.route("**/api/config/raw_paths", (route) =>
+    route.fulfill({
+      json: { cameras: { [CAMERA]: { ffmpeg: { inputs: [] } } } },
+    }),
+  );
+  await page.route("**/api/config/schema.json", (route) =>
+    route.fulfill({ json: CONFIG_SCHEMA }),
+  );
+  await page.route("**/api/config", (route) => {
+    if (route.request().method() === "GET") {
+      return route.fulfill({ json: config });
+    }
+    return route.fulfill({ json: { success: true } });
+  });
+}
+
+test.describe("runtime overrides @medium", () => {
+  test("a runtime-only toggle gets its own wording and a field marker", async ({
+    frigateApp,
+  }) => {
+    await installRoutes(frigateApp.page, {
+      enabled: false,
+      enabled_in_config: true,
+    });
+
+    await frigateApp.goto(TRANSCRIPTION_URL);
+    await expect(frigateApp.page.getByText(RUNTIME_DISABLED)).toBeVisible();
+    await expect(frigateApp.page.getByText(CONFIG_DISABLED)).toBeHidden();
+
+    // The audio section still shows the saved value, so the switch stays on and
+    // the marker carries the live state.
+    await frigateApp.goto(AUDIO_URL);
+    await expect(
+      frigateApp.page.getByRole("switch", { name: "Enable audio detection" }),
+    ).toHaveAttribute("data-state", "checked");
+    // the boolean layout renders a mobile and a desktop label block, so only
+    // one of the two badges is on screen
+    await expect(
+      frigateApp.page.getByText("Overridden (Live)").filter({ visible: true }),
+    ).toHaveCount(1);
+  });
+
+  test("a config-disabled section keeps the original wording", async ({
+    frigateApp,
+  }) => {
+    await installRoutes(frigateApp.page, {
+      enabled: false,
+      enabled_in_config: false,
+    });
+
+    await frigateApp.goto(TRANSCRIPTION_URL);
+    await expect(frigateApp.page.getByText(CONFIG_DISABLED)).toBeVisible();
+    await expect(frigateApp.page.getByText(RUNTIME_DISABLED)).toBeHidden();
+
+    await frigateApp.goto(AUDIO_URL);
+    await expect(
+      frigateApp.page.getByRole("switch", { name: "Enable audio detection" }),
+    ).toHaveAttribute("data-state", "unchecked");
+    await expect(
+      frigateApp.page.getByText("Overridden (Live)").filter({ visible: true }),
+    ).toHaveCount(0);
+  });
+});

web/e2e/specs/system.spec.ts

@@ -72,7 +72,15 @@ test.describe("System — tabs @medium", () => {
       { timeout: 15_000 },
     );
     await expect(frigateApp.page.getByText("0.15.0-test")).toBeVisible();
-    await expect(frigateApp.page.getByText(/Last refreshed/)).toBeVisible();
+
+    if (frigateApp.isMobile) {
+      // the "Last refreshed" label is dropped on mobile so the timestamp
+      // clears the centered logo
+      await expect(frigateApp.page.getByText(/Last refreshed/)).toHaveCount(0);
+      await expect(frigateApp.page.getByText(/Just now|ago/)).toBeVisible();
+    } else {
+      await expect(frigateApp.page.getByText(/Last refreshed/)).toBeVisible();
+    }
   });
 
   test("storage tab renders content after switching", async ({
@@ -234,4 +242,45 @@ test.describe("System — mobile @medium @mobile", () => {
       { timeout: 5_000 },
     );
   });
+
+  test("header controls leave the logo uncovered on a narrow phone", async ({
+    frigateApp,
+  }) => {
+    await frigateApp.goto("/system#general");
+    await expect(frigateApp.page.getByLabel("Select general")).toHaveAttribute(
+      "data-state",
+      "on",
+      { timeout: 15_000 },
+    );
+    await frigateApp.page.setViewportSize({ width: 320, height: 740 });
+
+    const logo = frigateApp.page.locator("svg.fill-current").first();
+    const tabs = frigateApp.page
+      .locator("[data-radix-scroll-area-viewport]")
+      .filter({ has: frigateApp.page.getByLabel("Select general") });
+    const refreshed = frigateApp.page.getByText(/Just now|ago/);
+
+    const logoBox = await logo.boundingBox();
+    const tabsBox = await tabs.boundingBox();
+    const refreshedBox = await refreshed.boundingBox();
+
+    expect(tabsBox!.x + tabsBox!.width).toBeLessThanOrEqual(logoBox!.x + 1);
+    expect(refreshedBox!.x).toBeGreaterThanOrEqual(logoBox!.x + logoBox!.width);
+
+    // the clipped tabs stay reachable by scrolling
+    const overflow = await tabs.evaluate((el) => ({
+      scroll: el.scrollWidth,
+      client: el.clientWidth,
+    }));
+    expect(overflow.scroll).toBeGreaterThan(overflow.client);
+    await tabs.evaluate((el) => {
+      el.scrollLeft = el.scrollWidth;
+    });
+    await frigateApp.page.getByLabel("Select cameras").click();
+    await expect(frigateApp.page.getByLabel("Select cameras")).toHaveAttribute(
+      "data-state",
+      "on",
+      { timeout: 5_000 },
+    );
+  });
 });

web/public/locales/en/config/cameras.json

@@ -657,6 +657,10 @@
         "label": "Preferred language",
         "description": "Preferred language to request from the GenAI provider for generated responses."
       },
+      "frame_mode": {
+        "label": "Frame mode",
+        "description": "How frames are presented to the model. 'frames' sends the prompt followed by the frames, which suits models that track a sequence well on their own. 'annotated_frames' labels each frame and interleaves notes derived from object tracking, which helps models that lose track of activity that repeats or reverses."
+      },
       "response_style": {
         "label": "Response style",
         "description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."

web/public/locales/en/config/global.json

@@ -348,7 +348,7 @@
     },
     "roles": {
       "label": "Roles",
-      "description": "GenAI roles (chat, descriptions, embeddings); one provider per role."
+      "description": "GenAI roles (chat, descriptions, embeddings, transcribe); one provider per role. Only chat, descriptions, and embeddings are granted by default; transcribe must be listed explicitly."
     },
     "provider_options": {
       "label": "Provider options",
@@ -1028,6 +1028,10 @@
         "label": "Preferred language",
         "description": "Preferred language to request from the GenAI provider for generated responses."
       },
+      "frame_mode": {
+        "label": "Frame mode",
+        "description": "How frames are presented to the model. 'frames' sends the prompt followed by the frames, which suits models that track a sequence well on their own. 'annotated_frames' labels each frame and interleaves notes derived from object tracking, which helps models that lose track of activity that repeats or reverses."
+      },
       "response_style": {
         "label": "Response style",
         "description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."
@@ -1127,7 +1131,11 @@
     },
     "language": {
       "label": "Transcription language",
-      "description": "Language code used for transcription/translation (for example 'en' for English). See https://whisper-api.com/docs/languages/ for supported language codes."
+      "description": "Language code used for transcription/translation (for example 'en' for English), or 'auto' to let the model detect it. See https://whisper-api.com/docs/languages/ for supported language codes."
+    },
+    "model": {
+      "label": "Audio transcription model or GenAI provider name",
+      "description": "The transcription backend: 'whisper' for Frigate's built-in local models, or the name of a GenAI provider with the transcribe role."
     },
     "device": {
       "label": "Transcription device",

web/public/locales/en/views/settings.json

@@ -23,6 +23,9 @@
     "overriddenGlobalHeading_one": "This camera overrides {{count}} field from the global config:",
     "overriddenGlobalHeading_other": "This camera overrides {{count}} fields from the global config:",
     "overriddenGlobalNoDeltas": "This camera overrides the global config, but no field values differ.",
+    "overriddenLive": "Overridden (Live)",
+    "overriddenLiveTooltip": "This camera is running with a different value than the one saved in your config, which is shown here. The live view, MQTT, or an active profile can change it while Frigate is running.",
+    "overriddenLiveValue": "Running value: {{value}}",
     "overriddenBaseConfig": "Overridden (Base Config)",
     "overriddenBaseConfigTooltip": "The {{profile}} profile overrides configuration settings in this section",
     "overriddenBaseConfigHeading_one": "The {{profile}} profile overrides {{count}} field from the base config:",
@@ -1693,7 +1696,8 @@
       "options": {
         "embeddings": "Embedding",
         "descriptions": "Descriptions",
-        "chat": "Chat"
+        "chat": "Chat",
+        "transcribe": "Transcription"
       }
     },
     "semanticSearchModel": {
@@ -1742,6 +1746,14 @@
       "admin": "Admin",
       "viewer": "Viewer",
       "none": "None (deny access)"
+    },
+    "audioTranscriptionModel": {
+      "placeholder": "Select model…",
+      "builtIn": "Built-in Models",
+      "genaiProviders": "GenAI Providers"
+    },
+    "audioTranscriptionModelSize": {
+      "notApplicable": "Not applicable for GenAI providers"
     }
   },
   "globalConfig": {
@@ -1912,6 +1924,10 @@
     "imageSource": {
       "recordings": "Recordings",
       "previews": "Previews"
+    },
+    "frameMode": {
+      "frames": "Frames",
+      "annotated_frames": "Annotated frames"
     }
   },
   "logger": {
@@ -1941,19 +1957,25 @@
   "configMessages": {
     "review": {
       "recordDisabled": "Recording is disabled, review items will not be generated.",
+      "recordRuntimeDisabled": "Recording is enabled in your config, but it is currently turned off for this camera, so review items will not be generated. Turn it back on from the camera's live view, or check whether an active profile is disabling it.",
       "detectDisabled": "Object detection is disabled. Review items require detected objects to categorize alerts and detections.",
+      "detectRuntimeDisabled": "Object detection is enabled in your config, but it is currently turned off for this camera. Review items require detected objects to categorize alerts and detections. Turn it back on from the camera's live view, or check whether an active profile is disabling it.",
       "allNonAlertDetections": "All non-alert activity will be included as detections.",
-      "genaiImageSourceRecordingsRecordDisabled": "Image source is set to 'recordings', but recording is disabled. Frigate will fall back to preview images."
+      "genaiImageSourceRecordingsRecordDisabled": "Image source is set to 'recordings', but recording is disabled. Frigate will fall back to preview images.",
+      "genaiImageSourceRecordingsRecordRuntimeDisabled": "Image source is set to 'recordings', but recording is currently turned off for this camera even though your config enables it. Frigate will fall back to preview images."
     },
     "audio": {
       "noAudioRole": "No streams have the audio role defined. You must enable the audio role for audio detection to function."
     },
     "audioTranscription": {
-      "audioDetectionDisabled": "Audio detection is not enabled for this camera. Audio transcription requires audio detection to be active."
+      "audioDetectionDisabled": "Audio detection is not enabled for this camera. Audio transcription requires audio detection to be active.",
+      "audioDetectionRuntimeDisabled": "Audio detection is enabled in your config, but it is currently turned off for this camera, so audio transcription will not run. Turn it back on from the camera's live view, or check whether an active profile is disabling it.",
+      "genaiProviderSelected": "A GenAI provider is selected, so the device and model size settings are ignored."
     },
     "detect": {
       "fpsGreaterThanFive": "Setting the detect FPS higher than 5 is not recommended. Higher values may cause performance issues and will not provide any benefit.",
       "disabled": "Object detection is disabled. Snapshots, review items, and enrichments such as face recognition, license plate recognition, and Generative AI will not function.",
+      "runtimeDisabled": "Object detection is enabled in your config, but it is currently turned off for this camera. Snapshots, review items, and enrichments such as face recognition, license plate recognition, and Generative AI will not function until it is turned back on from the camera's live view, or until the active profile stops disabling it.",
       "sceneWithoutModel": "No detection model is configured for this scene, so this camera falls back to the model with a scene of 'All cameras'. Add a model for this scene to give the camera its own.",
       "resolutionShouldBeMultipleOfFour": "For best results, detect width and height should be multiples of 4. Other even values may produce visual artifacts or slight distortion in the detect stream.",
       "aspectRatioMismatch": "The width and height you've entered don't match the aspect ratio of your current detect resolution. This may produce a stretched or distorted image.",
@@ -1988,10 +2010,12 @@
       "noRecordSubRole": "No streams have the record_sub role defined. Sub stream recording will not function."
     },
     "birdseye": {
-      "objectTrackingDetectDisabled": "Birdseye includes tracked objects, but object detection is disabled for this camera. The camera will not appear in Birdseye."
+      "objectTrackingDetectDisabled": "Birdseye includes tracked objects, but object detection is disabled for this camera. The camera will not appear in Birdseye.",
+      "objectTrackingDetectRuntimeDisabled": "Birdseye includes tracked objects, but object detection is currently turned off for this camera even though your config enables it. The camera will not appear in Birdseye until it is turned back on from the camera's live view, or until the active profile stops disabling it."
     },
     "snapshots": {
-      "detectDisabled": "Object detection is disabled. Snapshots are generated from tracked objects and will not be created."
+      "detectDisabled": "Object detection is disabled. Snapshots are generated from tracked objects and will not be created.",
+      "detectRuntimeDisabled": "Object detection is enabled in your config, but it is currently turned off for this camera, so snapshots will not be created. Turn it back on from the camera's live view, or check whether an active profile is disabling it."
     },
     "semanticSearch": {
       "jinav2SmallModelSize": "The 'small' size with the Jina V2 model has high RAM and inference cost. The 'large' model with a discrete GPU is recommended."

web/src/components/chat/ChatMessage.tsx

@@ -133,7 +133,7 @@ export function MessageBubble({
             variant="select"
             size="icon"
             className="size-9 rounded-full"
-            disabled={!draftContent.trim()}
+            disabled={!draftContent.trim() || onEditSubmit == null}
             onClick={handleEditSubmit}
             aria-label={t("send")}
           >

web/src/components/config-form/section-configs/audio_transcription.ts

@@ -9,6 +9,11 @@ const audioTranscription: SectionConfigOverrides = {
         health: (ctx) =>
           ctx.fullCameraConfig?.audio_transcription?.enabled === true,
         messageKey: "configMessages.audioTranscription.audioDetectionDisabled",
+        runtimeOverride: {
+          section: "audio",
+          messageKey:
+            "configMessages.audioTranscription.audioDetectionRuntimeDisabled",
+        },
         severity: "warning",
         condition: (ctx) => {
           if (ctx.level === "camera" && ctx.fullCameraConfig) {
@@ -34,9 +39,32 @@ const audioTranscription: SectionConfigOverrides = {
     },
   },
   global: {
-    fieldOrder: ["enabled", "language", "device", "model_size"],
+    fieldOrder: ["enabled", "model", "language", "device", "model_size"],
     advancedFields: ["language", "device", "model_size"],
-    restartRequired: ["enabled", "language", "device", "model_size"],
+    restartRequired: ["enabled", "model", "language", "device", "model_size"],
+    fieldMessages: [
+      {
+        key: "genai-provider-ignores-local-settings",
+        health: (ctx) => ctx.fullConfig.audio_transcription?.enabled === true,
+        field: "device",
+        messageKey: "configMessages.audioTranscription.genaiProviderSelected",
+        severity: "info",
+        position: "after",
+        condition: (ctx) =>
+          typeof ctx.formData?.model === "string" &&
+          ctx.formData.model !== "" &&
+          ctx.formData.model !== "whisper",
+      },
+    ],
+    uiSchema: {
+      model: {
+        "ui:widget": "audioTranscriptionModel",
+      },
+      model_size: {
+        "ui:widget": "audioTranscriptionModelSize",
+        "ui:options": { size: "xs", enumI18nPrefix: "modelSize" },
+      },
+    },
   },
 };
 

web/src/components/config-form/section-configs/birdseye.ts

@@ -7,6 +7,11 @@ const birdseye: SectionConfigOverrides = {
       {
         key: "object-tracking-detect-disabled",
         messageKey: "configMessages.birdseye.objectTrackingDetectDisabled",
+        runtimeOverride: {
+          section: "detect",
+          messageKey:
+            "configMessages.birdseye.objectTrackingDetectRuntimeDisabled",
+        },
         severity: "info",
         condition: (ctx) => {
           if (ctx.level !== "camera" || !ctx.fullCameraConfig) return false;

web/src/components/config-form/section-configs/objects.ts

@@ -63,6 +63,10 @@ const objects: SectionConfigOverrides = {
       {
         key: "detect-disabled",
         messageKey: "configMessages.detect.disabled",
+        runtimeOverride: {
+          section: "detect",
+          messageKey: "configMessages.detect.runtimeDisabled",
+        },
         severity: "info",
         condition: (ctx) =>
           ctx.level === "camera" &&

web/src/components/config-form/section-configs/review.ts

@@ -7,6 +7,10 @@ const review: SectionConfigOverrides = {
       {
         key: "record-disabled",
         messageKey: "configMessages.review.recordDisabled",
+        runtimeOverride: {
+          section: "record",
+          messageKey: "configMessages.review.recordRuntimeDisabled",
+        },
         severity: "warning",
         condition: (ctx) => {
           if (ctx.level === "camera" && ctx.fullCameraConfig) {
@@ -18,6 +22,10 @@ const review: SectionConfigOverrides = {
       {
         key: "detect-disabled",
         messageKey: "configMessages.review.detectDisabled",
+        runtimeOverride: {
+          section: "detect",
+          messageKey: "configMessages.review.detectRuntimeDisabled",
+        },
         severity: "info",
         condition: (ctx) => {
           if (ctx.level === "camera" && ctx.fullCameraConfig) {
@@ -64,6 +72,11 @@ const review: SectionConfigOverrides = {
         field: "genai.image_source",
         messageKey:
           "configMessages.review.genaiImageSourceRecordingsRecordDisabled",
+        runtimeOverride: {
+          section: "record",
+          messageKey:
+            "configMessages.review.genaiImageSourceRecordingsRecordRuntimeDisabled",
+        },
         severity: "warning",
         position: "after",
         condition: (ctx) => {
@@ -82,6 +95,7 @@ const review: SectionConfigOverrides = {
       "detections.labels": "/configuration/review/#alerts-and-detections",
       genai: "/configuration/genai/genai_review",
       "genai.image_source": "/configuration/genai/genai_review#image-source",
+      "genai.frame_mode": "/configuration/genai/genai_review#frame-mode",
       "genai.additional_concerns":
         "/configuration/genai/genai_review#additional-concerns",
     },
@@ -138,6 +152,11 @@ const review: SectionConfigOverrides = {
             enumI18nPrefix: "review.imageSource",
           },
         },
+        frame_mode: {
+          "ui:options": {
+            enumI18nPrefix: "review.frameMode",
+          },
+        },
       },
     },
   },

web/src/components/config-form/section-configs/snapshots.ts

@@ -7,6 +7,10 @@ const snapshots: SectionConfigOverrides = {
       {
         key: "detect-disabled",
         messageKey: "configMessages.snapshots.detectDisabled",
+        runtimeOverride: {
+          section: "detect",
+          messageKey: "configMessages.snapshots.detectRuntimeDisabled",
+        },
         severity: "info",
         condition: (ctx) => {
           if (ctx.level !== "camera" || !ctx.fullCameraConfig) return false;

web/src/components/config-form/section-configs/types.ts

@@ -28,6 +28,17 @@ export type ConditionalMessage = {
   values?: Record<string, unknown>;
   /** Optional documentation path (e.g. "/configuration/object_detectors#model"). */
   docLink?: string;
+  /**
+   * Alternate wording for when the section this message depends on is enabled
+   * in the config but turned off on the running camera. Without it the message
+   * reads as a contradiction, since the form shows the saved config value.
+   */
+  runtimeOverride?: {
+    /** Camera section whose runtime state explains the message, e.g. "audio". */
+    section: string;
+    /** Translation key used in place of `messageKey`. */
+    messageKey: string;
+  };
   /**
    * Whether the Health tab evaluates this message against the saved config.
    * Absent or false: form only. true: shown whenever condition() holds. A

web/src/components/config-form/sections/BaseSection.tsx

@@ -1022,6 +1022,7 @@ export function ConfigSection({
             formContext={{
               level: effectiveLevel,
               cameraName,
+              sectionPath,
               globalValue,
               cameraValue,
               hasChanges,

web/src/components/config-form/theme/frigateTheme.ts

@@ -33,6 +33,8 @@ import { CameraPathWidget } from "./widgets/CameraPathWidget";
 import { OptionalFieldWidget } from "./widgets/OptionalFieldWidget";
 import { SemanticSearchModelWidget } from "./widgets/SemanticSearchModelWidget";
 import { SemanticSearchModelSizeWidget } from "./widgets/SemanticSearchModelSizeWidget";
+import { AudioTranscriptionModelWidget } from "./widgets/AudioTranscriptionModelWidget";
+import { AudioTranscriptionModelSizeWidget } from "./widgets/AudioTranscriptionModelSizeWidget";
 import { OnvifProfileWidget } from "./widgets/OnvifProfileWidget";
 import { PTZPresetsWidget } from "./widgets/PTZPresetsWidget";
 import { DefaultRoleWidget } from "./widgets/DefaultRoleWidget";
@@ -93,6 +95,8 @@ export const frigateTheme: FrigateTheme = {
     optionalField: OptionalFieldWidget,
     semanticSearchModel: SemanticSearchModelWidget,
     semanticSearchModelSize: SemanticSearchModelSizeWidget,
+    audioTranscriptionModel: AudioTranscriptionModelWidget,
+    audioTranscriptionModelSize: AudioTranscriptionModelSizeWidget,
     onvifProfile: OnvifProfileWidget,
     ptzPresets: PTZPresetsWidget,
     defaultRole: DefaultRoleWidget,

web/src/components/config-form/theme/templates/FieldTemplate.tsx

@@ -19,6 +19,8 @@ import { LuExternalLink } from "react-icons/lu";
 import { useDocDomain } from "@/hooks/use-doc-domain";
 import { requiresRestartForFieldPath } from "@/utils/configUtil";
 import RestartRequiredIndicator from "@/components/indicators/RestartRequiredIndicator";
+import RuntimeOverrideIndicator from "@/components/indicators/RuntimeOverrideIndicator";
+import { getRuntimeOverride } from "@/utils/runtimeOverrides";
 import {
   buildTranslationPath,
   resolveConfigTranslation,
@@ -211,6 +213,18 @@ export function FieldTemplate(props: FieldTemplateProps) {
     defaultRequiresRestart,
   );
 
+  // The form shows saved config values, so flag any field the running camera
+  // currently disagrees with. Profile editing shows that profile's overrides
+  // instead, where the comparison does not apply.
+  const runtimeOverride =
+    isCameraLevel && !formContext?.isProfile
+      ? getRuntimeOverride(
+          formContext?.fullCameraConfig,
+          formContext?.sectionPath,
+          pathSegments.join("."),
+        )
+      : undefined;
+
   // Use schema title/description as primary source (from JSON Schema)
   const schemaTitle = schema.title;
   const schemaDescription = schema.description;
@@ -502,6 +516,12 @@ export function FieldTemplate(props: FieldTemplateProps) {
         {finalLabel}
         {required && <span className="ml-1 text-destructive">*</span>}
         {fieldRequiresRestart && <RestartRequiredIndicator className="ml-2" />}
+        {runtimeOverride && (
+          <RuntimeOverrideIndicator
+            runtimeValue={runtimeOverride.runtime}
+            className="ml-2"
+          />
+        )}
       </Label>
     );
   };
@@ -519,6 +539,12 @@ export function FieldTemplate(props: FieldTemplateProps) {
         {finalLabel}
         {required && <span className="ml-1 text-destructive">*</span>}
         {fieldRequiresRestart && <RestartRequiredIndicator className="ml-2" />}
+        {runtimeOverride && (
+          <RuntimeOverrideIndicator
+            runtimeValue={runtimeOverride.runtime}
+            className="ml-2"
+          />
+        )}
       </Label>
     );
   };
@@ -540,6 +566,12 @@ export function FieldTemplate(props: FieldTemplateProps) {
         {finalLabel}
         {required && <span className="ml-1 text-destructive">*</span>}
         {fieldRequiresRestart && <RestartRequiredIndicator className="ml-2" />}
+        {runtimeOverride && (
+          <RuntimeOverrideIndicator
+            runtimeValue={runtimeOverride.runtime}
+            className="ml-2"
+          />
+        )}
       </Label>
     );
   };

web/src/components/config-form/theme/widgets/AudioTranscriptionModelSizeWidget.tsx

@@ -0,0 +1,17 @@
+// audio_transcription.model_size. See GenAIBackedModelSizeWidget for the shared
+// implementation, including the clear-vs-default handling.
+import type { WidgetProps } from "@rjsf/utils";
+import { GenAIBackedModelSizeWidget } from "./GenAIBackedModelSizeWidget";
+
+export function AudioTranscriptionModelSizeWidget(props: WidgetProps) {
+  return (
+    <GenAIBackedModelSizeWidget
+      {...props}
+      options={{
+        ...props.options,
+        builtInModels: ["whisper"],
+        i18nPrefix: "audioTranscriptionModelSize",
+      }}
+    />
+  );
+}

web/src/components/config-form/theme/widgets/AudioTranscriptionModelWidget.tsx

@@ -0,0 +1,18 @@
+// audio_transcription.model: the built-in whisper backend plus GenAI providers
+// with the transcribe role. See GenAIBackedModelWidget for the shared
+// implementation.
+import type { WidgetProps } from "@rjsf/utils";
+import { GenAIBackedModelWidget } from "./GenAIBackedModelWidget";
+
+export function AudioTranscriptionModelWidget(props: WidgetProps) {
+  return (
+    <GenAIBackedModelWidget
+      {...props}
+      options={{
+        ...props.options,
+        role: "transcribe",
+        i18nPrefix: "audioTranscriptionModel",
+      }}
+    />
+  );
+}

web/src/components/config-form/theme/widgets/GenAIBackedModelSizeWidget.tsx

@@ -0,0 +1,65 @@
+// Disables model_size and shows "N/A" when a GenAI provider is selected in the
+// companion model field. Reads model via LiveFormDataContext so it re-runs even
+// when RJSF's SchemaField memoization would skip this widget. The built-in model
+// names and the i18n key prefix come from ui:options.
+import type { WidgetProps } from "@rjsf/utils";
+import { useContext, useEffect } from "react";
+import { useTranslation } from "react-i18next";
+import {
+  Select,
+  SelectContent,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { LiveFormDataContext } from "../../LiveFormDataContext";
+import { getSizedFieldClassName } from "../utils";
+import { SelectWidget } from "./SelectWidget";
+
+export function GenAIBackedModelSizeWidget(props: WidgetProps) {
+  const { t } = useTranslation(["views/settings"]);
+  const liveFormData = useContext(LiveFormDataContext);
+  const model = liveFormData?.model;
+
+  const builtInModels = (props.options?.builtInModels as string[]) ?? [];
+  const i18nPrefix =
+    (props.options?.i18nPrefix as string | undefined) ??
+    "semanticSearchModelSize";
+
+  const isProvider =
+    typeof model === "string" && model !== "" && !builtInModels.includes(model);
+
+  // model_size is unused on a GenAI provider. Only clear it (which the backend
+  // treats as "remove") for a non-default value, which can only come from the
+  // config file. A defaulted value is indistinguishable from unset in the
+  // resolved config, so clearing it would falsely dirty the field and delete a
+  // YAML key that isn't there. Restore the default when returning to a built-in model.
+  const { value, onChange, schema } = props;
+  const schemaDefault = schema?.default as string | undefined;
+  useEffect(() => {
+    if (isProvider) {
+      if (value !== undefined && value !== schemaDefault) {
+        onChange(undefined);
+      }
+    } else if (value === undefined && schemaDefault) {
+      onChange(schemaDefault);
+    }
+  }, [isProvider, value, onChange, schemaDefault]);
+
+  if (isProvider) {
+    const fieldClassName = getSizedFieldClassName(props.options ?? {}, "sm");
+    return (
+      <Select value="" disabled>
+        <SelectTrigger className={fieldClassName}>
+          <SelectValue
+            placeholder={t(`configForm.${i18nPrefix}.notApplicable`, {
+              defaultValue: "Not applicable for GenAI providers",
+            })}
+          />
+        </SelectTrigger>
+        <SelectContent />
+      </Select>
+    );
+  }
+
+  return <SelectWidget {...props} />;
+}

web/src/components/config-form/theme/widgets/GenAIBackedModelWidget.tsx

@@ -0,0 +1,164 @@
+// Combobox for a "local model or GenAI provider" field (semantic_search.model,
+// audio_transcription.model). Shows the built-in model enum values alongside the
+// GenAI providers holding the relevant role. The role and the i18n key prefix
+// come from ui:options so each field can reuse this with its own wording.
+import { useState, useMemo } from "react";
+import type { WidgetProps } from "@rjsf/utils";
+import { useTranslation } from "react-i18next";
+import { Check, ChevronsUpDown } from "lucide-react";
+import { cn } from "@/lib/utils";
+import { Button } from "@/components/ui/button";
+import {
+  Command,
+  CommandGroup,
+  CommandItem,
+  CommandList,
+} from "@/components/ui/command";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/components/ui/popover";
+import type { ConfigFormContext } from "@/types/configForm";
+import { getSizedFieldClassName } from "../utils";
+
+interface ProviderOption {
+  value: string;
+  label: string;
+}
+
+export function GenAIBackedModelWidget(props: WidgetProps) {
+  const { id, value, disabled, readonly, onChange, schema, registry, options } =
+    props;
+  const { t } = useTranslation(["views/settings"]);
+  const [open, setOpen] = useState(false);
+
+  const formContext = registry?.formContext as ConfigFormContext | undefined;
+  const fieldClassName = getSizedFieldClassName(options, "sm");
+  const role = (options?.role as string | undefined) ?? "embeddings";
+  const i18nPrefix =
+    (options?.i18nPrefix as string | undefined) ?? "semanticSearchModel";
+
+  // Built-in model options from schema.examples (populated by transformer
+  // collapsing the anyOf enum+string union)
+  const builtInModels: ProviderOption[] = useMemo(() => {
+    const examples = (schema as Record<string, unknown>).examples;
+    if (!Array.isArray(examples)) return [];
+    return examples
+      .filter((v): v is string => typeof v === "string")
+      .map((v) => ({ value: v, label: v }));
+  }, [schema]);
+
+  // GenAI providers that have the role this field is backed by
+  const roleProviders: ProviderOption[] = useMemo(() => {
+    const genai = (
+      formContext?.fullConfig as Record<string, unknown> | undefined
+    )?.genai;
+    if (!genai || typeof genai !== "object" || Array.isArray(genai)) return [];
+
+    const providers: ProviderOption[] = [];
+    for (const [key, config] of Object.entries(
+      genai as Record<string, unknown>,
+    )) {
+      if (!config || typeof config !== "object" || Array.isArray(config))
+        continue;
+      const roles = (config as Record<string, unknown>).roles;
+      if (Array.isArray(roles) && roles.includes(role)) {
+        providers.push({ value: key, label: key });
+      }
+    }
+    return providers;
+  }, [formContext?.fullConfig, role]);
+
+  const currentLabel =
+    builtInModels.find((m) => m.value === value)?.label ??
+    roleProviders.find((p) => p.value === value)?.label ??
+    (typeof value === "string" && value ? value : undefined);
+
+  return (
+    <Popover open={open} onOpenChange={setOpen}>
+      <PopoverTrigger asChild>
+        <Button
+          id={id}
+          type="button"
+          variant="outline"
+          role="combobox"
+          aria-expanded={open}
+          disabled={disabled || readonly}
+          className={cn(
+            "justify-between font-normal",
+            !currentLabel && "text-muted-foreground",
+            fieldClassName,
+          )}
+        >
+          {currentLabel ??
+            t(`configForm.${i18nPrefix}.placeholder`, {
+              ns: "views/settings",
+              defaultValue: "Select model…",
+            })}
+          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-[--radix-popover-trigger-width] p-0">
+        <Command>
+          <CommandList>
+            {builtInModels.length > 0 && (
+              <CommandGroup
+                heading={t(`configForm.${i18nPrefix}.builtIn`, {
+                  ns: "views/settings",
+                  defaultValue: "Built-in Models",
+                })}
+              >
+                {builtInModels.map((model) => (
+                  <CommandItem
+                    key={model.value}
+                    value={model.value}
+                    onSelect={() => {
+                      onChange(model.value);
+                      setOpen(false);
+                    }}
+                  >
+                    <Check
+                      className={cn(
+                        "mr-2 h-4 w-4",
+                        value === model.value ? "opacity-100" : "opacity-0",
+                      )}
+                    />
+                    {model.label}
+                  </CommandItem>
+                ))}
+              </CommandGroup>
+            )}
+            {roleProviders.length > 0 && (
+              <CommandGroup
+                heading={t(`configForm.${i18nPrefix}.genaiProviders`, {
+                  ns: "views/settings",
+                  defaultValue: "GenAI Providers",
+                })}
+              >
+                {roleProviders.map((provider) => (
+                  <CommandItem
+                    key={provider.value}
+                    value={provider.value}
+                    onSelect={() => {
+                      onChange(provider.value);
+                      setOpen(false);
+                    }}
+                  >
+                    <Check
+                      className={cn(
+                        "mr-2 h-4 w-4",
+                        value === provider.value ? "opacity-100" : "opacity-0",
+                      )}
+                    />
+                    {provider.label}
+                  </CommandItem>
+                ))}
+              </CommandGroup>
+            )}
+          </CommandList>
+        </Command>
+      </PopoverContent>
+    </Popover>
+  );
+}

web/src/components/config-form/theme/widgets/GenAIModelWidget.tsx

@@ -59,20 +59,30 @@ export function GenAIModelWidget(props: WidgetProps) {
 
   const formContext = registry?.formContext as ConfigFormContext | undefined;
 
-  // Build a fingerprint from the saved config's provider + base_url so the
-  // SWR key changes (and models are refetched) whenever those fields are saved.
-  const configFingerprint = useMemo(() => {
-    if (!providerKey) return "";
+  const savedEntry = useMemo<Record<string, unknown> | null>(() => {
+    if (!providerKey) return null;
     const genai = (
       formContext?.fullConfig as Record<string, unknown> | undefined
     )?.genai;
-    if (!genai || typeof genai !== "object" || Array.isArray(genai)) return "";
+    if (!genai || typeof genai !== "object" || Array.isArray(genai)) {
+      return null;
+    }
     const entry = (genai as Record<string, unknown>)[providerKey];
-    if (!entry || typeof entry !== "object" || Array.isArray(entry)) return "";
-    const e = entry as Record<string, unknown>;
-    return `${e.provider ?? ""}|${e.base_url ?? ""}`;
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+      return null;
+    }
+    return entry as Record<string, unknown>;
   }, [providerKey, formContext?.fullConfig]);
 
+  const savedProvider =
+    typeof savedEntry?.provider === "string" ? savedEntry.provider : null;
+
+  // Build a fingerprint from the saved config's provider + base_url so the
+  // SWR key changes (and models are refetched) whenever those fields are saved.
+  const configFingerprint = savedEntry
+    ? `${savedEntry.provider ?? ""}|${savedEntry.base_url ?? ""}`
+    : "";
+
   const { data: allModels, mutate: mutateModels } = useSWR<GenAIModelsResponse>(
     "genai/models",
     {
@@ -148,6 +158,17 @@ export function GenAIModelWidget(props: WidgetProps) {
     typeof formEntry?.provider === "string" ? formEntry.provider : null;
   const canProbe = Boolean(formProvider) && !probing;
 
+  // A model name belongs to its provider, so switching provider clears it.
+  // Returning to the saved provider (including a form reset) leaves it alone.
+  const prevFormProvider = useRef(formProvider);
+  useEffect(() => {
+    const previous = prevFormProvider.current;
+    prevFormProvider.current = formProvider;
+
+    if (previous === formProvider || formProvider === savedProvider) return;
+    if (typeof value === "string" && value) onChange("");
+  }, [formProvider, savedProvider, value, onChange]);
+
   const probe = async () => {
     if (!formEntry || !formProvider) return;
     if (probeSuccessTimerRef.current) {

web/src/components/config-form/theme/widgets/GenAIRolesWidget.tsx

@@ -4,9 +4,14 @@ import { useTranslation } from "react-i18next";
 import useSWR from "swr";
 import { Switch } from "@/components/ui/switch";
 import type { ConfigFormContext } from "@/types/configForm";
-import type { GenAIModelsResponse } from "@/types/chat";
+import type { GenAIModelCapabilities, GenAIModelsResponse } from "@/types/chat";
 
-const GENAI_ROLES = ["embeddings", "descriptions", "chat"] as const;
+const GENAI_ROLES = [
+  "embeddings",
+  "descriptions",
+  "chat",
+  "transcribe",
+] as const;
 
 function normalizeValue(value: unknown): string[] {
   if (Array.isArray(value)) {
@@ -20,6 +25,22 @@ function normalizeValue(value: unknown): string[] {
   return [];
 }
 
+function getString(value: unknown): string | undefined {
+  return typeof value === "string" && value ? value : undefined;
+}
+
+function getEntry(
+  entries: unknown,
+  providerKey: string | undefined,
+): Record<string, unknown> | undefined {
+  if (!providerKey || !entries || typeof entries !== "object") return undefined;
+  const entry = (entries as Record<string, unknown>)[providerKey];
+  if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+    return undefined;
+  }
+  return entry as Record<string, unknown>;
+}
+
 function getProviderKey(widgetId: string): string | undefined {
   const prefix = "root_";
   const suffix = "_roles";
@@ -43,18 +64,65 @@ export function GenAIRolesWidget(props: WidgetProps) {
     revalidateOnFocus: false,
   });
 
-  const embeddingsSupported = useMemo(() => {
-    if (!providerKey) return true;
-    const info = genaiInfo?.[providerKey];
-    return info ? info.supports_embeddings : true;
-  }, [genaiInfo, providerKey]);
+  // The model currently chosen in the form, which is what the roles have to
+  // reflect. Reading the saved config instead would keep reporting the previous
+  // model's capabilities until a save and a refetch.
+  const formEntry = useMemo(
+    () => getEntry(formContext?.formData, providerKey),
+    [formContext?.formData, providerKey],
+  );
+  const savedEntry = useMemo(
+    () => getEntry(formContext?.fullConfig?.genai, providerKey),
+    [formContext?.fullConfig?.genai, providerKey],
+  );
 
+  const selectedModel = getString(formEntry?.model);
+
+  // The entry-level capability flags describe the saved provider and model
+  // only, so they apply while the form still matches the saved entry.
+  const matchesSaved =
+    savedEntry !== undefined &&
+    getString(formEntry?.provider) === getString(savedEntry.provider) &&
+    selectedModel === getString(savedEntry.model);
+
+  // Capabilities the provider reported for that specific model. Absent when the
+  // provider cannot describe a model it has not loaded.
+  const modelCapabilities: GenAIModelCapabilities | undefined = useMemo(() => {
+    if (!providerKey || !selectedModel) return undefined;
+    return genaiInfo?.[providerKey]?.model_capabilities?.[selectedModel];
+  }, [genaiInfo, providerKey, selectedModel]);
+
+  const capabilityOf = (
+    key: "supports_embeddings" | "supports_transcription",
+  ): boolean => {
+    const perModel = modelCapabilities?.[key];
+    if (perModel !== undefined) return perModel;
+    if (!providerKey || !matchesSaved) return true;
+    const info = genaiInfo?.[providerKey];
+    // assume supported when nothing is known, so a role is never hidden on
+    // missing information alone
+    return info ? info[key] : true;
+  };
+
+  const embeddingsSupported = capabilityOf("supports_embeddings");
+  const transcriptionSupported = capabilityOf("supports_transcription");
+
+  const unsupportedRoles = useMemo(() => {
+    const unsupported = new Set<string>();
+
+    if (!embeddingsSupported) unsupported.add("embeddings");
+    if (!transcriptionSupported) unsupported.add("transcribe");
+
+    return unsupported;
+  }, [embeddingsSupported, transcriptionSupported]);
+
+  // a selected role stays visible so it can still be switched off
   const availableRoles = useMemo(
     () =>
-      embeddingsSupported
-        ? GENAI_ROLES
-        : GENAI_ROLES.filter((role) => role !== "embeddings"),
-    [embeddingsSupported],
+      GENAI_ROLES.filter(
+        (role) => !unsupportedRoles.has(role) || selectedRoles.includes(role),
+      ),
+    [unsupportedRoles, selectedRoles],
   );
 
   const occupiedRoles = useMemo(() => {
@@ -80,11 +148,16 @@ export function GenAIRolesWidget(props: WidgetProps) {
     return occupied;
   }, [formContext?.formData, providerKey]);
 
+  // Strip every unsupported role in a single onChange; two effects each
+  // rewriting the same value would race and lose one of the edits. Only a
+  // model or provider picked in the form can rule a role out, so capability
+  // data arriving for the saved entry never edits the form on its own.
   useEffect(() => {
-    if (!embeddingsSupported && selectedRoles.includes("embeddings")) {
-      onChange(selectedRoles.filter((role) => role !== "embeddings"));
-    }
-  }, [embeddingsSupported, selectedRoles, onChange]);
+    if (matchesSaved) return;
+    if (!selectedRoles.some((role) => unsupportedRoles.has(role))) return;
+
+    onChange(selectedRoles.filter((role) => !unsupportedRoles.has(role)));
+  }, [matchesSaved, unsupportedRoles, selectedRoles, onChange]);
 
   const toggleRole = (role: string, enabled: boolean) => {
     if (enabled) {

web/src/components/config-form/theme/widgets/SemanticSearchModelSizeWidget.tsx

@@ -1,61 +1,17 @@
-// Disables model_size and shows "N/A" when a GenAI provider is selected.
-// Reads model via LiveFormDataContext so it re-runs even when RJSF's
-// SchemaField memoization would skip this widget.
+// semantic_search.model_size. See GenAIBackedModelSizeWidget for the shared
+// implementation, including the clear-vs-default handling.
 import type { WidgetProps } from "@rjsf/utils";
-import { useContext, useEffect } from "react";
-import { useTranslation } from "react-i18next";
-import {
-  Select,
-  SelectContent,
-  SelectTrigger,
-  SelectValue,
-} from "@/components/ui/select";
-import { LiveFormDataContext } from "../../LiveFormDataContext";
-import { getSizedFieldClassName } from "../utils";
-import { SelectWidget } from "./SelectWidget";
+import { GenAIBackedModelSizeWidget } from "./GenAIBackedModelSizeWidget";
 
 export function SemanticSearchModelSizeWidget(props: WidgetProps) {
-  const { t } = useTranslation(["views/settings"]);
-  const liveFormData = useContext(LiveFormDataContext);
-  const model = liveFormData?.model;
-  const isProvider =
-    typeof model === "string" &&
-    model !== "" &&
-    model !== "jinav1" &&
-    model !== "jinav2";
-
-  // model_size is unused on a GenAI provider. Only clear it (which the backend
-  // treats as "remove") for a non-default value, which can only come from the
-  // config file. A defaulted value is indistinguishable from unset in the
-  // resolved config, so clearing it would falsely dirty the field and delete a
-  // YAML key that isn't there. Restore the default when returning to a Jina model.
-  const { value, onChange, schema } = props;
-  const schemaDefault = schema?.default as string | undefined;
-  useEffect(() => {
-    if (isProvider) {
-      if (value !== undefined && value !== schemaDefault) {
-        onChange(undefined);
-      }
-    } else if (value === undefined && schemaDefault) {
-      onChange(schemaDefault);
-    }
-  }, [isProvider, value, onChange, schemaDefault]);
-
-  if (isProvider) {
-    const fieldClassName = getSizedFieldClassName(props.options ?? {}, "sm");
-    return (
-      <Select value="" disabled>
-        <SelectTrigger className={fieldClassName}>
-          <SelectValue
-            placeholder={t("configForm.semanticSearchModelSize.notApplicable", {
-              defaultValue: "Not applicable for GenAI providers",
-            })}
-          />
-        </SelectTrigger>
-        <SelectContent />
-      </Select>
-    );
-  }
-
-  return <SelectWidget {...props} />;
+  return (
+    <GenAIBackedModelSizeWidget
+      {...props}
+      options={{
+        ...props.options,
+        builtInModels: ["jinav1", "jinav2"],
+        i18nPrefix: "semanticSearchModelSize",
+      }}
+    />
+  );
 }

web/src/components/config-form/theme/widgets/SemanticSearchModelWidget.tsx

@@ -1,159 +1,17 @@
-// Combobox widget for semantic_search.model field.
-// Shows built-in model enum values and GenAI providers with the embeddings role.
-import { useState, useMemo } from "react";
+// semantic_search.model: built-in Jina models plus GenAI providers with the
+// embeddings role. See GenAIBackedModelWidget for the shared implementation.
 import type { WidgetProps } from "@rjsf/utils";
-import { useTranslation } from "react-i18next";
-import { Check, ChevronsUpDown } from "lucide-react";
-import { cn } from "@/lib/utils";
-import { Button } from "@/components/ui/button";
-import {
-  Command,
-  CommandGroup,
-  CommandItem,
-  CommandList,
-} from "@/components/ui/command";
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from "@/components/ui/popover";
-import type { ConfigFormContext } from "@/types/configForm";
-import { getSizedFieldClassName } from "../utils";
-
-interface ProviderOption {
-  value: string;
-  label: string;
-}
+import { GenAIBackedModelWidget } from "./GenAIBackedModelWidget";
 
 export function SemanticSearchModelWidget(props: WidgetProps) {
-  const { id, value, disabled, readonly, onChange, schema, registry, options } =
-    props;
-  const { t } = useTranslation(["views/settings"]);
-  const [open, setOpen] = useState(false);
-
-  const formContext = registry?.formContext as ConfigFormContext | undefined;
-  const fieldClassName = getSizedFieldClassName(options, "sm");
-
-  // Built-in model options from schema.examples (populated by transformer
-  // collapsing the anyOf enum+string union)
-  const builtInModels: ProviderOption[] = useMemo(() => {
-    const examples = (schema as Record<string, unknown>).examples;
-    if (!Array.isArray(examples)) return [];
-    return examples
-      .filter((v): v is string => typeof v === "string")
-      .map((v) => ({ value: v, label: v }));
-  }, [schema]);
-
-  // GenAI providers that have the "embeddings" role
-  const embeddingsProviders: ProviderOption[] = useMemo(() => {
-    const genai = (
-      formContext?.fullConfig as Record<string, unknown> | undefined
-    )?.genai;
-    if (!genai || typeof genai !== "object" || Array.isArray(genai)) return [];
-
-    const providers: ProviderOption[] = [];
-    for (const [key, config] of Object.entries(
-      genai as Record<string, unknown>,
-    )) {
-      if (!config || typeof config !== "object" || Array.isArray(config))
-        continue;
-      const roles = (config as Record<string, unknown>).roles;
-      if (Array.isArray(roles) && roles.includes("embeddings")) {
-        providers.push({ value: key, label: key });
-      }
-    }
-    return providers;
-  }, [formContext?.fullConfig]);
-
-  const currentLabel =
-    builtInModels.find((m) => m.value === value)?.label ??
-    embeddingsProviders.find((p) => p.value === value)?.label ??
-    (typeof value === "string" && value ? value : undefined);
-
   return (
-    <Popover open={open} onOpenChange={setOpen}>
-      <PopoverTrigger asChild>
-        <Button
-          id={id}
-          type="button"
-          variant="outline"
-          role="combobox"
-          aria-expanded={open}
-          disabled={disabled || readonly}
-          className={cn(
-            "justify-between font-normal",
-            !currentLabel && "text-muted-foreground",
-            fieldClassName,
-          )}
-        >
-          {currentLabel ??
-            t("configForm.semanticSearchModel.placeholder", {
-              ns: "views/settings",
-              defaultValue: "Select model…",
-            })}
-          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
-        </Button>
-      </PopoverTrigger>
-      <PopoverContent className="w-[--radix-popover-trigger-width] p-0">
-        <Command>
-          <CommandList>
-            {builtInModels.length > 0 && (
-              <CommandGroup
-                heading={t("configForm.semanticSearchModel.builtIn", {
-                  ns: "views/settings",
-                  defaultValue: "Built-in Models",
-                })}
-              >
-                {builtInModels.map((model) => (
-                  <CommandItem
-                    key={model.value}
-                    value={model.value}
-                    onSelect={() => {
-                      onChange(model.value);
-                      setOpen(false);
-                    }}
-                  >
-                    <Check
-                      className={cn(
-                        "mr-2 h-4 w-4",
-                        value === model.value ? "opacity-100" : "opacity-0",
-                      )}
-                    />
-                    {model.label}
-                  </CommandItem>
-                ))}
-              </CommandGroup>
-            )}
-            {embeddingsProviders.length > 0 && (
-              <CommandGroup
-                heading={t("configForm.semanticSearchModel.genaiProviders", {
-                  ns: "views/settings",
-                  defaultValue: "GenAI Providers",
-                })}
-              >
-                {embeddingsProviders.map((provider) => (
-                  <CommandItem
-                    key={provider.value}
-                    value={provider.value}
-                    onSelect={() => {
-                      onChange(provider.value);
-                      setOpen(false);
-                    }}
-                  >
-                    <Check
-                      className={cn(
-                        "mr-2 h-4 w-4",
-                        value === provider.value ? "opacity-100" : "opacity-0",
-                      )}
-                    />
-                    {provider.label}
-                  </CommandItem>
-                ))}
-              </CommandGroup>
-            )}
-          </CommandList>
-        </Command>
-      </PopoverContent>
-    </Popover>
+    <GenAIBackedModelWidget
+      {...props}
+      options={{
+        ...props.options,
+        role: "embeddings",
+        i18nPrefix: "semanticSearchModel",
+      }}
+    />
   );
 }

web/src/components/indicators/RuntimeOverrideIndicator.tsx

@@ -0,0 +1,55 @@
+import { useTranslation } from "react-i18next";
+import { Badge } from "@/components/ui/badge";
+import { cn } from "@/lib/utils";
+import { Tooltip, TooltipContent } from "../ui/tooltip";
+import { TooltipTrigger } from "@radix-ui/react-tooltip";
+
+type RuntimeOverrideIndicatorProps = {
+  /** The value the camera is running with, which differs from the saved config. */
+  runtimeValue: unknown;
+  className?: string;
+};
+
+/**
+ * Field-level companion to the section override badges. Marks a field the
+ * running camera has drifted from, so the saved value on screen never reads as
+ * the live one.
+ */
+export default function RuntimeOverrideIndicator({
+  runtimeValue,
+  className,
+}: RuntimeOverrideIndicatorProps) {
+  const { t } = useTranslation(["views/settings", "common"]);
+
+  const displayValue =
+    typeof runtimeValue === "boolean"
+      ? t(runtimeValue ? "button.on" : "button.off", { ns: "common" })
+      : Array.isArray(runtimeValue)
+        ? runtimeValue.join(", ")
+        : String(runtimeValue);
+
+  return (
+    <Tooltip>
+      <TooltipTrigger asChild>
+        <Badge
+          variant="secondary"
+          className={cn(
+            "cursor-default border-2 border-selected text-center align-middle text-xs font-normal text-primary-variant",
+            className,
+          )}
+        >
+          {t("button.overriddenLive", { ns: "views/settings" })}
+        </Badge>
+      </TooltipTrigger>
+      <TooltipContent className="max-w-72">
+        <p>{t("button.overriddenLiveTooltip", { ns: "views/settings" })}</p>
+        <p className="mt-1">
+          {t("button.overriddenLiveValue", {
+            ns: "views/settings",
+            value: displayValue,
+          })}
+        </p>
+      </TooltipContent>
+    </Tooltip>
+  );
+}

web/src/hooks/use-config-messages.ts

@@ -4,6 +4,7 @@ import type {
   FieldConditionalMessage,
   MessageConditionContext,
 } from "@/components/config-form/section-configs/types";
+import { resolveMessageKey } from "@/utils/runtimeOverrides";
 
 export function useConfigMessages(
   messages: ConditionalMessage[] | undefined,
@@ -15,12 +16,16 @@ export function useConfigMessages(
 } {
   const activeMessages = useMemo(() => {
     if (!messages || !context) return [];
-    return messages.filter((msg) => msg.condition(context));
+    return messages
+      .filter((msg) => msg.condition(context))
+      .map((msg) => ({ ...msg, messageKey: resolveMessageKey(msg, context) }));
   }, [messages, context]);
 
   const activeFieldMessages = useMemo(() => {
     if (!fieldMessages || !context) return [];
-    return fieldMessages.filter((msg) => msg.condition(context));
+    return fieldMessages
+      .filter((msg) => msg.condition(context))
+      .map((msg) => ({ ...msg, messageKey: resolveMessageKey(msg, context) }));
   }, [fieldMessages, context]);
 
   return { activeMessages, activeFieldMessages };

web/src/hooks/use-keyboard-listener.tsx

@@ -7,6 +7,16 @@ export type KeyModifiers = {
   shift: boolean;
 };
 
+const handledByShortcut = new WeakSet<Event>();
+
+// Radix dismisses a dialog or menu on Escape from a capture-phase listener and
+// calls preventDefault() without stopping propagation, so a page shortcut would
+// otherwise act on the same press. Keys another shortcut hook handled still get
+// through, since their listener order changes with every render.
+function handledElsewhere(event: KeyboardEvent): boolean {
+  return event.defaultPrevented && !handledByShortcut.has(event);
+}
+
 export default function useKeyboardListener(
   keys: string[],
   listener?: (key: string | null, modifiers: KeyModifiers) => boolean,
@@ -27,6 +37,10 @@ export default function useKeyboardListener(
         return;
       }
 
+      if (handledElsewhere(e)) {
+        return;
+      }
+
       const modifiers = {
         down: true,
         repeat: e.repeat,
@@ -63,7 +77,10 @@ export default function useKeyboardListener(
         }
       } else if (keys.includes(e.key) && listener) {
         const preventDefault = listener(e.key, modifiers);
-        if (preventDefault) e.preventDefault();
+        if (preventDefault) {
+          e.preventDefault();
+          handledByShortcut.add(e);
+        }
       } else if (
         listener &&
         (e.key === "Shift" || e.key === "Control" || e.key === "Meta")

web/src/pages/Chat.tsx

@@ -362,7 +362,7 @@ export default function ChatPage() {
             role="user"
             content={msg.content}
             messageIndex={i}
-            onEditSubmit={handleEditSubmit}
+            onEditSubmit={isLoading ? undefined : handleEditSubmit}
             isComplete
             showStats={showStats}
           />

web/src/pages/System.tsx

@@ -24,6 +24,8 @@ import HealthMetrics from "@/views/system/HealthMetrics";
 import NoticeFilterButton from "@/components/health/NoticeFilterButton";
 import { DEFAULT_NOTICE_FILTER, NoticeFilter } from "@/types/health";
 import { useTranslation } from "react-i18next";
+import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
+import { cn } from "@/lib/utils";
 
 const allMetrics = [
   "health",
@@ -96,35 +98,40 @@ function System() {
         {isMobile && (
           <Logo className="absolute inset-x-1/2 h-8 -translate-x-1/2" />
         )}
-        <ToggleGroup
-          className="*:rounded-md *:px-3 *:py-4"
-          type="single"
-          size="sm"
-          value={pageToggle}
-          onValueChange={(value: SystemMetric) => {
-            if (value) {
-              setPageToggle(value);
-            }
-          }} // don't allow the severity to be unselected
-        >
-          {Object.values(metrics).map((item) => (
-            <ToggleGroupItem
-              key={item}
-              className={`flex items-center justify-between gap-2 ${pageToggle == item ? "" : "*:text-muted-foreground"}`}
-              value={item}
-              aria-label={`Select ${item}`}
+        <ScrollArea className={cn("whitespace-nowrap", isMobile && "w-[45%]")}>
+          <div className="flex flex-row">
+            <ToggleGroup
+              className="*:rounded-md *:px-3 *:py-4"
+              type="single"
+              size="sm"
+              value={pageToggle}
+              onValueChange={(value: SystemMetric) => {
+                if (value) {
+                  setPageToggle(value);
+                }
+              }} // don't allow the severity to be unselected
             >
-              {item == "health" && <LuHeartPulse className="size-4" />}
-              {item == "general" && <LuActivity className="size-4" />}
-              {item == "enrichments" && <LuSearchCode className="size-4" />}
-              {item == "storage" && <LuHardDrive className="size-4" />}
-              {item == "cameras" && <FaVideo className="size-4" />}
-              {isDesktop && (
-                <div className="smart-capitalize">{t(item + ".title")}</div>
-              )}
-            </ToggleGroupItem>
-          ))}
-        </ToggleGroup>
+              {Object.values(metrics).map((item) => (
+                <ToggleGroupItem
+                  key={item}
+                  className={`flex items-center justify-between gap-2 ${pageToggle == item ? "" : "*:text-muted-foreground"}`}
+                  value={item}
+                  aria-label={`Select ${item}`}
+                >
+                  {item == "health" && <LuHeartPulse className="size-4" />}
+                  {item == "general" && <LuActivity className="size-4" />}
+                  {item == "enrichments" && <LuSearchCode className="size-4" />}
+                  {item == "storage" && <LuHardDrive className="size-4" />}
+                  {item == "cameras" && <FaVideo className="size-4" />}
+                  {isDesktop && (
+                    <div className="smart-capitalize">{t(item + ".title")}</div>
+                  )}
+                </ToggleGroupItem>
+              ))}
+            </ToggleGroup>
+            <ScrollBar orientation="horizontal" className="h-0" />
+          </div>
+        </ScrollArea>
 
         <div className="flex h-full items-center">
           {pageToggle == "health" && (
@@ -135,7 +142,7 @@ function System() {
           )}
           {lastUpdated && pageToggle != "health" && (
             <div className="h-full content-center text-sm text-muted-foreground">
-              {t("lastRefreshed")}
+              {isDesktop && t("lastRefreshed")}
               <TimeAgo time={lastUpdated * 1000} dense />
             </div>
           )}

web/src/types/chat.ts

@@ -50,11 +50,23 @@ export type ChatStats = {
 
 export type ShowStatsMode = "while_generating" | "always";
 
+// Capability flags a provider can report for a model it has not loaded.
+// Keyed by model name (and alias) in GenAIProviderInfo.model_capabilities.
+export type GenAIModelCapabilities = {
+  supports_vision?: boolean;
+  supports_embeddings?: boolean;
+  supports_transcription?: boolean;
+};
+
 export type GenAIProviderInfo = {
   models: string[];
   roles: string[];
   supports_toggleable_thinking: boolean;
   supports_embeddings: boolean;
+  supports_transcription: boolean;
+  // Per-model capabilities, when the provider can report them without loading
+  // the model. The top-level flags above describe the configured model only.
+  model_capabilities?: Record<string, GenAIModelCapabilities>;
 };
 
 export type GenAIModelsResponse = Record<string, GenAIProviderInfo>;

web/src/types/configForm.ts

@@ -30,6 +30,8 @@ export type HiddenFieldEntry = string | ((ctx: HiddenFieldContext) => string[]);
 export type ConfigFormContext = {
   level?: "global" | "camera";
   cameraName?: string;
+  /** Config section being edited, e.g. "audio" or "review". */
+  sectionPath?: string;
   globalValue?: JsonValue;
   cameraValue?: JsonValue;
   overrides?: JsonValue;

web/src/types/frigateConfig.ts

@@ -394,7 +394,7 @@ export type AllGroupsStreamingSettings = {
   [groupName: string]: GroupStreamingSettings;
 };
 
-export type GenAIRole = "chat" | "descriptions" | "embeddings";
+export type GenAIRole = "chat" | "descriptions" | "embeddings" | "transcribe";
 
 export type GenAIAgentConfig = {
   api_key?: string;

web/src/utils/configHealth.ts

@@ -10,6 +10,7 @@ import type { FrigateConfig } from "@/types/frigateConfig";
 import type { HealthProblem } from "@/types/health";
 import { getSectionConfig } from "@/utils/configUtil";
 import { activeCameras } from "@/utils/health";
+import { resolveMessageKey } from "@/utils/runtimeOverrides";
 
 function healthMessages(
   section: string,
@@ -47,7 +48,7 @@ function toProblem(
     severity: message.severity,
     scope,
     scopeIsCamera,
-    text: t(message.messageKey, {
+    text: t(resolveMessageKey(message, ctx), {
       ns: "views/settings",
       ...(message.values ?? {}),
     }),

web/src/utils/configUtil.ts

@@ -13,6 +13,7 @@ import set from "lodash/set";
 import { isJsonObject } from "@/lib/utils";
 import { REDACTED_CREDENTIAL_SENTINEL } from "@/lib/const";
 import { applySchemaDefaults } from "@/lib/config-schema";
+import { applyConfiguredToggles } from "@/utils/runtimeOverrides";
 import { normalizeConfigValue } from "@/hooks/use-config-override";
 import {
   modifySchemaForSection,
@@ -103,11 +104,13 @@ export const globalCameraDefaultSections = new Set([
 // ---------------------------------------------------------------------------
 
 /**
- * Get the base (pre-profile) value for a camera section.
+ * Get the saved-config value for a camera section, which is what the settings
+ * form edits.
  *
- * When a profile is active the API populates `base_config` with original
- * section values.  This helper returns that value when available, falling
- * back to the top-level (effective) value otherwise.
+ * Two things move the top-level (effective) value away from yaml. A profile
+ * merges its overrides into it, and the API then populates `base_config` with
+ * the originals. Runtime toggles from the live view, MQTT, or Home Assistant
+ * change it in place, and `applyConfiguredToggles` puts those fields back.
  */
 export function getBaseCameraSectionValue(
   config: FrigateConfig | undefined,
@@ -118,7 +121,11 @@ export function getBaseCameraSectionValue(
   const cam = config.cameras?.[cameraName];
   if (!cam) return undefined;
   const base = cam.base_config?.[sectionPath];
-  return base !== undefined ? base : get(cam, sectionPath);
+  return applyConfiguredToggles(
+    cam,
+    sectionPath,
+    base !== undefined ? base : get(cam, sectionPath),
+  );
 }
 
 // mergeWith customizer that replaces arrays wholesale instead of merging them

web/src/utils/runtimeOverrides.ts

@@ -0,0 +1,148 @@
+import get from "lodash/get";
+import isEqual from "lodash/isEqual";
+import set from "lodash/set";
+import cloneDeep from "lodash/cloneDeep";
+import type { CameraConfig } from "@/types/frigateConfig";
+import type {
+  ConditionalMessage,
+  MessageConditionContext,
+} from "@/components/config-form/section-configs/types";
+
+/**
+ * Camera fields the dispatcher can change at runtime from the live view, MQTT,
+ * or Home Assistant. Mirrors the camera command handlers in
+ * frigate/comms/dispatcher.py. Runtime changes persist across restarts and win
+ * over yaml until the field is saved again, so the settings form (which edits
+ * yaml) has to show the config value and flag the divergence.
+ *
+ * Paths are relative to the section.
+ */
+export const RUNTIME_TOGGLEABLE_FIELDS: Record<string, string[]> = {
+  audio: ["enabled"],
+  birdseye: ["enabled", "modes"],
+  detect: ["enabled"],
+  motion: ["enabled", "improve_contrast", "threshold", "contour_area"],
+  notifications: ["enabled"],
+  objects: ["genai.enabled"],
+  onvif: ["autotracking.enabled"],
+  record: ["enabled"],
+  review: ["alerts.enabled", "detections.enabled", "genai.enabled"],
+  snapshots: ["enabled"],
+};
+
+/**
+ * The value a field holds in yaml, or undefined when the backend exposes no
+ * config-side copy of it.
+ *
+ * Two sources carry it. `base_config` is the pre-profile snapshot, sent only
+ * while a profile is active. The `<field>_in_config` siblings are always sent,
+ * but only exist for the toggles the backend tracks that way (`detect`,
+ * `snapshots`, and `birdseye` have none).
+ */
+export function getConfiguredFieldValue(
+  cameraConfig: CameraConfig | undefined,
+  sectionPath: string,
+  fieldPath: string,
+): unknown {
+  if (!cameraConfig) return undefined;
+
+  const inConfig = get(cameraConfig, `${sectionPath}.${fieldPath}_in_config`);
+  if (inConfig !== undefined && inConfig !== null) {
+    return inConfig;
+  }
+
+  const base = cameraConfig.base_config?.[sectionPath];
+  return base !== undefined ? get(base, fieldPath) : undefined;
+}
+
+export type RuntimeOverride = {
+  /** The value saved in yaml. */
+  configured: unknown;
+  /** The value the camera is running with right now. */
+  runtime: unknown;
+};
+
+/**
+ * Describes a field whose live value has drifted from the saved config, or
+ * undefined when the two agree or the config value can't be read.
+ */
+export function getRuntimeOverride(
+  cameraConfig: CameraConfig | undefined,
+  sectionPath: string | undefined,
+  fieldPath: string,
+): RuntimeOverride | undefined {
+  if (!cameraConfig || !sectionPath) return undefined;
+  if (!RUNTIME_TOGGLEABLE_FIELDS[sectionPath]?.includes(fieldPath)) {
+    return undefined;
+  }
+
+  const configured = getConfiguredFieldValue(
+    cameraConfig,
+    sectionPath,
+    fieldPath,
+  );
+  if (configured === undefined) return undefined;
+
+  const runtime = get(cameraConfig, `${sectionPath}.${fieldPath}`);
+  if (runtime === undefined || isEqual(configured, runtime)) return undefined;
+
+  return { configured, runtime };
+}
+
+/**
+ * Whether a section is enabled in yaml but turned off on the running camera.
+ * This is the state that makes a dependent warning read as a contradiction,
+ * since the form shows the section switch on.
+ */
+export function isSectionRuntimeDisabled(
+  cameraConfig: CameraConfig | undefined,
+  sectionPath: string,
+): boolean {
+  const override = getRuntimeOverride(cameraConfig, sectionPath, "enabled");
+  return override?.configured === true && override.runtime === false;
+}
+
+/**
+ * Overlays the saved config values onto a section so the form edits yaml
+ * rather than live state. Returns the section unchanged when nothing drifted.
+ */
+export function applyConfiguredToggles(
+  cameraConfig: CameraConfig | undefined,
+  sectionPath: string,
+  sectionValue: unknown,
+): unknown {
+  const fields = RUNTIME_TOGGLEABLE_FIELDS[sectionPath];
+  if (!fields || !sectionValue || typeof sectionValue !== "object") {
+    return sectionValue;
+  }
+
+  let result = sectionValue;
+  for (const field of fields) {
+    const override = getRuntimeOverride(cameraConfig, sectionPath, field);
+    if (!override) continue;
+
+    if (result === sectionValue) {
+      result = cloneDeep(sectionValue);
+    }
+    set(result as object, field, override.configured);
+  }
+
+  return result;
+}
+
+/**
+ * Picks the wording for a message. A message that depends on another section
+ * being on switches to its runtime wording when the config has that section
+ * enabled but the running camera has it off.
+ */
+export function resolveMessageKey(
+  message: Pick<ConditionalMessage, "messageKey" | "runtimeOverride">,
+  ctx: MessageConditionContext | undefined,
+): string {
+  const runtime = message.runtimeOverride;
+  if (!runtime || !ctx || ctx.level !== "camera") return message.messageKey;
+
+  return isSectionRuntimeDisabled(ctx.fullCameraConfig, runtime.section)
+    ? runtime.messageKey
+    : message.messageKey;
+}

web/src/views/classification/ModelTrainingView.tsx

@@ -899,6 +899,14 @@ type TrainGridProps = {
   onRefresh: () => void;
   onDelete: (ids: string[]) => void;
 };
+
+// the backend writes a class with a "-" as "_" in train file names, since it
+// splits those names on "-", so a dataset class may still carry the dash
+function matchesTrainClass(classes: string[], name: string): boolean {
+  const target = name.replaceAll("-", "_");
+  return classes.some((item) => item.replaceAll("-", "_") === target);
+}
+
 function TrainGrid({
   model,
   contentRef,
@@ -936,7 +944,10 @@ function TrainGrid({
             return true;
           }
 
-          if (trainFilter.classes && !trainFilter.classes.includes(data.name)) {
+          if (
+            trainFilter.classes &&
+            !matchesTrainClass(trainFilter.classes, data.name)
+          ) {
             return false;
           }