Support using GenAI for audio transcription (#24396)

* Add support for running transcription with GenAI

* Improve audio joining

* Fix GenAI model capability reporting

* Support language correctly

* Migrate existing users to keep english selected

* Fix models

* Fix tests

* Fix accepted null model

* Handle slwo providers

docs/docs/configuration/advanced/reference.md

@@ -800,7 +800,7 @@ lpr:
 # to Google or OpenAI's LLMs to generate descriptions. GenAI features can be configured at
 # the camera level to enhance privacy for indoor cameras.
 # NOTE: genai is a map of named providers. Each key is a name you choose for the provider,
-#       and each role (chat, descriptions, embeddings) may be assigned to exactly one provider.
+#       and each role (chat, descriptions, embeddings, transcribe) may be assigned to exactly one provider.
 genai:
   # Required: name of the provider (chosen by you, used to reference it elsewhere)
   my_provider:
@@ -813,11 +813,13 @@ genai:
     # Required: The model to use with the provider.
     model: gemini-1.5-flash
     # Optional: Roles this provider handles (default: shown below)
-    # Each role (chat, descriptions, embeddings) must be assigned to exactly one provider.
+    # Each role (chat, descriptions, embeddings, transcribe) must be assigned to exactly
+    # one provider.
     roles:
       - chat
       - descriptions
       - embeddings
+      - transcribe
     # Optional additional args to pass to the GenAI Provider (default: None)
     provider_options:
       keep_alive: -1
@@ -830,13 +832,19 @@ genai:
 audio_transcription:
   # Optional: Enable live and speech event audio transcription (default: shown below)
   enabled: False
+  # Optional: The transcription backend (default: shown below)
+  # Either 'whisper' for Frigate's built-in local models, or the name of a genai
+  # provider that has 'transcribe' in its roles. device and model_size are ignored
+  # when a genai provider is named.
+  model: whisper
   # Optional: The device to run the models on for live transcription. (default: shown below)
   device: CPU
   # Optional: Set the model size used for live transcription. (default: shown below)
   model_size: small
   # Optional: Set the language used for transcription translation. (default: shown below)
-  # List of language codes: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
-  language: en
+  # Use 'auto' to let the model detect the language, or a language code from
+  # https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
+  language: auto
 
 # Optional: Configuration for classification models
 classification:

docs/docs/configuration/audio_detectors.md

@@ -204,7 +204,7 @@ Frequently-heard labels like `speech` can generate a lot of events, and each eve
 
 ### Audio Transcription
 
-Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`. The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
+Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`, and can alternatively offload transcription to a [GenAI provider](#genai-provider). The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
 
 :::info
 
@@ -224,6 +224,7 @@ To enable transcription, configure it globally and optionally disable for specif
 **Global:** Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
 
 - Set **Enable audio transcription** to on
+- Set **Audio transcription model or GenAI provider name** to `whisper` for Frigate's built-in local models, or to the name of a GenAI provider
 - Set **Transcription device** to the desired device
 - Set **Model size** to the desired size
 
@@ -235,6 +236,7 @@ To enable transcription, configure it globally and optionally disable for specif
 ```yaml
 audio_transcription:
   enabled: True
+  model: whisper
   device: ...
   model_size: ...
 ```
@@ -263,20 +265,88 @@ The optional config parameters that can be set at the global level include:
 - **`enabled`**: Enable or disable the audio transcription feature.
   - Default: `False`
   - It is recommended to only configure the features at the global level, and enable it at the individual camera level.
+- **`model`**: The transcription backend.
+  - Default: `whisper`
+  - `whisper` uses Frigate's built-in local models, described by `device` and `model_size` below.
+  - Any other value must name a key in your `genai` config whose entry has `transcribe` in its `roles`. See [GenAI Provider](#genai-provider).
 - **`device`**: Device to use to run transcription and translation models.
   - Default: `CPU`
   - This can be `CPU` or `GPU`. The `sherpa-onnx` models are lightweight and run on the CPU only. The `whisper` models can run on GPU but are only supported on CUDA hardware.
+  - Ignored when `model` names a GenAI provider.
 - **`model_size`**: The size of the model used for live transcription.
   - Default: `small`
   - This can be `small` or `large`. The `small` setting uses `sherpa-onnx` models that are fast, lightweight, and always run on the CPU but are not as accurate as the `whisper` model.
-  - This config option applies to **live transcription only**. Recorded `speech` events will always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
-- **`language`**: Defines the language used by `whisper` to translate `speech` audio events (and live audio only if using the `large` model).
-  - Default: `en`
-  - You must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
+  - This config option applies to **live transcription only**. With `model: whisper`, recorded `speech` events always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
+  - Ignored when `model` names a GenAI provider.
+- **`language`**: Defines the language used to transcribe and translate `speech` audio events (and live audio only if using the `large` model or a GenAI provider).
+  - Default: `auto`
+  - `auto` lets the model detect the language itself, which most models do well. Set an explicit language only if detection is picking the wrong one.
+  - Otherwise you must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
   - Transcriptions for `speech` events are translated.
   - Live audio is translated only if you are using the `large` model. The `small` `sherpa-onnx` model is English-only.
 
-The only field that is valid at the camera level is `enabled`.
+The only field that is valid at the camera level is `enabled`. In particular `model` is global only: the transcription backend is a process-wide resource shared by every camera.
+
+#### GenAI Provider
+
+Frigate can send audio to a GenAI provider for transcription when that provider has the `transcribe` role. This is useful if you already run a GenAI provider, or if you do not have the CPU/GPU headroom for a local whisper model. Supported providers are **OpenAI**, **Azure OpenAI**, **Gemini**, and **llama.cpp** with an audio-capable model (a dedicated ASR model such as Qwen3-ASR, or a general multimodal model that accepts audio). Ollama is not supported as it has no audio input.
+
+To use a GenAI provider for audio transcription:
+
+1. Configure a GenAI provider with `transcribe` in its `roles`.
+2. Set the audio transcription model to that GenAI config key (e.g. `whisper_cloud`).
+
+<ConfigTabs>
+<TabItem value="ui">
+
+Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
+
+| Field                                                | Description                                                                                              |
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
+| **Audio transcription model or GenAI provider name** | Set to the GenAI config key (e.g. `whisper_cloud`) to use a configured GenAI provider for transcription |
+
+The GenAI provider must also be configured with the `transcribe` role under <NavPath path="Settings > Enrichments > Generative AI" />.
+
+</TabItem>
+<TabItem value="yaml">
+
+```yaml
+genai:
+  whisper_cloud:
+    provider: openai
+    api_key: your-api-key
+    model: gpt-transcribe
+    roles:
+      - transcribe
+
+audio_transcription:
+  enabled: True
+  model: whisper_cloud
+  language: en
+```
+
+</TabItem>
+</ConfigTabs>
+
+:::warning
+
+**Give `transcribe` its own `genai` entry.** A `genai` entry has a single `model` string that is shared by every role it holds, so `roles: [descriptions, transcribe]` would send the same model name to both the chat endpoint and the transcription endpoint. Transcription models and chat models are almost never the same model, so define a dedicated entry as shown above.
+
+:::
+
+:::warning
+
+**Live transcription against a metered provider is billed continuously.** In live mode Frigate uploads an overlapping ~2 second window of audio roughly once per second, per camera, for as long as audio stays above that camera's `audio.min_volume`. Windows below that threshold are never uploaded, which is what keeps a quiet camera near zero requests, but a camera pointed at a busy street will keep sending.
+
+Three things keep this opt-in: `transcribe` is not one of the default roles, live transcription is off by default, and the volume gate suppresses silence. Transcription of recorded `speech` events is unaffected - it remains a manual, one-request-per-event action.
+
+:::
+
+`device` and `model_size` have no effect on this path and no local model is ever downloaded.
+
+`language` defaults to `auto`, which sends no language hint and lets the model detect it. Most audio models detect language well, so leave it on `auto` unless detection is picking the wrong one.
+
+When set explicitly, it is sent as the transcription endpoint's native `language` parameter for OpenAI, Azure, and llama.cpp, and as part of the prompt for Gemini. This matters for dedicated ASR models such as Qwen3-ASR: they read the prompt as contextual biasing rather than as an instruction, so a language named in the prompt is ignored, while the endpoint parameter is honored.
 
 #### Live transcription
 
@@ -292,6 +362,8 @@ Results can be error-prone due to a number of factors, including:
 
 For speech sources close to the camera with minimal background noise, use the `small` model.
 
+A [GenAI provider](#genai-provider) is generally the most accurate option for live transcription, at the cost of a network round trip per window. That round trip has to stay under about a second to keep up with the audio; if it does not, Frigate drops the oldest buffered audio rather than letting the backlog grow.
+
 If you have CUDA hardware, you can experiment with the `large` `whisper` model on GPU. Performance is not quite as fast as the `sherpa-onnx` `small` model, but live transcription is far more accurate. Using the `large` model with CPU will likely be too slow for real-time transcription.
 
 #### Transcription and translation of `speech` audio events
@@ -308,7 +380,7 @@ Only one `speech` event may be transcribed at a time. Frigate does not automatic
 
 :::
 
-Recorded `speech` events will always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient.
+With `model: whisper`, recorded `speech` events always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient. With a [GenAI provider](#genai-provider), the recorded clip is sent to the provider instead and no local model is used.
 
 #### FAQ
 

docs/docs/configuration/genai/config.md

@@ -43,7 +43,7 @@ genai:
 
 The examples on this page all use `my_provider`, but the name is arbitrary and is only used to reference the provider elsewhere in the config (for example, `semantic_search.model`).
 
-Each provider handles one or more **roles**: `chat`, `descriptions`, and `embeddings`. A provider handles all three by default, and each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
+Each provider handles one or more **roles**: `chat`, `descriptions`, `embeddings`, and `transcribe`. A provider handles the first three by default; `transcribe` must always be listed explicitly, and is not available on Ollama, which has no audio input. Each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
 
 If the provider you choose requires an API key, you may either directly paste it in your configuration, or store it in an environment variable prefixed with `FRIGATE_`.
 
@@ -77,6 +77,17 @@ The `embeddings` role needs a different kind of model. Text queries are matched
 | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `qwen3-vl-embedding` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Must be served by llama.cpp started with `--embeddings` and `--mmproj`. |
 
+#### Transcription models
+
+The `transcribe` role needs a model that accepts audio input. A text-only or vision-only model cannot serve this role. The following are recommended for local deployment of the `transcribe` role:
+
+| Model       | Notes                                                                                                                                                                                                      |
+| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `qwen3-asr` | Dedicated speech recognition model covering 30 languages, and the better choice for transcription quality. It only transcribes, so it cannot be shared with the `descriptions` or `chat` roles.            |
+| `gemma4`    | General multimodal model that accepts audio as well as images, so one served model can cover `transcribe` alongside the other roles. Transcript quality is below `qwen3-asr`, particularly on noisy audio. |
+
+Both must be served by llama.cpp started with the matching audio `--mmproj`. llama.cpp only reports audio support when an audio projector is loaded. Without it Frigate sees the model as text-only and the `transcribe` role is unavailable in the UI. Frigate transcribes through the server's `/v1/audio/transcriptions` route, which llama.cpp serves for any audio-capable model.
+
 :::info
 
 Each model is available in multiple parameter sizes (3b, 4b, 8b, etc.). Larger sizes are more capable of complex tasks and understanding of situations, but requires more memory and computational resources. It is recommended to try multiple models and experiment to see which performs best.

frigate/config/camera/genai.py

@@ -1,7 +1,7 @@
 from enum import Enum
-from typing import Any
+from typing import Any, Self
 
-from pydantic import Field
+from pydantic import Field, model_validator
 
 from ..base import FrigateBaseModel
 from ..env import EnvString
@@ -21,6 +21,17 @@ class GenAIRoleEnum(str, Enum):
     chat = "chat"
     descriptions = "descriptions"
     embeddings = "embeddings"
+    transcribe = "transcribe"
+
+
+# Providers that can accept audio input for the transcribe role. Ollama has no
+# audio input support, so claiming the role there would fail at request time.
+TRANSCRIBE_CAPABLE_PROVIDERS = {
+    GenAIProviderEnum.openai,
+    GenAIProviderEnum.azure_openai,
+    GenAIProviderEnum.gemini,
+    GenAIProviderEnum.llamacpp,
+}
 
 
 class GenAIConfig(FrigateBaseModel):
@@ -52,7 +63,7 @@ class GenAIConfig(FrigateBaseModel):
             GenAIRoleEnum.chat,
         ],
         title="Roles",
-        description="GenAI roles (chat, descriptions, embeddings); one provider per role.",
+        description="GenAI roles (chat, descriptions, embeddings, transcribe); one provider per role. Only chat, descriptions, and embeddings are granted by default; transcribe must be listed explicitly.",
     )
     provider_options: dict[str, Any] = Field(
         default={},
@@ -66,3 +77,17 @@ class GenAIConfig(FrigateBaseModel):
         description="Runtime options passed to the provider for each inference call.",
         json_schema_extra={"additionalProperties": {}},
     )
+
+    @model_validator(mode="after")
+    def validate_transcribe_provider(self) -> Self:
+        """Reject the transcribe role on providers that cannot accept audio input."""
+        if (
+            GenAIRoleEnum.transcribe in self.roles
+            and self.provider not in TRANSCRIBE_CAPABLE_PROVIDERS
+        ):
+            raise ValueError(
+                f"GenAI provider '{self.provider.value}' does not support audio input "
+                "and cannot be given the 'transcribe' role."
+            )
+
+        return self

frigate/config/classification.py

@@ -5,6 +5,7 @@ from pydantic import ConfigDict, Field, field_validator
 from .base import FrigateBaseModel
 
 __all__ = [
+    "AudioTranscriptionModelEnum",
     "CameraFaceRecognitionConfig",
     "CameraLicensePlateRecognitionConfig",
     "CameraAudioTranscriptionConfig",
@@ -20,6 +21,10 @@ class SemanticSearchModelEnum(str, Enum):
     jinav2 = "jinav2"
 
 
+class AudioTranscriptionModelEnum(str, Enum):
+    whisper = "whisper"
+
+
 class EnrichmentsDeviceEnum(str, Enum):
     GPU = "GPU"
     CPU = "CPU"
@@ -53,10 +58,35 @@ class AudioTranscriptionConfig(FrigateBaseModel):
         description="Enable or disable automatic audio transcription for all cameras; can be overridden per-camera.",
     )
     language: str = Field(
-        default="en",
+        default="auto",
         title="Transcription language",
-        description="Language code used for transcription/translation (for example 'en' for English). See https://whisper-api.com/docs/languages/ for supported language codes.",
+        description="Language code used for transcription/translation (for example 'en' for English), or 'auto' to let the model detect it. See https://whisper-api.com/docs/languages/ for supported language codes.",
     )
+    model: AudioTranscriptionModelEnum | str | None = Field(
+        default=AudioTranscriptionModelEnum.whisper,
+        title="Audio transcription model or GenAI provider name",
+        description="The transcription backend: 'whisper' for Frigate's built-in local models, or the name of a GenAI provider with the transcribe role.",
+    )
+
+    @field_validator("model", mode="before")
+    @classmethod
+    def coerce_model_enum(cls, v):
+        # An absent value ("model:" with nothing after it, or an explicit null)
+        # means unspecified, so fall back to the built-in backend. Left as None
+        # it would pass the GenAI-provider validation, which only inspects
+        # strings, and then be treated as a provider name that resolves to no
+        # client, turning transcription into a silent no-op.
+        if v is None or (isinstance(v, str) and not v.strip()):
+            return AudioTranscriptionModelEnum.whisper
+
+        if isinstance(v, str):
+            try:
+                return AudioTranscriptionModelEnum(v)
+            except ValueError:
+                return v
+
+        return v
+
     device: EnrichmentsDeviceEnum = Field(
         default=EnrichmentsDeviceEnum.CPU,
         title="Transcription device",

frigate/config/config.py

@@ -56,6 +56,7 @@ from .camera.timestamp import TimestampStyleConfig
 from .camera_group import CameraGroupConfig
 from .classification import (
     AudioTranscriptionConfig,
+    AudioTranscriptionModelEnum,
     ClassificationConfig,
     FaceRecognitionConfig,
     LicensePlateRecognitionConfig,
@@ -884,7 +885,7 @@ class FrigateConfig(FrigateBaseModel):
         # set notifications state
         self.notifications.enabled_in_config = self.notifications.enabled
 
-        # validate genai: each role (chat, descriptions, embeddings) at most once
+        # validate genai: each role (chat, descriptions, embeddings, transcribe) at most once
         role_to_name: dict[GenAIRoleEnum, str] = {}
         for name, genai_cfg in self.genai.items():
             for role in genai_cfg.roles:
@@ -1245,6 +1246,35 @@ class FrigateConfig(FrigateBaseModel):
         for model in self.models:
             model.create_colormap(colored_labels)
 
+        # validate audio_transcription.model when it is a GenAI provider name.
+        # this runs here rather than beside the semantic_search check because the
+        # global->camera merge above is what resolves camera-level enablement.
+        transcription_active = self.audio_transcription.enabled or any(
+            camera.audio_transcription.enabled for camera in self.cameras.values()
+        )
+
+        if (
+            transcription_active
+            and isinstance(self.audio_transcription.model, str)
+            and not isinstance(
+                self.audio_transcription.model, AudioTranscriptionModelEnum
+            )
+        ):
+            if self.audio_transcription.model not in self.genai:
+                raise ValueError(
+                    f"audio_transcription.model '{self.audio_transcription.model}' is not a "
+                    "valid GenAI config key. Must match a key in genai config."
+                )
+
+            if (
+                GenAIRoleEnum.transcribe
+                not in self.genai[self.audio_transcription.model].roles
+            ):
+                raise ValueError(
+                    f"GenAI provider '{self.audio_transcription.model}' must have "
+                    "'transcribe' in its roles for audio transcription."
+                )
+
         # Check audio transcription and audio detection requirements
         if self.audio_transcription.enabled:
             # If audio transcription is enabled globally, at least one camera must have audio detection enabled

frigate/data_processing/post/audio_transcription.py

@@ -10,6 +10,7 @@ from peewee import DoesNotExist
 
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
+from frigate.config.classification import AudioTranscriptionModelEnum
 from frigate.const import (
     CACHE_DIR,
     MODEL_CACHE_DIR,
@@ -18,8 +19,13 @@ from frigate.const import (
 )
 from frigate.data_processing.types import PostProcessDataEnum
 from frigate.embeddings.embeddings import Embeddings
+from frigate.genai.manager import GenAIClientManager
 from frigate.types import TrackedObjectUpdateTypesEnum
-from frigate.util.audio import get_audio_from_recording
+from frigate.util.audio import (
+    clean_transcript,
+    get_audio_from_recording,
+    resolve_language,
+)
 
 from ..types import DataProcessorMetrics
 from .api import PostProcessorApi
@@ -34,15 +40,25 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
         requestor: InterProcessRequestor,
         embeddings: Embeddings,
         metrics: DataProcessorMetrics,
+        genai_manager: GenAIClientManager | None = None,
     ):
         super().__init__(config, metrics, None)
         self.config = config
         self.requestor = requestor
         self.embeddings = embeddings
+        self.genai_manager = genai_manager
         self.recognizer = None
         self.transcription_lock = threading.Lock()
         self.transcription_thread: threading.Thread | None = None
         self.transcription_running = False
+        self._use_genai = not isinstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+
+        if self._use_genai:
+            # never build the local recognizer on the GenAI path; WhisperModel
+            # downloads several hundred MB on first use
+            return
 
         # faster-whisper handles model downloading automatically
         self.model_path = os.path.join(MODEL_CACHE_DIR, "whisper")
@@ -147,6 +163,31 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
             logger.error(f"Error in audio transcription post-processing: {e}")
 
     def __transcribe_audio(self, audio_data: bytes) -> str | None:
+        """Transcribe WAV audio data with the configured backend."""
+        if self._use_genai:
+            return self.__transcribe_audio_genai(audio_data)
+
+        return self.__transcribe_audio_whisper(audio_data)
+
+    def __transcribe_audio_genai(self, audio_data: bytes) -> str | None:
+        """Hand the WAV bytes to the GenAI provider holding the transcribe role."""
+        client = self.genai_manager.transcribe_client if self.genai_manager else None
+
+        if not client:
+            logger.error(
+                "audio_transcription.model is '%s' (GenAI provider) but no transcribe "
+                "client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
+                self.config.audio_transcription.model,
+            )
+            return None
+
+        text = client.transcribe(
+            audio_data,
+            language=resolve_language(self.config.audio_transcription.language),
+        )
+        return clean_transcript(text) or None
+
+    def __transcribe_audio_whisper(self, audio_data: bytes) -> str | None:
         """Transcribe WAV audio data using faster-whisper."""
         if not self.recognizer:
             logger.debug("Recognizer not initialized")
@@ -160,7 +201,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
 
             segments, info = self.recognizer.transcribe(
                 temp_wav,
-                language=self.config.audio_transcription.language,
+                language=resolve_language(self.config.audio_transcription.language),
                 beam_size=5,
             )
 

frigate/data_processing/real_time/audio_transcription.py

@@ -1,16 +1,19 @@
 """Handle processing audio for speech transcription using sherpa-onnx with FFmpeg pipe."""
 
+import collections
 import logging
 import os
 import queue
 import threading
-from typing import Any
+import time
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import CameraConfig, FrigateConfig
-from frigate.const import MODEL_CACHE_DIR
+from frigate.config.classification import AudioTranscriptionModelEnum
+from frigate.const import AUDIO_DURATION, MODEL_CACHE_DIR
 from frigate.data_processing.common.audio_transcription.model import (
     AudioTranscriptionModelRunner,
 )
@@ -18,12 +21,38 @@ from frigate.data_processing.real_time.whisper_online import (
     FasterWhisperASR,
     OnlineASRProcessor,
 )
+from frigate.util.audio import (
+    clean_transcript,
+    pcm16_to_wav,
+    resolve_language,
+    stitch_transcripts,
+)
 
 from ..types import DataProcessorMetrics
 from .api import RealTimeProcessorApi
 
+if TYPE_CHECKING:
+    # importing frigate.genai eagerly would pull the provider SDKs into the
+    # audio process even when transcription runs on a local model
+    from frigate.genai.manager import GenAIClientManager
+
 logger = logging.getLogger(__name__)
 
+# Number of ~0.975s audio detector chunks per GenAI request. The window advances
+# one chunk at a time, so two chunks means a 50% overlap: every word lands whole
+# in at least one window, which whisper-family models need to avoid hallucinating
+# on a clipped clip. The cadence is fixed by the audio detector's frame size, so
+# this is a constant rather than a config knob.
+GENAI_WINDOW_CHUNKS = 2
+
+# Bound the queue at ~30s of audio so a slow or hung provider cannot grow it
+# without limit. The producer is the ffmpeg read thread and must never block.
+AUDIO_QUEUE_MAXSIZE = int(30 / AUDIO_DURATION)
+
+# A backed-up queue drops a chunk per cycle, so warning on each one would spam
+# the log once a second per camera for as long as the provider stays slow.
+AUDIO_DROP_WARN_INTERVAL = 10.0
+
 
 class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
     def __init__(
@@ -31,9 +60,10 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
         config: FrigateConfig,
         camera_config: CameraConfig,
         requestor: InterProcessRequestor,
-        model_runner: AudioTranscriptionModelRunner,
+        model_runner: AudioTranscriptionModelRunner | None,
         metrics: DataProcessorMetrics,
         stop_event: threading.Event,
+        genai_manager: "GenAIClientManager | None" = None,
     ):
         super().__init__(config, metrics)
         self.config = config
@@ -42,11 +72,31 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
         self.stream: Any = None
         self.whisper_model: FasterWhisperASR | None = None
         self.model_runner = model_runner
+        self.genai_manager = genai_manager
         self.transcription_segments: list[str] = []
-        self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue()
+        self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue(
+            maxsize=AUDIO_QUEUE_MAXSIZE
+        )
         self.stop_event = stop_event
+        self._use_genai = not isinstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+        # sliding window of raw int16 chunks; the deque's maxlen is what evicts
+        # the oldest chunk and so produces the overlap
+        self._genai_window: collections.deque[np.ndarray] = collections.deque(
+            maxlen=GENAI_WINDOW_CHUNKS
+        )
+        self._genai_committed = ""
+        # set by the producer when it discards a chunk, so the consumer knows the
+        # audio it is about to receive is not contiguous with what it buffered
+        self._audio_dropped = threading.Event()
+        self._last_drop_warning = 0.0
 
     def __build_recognizer(self) -> None:
+        if self._use_genai:
+            # nothing local to load; never import sherpa or FasterWhisperASR
+            return
+
         try:
             if self.config.audio_transcription.model_size == "large":
                 # Whisper models need to be per-process and can only run one stream at a time
@@ -64,7 +114,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
                 self.stream = OnlineASRProcessor(
                     asr=self.whisper_model,
                 )
-            else:
+            elif self.model_runner is not None:
                 logger.debug(f"Loading sherpa stream for {self.camera_config.name}")
                 self.stream = self.model_runner.model.create_stream()
             logger.debug(
@@ -76,6 +126,15 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             )
 
     def __process_audio_stream(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
+        # must precede both the model_runner guard (model_runner is None on this
+        # path) and the float32 normalization below (GenAI wants untouched int16)
+        if self._use_genai:
+            return self.__process_audio_genai(audio_data)
+
+        if self.model_runner is None:
+            logger.debug("Audio transcription (live) model runner not initialized")
+            return None
+
         if (
             self.model_runner.model is None
             and self.config.audio_transcription.model_size == "small"
@@ -140,6 +199,82 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             logger.error(f"Error processing audio stream: {e}")
             return None
 
+    def __process_audio_genai(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
+        """Transcribe a sliding overlapped window through the GenAI provider."""
+        client = self.genai_manager.transcribe_client if self.genai_manager else None
+
+        if not client:
+            logger.error(
+                "audio_transcription.model is '%s' (GenAI provider) but no transcribe "
+                "client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
+                self.config.audio_transcription.model,
+            )
+            return None
+
+        if self._audio_dropped.is_set():
+            self._audio_dropped.clear()
+
+            # Chunks were discarded between what is buffered and this one, so
+            # concatenating them would splice non-adjacent audio into one window
+            # and destroy the overlap the stitcher depends on.
+            self._genai_window.clear()
+
+            if self._genai_committed:
+                # the transcript has a gap in it; close the utterance out rather
+                # than stitching across missing speech
+                return self.__end_genai_utterance()
+
+        self._genai_window.append(audio_data)
+
+        if len(self._genai_window) < GENAI_WINDOW_CHUNKS:
+            # wait for a full window so the first request is never a clipped clip
+            return None
+
+        window = np.concatenate(list(self._genai_window))
+
+        # Silence gate, using the same threshold audio detection uses. Gate the
+        # whole window rather than individual chunks; this is the primary cost
+        # and privacy brake and is what keeps a quiet camera near zero requests.
+        window_as_float = window.astype(np.float32)
+        rms = float(np.sqrt(np.mean(np.absolute(np.square(window_as_float)))))
+
+        if rms < self.camera_config.audio.min_volume:
+            logger.debug(
+                f"Window RMS {rms:.1f} below min_volume, skipping transcription"
+            )
+            return self.__end_genai_utterance()
+
+        text = client.transcribe(
+            pcm16_to_wav(window),
+            language=resolve_language(self.config.audio_transcription.language),
+        )
+
+        # cleaning has to come first: a silent window often comes back as the
+        # model's preamble alone, which is silence, not a word to commit
+        cleaned = clean_transcript(text)
+
+        if not cleaned:
+            return self.__end_genai_utterance()
+
+        self._genai_committed = stitch_transcripts(self._genai_committed, cleaned)
+
+        # no VAD on this path, so mirror the whisper branch's heuristic endpoint
+        is_endpoint = (
+            self._genai_committed.endswith((".", "!", "?"))
+            and len(self._genai_committed) > 300
+        )
+
+        logger.debug(f"GenAI transcription: '{self._genai_committed}'")
+
+        return self._genai_committed, is_endpoint
+
+    def __end_genai_utterance(self) -> tuple[str, bool] | None:
+        """Close out the current utterance when a window carries no speech."""
+        if not self._genai_committed:
+            return None
+
+        return self._genai_committed, True
+
     def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
         pass
 
@@ -148,8 +283,38 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             logger.debug("No audio data provided for transcription")
             return None
 
-        # enqueue audio data for processing in the thread
-        self.audio_queue.put((obj_data, audio))
+        # enqueue audio data for processing in the thread. never block: the
+        # producer is the ffmpeg read thread that audio detection depends on,
+        # so on a backlog drop the oldest chunk instead.
+        try:
+            self.audio_queue.put_nowait((obj_data, audio))
+        except queue.Full:
+            try:
+                self.audio_queue.get_nowait()
+                self.audio_queue.task_done()
+            except queue.Empty:
+                pass
+
+            # the stream now has a hole in it, which the consumer has to know
+            # about before it splices the next chunk onto what it already holds
+            self._audio_dropped.set()
+
+            now = time.monotonic()
+
+            if now - self._last_drop_warning >= AUDIO_DROP_WARN_INTERVAL:
+                self._last_drop_warning = now
+                logger.warning(
+                    "Audio transcription queue for %s is full, dropping audio. The "
+                    "provider is not keeping up with the %.2fs chunk rate",
+                    self.camera_config.name,
+                    AUDIO_DURATION,
+                )
+
+            try:
+                self.audio_queue.put_nowait((obj_data, audio))
+            except queue.Full:
+                pass
+
         return None
 
     def run(self) -> None:
@@ -205,6 +370,14 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
                 break
 
     def reset(self) -> None:
+        if self._use_genai:
+            self._genai_committed = ""
+            # stale audio carried across an utterance boundary would be
+            # re-transcribed into the next one
+            self._genai_window.clear()
+            logger.debug("Stream reset")
+            return
+
         if self.config.audio_transcription.model_size == "large":
             # get final output from whisper
             output = self.stream.finish()
@@ -218,7 +391,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
             # reset whisper
             self.stream.init()
             self.transcription_segments = []
-        else:
+        elif self.model_runner is not None:
             # reset sherpa
             self.model_runner.model.reset(self.stream)
 
@@ -226,6 +399,24 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
 
     def check_unload_model(self) -> None:
         # regularly called in the loop in audio maintainer
+        if self._use_genai:
+            # no model to unload, but this is the hook that fires when
+            # live_enabled flips off. guard on emptiness: called ~1x/s per camera.
+            if self._genai_committed or self._genai_window:
+                logger.debug(
+                    f"Clearing GenAI transcription state for {self.camera_config.name}"
+                )
+                self.clear_audio_queue()
+                self._genai_committed = ""
+                self._genai_window.clear()
+
+                self.requestor.send_data(
+                    f"{self.camera_config.name}/audio/transcription",
+                    "",
+                )
+
+            return
+
         if (
             self.config.audio_transcription.model_size == "large"
             and self.whisper_model is not None
@@ -270,6 +461,10 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
         self, topic: str, request_data: dict[str, Any]
     ) -> dict[str, Any] | None:
         if topic == "clear_audio_recognizer":
+            if self._use_genai:
+                self.reset()
+                return {"message": "Audio transcription state cleared", "success": True}
+
             self.stream = None
             self.__build_recognizer()
             return {"message": "Audio recognizer cleared and rebuilt", "success": True}

frigate/embeddings/maintainer.py

@@ -251,7 +251,11 @@ class EmbeddingMaintainer(threading.Thread):
         ):
             self.post_processors.append(
                 AudioTranscriptionPostProcessor(
-                    self.config, self.requestor, self.embeddings, metrics
+                    self.config,
+                    self.requestor,
+                    self.embeddings,
+                    metrics,
+                    self.genai_manager,
                 )
             )
 

frigate/events/audio.py

@@ -19,6 +19,7 @@ from frigate.config.camera.updater import (
     CameraConfigUpdateEnum,
     CameraConfigUpdateSubscriber,
 )
+from frigate.config.classification import AudioTranscriptionModelEnum
 from frigate.const import (
     AUDIO_DURATION,
     AUDIO_FORMAT,
@@ -112,18 +113,31 @@ class AudioProcessor(FrigateProcess):
 
         threading.current_thread().name = "process:audio_manager"
 
+        self.transcription_model_runner: AudioTranscriptionModelRunner | None = None
+        self.genai_manager: Any = None
+
         if any(
             c.enabled_in_config and c.audio_transcription.enabled
             for c in self.config.cameras.values()
         ):
-            self.transcription_model_runner: AudioTranscriptionModelRunner | None = (
-                AudioTranscriptionModelRunner(
+            if isinstance(
+                self.config.audio_transcription.model, AudioTranscriptionModelEnum
+            ):
+                # AudioTranscriptionModelRunner.__init__ unconditionally fetches
+                # sherpa-onnx or whisper weights, so only build it on the local path
+                self.transcription_model_runner = AudioTranscriptionModelRunner(
                     self.config.audio_transcription.device or "AUTO",
                     self.config.audio_transcription.model_size,
                 )
-            )
-        else:
-            self.transcription_model_runner = None
+            else:
+                # imported here rather than at module scope: frigate.genai pulls in
+                # numpy, the provider SDKs, frigate.models, and the prompt builders,
+                # and this process runs at PROCESS_PRIORITY_HIGH. built after the
+                # fork because SDK clients hold sockets and TLS state that must not
+                # cross it; clients themselves stay lazy behind the role property.
+                from frigate.genai.manager import GenAIClientManager
+
+                self.genai_manager = GenAIClientManager(self.config)
 
         config_subscriber = CameraConfigUpdateSubscriber(
             self.config,
@@ -151,6 +165,7 @@ class AudioProcessor(FrigateProcess):
                 self.camera_metrics,
                 self.transcription_model_runner,
                 self.stop_event,  # type: ignore[arg-type]
+                self.genai_manager,
             )
             self.audio_threads[name] = thread
             thread.start()
@@ -200,6 +215,7 @@ class AudioEventMaintainer(threading.Thread):
         camera_metrics: DictProxy,
         audio_transcription_model_runner: AudioTranscriptionModelRunner | None,
         stop_event: threading.Event,
+        genai_manager: Any = None,
     ) -> None:
         super().__init__(name=f"{camera.name}_audio_event_processor")
 
@@ -222,6 +238,7 @@ class AudioEventMaintainer(threading.Thread):
         self.logpipe = LogPipe(f"ffmpeg.{self.camera_config.name}.audio")
         self.audio_listener: subprocess.Popen[Any] | None = None
         self.audio_transcription_model_runner = audio_transcription_model_runner
+        self.genai_manager = genai_manager
         self.transcription_processor = None
         self.transcription_thread = None
 
@@ -238,9 +255,9 @@ class AudioEventMaintainer(threading.Thread):
         )
         self.detection_publisher = DetectionPublisher(DetectionTypeEnum.audio.value)
 
-        if (
-            self.camera_config.audio_transcription.enabled
-            and self.audio_transcription_model_runner is not None
+        if self.camera_config.audio_transcription.enabled and (
+            self.audio_transcription_model_runner is not None
+            or self.genai_manager is not None
         ):
             # init the transcription processor for this camera
             self.transcription_processor = AudioTranscriptionRealTimeProcessor(
@@ -250,6 +267,7 @@ class AudioEventMaintainer(threading.Thread):
                 model_runner=self.audio_transcription_model_runner,
                 metrics=self.camera_metrics[self.camera_config.name],
                 stop_event=self.stop_event,
+                genai_manager=self.genai_manager,
             )
 
             self.transcription_thread = threading.Thread(

frigate/genai/__init__.py

@@ -338,6 +338,11 @@ class GenAIClient:
         """Whether the configured model can generate embeddings via embed()."""
         return False
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Whether the configured model can transcribe audio via transcribe()."""
+        return False
+
     def list_models(self) -> list[str]:
         """Return the list of model names available from this provider.
 
@@ -345,6 +350,21 @@ class GenAIClient:
         """
         return []
 
+    def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
+        """Return capability flags for each model the provider serves.
+
+        Only providers whose backend advertises capabilities per model can
+        populate this; llama.cpp reports input modalities for every model it
+        serves, so one request describes them all. An empty mapping means "no
+        per-model information available", and callers fall back to this
+        client's own capability properties, which describe only the configured
+        model. A model absent from a non-empty mapping means the same thing.
+
+        Returns:
+            Model name (including aliases) to its capability flags
+        """
+        return {}
+
     def get_context_size(self) -> int:
         """Get the context window size for this provider in tokens."""
         return 4096
@@ -376,6 +396,33 @@ class GenAIClient:
         )
         return []
 
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe speech audio to text.
+
+        Audio is passed as a self-describing blob rather than raw samples so
+        every provider receives a container it can declare, and WAV framing
+        lives in one place instead of in each plugin.
+
+        Args:
+            audio: The encoded audio payload (WAV bytes by default)
+            language: Optional ISO language hint for the provider
+            mime_type: Media type of ``audio``
+
+        Returns:
+            The transcript, or None when the provider cannot produce one
+        """
+        logger.warning(
+            "%s does not support transcription. "
+            "This method should be overridden by the provider implementation.",
+            self.__class__.__name__,
+        )
+        return None
+
     def chat_with_tools(
         self,
         messages: list[dict[str, Any]],

frigate/genai/manager.py

@@ -110,6 +110,12 @@ class GenAIClientManager:
         name = self._role_map.get(GenAIRoleEnum.embeddings)
         return self._get_client(name) if name else None
 
+    @property
+    def transcribe_client(self) -> "GenAIClient | None":
+        """Client configured for the transcribe role."""
+        name = self._role_map.get(GenAIRoleEnum.transcribe)
+        return self._get_client(name) if name else None
+
     def role_info(self) -> dict[str, dict[str, Any]]:
         """Return the model selected for each configured role and its context size.
 
@@ -144,5 +150,11 @@ class GenAIClientManager:
                 "roles": [r.value for r in genai_cfg.roles],
                 "supports_toggleable_thinking": client.supports_toggleable_thinking,
                 "supports_embeddings": client.supports_embeddings,
+                "supports_transcription": client.supports_transcription,
+                # Capabilities of the configured model are above; this maps every
+                # model the provider serves to its own, so the UI can react to a
+                # model selected but not yet saved. Empty when the provider
+                # cannot report capabilities without loading a model.
+                "model_capabilities": client.list_model_capabilities(),
             }
         return result

frigate/genai/plugins/azure-openai.py

@@ -13,6 +13,19 @@ overrides what is genuinely Azure-specific:
 - Context size: Azure does not expose a per-model ``max_model_len`` field
   reliably, so we keep the historical 128K default rather than the
   model-name heuristic used by OpenAI.
+
+Transcription is inherited too: :class:`openai.AzureOpenAI` exposes the same
+``audio.transcriptions.create``. Two Azure-specific caveats apply when using
+the ``transcribe`` role:
+
+- ``model`` must be the Azure *deployment* name, not the underlying model name.
+- The ``api-version`` parsed from ``base_url`` must be 2024-06-01 or later;
+  earlier versions have no transcriptions route and the 404 surfaces only as a
+  generic provider error.
+- Because ``model`` is a deployment name, the inherited check that picks
+  ``languages`` over ``language`` for gpt-transcribe cannot fire unless the
+  deployment happens to be named after the model. Name the deployment
+  ``gpt-transcribe`` to get the right field, or leave the language on ``auto``.
 """
 
 import logging

frigate/genai/plugins/gemini.py

@@ -17,6 +17,10 @@ from frigate.genai.utils import interleave_images
 
 logger = logging.getLogger(__name__)
 
+# Gemini requests carrying inline data are capped at ~20 MB total; stay well
+# under it so the request fails as a log line rather than a 400.
+GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024
+
 
 def _decode_thought_signature(value: Any) -> bytes | None:
     """Decode a base64-encoded thought_signature carried across conversation turns."""
@@ -163,6 +167,58 @@ class GeminiClient(GenAIClient):
             return None
         return description
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Gemini models accept inline audio parts."""
+        return True
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio by sending it as an inline part alongside a prompt."""
+        if len(audio) > GEMINI_MAX_INLINE_BYTES:
+            logger.warning(
+                "Audio payload of %d bytes exceeds the Gemini inline limit; skipping transcription",
+                len(audio),
+            )
+            return None
+
+        prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
+
+        if language:
+            prompt += f" The speech is in language '{language}'."
+
+        try:
+            contents: list[Any] = [
+                prompt,
+                types.Part.from_bytes(data=audio, mime_type=mime_type),
+            ]
+            response = self.provider.models.generate_content(
+                model=self.genai_config.model,
+                contents=contents,
+                config=types.GenerateContentConfig(candidate_count=1),
+            )
+        except errors.APIError as e:
+            logger.warning("Gemini returned an error: %s", str(e))
+            return None
+        except Exception as e:
+            logger.warning("An unexpected error occurred with Gemini: %s", str(e))
+            return None
+
+        try:
+            if response.text is None:
+                return None
+
+            transcript = response.text.strip()
+        except (ValueError, AttributeError):
+            # No transcript was generated
+            return None
+
+        return transcript or None
+
     def list_models(self) -> list[str]:
         """Return available model names from Gemini."""
         try:

frigate/genai/plugins/llama_cpp.py

@@ -408,6 +408,126 @@ class LlamaCppClient(GenAIClient):
         """Whether the loaded model supports audio input."""
         return self._supports_audio
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Audio-capable models can transcribe through chat completions."""
+        return self._supports_audio
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio through the OpenAI-compatible transcriptions route.
+
+        llama.cpp serves /v1/audio/transcriptions for any audio-capable model,
+        not only a separately loaded whisper (ggml-org/llama.cpp#21863), so it
+        covers exactly the models supports_transcription detects. It takes the
+        language as a native multipart field, which is the only thing dedicated
+        ASR models honor: they read the chat prompt as contextual biasing, so
+        asking one there to use a language does nothing.
+
+        Falls back to chat completions when the server predates that route.
+        """
+        if self.provider is None:
+            logger.warning(
+                "llama.cpp provider has not been initialized, audio will not be transcribed. Check your llama.cpp configuration."
+            )
+            return None
+
+        if not self._supports_audio:
+            logger.warning(
+                "llama.cpp model '%s' does not accept audio input",
+                self.genai_config.model,
+            )
+            return None
+
+        try:
+            data = {"model": self.genai_config.model, "response_format": "json"}
+
+            if language:
+                data["language"] = language
+
+            response = self._post(
+                f"{self.provider}/v1/audio/transcriptions",
+                files={"file": ("audio.wav", audio, mime_type)},
+                data=data,
+                timeout=self.timeout,
+            )
+
+            if response.status_code == 404:
+                logger.debug(
+                    "llama.cpp server has no /v1/audio/transcriptions route, using chat completions"
+                )
+                return self._transcribe_via_chat(audio, language)
+
+            response.raise_for_status()
+            result = response.json()
+            text = result.get("text") if isinstance(result, dict) else None
+
+            return str(text).strip() or None if text else None
+        except Exception as e:
+            logger.warning("llama.cpp returned an error: %s", str(e))
+            return None
+
+    def _transcribe_via_chat(self, audio: bytes, language: str | None) -> str | None:
+        """Transcribe through /v1/chat/completions, for servers without the
+        transcriptions route.
+
+        The _media_marker / multimodal_data convention is an /embeddings-only
+        protocol, so no marker-refresh retry is needed here.
+        """
+        prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
+
+        if language:
+            prompt += f" The speech is in language '{language}'."
+
+        try:
+            encoded_audio = base64.b64encode(audio).decode("utf-8")
+            payload: dict[str, Any] = {
+                "model": self.genai_config.model,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "input_audio",
+                                "input_audio": {
+                                    "data": encoded_audio,
+                                    "format": "wav",
+                                },
+                            },
+                        ],
+                    },
+                ],
+                **self.provider_options,
+            }
+
+            response = self._post(
+                f"{self.provider}/v1/chat/completions",
+                json=payload,
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            if (
+                result is not None
+                and "choices" in result
+                and len(result["choices"]) > 0
+            ):
+                choice = result["choices"][0]
+
+                if "message" in choice and choice["message"].get("content"):
+                    return str(choice["message"]["content"].strip()) or None
+
+            return None
+        except Exception as e:
+            logger.warning("llama.cpp returned an error: %s", str(e))
+            return None
+
     @property
     def supports_tools(self) -> bool:
         """Whether the loaded model supports tool/function calling."""
@@ -417,28 +537,74 @@ class LlamaCppClient(GenAIClient):
     def supports_toggleable_thinking(self) -> bool:
         return self._supports_reasoning
 
-    def list_models(self) -> list[str]:
-        """Return available model IDs from the llama.cpp server."""
+    def _fetch_models_data(self) -> list[dict[str, Any]]:
+        """Return the raw /v1/models entries, or an empty list if unreachable."""
         base_url = self.provider or (
             self.genai_config.base_url.rstrip("/")
             if self.genai_config.base_url
             else None
         )
+
         if base_url is None:
             return []
+
         try:
             response = self._get(f"{base_url}/v1/models", timeout=10)
             response.raise_for_status()
-            models = []
-            for m in response.json().get("data", []):
-                models.append(m.get("id", "unknown"))
-                for alias in m.get("aliases", []):
-                    models.append(alias)
-            return sorted(models)
+            data = response.json().get("data", [])
         except Exception as e:
             logger.warning("Failed to list llama.cpp models: %s", e)
             return []
 
+        return data if isinstance(data, list) else []
+
+    def list_models(self) -> list[str]:
+        """Return available model IDs from the llama.cpp server."""
+        models = []
+
+        for m in self._fetch_models_data():
+            models.append(m.get("id", "unknown"))
+
+            for alias in m.get("aliases", []):
+                models.append(alias)
+
+        return sorted(models)
+
+    def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
+        """Report input modalities for every model the server serves.
+
+        Since ggml-org/llama.cpp#22952 each /v1/models entry carries
+        architecture.input_modalities, so a single request describes every
+        model rather than just the configured one. That is what lets the UI
+        answer "can the model I just picked transcribe" before the config is
+        saved and a client for it exists.
+
+        Models whose entry predates that field are omitted rather than reported
+        as incapable, so an older server falls back to the /props probe instead
+        of silently losing capabilities it actually has.
+        """
+        capabilities: dict[str, dict[str, bool]] = {}
+
+        for model in self._fetch_models_data():
+            architecture = model.get("architecture") or {}
+            modalities = architecture.get("input_modalities")
+
+            if not isinstance(modalities, list) or not modalities:
+                continue
+
+            flags = {
+                "supports_vision": "image" in modalities,
+                "supports_transcription": "audio" in modalities,
+            }
+
+            names = [model.get("id"), *(model.get("aliases") or [])]
+
+            for name in names:
+                if isinstance(name, str) and name:
+                    capabilities[name] = flags
+
+        return capabilities
+
     def get_context_size(self) -> int:
         """Get the context window size for llama.cpp.
 

frigate/genai/plugins/openai.py

@@ -15,6 +15,12 @@ from frigate.genai.utils import interleave_images
 
 logger = logging.getLogger(__name__)
 
+# gpt-transcribe replaced the singular `language` field with a `languages` array
+# and rejects a request that sends both. Older transcription models
+# (gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1) still take the singular
+# form. https://developers.openai.com/api/docs/guides/speech-to-text
+_LANGUAGES_ARRAY_MODEL_PREFIX = "gpt-transcribe"
+
 
 def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None:
     """Build a stats dict from an OpenAI-compatible usage object."""
@@ -134,6 +140,51 @@ class OpenAIClient(GenAIClient):
             logger.warning("OpenAI returned an error: %s", str(e))
             return None
 
+    @property
+    def supports_transcription(self) -> bool:
+        """OpenAI exposes /v1/audio/transcriptions for its speech models."""
+        return True
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio via the OpenAI audio transcriptions endpoint."""
+        try:
+            # runtime_options are chat-completion parameters; the transcriptions
+            # endpoint rejects unknown fields, so they are deliberately not splatted
+            # in here the way _send() does.
+            request_params: dict[str, Any] = {
+                "model": self.genai_config.model,
+                "file": ("audio.wav", audio, mime_type),
+                "response_format": "text",
+                "timeout": self.timeout,
+            }
+
+            if language:
+                if (
+                    self.genai_config.model.strip()
+                    .lower()
+                    .startswith(_LANGUAGES_ARRAY_MODEL_PREFIX)
+                ):
+                    # not a typed parameter on the SDK method, so it has to ride
+                    # along in extra_body
+                    request_params["extra_body"] = {"languages": [language]}
+                else:
+                    request_params["language"] = language
+
+            result = self.provider.audio.transcriptions.create(**request_params)
+        except (TimeoutException, Exception) as e:
+            logger.warning("OpenAI returned an error: %s", str(e))
+            return None
+
+        # response_format="text" yields a bare string, but some compatible
+        # servers still return the object form
+        text = result if isinstance(result, str) else getattr(result, "text", None)
+        return text.strip() if text else None
+
     def list_models(self) -> list[str]:
         """Return available model IDs from the OpenAI-compatible API."""
         try:

frigate/test/test_audio_transcription_genai.py

@@ -0,0 +1,286 @@
+"""Config validation and the historical path for the audio_transcription GenAI backend."""
+
+import unittest
+from copy import deepcopy
+from unittest.mock import MagicMock, patch
+
+from pydantic import ValidationError
+
+from frigate.config import FrigateConfig
+from frigate.config.camera.genai import GenAIConfig, GenAIRoleEnum
+from frigate.config.classification import AudioTranscriptionModelEnum
+from frigate.const import UPDATE_EVENT_DESCRIPTION
+from frigate.data_processing.post.audio_transcription import (
+    AudioTranscriptionPostProcessor,
+)
+from frigate.data_processing.types import PostProcessDataEnum
+
+
+class TestAudioTranscriptionGenAIConfig(unittest.TestCase):
+    def setUp(self):
+        self.base = {
+            "mqtt": {"host": "mqtt"},
+            "cameras": {
+                "back": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {
+                                "path": "rtsp://10.0.0.1:554/video",
+                                "roles": ["detect", "audio"],
+                            }
+                        ]
+                    },
+                    "detect": {"height": 1080, "width": 1920, "fps": 5},
+                    "audio": {"enabled": True},
+                }
+            },
+        }
+
+    def _config(self, **overrides) -> dict:
+        config = deepcopy(self.base)
+        config.update(deepcopy(overrides))
+        return config
+
+    def _provider(self, roles: list[str]) -> dict:
+        return {
+            "whisper_cloud": {
+                "provider": "openai",
+                "model": "gpt-4o-transcribe",
+                "api_key": "k",
+                "roles": roles,
+            }
+        }
+
+    def test_default_model_is_whisper_enum(self):
+        config = FrigateConfig(**self._config())
+        self.assertEqual(
+            config.audio_transcription.model, AudioTranscriptionModelEnum.whisper
+        )
+
+    def test_whisper_string_coerces_to_enum(self):
+        config = FrigateConfig(
+            **self._config(audio_transcription={"enabled": True, "model": "whisper"})
+        )
+        self.assertIsInstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+
+    def test_provider_name_stays_a_string(self):
+        config = FrigateConfig(
+            **self._config(
+                genai=self._provider(["transcribe"]),
+                audio_transcription={"enabled": True, "model": "whisper_cloud"},
+            )
+        )
+        self.assertNotIsInstance(
+            config.audio_transcription.model, AudioTranscriptionModelEnum
+        )
+        self.assertEqual(config.audio_transcription.model, "whisper_cloud")
+
+    def test_unspecified_model_falls_back_to_whisper(self):
+        """An empty value must not read as a GenAI provider that resolves to no client."""
+        for value in (None, "", "   "):
+            with self.subTest(repr(value)):
+                config = FrigateConfig(
+                    **self._config(
+                        audio_transcription={"enabled": True, "model": value}
+                    )
+                )
+                self.assertIs(
+                    config.audio_transcription.model,
+                    AudioTranscriptionModelEnum.whisper,
+                )
+
+    def test_missing_genai_key_raises(self):
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(
+                **self._config(
+                    audio_transcription={"enabled": True, "model": "nope"},
+                )
+            )
+        self.assertIn("is not a valid GenAI config key", str(ctx.exception))
+
+    def test_provider_without_role_raises(self):
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(
+                **self._config(
+                    genai=self._provider(["descriptions"]),
+                    audio_transcription={"enabled": True, "model": "whisper_cloud"},
+                )
+            )
+        self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
+
+    def test_global_off_camera_on_still_validates(self):
+        """Global-off/camera-on is a supported deployment and must not skip the check."""
+        config = self._config(
+            genai=self._provider(["descriptions"]),
+            audio_transcription={"enabled": False, "model": "whisper_cloud"},
+        )
+        config["cameras"]["back"]["audio_transcription"] = {"enabled": True}
+
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(**config)
+
+        self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
+
+    def test_disabled_transcription_skips_validation(self):
+        config = FrigateConfig(
+            **self._config(audio_transcription={"enabled": False, "model": "nope"})
+        )
+        self.assertEqual(config.audio_transcription.model, "nope")
+
+    def test_camera_level_model_is_rejected(self):
+        config = self._config()
+        config["cameras"]["back"]["audio_transcription"] = {
+            "enabled": True,
+            "model": "whisper",
+        }
+
+        with self.assertRaises(ValidationError):
+            FrigateConfig(**config)
+
+    def test_default_roles_do_not_include_transcribe(self):
+        """Backward compatibility: existing providers must not silently claim it."""
+        genai = GenAIConfig(provider="openai", model="gpt-4o")
+        self.assertNotIn(GenAIRoleEnum.transcribe, genai.roles)
+
+    def test_two_providers_claiming_transcribe_raises(self):
+        genai = self._provider(["transcribe"])
+        genai["other"] = {
+            "provider": "gemini",
+            "model": "gemini-2.0-flash",
+            "api_key": "k",
+            "roles": ["transcribe"],
+        }
+
+        with self.assertRaises(ValidationError) as ctx:
+            FrigateConfig(
+                **self._config(
+                    genai=genai,
+                    audio_transcription={"enabled": True, "model": "whisper_cloud"},
+                )
+            )
+
+        self.assertIn("each role must have", str(ctx.exception))
+
+    def test_transcribe_rejected_on_provider_without_audio_input(self):
+        with self.assertRaises(ValidationError) as ctx:
+            GenAIConfig(provider="ollama", model="llava", roles=["transcribe"])
+
+        self.assertIn("does not support audio input", str(ctx.exception))
+
+
+class TestAudioTranscriptionPostProcessorGenAI(unittest.TestCase):
+    """The recorded-speech path must reach the provider and skip the local model."""
+
+    def setUp(self):
+        self.config = FrigateConfig(
+            **{
+                "mqtt": {"host": "mqtt"},
+                "genai": {
+                    "whisper_cloud": {
+                        "provider": "openai",
+                        "model": "gpt-4o-transcribe",
+                        "api_key": "k",
+                        "roles": ["transcribe"],
+                    }
+                },
+                "audio_transcription": {
+                    "enabled": True,
+                    "model": "whisper_cloud",
+                    "language": "en",
+                },
+                "cameras": {
+                    "back": {
+                        "ffmpeg": {
+                            "inputs": [
+                                {
+                                    "path": "rtsp://10.0.0.1:554/video",
+                                    "roles": ["detect", "audio"],
+                                }
+                            ]
+                        },
+                        "detect": {"height": 1080, "width": 1920, "fps": 5},
+                        "audio": {"enabled": True},
+                    }
+                },
+            }
+        )
+
+        self.client = MagicMock()
+        self.client.transcribe.return_value = "recorded speech"
+        self.manager = MagicMock()
+        self.manager.transcribe_client = self.client
+        self.requestor = MagicMock()
+
+        self.processor = AudioTranscriptionPostProcessor(
+            self.config,
+            self.requestor,
+            MagicMock(),
+            MagicMock(),
+            self.manager,
+        )
+
+    def _process(self):
+        self.processor.process_data(
+            {
+                "event_id": "1234.5-abc",
+                "camera": "back",
+                "event": {
+                    "id": "1234.5-abc",
+                    "camera": "back",
+                    "start_time": 100.0,
+                    "end_time": 110.0,
+                    "data": {},
+                },
+            },
+            PostProcessDataEnum.tracked_object,
+        )
+
+    def test_local_recognizer_is_never_built(self):
+        self.assertTrue(self.processor._use_genai)
+        self.assertIsNone(self.processor.recognizer)
+
+    def test_audio_bytes_and_language_reach_the_client(self):
+        with patch(
+            "frigate.data_processing.post.audio_transcription.get_audio_from_recording",
+            return_value=b"RIFF....WAVE",
+        ):
+            self._process()
+
+        self.client.transcribe.assert_called_once()
+        self.assertEqual(self.client.transcribe.call_args.args[0], b"RIFF....WAVE")
+        self.assertEqual(self.client.transcribe.call_args.kwargs["language"], "en")
+
+    def test_transcript_is_published_as_the_description(self):
+        with patch(
+            "frigate.data_processing.post.audio_transcription.get_audio_from_recording",
+            return_value=b"RIFF....WAVE",
+        ):
+            self._process()
+
+        topics = [call.args[0] for call in self.requestor.send_data.call_args_list]
+        self.assertIn(UPDATE_EVENT_DESCRIPTION, topics)
+
+        payload = next(
+            call.args[1]
+            for call in self.requestor.send_data.call_args_list
+            if call.args[0] == UPDATE_EVENT_DESCRIPTION
+        )
+        self.assertEqual(payload["description"], "recorded speech")
+        self.assertEqual(payload["id"], "1234.5-abc")
+
+    def test_missing_client_publishes_nothing(self):
+        self.manager.transcribe_client = None
+
+        with patch(
+            "frigate.data_processing.post.audio_transcription.get_audio_from_recording",
+            return_value=b"RIFF....WAVE",
+        ):
+            self._process()
+
+        self.requestor.send_data.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/test/test_audio_transcription_live_buffer.py

@@ -0,0 +1,288 @@
+"""Live GenAI transcription: sliding overlapped windows and their lifecycle."""
+
+import io
+import threading
+import unittest
+import wave
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+
+from frigate.config import FrigateConfig
+from frigate.const import AUDIO_DURATION, AUDIO_SAMPLE_RATE
+from frigate.data_processing.real_time.audio_transcription import (
+    GENAI_WINDOW_CHUNKS,
+    AudioTranscriptionRealTimeProcessor,
+)
+
+CHUNK_SAMPLES = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE))
+
+
+def _chunk(amplitude: int) -> np.ndarray:
+    """One audio-detector-sized chunk of int16 samples at a constant amplitude."""
+    return np.full(CHUNK_SAMPLES, amplitude, dtype=np.int16)
+
+
+class TestLiveGenAITranscription(unittest.TestCase):
+    def setUp(self):
+        self.config = FrigateConfig(
+            **{
+                "mqtt": {"host": "mqtt"},
+                "genai": {
+                    "whisper_cloud": {
+                        "provider": "openai",
+                        "model": "gpt-4o-transcribe",
+                        "api_key": "k",
+                        "roles": ["transcribe"],
+                    }
+                },
+                "audio_transcription": {
+                    "enabled": True,
+                    "model": "whisper_cloud",
+                    "language": "en",
+                },
+                "cameras": {
+                    "back": {
+                        "ffmpeg": {
+                            "inputs": [
+                                {
+                                    "path": "rtsp://10.0.0.1:554/video",
+                                    "roles": ["detect", "audio"],
+                                }
+                            ]
+                        },
+                        "detect": {"height": 1080, "width": 1920, "fps": 5},
+                        "audio": {"enabled": True},
+                    }
+                },
+            }
+        )
+
+        self.client = MagicMock()
+        self.client.transcribe.return_value = "hello"
+        self.manager = MagicMock()
+        self.manager.transcribe_client = self.client
+        self.requestor = MagicMock()
+
+        self.processor = AudioTranscriptionRealTimeProcessor(
+            config=self.config,
+            camera_config=self.config.cameras["back"],
+            requestor=self.requestor,
+            model_runner=None,
+            metrics=MagicMock(),
+            stop_event=threading.Event(),
+            genai_manager=self.manager,
+        )
+
+    def _feed(self, chunk: np.ndarray):
+        return (
+            self.processor._AudioTranscriptionRealTimeProcessor__process_audio_stream(
+                chunk
+            )
+        )
+
+    def _sent_wav(self, call_index: int) -> wave.Wave_read:
+        payload = self.client.transcribe.call_args_list[call_index].args[0]
+        return wave.open(io.BytesIO(payload), "rb")
+
+    def test_uses_genai_path(self):
+        self.assertTrue(self.processor._use_genai)
+
+    def test_first_chunk_does_not_transcribe(self):
+        self.assertIsNone(self._feed(_chunk(4000)))
+        self.client.transcribe.assert_not_called()
+
+    def test_full_window_transcribes_two_chunks(self):
+        self._feed(_chunk(4000))
+        result = self._feed(_chunk(4000))
+
+        self.assertEqual(result, ("hello", False))
+        self.client.transcribe.assert_called_once()
+        self.assertEqual(
+            self.client.transcribe.call_args.kwargs["language"],
+            "en",
+        )
+
+        with self._sent_wav(0) as wav:
+            self.assertEqual(wav.getnchannels(), 1)
+            self.assertEqual(wav.getsampwidth(), 2)
+            self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
+            self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
+
+    def test_window_slides_with_overlap(self):
+        """The third chunk's window is chunks 2+3, not 3 alone and not 1+2+3."""
+        self.client.transcribe.side_effect = ["one two", "two three"]
+
+        self._feed(_chunk(1000))
+        self._feed(_chunk(2000))
+        result = self._feed(_chunk(3000))
+
+        self.assertEqual(self.client.transcribe.call_count, 2)
+
+        with self._sent_wav(1) as wav:
+            self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
+            samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        self.assertEqual(samples[0], 2000)
+        self.assertEqual(samples[-1], 3000)
+
+        # the shared "two" appears once
+        self.assertEqual(result, ("one two three", False))
+
+    def test_silent_window_is_not_uploaded(self):
+        self._feed(_chunk(0))
+        self.assertIsNone(self._feed(_chunk(0)))
+        self.client.transcribe.assert_not_called()
+
+    def test_silent_window_ends_a_pending_utterance(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        # the gate covers the whole window, so it takes GENAI_WINDOW_CHUNKS
+        # silent chunks to push the last speech out of it
+        self._feed(_chunk(0))
+
+        self.assertEqual(self._feed(_chunk(0)), ("hello", True))
+
+    def test_empty_transcript_ends_a_pending_utterance(self):
+        self.client.transcribe.side_effect = ["hello", ""]
+
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
+
+    def test_empty_transcript_with_nothing_pending_returns_none(self):
+        self.client.transcribe.return_value = ""
+
+        self._feed(_chunk(4000))
+        self.assertIsNone(self._feed(_chunk(4000)))
+
+    def test_reset_clears_committed_text_and_window(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        self.processor.reset()
+
+        self.assertEqual(self.processor._genai_committed, "")
+        self.assertEqual(len(self.processor._genai_window), 0)
+
+        # a fresh window is required again before the next request
+        self.client.transcribe.reset_mock()
+        self._feed(_chunk(4000))
+        self.client.transcribe.assert_not_called()
+
+    def test_check_unload_model_clears_once_then_is_idempotent(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        self.processor.check_unload_model()
+
+        self.requestor.send_data.assert_called_once_with("back/audio/transcription", "")
+        self.assertEqual(self.processor._genai_committed, "")
+        self.assertEqual(len(self.processor._genai_window), 0)
+
+        self.processor.check_unload_model()
+        self.requestor.send_data.assert_called_once()
+
+    def test_build_recognizer_never_loads_a_local_model(self):
+        with patch(
+            "frigate.data_processing.real_time.audio_transcription.FasterWhisperASR"
+        ) as whisper:
+            self.processor._AudioTranscriptionRealTimeProcessor__build_recognizer()
+
+        whisper.assert_not_called()
+        self.assertIsNone(self.processor.stream)
+
+    def test_clear_audio_recognizer_request_only_resets(self):
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+
+        with patch.object(
+            self.processor,
+            "_AudioTranscriptionRealTimeProcessor__build_recognizer",
+        ) as build:
+            result = self.processor.handle_request("clear_audio_recognizer", {})
+
+        build.assert_not_called()
+        self.assertTrue(result["success"])
+        self.assertEqual(self.processor._genai_committed, "")
+
+    def test_missing_client_logs_and_returns_none(self):
+        self.manager.transcribe_client = None
+
+        self.assertIsNone(self._feed(_chunk(4000)))
+        self.assertIsNone(self._feed(_chunk(4000)))
+
+    def test_dropped_audio_discards_the_buffered_window(self):
+        """A gap in the stream must not be spliced into a single window.
+
+        Dropping a queued chunk leaves the next one non-adjacent to what is
+        buffered, so concatenating them would hand the provider audio with a
+        hole in it and break the 50% overlap the stitcher relies on.
+        """
+        self._feed(_chunk(4000))
+        self.assertEqual(len(self.processor._genai_window), 1)
+
+        # the producer discards a chunk while the consumer is blocked
+        self.processor._audio_dropped.set()
+
+        self._feed(_chunk(5000))
+
+        # the buffered chunk was discarded, so this one starts a fresh window
+        self.assertEqual(len(self.processor._genai_window), 1)
+        self.client.transcribe.assert_not_called()
+
+        # and the window that does go out holds only contiguous audio
+        self._feed(_chunk(5000))
+        self.client.transcribe.assert_called_once()
+
+        with self._sent_wav(0) as wav:
+            samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
+        self.assertTrue((samples == 5000).all(), "window spliced across the gap")
+
+    def test_dropped_audio_ends_a_pending_utterance(self):
+        """Committed text cannot be stitched across missing speech."""
+        self._feed(_chunk(4000))
+        self._feed(_chunk(4000))
+        self.assertEqual(self.processor._genai_committed, "hello")
+
+        self.processor._audio_dropped.set()
+
+        self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
+        self.assertEqual(len(self.processor._genai_window), 0)
+
+    def test_drop_flag_is_consumed_once(self):
+        self.processor._audio_dropped.set()
+        self._feed(_chunk(4000))
+
+        self.assertFalse(self.processor._audio_dropped.is_set())
+
+    def test_full_queue_flags_a_drop(self):
+        for i in range(self.processor.audio_queue.maxsize + 1):
+            self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
+
+        self.assertTrue(self.processor._audio_dropped.is_set())
+
+    def test_queue_is_bounded_and_drops_oldest(self):
+        maxsize = self.processor.audio_queue.maxsize
+        self.assertGreater(maxsize, 0)
+
+        for i in range(maxsize + 5):
+            self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
+
+        self.assertEqual(self.processor.audio_queue.qsize(), maxsize)
+
+        # the newest chunk survived, the oldest did not
+        remaining = []
+        while not self.processor.audio_queue.empty():
+            remaining.append(self.processor.audio_queue.get_nowait()[1][0])
+
+        self.assertEqual(remaining[-1], maxsize + 5)
+        self.assertNotIn(1, remaining)
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/test/test_audio_util.py

@@ -0,0 +1,176 @@
+"""Tests for the WAV helpers and transcript stitcher in frigate.util.audio."""
+
+import io
+import struct
+import unittest
+import wave
+
+import numpy as np
+
+from frigate.const import AUDIO_SAMPLE_RATE
+from frigate.util.audio import fix_wav_header, pcm16_to_wav, stitch_transcripts
+
+
+def _wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
+    buffer = io.BytesIO()
+
+    with wave.open(buffer, "wb") as out:
+        out.setnchannels(1)
+        out.setsampwidth(2)
+        out.setframerate(sample_rate)
+        out.writeframes(samples.tobytes())
+
+    return buffer.getvalue()
+
+
+class TestPcm16ToWav(unittest.TestCase):
+    def test_round_trips_through_wave(self):
+        samples = np.arange(-1000, 1000, dtype=np.int16)
+
+        with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
+            self.assertEqual(wav.getnchannels(), 1)
+            self.assertEqual(wav.getsampwidth(), 2)
+            self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
+            self.assertEqual(wav.getnframes(), samples.size)
+            decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        np.testing.assert_array_equal(decoded, samples)
+
+    def test_casts_non_int16_input(self):
+        samples = np.array([0.0, 100.0, -100.0], dtype=np.float32)
+
+        with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
+            decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
+
+        np.testing.assert_array_equal(decoded, np.array([0, 100, -100], np.int16))
+
+    def test_honors_sample_rate(self):
+        with wave.open(
+            io.BytesIO(pcm16_to_wav(np.zeros(4, np.int16), 8000)), "rb"
+        ) as w:
+            self.assertEqual(w.getframerate(), 8000)
+
+
+class TestFixWavHeader(unittest.TestCase):
+    def test_rewrites_placeholder_sizes(self):
+        samples = np.arange(64, dtype=np.int16)
+        data = bytearray(_wav(samples))
+
+        # ffmpeg piping to non-seekable stdout leaves both sizes unpatched
+        struct.pack_into("<I", data, 4, 0xFFFFFFFF)
+        data_offset = data.index(b"data")
+        struct.pack_into("<I", data, data_offset + 4, 0xFFFFFFFF)
+
+        fixed = fix_wav_header(bytes(data))
+
+        self.assertEqual(struct.unpack_from("<I", fixed, 4)[0], len(fixed) - 8)
+        self.assertEqual(
+            struct.unpack_from("<I", fixed, data_offset + 4)[0],
+            len(fixed) - (data_offset + 8),
+        )
+
+        with wave.open(io.BytesIO(fixed), "rb") as wav:
+            self.assertEqual(wav.getnframes(), samples.size)
+
+    def test_leaves_a_well_formed_header_alone(self):
+        data = _wav(np.arange(32, dtype=np.int16))
+        self.assertEqual(fix_wav_header(data), data)
+
+    def test_non_riff_payload_passes_through(self):
+        self.assertEqual(fix_wav_header(b"not a wav"), b"not a wav")
+        self.assertEqual(fix_wav_header(b""), b"")
+
+
+class TestStitchTranscripts(unittest.TestCase):
+    def test_table(self):
+        cases = [
+            # (committed, incoming, expected, description)
+            (
+                "the quick brown",
+                "brown fox jumps",
+                "the quick brown fox jumps",
+                "one word",
+            ),
+            (
+                "and then the quick brown",
+                "the quick brown fox",
+                "and then the quick brown fox",
+                "multi word",
+            ),
+            (
+                "hello there",
+                "general kenobi",
+                "hello there general kenobi",
+                "no overlap",
+            ),
+            ("the quick brown fox", "brown fox", "the quick brown fox", "contained"),
+            ("", "first words", "first words", "empty committed"),
+            ("already here", "", "already here", "empty incoming"),
+            (
+                "  spaced   out  ",
+                "out again",
+                "spaced out again",
+                "whitespace normalized",
+            ),
+        ]
+
+        for committed, incoming, expected, description in cases:
+            with self.subTest(description):
+                self.assertEqual(stitch_transcripts(committed, incoming), expected)
+
+    def test_overlap_found_mid_window(self):
+        """The shared run is rarely at the start of the new window.
+
+        The provider re-transcribes the overlapping audio independently and
+        often renders its first word differently, so anchoring the match to the
+        start of the incoming window duplicates the whole phrase.
+        """
+        self.assertEqual(
+            stitch_transcripts(
+                "this is just gonna be a fun time", "It's gonna be a fun time."
+            ),
+            "this is just gonna be a fun time",
+        )
+
+    def test_overlap_longer_than_five_words(self):
+        """The cap is bounded by window duration, not by the old 5-word n-gram."""
+        self.assertEqual(
+            stitch_transcripts(
+                "well anyway one two three four five six",
+                "one two three four five six seven",
+            ),
+            "well anyway one two three four five six seven",
+        )
+
+    def test_repeated_phrase_keeps_its_second_utterance(self):
+        """Preferring the earliest match is what protects a real repeat."""
+        self.assertEqual(
+            stitch_transcripts("a b c fun time", "fun time fun time"),
+            "a b c fun time fun time",
+        )
+
+    def test_window_wholly_repeating_the_tail_is_dropped(self):
+        """The accepted trade-off: an entirely redundant window adds nothing."""
+        self.assertEqual(stitch_transcripts("go go go", "go go go"), "go go go")
+
+    def test_revises_a_mistranscribed_tail(self):
+        """A wrong last word would otherwise block every alignment.
+
+        Those words came from the newest audio, which the next window re-covers,
+        so replacing them is better than duplicating the phrase behind them.
+        """
+        self.assertEqual(
+            stitch_transcripts("Yeah. this is Jessica.", "This is just gonna be fun."),
+            "Yeah. this is just gonna be fun.",
+        )
+
+    def test_revision_needs_more_than_one_shared_word(self):
+        """A revision deletes published text, so it takes real evidence."""
+        self.assertEqual(
+            stitch_transcripts("the cat sat on a mat", "a dog barked"),
+            "the cat sat on a mat a dog barked",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/test/test_genai_providers.py

@@ -581,5 +581,271 @@ class TestLlamaCppProvider(unittest.TestCase):
         self.assertEqual(client.get_context_size(), 32768)
 
 
+# ---------------------------------------------------------------------------
+# transcribe role
+# ---------------------------------------------------------------------------
+WAV_BYTES = b"RIFF$\x00\x00\x00WAVEfmt "
+
+
+class TestOpenAITranscribe(unittest.TestCase):
+    def _client(self):
+        return _make_client(
+            "openai",
+            model="gpt-4o-transcribe",
+            api_key="k",
+            base_url="http://localhost:9999/v1",
+            runtime_options={"temperature": 0.7},
+        )
+
+    def test_supports_transcription(self):
+        self.assertTrue(self._client().supports_transcription)
+
+    def test_passes_file_tuple_and_language(self):
+        client = self._client()
+        create = MagicMock(return_value="  hello there  ")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "hello there")
+
+        kwargs = create.call_args.kwargs
+        self.assertEqual(kwargs["model"], "gpt-4o-transcribe")
+        self.assertEqual(kwargs["file"], ("audio.wav", WAV_BYTES, "audio/wav"))
+        self.assertEqual(kwargs["language"], "en")
+        self.assertEqual(kwargs["response_format"], "text")
+
+    def test_does_not_forward_runtime_options(self):
+        """runtime_options are chat parameters; /audio/transcriptions rejects them."""
+        client = self._client()
+        create = MagicMock(return_value="hi")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        client.transcribe(WAV_BYTES)
+
+        self.assertNotIn("temperature", create.call_args.kwargs)
+
+    def test_gpt_transcribe_uses_languages_array(self):
+        """gpt-transcribe replaced `language` with a `languages` array."""
+        client = _make_client("openai", model="gpt-transcribe", api_key="k")
+        create = MagicMock(return_value="hi")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        client.transcribe(WAV_BYTES, language="en")
+
+        kwargs = create.call_args.kwargs
+        self.assertEqual(kwargs["extra_body"], {"languages": ["en"]})
+        # sending both fields is rejected by the API
+        self.assertNotIn("language", kwargs)
+
+    def test_older_models_use_singular_language(self):
+        for model in ("gpt-4o-transcribe", "whisper-1"):
+            with self.subTest(model):
+                client = _make_client("openai", model=model, api_key="k")
+                create = MagicMock(return_value="hi")
+                client.provider = SimpleNamespace(
+                    audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+                )
+
+                client.transcribe(WAV_BYTES, language="en")
+
+                kwargs = create.call_args.kwargs
+                self.assertEqual(kwargs["language"], "en")
+                self.assertNotIn("extra_body", kwargs)
+
+    def test_object_response_form(self):
+        client = self._client()
+        create = MagicMock(return_value=SimpleNamespace(text="hi"))
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES), "hi")
+
+    def test_error_returns_none(self):
+        client = self._client()
+        create = MagicMock(side_effect=RuntimeError("boom"))
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertIsNone(client.transcribe(WAV_BYTES))
+
+
+class TestAzureOpenAITranscribe(unittest.TestCase):
+    def _client(self):
+        return _make_client(
+            "azure_openai",
+            model="my-deployment",
+            api_key="k",
+            base_url="https://example.openai.azure.com/?api-version=2024-06-01",
+        )
+
+    def test_routes_through_azure_client(self):
+        from openai import AzureOpenAI
+
+        client = self._client()
+        self.assertIsInstance(client.provider, AzureOpenAI)
+        self.assertTrue(client.supports_transcription)
+
+    def test_transcribe_inherited(self):
+        client = self._client()
+        create = MagicMock(return_value="azure text")
+        client.provider = SimpleNamespace(
+            audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES, language="fr"), "azure text")
+        self.assertEqual(create.call_args.kwargs["model"], "my-deployment")
+
+
+class TestGeminiTranscribe(unittest.TestCase):
+    def _client(self):
+        return _make_client("gemini", model="gemini-2.0-flash", api_key="k")
+
+    def test_supports_transcription(self):
+        self.assertTrue(self._client().supports_transcription)
+
+    def test_sends_audio_part(self):
+        client = self._client()
+        generate = MagicMock(return_value=SimpleNamespace(text=" spoken words "))
+        client.provider = SimpleNamespace(
+            models=SimpleNamespace(generate_content=generate)
+        )
+
+        self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "spoken words")
+
+        contents = generate.call_args.kwargs["contents"]
+        audio_parts = [
+            p for p in contents if getattr(p, "inline_data", None) is not None
+        ]
+        self.assertEqual(len(audio_parts), 1)
+        self.assertEqual(audio_parts[0].inline_data.mime_type, "audio/wav")
+        self.assertEqual(audio_parts[0].inline_data.data, WAV_BYTES)
+
+    def test_oversized_payload_is_skipped(self):
+        from frigate.genai.plugins.gemini import GEMINI_MAX_INLINE_BYTES
+
+        client = self._client()
+        generate = MagicMock()
+        client.provider = SimpleNamespace(
+            models=SimpleNamespace(generate_content=generate)
+        )
+
+        self.assertIsNone(client.transcribe(b"\x00" * (GEMINI_MAX_INLINE_BYTES + 1)))
+        generate.assert_not_called()
+
+
+class TestLlamaCppTranscribe(unittest.TestCase):
+    def _client(self, supports_audio: bool):
+        cfg = GenAIConfig(
+            provider="llamacpp",
+            model="m",
+            base_url="http://localhost:9999",
+        )
+        info = {
+            "context_size": 4096,
+            "supports_vision": False,
+            "supports_audio": supports_audio,
+            "supports_tools": False,
+            "supports_reasoning": False,
+            "media_marker": "<__media__>",
+        }
+        cls = PROVIDERS[GenAIProviderEnum.llamacpp]
+        with patch.object(cls, "_get_model_info", return_value=info):
+            return cls(cfg, timeout=5)
+
+    def test_supports_transcription_tracks_supports_audio(self):
+        self.assertTrue(self._client(True).supports_transcription)
+        self.assertFalse(self._client(False).supports_transcription)
+
+    @staticmethod
+    def _transcriptions_response(text: str = " transcript "):
+        response = MagicMock()
+        response.status_code = 200
+        response.json.return_value = {"text": text}
+        return response
+
+    @staticmethod
+    def _chat_response(content: str = " fallback transcript "):
+        response = MagicMock()
+        response.status_code = 200
+        response.json.return_value = {"choices": [{"message": {"content": content}}]}
+        return response
+
+    def test_posts_multipart_to_transcriptions(self):
+        client = self._client(True)
+
+        with patch.object(
+            client, "_post", return_value=self._transcriptions_response()
+        ) as post:
+            self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "transcript")
+
+        self.assertTrue(post.call_args.args[0].endswith("/v1/audio/transcriptions"))
+        self.assertEqual(
+            post.call_args.kwargs["files"]["file"],
+            ("audio.wav", WAV_BYTES, "audio/wav"),
+        )
+        self.assertEqual(post.call_args.kwargs["data"]["language"], "en")
+
+    def test_omits_language_when_not_set(self):
+        """An unset language is what lets the model detect one itself."""
+        client = self._client(True)
+
+        with patch.object(
+            client, "_post", return_value=self._transcriptions_response()
+        ) as post:
+            client.transcribe(WAV_BYTES)
+
+        self.assertNotIn("language", post.call_args.kwargs["data"])
+
+    def test_falls_back_to_chat_completions_on_404(self):
+        """Servers predating llama.cpp#21863 have no transcriptions route."""
+        client = self._client(True)
+        missing = MagicMock()
+        missing.status_code = 404
+
+        with patch.object(
+            client, "_post", side_effect=[missing, self._chat_response()]
+        ) as post:
+            self.assertEqual(
+                client.transcribe(WAV_BYTES, language="en"), "fallback transcript"
+            )
+
+        urls = [call.args[0] for call in post.call_args_list]
+        self.assertTrue(urls[0].endswith("/v1/audio/transcriptions"))
+        self.assertTrue(urls[1].endswith("/v1/chat/completions"))
+
+        payload = post.call_args_list[1].kwargs["json"]
+        content = payload["messages"][0]["content"]
+        audio_parts = [p for p in content if p["type"] == "input_audio"]
+        self.assertEqual(len(audio_parts), 1)
+        self.assertEqual(audio_parts[0]["input_audio"]["format"], "wav")
+        self.assertEqual(
+            base64.b64decode(audio_parts[0]["input_audio"]["data"]), WAV_BYTES
+        )
+
+    def test_audio_unsupported_returns_none(self):
+        client = self._client(False)
+
+        with patch.object(client, "_post") as post:
+            self.assertIsNone(client.transcribe(WAV_BYTES))
+
+        post.assert_not_called()
+
+
+class TestBaseClientTranscribe(unittest.TestCase):
+    """Providers that don't implement the role must be inert, not broken."""
+
+    def test_ollama_reports_and_returns_nothing(self):
+        client = _make_client("ollama", model="llava", base_url="http://localhost:9999")
+        self.assertFalse(client.supports_transcription)
+        self.assertIsNone(client.transcribe(WAV_BYTES, language="en"))
+
+
 if __name__ == "__main__":
     unittest.main()

frigate/util/audio.py

@@ -1,16 +1,61 @@
 """Utilities for creating and manipulating audio."""
 
+import io
 import logging
 import os
+import re
+import string
+import struct
 import subprocess as sp
+import wave
 
+import numpy as np
 from pathvalidate import sanitize_filename
 
-from frigate.const import CACHE_DIR, STREAM_TYPE_MAIN, STREAM_TYPE_SUB
+from frigate.const import (
+    AUDIO_SAMPLE_RATE,
+    CACHE_DIR,
+    STREAM_TYPE_MAIN,
+    STREAM_TYPE_SUB,
+)
 from frigate.models import Recordings
 
 logger = logging.getLogger(__name__)
 
+# Ceiling on the run of words the stitcher will treat as an overlap between two
+# consecutive windows. This is an audio-duration bound, not a linguistic one: a
+# window holds GENAI_WINDOW_CHUNKS * AUDIO_DURATION seconds of speech, so at a
+# fast talker's pace it tops out around this many words, and a whole window can
+# legitimately be redundant. The vendored whisper_streaming HypothesisBuffer
+# caps at 5, but there the n-gram is only a tie-break on top of word-level
+# timestamps; here it is the entire alignment, so 5 truncates real overlaps.
+# Sentinel meaning "let the model work out the language". The vendored
+# whisper_streaming code already uses this spelling, so it is the established
+# convention for the audio_transcription.language field.
+AUTO_LANGUAGE = "auto"
+
+MAX_STITCH_NGRAM = 16
+
+# How many trailing committed words the stitcher may discard to find an
+# alignment. Those words came from the newest audio, which the next window
+# re-covers, so when the provider got one of them wrong it blocks every
+# alignment and the whole phrase duplicates. Set to 0 to make committed text
+# strictly append-only.
+MAX_STITCH_REVISE = 3
+
+# A revision deletes text that was already published, so it has to clear a
+# higher bar than a plain append: a single coincidentally shared word is not
+# enough evidence to throw committed words away.
+MIN_STITCH_REVISE_RUN = 2
+
+# ASR models often wrap their output in control markup. Qwen3-ASR, for example,
+# answers "language English<asr_text>Yeah, that works." A structural opening tag
+# marks where the transcript starts, so anything before the last one is metadata.
+# Closing tags (</x>) and pipe-delimited special tokens (<|endoftext|>) are
+# excluded: those mark where the text ends, so text before them must be kept.
+_OPENING_TAG = re.compile(r"<(?![/|])[^<>]*>")
+_ANY_TAG = re.compile(r"<[^<>]*>")
+
 
 def _get_recordings_for_range(
     camera_name: str, start_ts: float, end_ts: float, stream_type: str
@@ -117,7 +162,9 @@ def get_audio_from_recording(
             logger.debug(
                 f"Successfully extracted audio for {camera_name} from {start_ts} to {end_ts}"
             )
-            return process.stdout
+            # ffmpeg writes to a pipe, so it cannot seek back to patch the chunk
+            # sizes it reserved; repair them before any strict consumer sees them
+            return fix_wav_header(process.stdout)
         else:
             logger.error(f"Failed to extract audio: {process.stderr.decode()}")
             return None
@@ -129,3 +176,217 @@ def get_audio_from_recording(
             os.unlink(file_path)
         except OSError:
             pass
+
+
+def fix_wav_header(data: bytes) -> bytes:
+    """Recompute the RIFF and data chunk sizes in a WAV header.
+
+    ffmpeg writing to a non-seekable pipe cannot go back and patch the sizes it
+    reserved, so it leaves 0xFFFFFFFF placeholders. PyAV-based demuxers ignore
+    them, but strict validators may reject the file or read zero frames.
+
+    Args:
+        data: The complete WAV payload
+
+    Returns:
+        The payload with both sizes corrected, or unchanged if it is not a
+        parseable RIFF/WAVE stream
+    """
+    if len(data) < 12 or data[0:4] != b"RIFF" or data[8:12] != b"WAVE":
+        return data
+
+    out = bytearray(data)
+
+    # RIFF size covers everything after the 8-byte RIFF header
+    struct.pack_into("<I", out, 4, len(out) - 8)
+
+    # walk the chunk list to find "data"; every chunk is padded to even length
+    pos = 12
+
+    while pos + 8 <= len(out):
+        chunk_id = bytes(out[pos : pos + 4])
+        (chunk_size,) = struct.unpack_from("<I", out, pos + 4)
+
+        if chunk_id == b"data":
+            struct.pack_into("<I", out, pos + 4, len(out) - (pos + 8))
+            return bytes(out)
+
+        if chunk_size == 0xFFFFFFFF:
+            # an unpatched size before the data chunk leaves nothing to walk
+            break
+
+        pos += 8 + chunk_size + (chunk_size % 2)
+
+    return bytes(out)
+
+
+def pcm16_to_wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
+    """Wrap mono int16 PCM samples in a WAV container.
+
+    Args:
+        samples: The audio samples; converted to int16 if they are not already
+        sample_rate: Sample rate to declare in the header
+
+    Returns:
+        WAV bytes suitable for upload to a GenAI provider
+    """
+    if samples.dtype != np.int16:
+        samples = samples.astype(np.int16)
+
+    buffer = io.BytesIO()
+
+    with wave.open(buffer, "wb") as wav:
+        wav.setnchannels(1)
+        wav.setsampwidth(2)
+        wav.setframerate(sample_rate)
+        wav.writeframes(samples.tobytes())
+
+    return buffer.getvalue()
+
+
+def stitch_transcripts(committed: str, incoming: str) -> str:
+    """Append *incoming* to *committed*, dropping the speech they share.
+
+    Consecutive overlapped transcription windows re-transcribe the same audio at
+    their seam, so the tail of one and the newest one name the same words. Find
+    the longest run that is a suffix of *committed* and occurs anywhere in
+    *incoming*, then keep only what follows that run.
+
+    Searching all of *incoming* rather than just its start is what makes this
+    work in practice. The provider re-transcribes the shared audio independently
+    and often gets its first word or two different ("just gonna" one window,
+    "It's gonna" the next), which leaves the real overlap sitting in the middle
+    of *incoming*. A prefix-anchored match sees no overlap at all there and
+    duplicates the entire phrase.
+
+    Text-level rather than timestamp-level because only some providers return
+    word timings, and this has to work across all of them.
+
+    Args:
+        committed: The transcript accumulated so far
+        incoming: The newest window's transcript
+
+    Returns:
+        The combined transcript
+    """
+    incoming_words = incoming.split()
+
+    if not incoming_words:
+        return committed
+
+    committed_words = committed.split()
+
+    if not committed_words:
+        return " ".join(incoming_words)
+
+    committed_keys = [_overlap_key(word) for word in committed_words]
+    incoming_keys = [_overlap_key(word) for word in incoming_words]
+
+    length, consumed = _find_overlap(committed_keys, incoming_keys)
+
+    if length:
+        return " ".join(committed_words + incoming_words[consumed:])
+
+    # Nothing aligns. Retry against a shortened committed tail: a single word the
+    # provider got wrong at the end of the previous window otherwise blocks every
+    # alignment, and the entire re-transcribed phrase duplicates behind it.
+    best: tuple[int, int, int] | None = None
+
+    for drop in range(1, min(MAX_STITCH_REVISE, len(committed_keys) - 1) + 1):
+        length, consumed = _find_overlap(committed_keys[:-drop], incoming_keys)
+
+        if length < MIN_STITCH_REVISE_RUN:
+            continue
+
+        # longest run wins; ties go to the smallest revision
+        if best is None or length > best[0]:
+            best = (length, drop, consumed)
+
+    if best is None:
+        return " ".join(committed_words + incoming_words)
+
+    _, drop, consumed = best
+
+    return " ".join(committed_words[:-drop] + incoming_words[consumed:])
+
+
+def _find_overlap(
+    committed_keys: list[str], incoming_keys: list[str]
+) -> tuple[int, int]:
+    """Locate the speech *incoming* shares with the end of *committed*.
+
+    Returns the length of the longest run that is a suffix of *committed_keys*
+    and occurs anywhere in *incoming_keys*, along with the index just past that
+    run in *incoming_keys*. Returns ``(0, 0)`` when nothing matches.
+
+    Prefers the longest run so a real overlap is not cut short, and within one
+    length the earliest position, so a phrase genuinely spoken twice keeps its
+    second utterance.
+    """
+    max_run = min(MAX_STITCH_NGRAM, len(committed_keys), len(incoming_keys))
+
+    for length in range(max_run, 0, -1):
+        tail = committed_keys[-length:]
+
+        for start in range(len(incoming_keys) - length + 1):
+            if incoming_keys[start : start + length] == tail:
+                return length, start + length
+
+    return 0, 0
+
+
+def clean_transcript(text: str | None) -> str:
+    """Strip provider control markup and any preamble from a raw transcript.
+
+    A window with no speech often still comes back as the preamble alone
+    ("language English<asr_text>"), which must reduce to an empty string so
+    callers treat it as silence rather than committing it as spoken words.
+
+    Args:
+        text: The provider's raw response
+
+    Returns:
+        The transcript with markup removed and whitespace collapsed
+    """
+    if not text:
+        return ""
+
+    # everything up to and including the last opening tag is metadata
+    openings = list(_OPENING_TAG.finditer(text))
+
+    if openings:
+        text = text[openings[-1].end() :]
+
+    # drop closing tags and special tokens wherever they landed
+    text = _ANY_TAG.sub(" ", text)
+
+    return " ".join(text.split())
+
+
+def _overlap_key(word: str) -> str:
+    """Comparison key for overlap matching.
+
+    Providers re-transcribe the shared audio at a window seam independently, so
+    the same word routinely comes back capitalized differently or with different
+    edge punctuation ("work." vs "Work"). Those differences must not defeat the
+    match, but the original spelling is what gets kept in the output.
+    """
+    key = word.strip(string.punctuation).casefold()
+
+    # a token that is nothing but punctuation would otherwise match any other
+    return key or word
+
+
+def resolve_language(language: str | None) -> str | None:
+    """Turn a configured language into an explicit code, or None for auto-detect.
+
+    Args:
+        language: The configured value, possibly AUTO_LANGUAGE
+
+    Returns:
+        An ISO language code, or None when the backend should detect it
+    """
+    if not language or language == AUTO_LANGUAGE:
+        return None
+
+    return language

frigate/util/config.py

@@ -829,6 +829,57 @@ def rename_hailo_detector(
     return new_config
 
 
+def _camera_enables_transcription(camera: dict[str, Any]) -> bool:
+    """Whether a camera or one of its profiles turns audio transcription on."""
+    sections = [camera.get("audio_transcription")]
+    profiles = camera.get("profiles")
+
+    if isinstance(profiles, dict):
+        for profile in profiles.values():
+            if isinstance(profile, dict):
+                sections.append(profile.get("audio_transcription"))
+
+    return any(
+        isinstance(section, dict) and section.get("enabled") for section in sections
+    )
+
+
+def _migrate_transcription_language(config: dict[str, Any]) -> None:
+    """Pin English for configs written before the language default became auto.
+
+    audio_transcription.language used to default to "en", so a config that
+    turned transcription on without naming a language was transcribing English.
+    The default is now "auto" (let the model detect), which is better for new
+    users but would silently change behavior for existing ones, so write the old
+    value explicitly for anyone actually using the feature.
+    """
+    transcription = config.get("audio_transcription")
+
+    if isinstance(transcription, dict) and "language" in transcription:
+        # named a language already, so nothing was relying on the default
+        return
+
+    enabled = isinstance(transcription, dict) and bool(transcription.get("enabled"))
+
+    if not enabled:
+        enabled = any(
+            _camera_enables_transcription(camera)
+            for camera in config.get("cameras", {}).values()
+            if isinstance(camera, dict)
+        )
+
+    if not enabled:
+        return
+
+    if not isinstance(transcription, dict):
+        # a camera enabled it without a global section, which still picked up
+        # the global default
+        transcription = {}
+        config["audio_transcription"] = transcription
+
+    transcription["language"] = "en"
+
+
 def migrate_019_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
     """Handle migrating Frigate config to 0.19-0."""
     new_config = rename_hailo_detector(config)
@@ -845,6 +896,8 @@ def migrate_019_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
 
         new_config["cameras"][name] = camera_config
 
+    _migrate_transcription_language(new_config)
+
     new_config["version"] = "0.19-0"
     return new_config
 

web/public/locales/en/config/global.json

@@ -348,7 +348,7 @@
     },
     "roles": {
       "label": "Roles",
-      "description": "GenAI roles (chat, descriptions, embeddings); one provider per role."
+      "description": "GenAI roles (chat, descriptions, embeddings, transcribe); one provider per role. Only chat, descriptions, and embeddings are granted by default; transcribe must be listed explicitly."
     },
     "provider_options": {
       "label": "Provider options",
@@ -1131,7 +1131,11 @@
     },
     "language": {
       "label": "Transcription language",
-      "description": "Language code used for transcription/translation (for example 'en' for English). See https://whisper-api.com/docs/languages/ for supported language codes."
+      "description": "Language code used for transcription/translation (for example 'en' for English), or 'auto' to let the model detect it. See https://whisper-api.com/docs/languages/ for supported language codes."
+    },
+    "model": {
+      "label": "Audio transcription model or GenAI provider name",
+      "description": "The transcription backend: 'whisper' for Frigate's built-in local models, or the name of a GenAI provider with the transcribe role."
     },
     "device": {
       "label": "Transcription device",

web/public/locales/en/views/settings.json

@@ -1693,7 +1693,8 @@
       "options": {
         "embeddings": "Embedding",
         "descriptions": "Descriptions",
-        "chat": "Chat"
+        "chat": "Chat",
+        "transcribe": "Transcription"
       }
     },
     "semanticSearchModel": {
@@ -1742,6 +1743,14 @@
       "admin": "Admin",
       "viewer": "Viewer",
       "none": "None (deny access)"
+    },
+    "audioTranscriptionModel": {
+      "placeholder": "Select model…",
+      "builtIn": "Built-in Models",
+      "genaiProviders": "GenAI Providers"
+    },
+    "audioTranscriptionModelSize": {
+      "notApplicable": "Not applicable for GenAI providers"
     }
   },
   "globalConfig": {
@@ -1953,7 +1962,8 @@
       "noAudioRole": "No streams have the audio role defined. You must enable the audio role for audio detection to function."
     },
     "audioTranscription": {
-      "audioDetectionDisabled": "Audio detection is not enabled for this camera. Audio transcription requires audio detection to be active."
+      "audioDetectionDisabled": "Audio detection is not enabled for this camera. Audio transcription requires audio detection to be active.",
+      "genaiProviderSelected": "A GenAI provider is selected, so the device and model size settings are ignored."
     },
     "detect": {
       "fpsGreaterThanFive": "Setting the detect FPS higher than 5 is not recommended. Higher values may cause performance issues and will not provide any benefit.",

web/src/components/config-form/section-configs/audio_transcription.ts

@@ -34,9 +34,32 @@ const audioTranscription: SectionConfigOverrides = {
     },
   },
   global: {
-    fieldOrder: ["enabled", "language", "device", "model_size"],
+    fieldOrder: ["enabled", "model", "language", "device", "model_size"],
     advancedFields: ["language", "device", "model_size"],
-    restartRequired: ["enabled", "language", "device", "model_size"],
+    restartRequired: ["enabled", "model", "language", "device", "model_size"],
+    fieldMessages: [
+      {
+        key: "genai-provider-ignores-local-settings",
+        health: (ctx) => ctx.fullConfig.audio_transcription?.enabled === true,
+        field: "device",
+        messageKey: "configMessages.audioTranscription.genaiProviderSelected",
+        severity: "info",
+        position: "after",
+        condition: (ctx) =>
+          typeof ctx.formData?.model === "string" &&
+          ctx.formData.model !== "" &&
+          ctx.formData.model !== "whisper",
+      },
+    ],
+    uiSchema: {
+      model: {
+        "ui:widget": "audioTranscriptionModel",
+      },
+      model_size: {
+        "ui:widget": "audioTranscriptionModelSize",
+        "ui:options": { size: "xs", enumI18nPrefix: "modelSize" },
+      },
+    },
   },
 };
 

web/src/components/config-form/theme/frigateTheme.ts

@@ -33,6 +33,8 @@ import { CameraPathWidget } from "./widgets/CameraPathWidget";
 import { OptionalFieldWidget } from "./widgets/OptionalFieldWidget";
 import { SemanticSearchModelWidget } from "./widgets/SemanticSearchModelWidget";
 import { SemanticSearchModelSizeWidget } from "./widgets/SemanticSearchModelSizeWidget";
+import { AudioTranscriptionModelWidget } from "./widgets/AudioTranscriptionModelWidget";
+import { AudioTranscriptionModelSizeWidget } from "./widgets/AudioTranscriptionModelSizeWidget";
 import { OnvifProfileWidget } from "./widgets/OnvifProfileWidget";
 import { PTZPresetsWidget } from "./widgets/PTZPresetsWidget";
 import { DefaultRoleWidget } from "./widgets/DefaultRoleWidget";
@@ -93,6 +95,8 @@ export const frigateTheme: FrigateTheme = {
     optionalField: OptionalFieldWidget,
     semanticSearchModel: SemanticSearchModelWidget,
     semanticSearchModelSize: SemanticSearchModelSizeWidget,
+    audioTranscriptionModel: AudioTranscriptionModelWidget,
+    audioTranscriptionModelSize: AudioTranscriptionModelSizeWidget,
     onvifProfile: OnvifProfileWidget,
     ptzPresets: PTZPresetsWidget,
     defaultRole: DefaultRoleWidget,

web/src/components/config-form/theme/widgets/AudioTranscriptionModelSizeWidget.tsx

@@ -0,0 +1,17 @@
+// audio_transcription.model_size. See GenAIBackedModelSizeWidget for the shared
+// implementation, including the clear-vs-default handling.
+import type { WidgetProps } from "@rjsf/utils";
+import { GenAIBackedModelSizeWidget } from "./GenAIBackedModelSizeWidget";
+
+export function AudioTranscriptionModelSizeWidget(props: WidgetProps) {
+  return (
+    <GenAIBackedModelSizeWidget
+      {...props}
+      options={{
+        ...props.options,
+        builtInModels: ["whisper"],
+        i18nPrefix: "audioTranscriptionModelSize",
+      }}
+    />
+  );
+}

web/src/components/config-form/theme/widgets/AudioTranscriptionModelWidget.tsx

@@ -0,0 +1,18 @@
+// audio_transcription.model: the built-in whisper backend plus GenAI providers
+// with the transcribe role. See GenAIBackedModelWidget for the shared
+// implementation.
+import type { WidgetProps } from "@rjsf/utils";
+import { GenAIBackedModelWidget } from "./GenAIBackedModelWidget";
+
+export function AudioTranscriptionModelWidget(props: WidgetProps) {
+  return (
+    <GenAIBackedModelWidget
+      {...props}
+      options={{
+        ...props.options,
+        role: "transcribe",
+        i18nPrefix: "audioTranscriptionModel",
+      }}
+    />
+  );
+}

web/src/components/config-form/theme/widgets/GenAIBackedModelSizeWidget.tsx

@@ -0,0 +1,65 @@
+// Disables model_size and shows "N/A" when a GenAI provider is selected in the
+// companion model field. Reads model via LiveFormDataContext so it re-runs even
+// when RJSF's SchemaField memoization would skip this widget. The built-in model
+// names and the i18n key prefix come from ui:options.
+import type { WidgetProps } from "@rjsf/utils";
+import { useContext, useEffect } from "react";
+import { useTranslation } from "react-i18next";
+import {
+  Select,
+  SelectContent,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { LiveFormDataContext } from "../../LiveFormDataContext";
+import { getSizedFieldClassName } from "../utils";
+import { SelectWidget } from "./SelectWidget";
+
+export function GenAIBackedModelSizeWidget(props: WidgetProps) {
+  const { t } = useTranslation(["views/settings"]);
+  const liveFormData = useContext(LiveFormDataContext);
+  const model = liveFormData?.model;
+
+  const builtInModels = (props.options?.builtInModels as string[]) ?? [];
+  const i18nPrefix =
+    (props.options?.i18nPrefix as string | undefined) ??
+    "semanticSearchModelSize";
+
+  const isProvider =
+    typeof model === "string" && model !== "" && !builtInModels.includes(model);
+
+  // model_size is unused on a GenAI provider. Only clear it (which the backend
+  // treats as "remove") for a non-default value, which can only come from the
+  // config file. A defaulted value is indistinguishable from unset in the
+  // resolved config, so clearing it would falsely dirty the field and delete a
+  // YAML key that isn't there. Restore the default when returning to a built-in model.
+  const { value, onChange, schema } = props;
+  const schemaDefault = schema?.default as string | undefined;
+  useEffect(() => {
+    if (isProvider) {
+      if (value !== undefined && value !== schemaDefault) {
+        onChange(undefined);
+      }
+    } else if (value === undefined && schemaDefault) {
+      onChange(schemaDefault);
+    }
+  }, [isProvider, value, onChange, schemaDefault]);
+
+  if (isProvider) {
+    const fieldClassName = getSizedFieldClassName(props.options ?? {}, "sm");
+    return (
+      <Select value="" disabled>
+        <SelectTrigger className={fieldClassName}>
+          <SelectValue
+            placeholder={t(`configForm.${i18nPrefix}.notApplicable`, {
+              defaultValue: "Not applicable for GenAI providers",
+            })}
+          />
+        </SelectTrigger>
+        <SelectContent />
+      </Select>
+    );
+  }
+
+  return <SelectWidget {...props} />;
+}

web/src/components/config-form/theme/widgets/GenAIBackedModelWidget.tsx

@@ -0,0 +1,164 @@
+// Combobox for a "local model or GenAI provider" field (semantic_search.model,
+// audio_transcription.model). Shows the built-in model enum values alongside the
+// GenAI providers holding the relevant role. The role and the i18n key prefix
+// come from ui:options so each field can reuse this with its own wording.
+import { useState, useMemo } from "react";
+import type { WidgetProps } from "@rjsf/utils";
+import { useTranslation } from "react-i18next";
+import { Check, ChevronsUpDown } from "lucide-react";
+import { cn } from "@/lib/utils";
+import { Button } from "@/components/ui/button";
+import {
+  Command,
+  CommandGroup,
+  CommandItem,
+  CommandList,
+} from "@/components/ui/command";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/components/ui/popover";
+import type { ConfigFormContext } from "@/types/configForm";
+import { getSizedFieldClassName } from "../utils";
+
+interface ProviderOption {
+  value: string;
+  label: string;
+}
+
+export function GenAIBackedModelWidget(props: WidgetProps) {
+  const { id, value, disabled, readonly, onChange, schema, registry, options } =
+    props;
+  const { t } = useTranslation(["views/settings"]);
+  const [open, setOpen] = useState(false);
+
+  const formContext = registry?.formContext as ConfigFormContext | undefined;
+  const fieldClassName = getSizedFieldClassName(options, "sm");
+  const role = (options?.role as string | undefined) ?? "embeddings";
+  const i18nPrefix =
+    (options?.i18nPrefix as string | undefined) ?? "semanticSearchModel";
+
+  // Built-in model options from schema.examples (populated by transformer
+  // collapsing the anyOf enum+string union)
+  const builtInModels: ProviderOption[] = useMemo(() => {
+    const examples = (schema as Record<string, unknown>).examples;
+    if (!Array.isArray(examples)) return [];
+    return examples
+      .filter((v): v is string => typeof v === "string")
+      .map((v) => ({ value: v, label: v }));
+  }, [schema]);
+
+  // GenAI providers that have the role this field is backed by
+  const roleProviders: ProviderOption[] = useMemo(() => {
+    const genai = (
+      formContext?.fullConfig as Record<string, unknown> | undefined
+    )?.genai;
+    if (!genai || typeof genai !== "object" || Array.isArray(genai)) return [];
+
+    const providers: ProviderOption[] = [];
+    for (const [key, config] of Object.entries(
+      genai as Record<string, unknown>,
+    )) {
+      if (!config || typeof config !== "object" || Array.isArray(config))
+        continue;
+      const roles = (config as Record<string, unknown>).roles;
+      if (Array.isArray(roles) && roles.includes(role)) {
+        providers.push({ value: key, label: key });
+      }
+    }
+    return providers;
+  }, [formContext?.fullConfig, role]);
+
+  const currentLabel =
+    builtInModels.find((m) => m.value === value)?.label ??
+    roleProviders.find((p) => p.value === value)?.label ??
+    (typeof value === "string" && value ? value : undefined);
+
+  return (
+    <Popover open={open} onOpenChange={setOpen}>
+      <PopoverTrigger asChild>
+        <Button
+          id={id}
+          type="button"
+          variant="outline"
+          role="combobox"
+          aria-expanded={open}
+          disabled={disabled || readonly}
+          className={cn(
+            "justify-between font-normal",
+            !currentLabel && "text-muted-foreground",
+            fieldClassName,
+          )}
+        >
+          {currentLabel ??
+            t(`configForm.${i18nPrefix}.placeholder`, {
+              ns: "views/settings",
+              defaultValue: "Select model…",
+            })}
+          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-[--radix-popover-trigger-width] p-0">
+        <Command>
+          <CommandList>
+            {builtInModels.length > 0 && (
+              <CommandGroup
+                heading={t(`configForm.${i18nPrefix}.builtIn`, {
+                  ns: "views/settings",
+                  defaultValue: "Built-in Models",
+                })}
+              >
+                {builtInModels.map((model) => (
+                  <CommandItem
+                    key={model.value}
+                    value={model.value}
+                    onSelect={() => {
+                      onChange(model.value);
+                      setOpen(false);
+                    }}
+                  >
+                    <Check
+                      className={cn(
+                        "mr-2 h-4 w-4",
+                        value === model.value ? "opacity-100" : "opacity-0",
+                      )}
+                    />
+                    {model.label}
+                  </CommandItem>
+                ))}
+              </CommandGroup>
+            )}
+            {roleProviders.length > 0 && (
+              <CommandGroup
+                heading={t(`configForm.${i18nPrefix}.genaiProviders`, {
+                  ns: "views/settings",
+                  defaultValue: "GenAI Providers",
+                })}
+              >
+                {roleProviders.map((provider) => (
+                  <CommandItem
+                    key={provider.value}
+                    value={provider.value}
+                    onSelect={() => {
+                      onChange(provider.value);
+                      setOpen(false);
+                    }}
+                  >
+                    <Check
+                      className={cn(
+                        "mr-2 h-4 w-4",
+                        value === provider.value ? "opacity-100" : "opacity-0",
+                      )}
+                    />
+                    {provider.label}
+                  </CommandItem>
+                ))}
+              </CommandGroup>
+            )}
+          </CommandList>
+        </Command>
+      </PopoverContent>
+    </Popover>
+  );
+}

web/src/components/config-form/theme/widgets/GenAIRolesWidget.tsx

@@ -4,9 +4,14 @@ import { useTranslation } from "react-i18next";
 import useSWR from "swr";
 import { Switch } from "@/components/ui/switch";
 import type { ConfigFormContext } from "@/types/configForm";
-import type { GenAIModelsResponse } from "@/types/chat";
+import type { GenAIModelCapabilities, GenAIModelsResponse } from "@/types/chat";
 
-const GENAI_ROLES = ["embeddings", "descriptions", "chat"] as const;
+const GENAI_ROLES = [
+  "embeddings",
+  "descriptions",
+  "chat",
+  "transcribe",
+] as const;
 
 function normalizeValue(value: unknown): string[] {
   if (Array.isArray(value)) {
@@ -43,18 +48,56 @@ export function GenAIRolesWidget(props: WidgetProps) {
     revalidateOnFocus: false,
   });
 
-  const embeddingsSupported = useMemo(() => {
+  // The model currently chosen in the form, which is what the roles have to
+  // reflect. Reading the saved config instead would keep reporting the previous
+  // model's capabilities until a save and a refetch.
+  const selectedModel = useMemo(() => {
+    if (!providerKey) return undefined;
+    const formData = formContext?.formData as
+      Record<string, unknown> | undefined;
+    const entry = formData?.[providerKey];
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+      return undefined;
+    }
+    const model = (entry as Record<string, unknown>).model;
+    return typeof model === "string" && model ? model : undefined;
+  }, [formContext?.formData, providerKey]);
+
+  // Capabilities the provider reported for that specific model. Absent when the
+  // provider cannot describe a model it has not loaded, in which case the
+  // entry-level flags (which describe the saved model) are the best available.
+  const modelCapabilities: GenAIModelCapabilities | undefined = useMemo(() => {
+    if (!providerKey || !selectedModel) return undefined;
+    return genaiInfo?.[providerKey]?.model_capabilities?.[selectedModel];
+  }, [genaiInfo, providerKey, selectedModel]);
+
+  const capabilityOf = (
+    key: "supports_embeddings" | "supports_transcription",
+  ): boolean => {
+    const perModel = modelCapabilities?.[key];
+    if (perModel !== undefined) return perModel;
     if (!providerKey) return true;
     const info = genaiInfo?.[providerKey];
-    return info ? info.supports_embeddings : true;
-  }, [genaiInfo, providerKey]);
+    // assume supported when nothing is known, so a role is never hidden on
+    // missing information alone
+    return info ? info[key] : true;
+  };
+
+  const embeddingsSupported = capabilityOf("supports_embeddings");
+  const transcriptionSupported = capabilityOf("supports_transcription");
+
+  const unsupportedRoles = useMemo(() => {
+    const unsupported = new Set<string>();
+
+    if (!embeddingsSupported) unsupported.add("embeddings");
+    if (!transcriptionSupported) unsupported.add("transcribe");
+
+    return unsupported;
+  }, [embeddingsSupported, transcriptionSupported]);
 
   const availableRoles = useMemo(
-    () =>
-      embeddingsSupported
-        ? GENAI_ROLES
-        : GENAI_ROLES.filter((role) => role !== "embeddings"),
-    [embeddingsSupported],
+    () => GENAI_ROLES.filter((role) => !unsupportedRoles.has(role)),
+    [unsupportedRoles],
   );
 
   const occupiedRoles = useMemo(() => {
@@ -80,11 +123,13 @@ export function GenAIRolesWidget(props: WidgetProps) {
     return occupied;
   }, [formContext?.formData, providerKey]);
 
+  // strip every unsupported role in a single onChange; two effects each
+  // rewriting the same value would race and lose one of the edits
   useEffect(() => {
-    if (!embeddingsSupported && selectedRoles.includes("embeddings")) {
-      onChange(selectedRoles.filter((role) => role !== "embeddings"));
-    }
-  }, [embeddingsSupported, selectedRoles, onChange]);
+    if (!selectedRoles.some((role) => unsupportedRoles.has(role))) return;
+
+    onChange(selectedRoles.filter((role) => !unsupportedRoles.has(role)));
+  }, [unsupportedRoles, selectedRoles, onChange]);
 
   const toggleRole = (role: string, enabled: boolean) => {
     if (enabled) {

web/src/components/config-form/theme/widgets/SemanticSearchModelSizeWidget.tsx

@@ -1,61 +1,17 @@
-// Disables model_size and shows "N/A" when a GenAI provider is selected.
-// Reads model via LiveFormDataContext so it re-runs even when RJSF's
-// SchemaField memoization would skip this widget.
+// semantic_search.model_size. See GenAIBackedModelSizeWidget for the shared
+// implementation, including the clear-vs-default handling.
 import type { WidgetProps } from "@rjsf/utils";
-import { useContext, useEffect } from "react";
-import { useTranslation } from "react-i18next";
-import {
-  Select,
-  SelectContent,
-  SelectTrigger,
-  SelectValue,
-} from "@/components/ui/select";
-import { LiveFormDataContext } from "../../LiveFormDataContext";
-import { getSizedFieldClassName } from "../utils";
-import { SelectWidget } from "./SelectWidget";
+import { GenAIBackedModelSizeWidget } from "./GenAIBackedModelSizeWidget";
 
 export function SemanticSearchModelSizeWidget(props: WidgetProps) {
-  const { t } = useTranslation(["views/settings"]);
-  const liveFormData = useContext(LiveFormDataContext);
-  const model = liveFormData?.model;
-  const isProvider =
-    typeof model === "string" &&
-    model !== "" &&
-    model !== "jinav1" &&
-    model !== "jinav2";
-
-  // model_size is unused on a GenAI provider. Only clear it (which the backend
-  // treats as "remove") for a non-default value, which can only come from the
-  // config file. A defaulted value is indistinguishable from unset in the
-  // resolved config, so clearing it would falsely dirty the field and delete a
-  // YAML key that isn't there. Restore the default when returning to a Jina model.
-  const { value, onChange, schema } = props;
-  const schemaDefault = schema?.default as string | undefined;
-  useEffect(() => {
-    if (isProvider) {
-      if (value !== undefined && value !== schemaDefault) {
-        onChange(undefined);
-      }
-    } else if (value === undefined && schemaDefault) {
-      onChange(schemaDefault);
-    }
-  }, [isProvider, value, onChange, schemaDefault]);
-
-  if (isProvider) {
-    const fieldClassName = getSizedFieldClassName(props.options ?? {}, "sm");
-    return (
-      <Select value="" disabled>
-        <SelectTrigger className={fieldClassName}>
-          <SelectValue
-            placeholder={t("configForm.semanticSearchModelSize.notApplicable", {
-              defaultValue: "Not applicable for GenAI providers",
-            })}
-          />
-        </SelectTrigger>
-        <SelectContent />
-      </Select>
-    );
-  }
-
-  return <SelectWidget {...props} />;
+  return (
+    <GenAIBackedModelSizeWidget
+      {...props}
+      options={{
+        ...props.options,
+        builtInModels: ["jinav1", "jinav2"],
+        i18nPrefix: "semanticSearchModelSize",
+      }}
+    />
+  );
 }

web/src/components/config-form/theme/widgets/SemanticSearchModelWidget.tsx

@@ -1,159 +1,17 @@
-// Combobox widget for semantic_search.model field.
-// Shows built-in model enum values and GenAI providers with the embeddings role.
-import { useState, useMemo } from "react";
+// semantic_search.model: built-in Jina models plus GenAI providers with the
+// embeddings role. See GenAIBackedModelWidget for the shared implementation.
 import type { WidgetProps } from "@rjsf/utils";
-import { useTranslation } from "react-i18next";
-import { Check, ChevronsUpDown } from "lucide-react";
-import { cn } from "@/lib/utils";
-import { Button } from "@/components/ui/button";
-import {
-  Command,
-  CommandGroup,
-  CommandItem,
-  CommandList,
-} from "@/components/ui/command";
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from "@/components/ui/popover";
-import type { ConfigFormContext } from "@/types/configForm";
-import { getSizedFieldClassName } from "../utils";
-
-interface ProviderOption {
-  value: string;
-  label: string;
-}
+import { GenAIBackedModelWidget } from "./GenAIBackedModelWidget";
 
 export function SemanticSearchModelWidget(props: WidgetProps) {
-  const { id, value, disabled, readonly, onChange, schema, registry, options } =
-    props;
-  const { t } = useTranslation(["views/settings"]);
-  const [open, setOpen] = useState(false);
-
-  const formContext = registry?.formContext as ConfigFormContext | undefined;
-  const fieldClassName = getSizedFieldClassName(options, "sm");
-
-  // Built-in model options from schema.examples (populated by transformer
-  // collapsing the anyOf enum+string union)
-  const builtInModels: ProviderOption[] = useMemo(() => {
-    const examples = (schema as Record<string, unknown>).examples;
-    if (!Array.isArray(examples)) return [];
-    return examples
-      .filter((v): v is string => typeof v === "string")
-      .map((v) => ({ value: v, label: v }));
-  }, [schema]);
-
-  // GenAI providers that have the "embeddings" role
-  const embeddingsProviders: ProviderOption[] = useMemo(() => {
-    const genai = (
-      formContext?.fullConfig as Record<string, unknown> | undefined
-    )?.genai;
-    if (!genai || typeof genai !== "object" || Array.isArray(genai)) return [];
-
-    const providers: ProviderOption[] = [];
-    for (const [key, config] of Object.entries(
-      genai as Record<string, unknown>,
-    )) {
-      if (!config || typeof config !== "object" || Array.isArray(config))
-        continue;
-      const roles = (config as Record<string, unknown>).roles;
-      if (Array.isArray(roles) && roles.includes("embeddings")) {
-        providers.push({ value: key, label: key });
-      }
-    }
-    return providers;
-  }, [formContext?.fullConfig]);
-
-  const currentLabel =
-    builtInModels.find((m) => m.value === value)?.label ??
-    embeddingsProviders.find((p) => p.value === value)?.label ??
-    (typeof value === "string" && value ? value : undefined);
-
   return (
-    <Popover open={open} onOpenChange={setOpen}>
-      <PopoverTrigger asChild>
-        <Button
-          id={id}
-          type="button"
-          variant="outline"
-          role="combobox"
-          aria-expanded={open}
-          disabled={disabled || readonly}
-          className={cn(
-            "justify-between font-normal",
-            !currentLabel && "text-muted-foreground",
-            fieldClassName,
-          )}
-        >
-          {currentLabel ??
-            t("configForm.semanticSearchModel.placeholder", {
-              ns: "views/settings",
-              defaultValue: "Select model…",
-            })}
-          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
-        </Button>
-      </PopoverTrigger>
-      <PopoverContent className="w-[--radix-popover-trigger-width] p-0">
-        <Command>
-          <CommandList>
-            {builtInModels.length > 0 && (
-              <CommandGroup
-                heading={t("configForm.semanticSearchModel.builtIn", {
-                  ns: "views/settings",
-                  defaultValue: "Built-in Models",
-                })}
-              >
-                {builtInModels.map((model) => (
-                  <CommandItem
-                    key={model.value}
-                    value={model.value}
-                    onSelect={() => {
-                      onChange(model.value);
-                      setOpen(false);
-                    }}
-                  >
-                    <Check
-                      className={cn(
-                        "mr-2 h-4 w-4",
-                        value === model.value ? "opacity-100" : "opacity-0",
-                      )}
-                    />
-                    {model.label}
-                  </CommandItem>
-                ))}
-              </CommandGroup>
-            )}
-            {embeddingsProviders.length > 0 && (
-              <CommandGroup
-                heading={t("configForm.semanticSearchModel.genaiProviders", {
-                  ns: "views/settings",
-                  defaultValue: "GenAI Providers",
-                })}
-              >
-                {embeddingsProviders.map((provider) => (
-                  <CommandItem
-                    key={provider.value}
-                    value={provider.value}
-                    onSelect={() => {
-                      onChange(provider.value);
-                      setOpen(false);
-                    }}
-                  >
-                    <Check
-                      className={cn(
-                        "mr-2 h-4 w-4",
-                        value === provider.value ? "opacity-100" : "opacity-0",
-                      )}
-                    />
-                    {provider.label}
-                  </CommandItem>
-                ))}
-              </CommandGroup>
-            )}
-          </CommandList>
-        </Command>
-      </PopoverContent>
-    </Popover>
+    <GenAIBackedModelWidget
+      {...props}
+      options={{
+        ...props.options,
+        role: "embeddings",
+        i18nPrefix: "semanticSearchModel",
+      }}
+    />
   );
 }

web/src/types/chat.ts

@@ -50,11 +50,23 @@ export type ChatStats = {
 
 export type ShowStatsMode = "while_generating" | "always";
 
+// Capability flags a provider can report for a model it has not loaded.
+// Keyed by model name (and alias) in GenAIProviderInfo.model_capabilities.
+export type GenAIModelCapabilities = {
+  supports_vision?: boolean;
+  supports_embeddings?: boolean;
+  supports_transcription?: boolean;
+};
+
 export type GenAIProviderInfo = {
   models: string[];
   roles: string[];
   supports_toggleable_thinking: boolean;
   supports_embeddings: boolean;
+  supports_transcription: boolean;
+  // Per-model capabilities, when the provider can report them without loading
+  // the model. The top-level flags above describe the configured model only.
+  model_capabilities?: Record<string, GenAIModelCapabilities>;
 };
 
 export type GenAIModelsResponse = Record<string, GenAIProviderInfo>;

web/src/types/frigateConfig.ts

@@ -394,7 +394,7 @@ export type AllGroupsStreamingSettings = {
   [groupName: string]: GroupStreamingSettings;
 };
 
-export type GenAIRole = "chat" | "descriptions" | "embeddings";
+export type GenAIRole = "chat" | "descriptions" | "embeddings" | "transcribe";
 
 export type GenAIAgentConfig = {
   api_key?: string;