Support using GenAI for audio transcription (#24396)

* Add support for running transcription with GenAI

* Improve audio joining

* Fix GenAI model capability reporting

* Support language correctly

* Migrate existing users to keep english selected

* Fix models

* Fix tests

* Fix accepted null model

* Handle slwo providers

frigate/genai/__init__.py

@@ -338,6 +338,11 @@ class GenAIClient:
         """Whether the configured model can generate embeddings via embed()."""
         return False
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Whether the configured model can transcribe audio via transcribe()."""
+        return False
+
     def list_models(self) -> list[str]:
         """Return the list of model names available from this provider.
 
@@ -345,6 +350,21 @@ class GenAIClient:
         """
         return []
 
+    def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
+        """Return capability flags for each model the provider serves.
+
+        Only providers whose backend advertises capabilities per model can
+        populate this; llama.cpp reports input modalities for every model it
+        serves, so one request describes them all. An empty mapping means "no
+        per-model information available", and callers fall back to this
+        client's own capability properties, which describe only the configured
+        model. A model absent from a non-empty mapping means the same thing.
+
+        Returns:
+            Model name (including aliases) to its capability flags
+        """
+        return {}
+
     def get_context_size(self) -> int:
         """Get the context window size for this provider in tokens."""
         return 4096
@@ -376,6 +396,33 @@ class GenAIClient:
         )
         return []
 
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe speech audio to text.
+
+        Audio is passed as a self-describing blob rather than raw samples so
+        every provider receives a container it can declare, and WAV framing
+        lives in one place instead of in each plugin.
+
+        Args:
+            audio: The encoded audio payload (WAV bytes by default)
+            language: Optional ISO language hint for the provider
+            mime_type: Media type of ``audio``
+
+        Returns:
+            The transcript, or None when the provider cannot produce one
+        """
+        logger.warning(
+            "%s does not support transcription. "
+            "This method should be overridden by the provider implementation.",
+            self.__class__.__name__,
+        )
+        return None
+
     def chat_with_tools(
         self,
         messages: list[dict[str, Any]],

frigate/genai/manager.py

@@ -110,6 +110,12 @@ class GenAIClientManager:
         name = self._role_map.get(GenAIRoleEnum.embeddings)
         return self._get_client(name) if name else None
 
+    @property
+    def transcribe_client(self) -> "GenAIClient | None":
+        """Client configured for the transcribe role."""
+        name = self._role_map.get(GenAIRoleEnum.transcribe)
+        return self._get_client(name) if name else None
+
     def role_info(self) -> dict[str, dict[str, Any]]:
         """Return the model selected for each configured role and its context size.
 
@@ -144,5 +150,11 @@ class GenAIClientManager:
                 "roles": [r.value for r in genai_cfg.roles],
                 "supports_toggleable_thinking": client.supports_toggleable_thinking,
                 "supports_embeddings": client.supports_embeddings,
+                "supports_transcription": client.supports_transcription,
+                # Capabilities of the configured model are above; this maps every
+                # model the provider serves to its own, so the UI can react to a
+                # model selected but not yet saved. Empty when the provider
+                # cannot report capabilities without loading a model.
+                "model_capabilities": client.list_model_capabilities(),
             }
         return result

frigate/genai/plugins/azure-openai.py

@@ -13,6 +13,19 @@ overrides what is genuinely Azure-specific:
 - Context size: Azure does not expose a per-model ``max_model_len`` field
   reliably, so we keep the historical 128K default rather than the
   model-name heuristic used by OpenAI.
+
+Transcription is inherited too: :class:`openai.AzureOpenAI` exposes the same
+``audio.transcriptions.create``. Two Azure-specific caveats apply when using
+the ``transcribe`` role:
+
+- ``model`` must be the Azure *deployment* name, not the underlying model name.
+- The ``api-version`` parsed from ``base_url`` must be 2024-06-01 or later;
+  earlier versions have no transcriptions route and the 404 surfaces only as a
+  generic provider error.
+- Because ``model`` is a deployment name, the inherited check that picks
+  ``languages`` over ``language`` for gpt-transcribe cannot fire unless the
+  deployment happens to be named after the model. Name the deployment
+  ``gpt-transcribe`` to get the right field, or leave the language on ``auto``.
 """
 
 import logging

frigate/genai/plugins/gemini.py

@@ -17,6 +17,10 @@ from frigate.genai.utils import interleave_images
 
 logger = logging.getLogger(__name__)
 
+# Gemini requests carrying inline data are capped at ~20 MB total; stay well
+# under it so the request fails as a log line rather than a 400.
+GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024
+
 
 def _decode_thought_signature(value: Any) -> bytes | None:
     """Decode a base64-encoded thought_signature carried across conversation turns."""
@@ -163,6 +167,58 @@ class GeminiClient(GenAIClient):
             return None
         return description
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Gemini models accept inline audio parts."""
+        return True
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio by sending it as an inline part alongside a prompt."""
+        if len(audio) > GEMINI_MAX_INLINE_BYTES:
+            logger.warning(
+                "Audio payload of %d bytes exceeds the Gemini inline limit; skipping transcription",
+                len(audio),
+            )
+            return None
+
+        prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
+
+        if language:
+            prompt += f" The speech is in language '{language}'."
+
+        try:
+            contents: list[Any] = [
+                prompt,
+                types.Part.from_bytes(data=audio, mime_type=mime_type),
+            ]
+            response = self.provider.models.generate_content(
+                model=self.genai_config.model,
+                contents=contents,
+                config=types.GenerateContentConfig(candidate_count=1),
+            )
+        except errors.APIError as e:
+            logger.warning("Gemini returned an error: %s", str(e))
+            return None
+        except Exception as e:
+            logger.warning("An unexpected error occurred with Gemini: %s", str(e))
+            return None
+
+        try:
+            if response.text is None:
+                return None
+
+            transcript = response.text.strip()
+        except (ValueError, AttributeError):
+            # No transcript was generated
+            return None
+
+        return transcript or None
+
     def list_models(self) -> list[str]:
         """Return available model names from Gemini."""
         try:

frigate/genai/plugins/llama_cpp.py

@@ -408,6 +408,126 @@ class LlamaCppClient(GenAIClient):
         """Whether the loaded model supports audio input."""
         return self._supports_audio
 
+    @property
+    def supports_transcription(self) -> bool:
+        """Audio-capable models can transcribe through chat completions."""
+        return self._supports_audio
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio through the OpenAI-compatible transcriptions route.
+
+        llama.cpp serves /v1/audio/transcriptions for any audio-capable model,
+        not only a separately loaded whisper (ggml-org/llama.cpp#21863), so it
+        covers exactly the models supports_transcription detects. It takes the
+        language as a native multipart field, which is the only thing dedicated
+        ASR models honor: they read the chat prompt as contextual biasing, so
+        asking one there to use a language does nothing.
+
+        Falls back to chat completions when the server predates that route.
+        """
+        if self.provider is None:
+            logger.warning(
+                "llama.cpp provider has not been initialized, audio will not be transcribed. Check your llama.cpp configuration."
+            )
+            return None
+
+        if not self._supports_audio:
+            logger.warning(
+                "llama.cpp model '%s' does not accept audio input",
+                self.genai_config.model,
+            )
+            return None
+
+        try:
+            data = {"model": self.genai_config.model, "response_format": "json"}
+
+            if language:
+                data["language"] = language
+
+            response = self._post(
+                f"{self.provider}/v1/audio/transcriptions",
+                files={"file": ("audio.wav", audio, mime_type)},
+                data=data,
+                timeout=self.timeout,
+            )
+
+            if response.status_code == 404:
+                logger.debug(
+                    "llama.cpp server has no /v1/audio/transcriptions route, using chat completions"
+                )
+                return self._transcribe_via_chat(audio, language)
+
+            response.raise_for_status()
+            result = response.json()
+            text = result.get("text") if isinstance(result, dict) else None
+
+            return str(text).strip() or None if text else None
+        except Exception as e:
+            logger.warning("llama.cpp returned an error: %s", str(e))
+            return None
+
+    def _transcribe_via_chat(self, audio: bytes, language: str | None) -> str | None:
+        """Transcribe through /v1/chat/completions, for servers without the
+        transcriptions route.
+
+        The _media_marker / multimodal_data convention is an /embeddings-only
+        protocol, so no marker-refresh retry is needed here.
+        """
+        prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
+
+        if language:
+            prompt += f" The speech is in language '{language}'."
+
+        try:
+            encoded_audio = base64.b64encode(audio).decode("utf-8")
+            payload: dict[str, Any] = {
+                "model": self.genai_config.model,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {
+                                "type": "input_audio",
+                                "input_audio": {
+                                    "data": encoded_audio,
+                                    "format": "wav",
+                                },
+                            },
+                        ],
+                    },
+                ],
+                **self.provider_options,
+            }
+
+            response = self._post(
+                f"{self.provider}/v1/chat/completions",
+                json=payload,
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            if (
+                result is not None
+                and "choices" in result
+                and len(result["choices"]) > 0
+            ):
+                choice = result["choices"][0]
+
+                if "message" in choice and choice["message"].get("content"):
+                    return str(choice["message"]["content"].strip()) or None
+
+            return None
+        except Exception as e:
+            logger.warning("llama.cpp returned an error: %s", str(e))
+            return None
+
     @property
     def supports_tools(self) -> bool:
         """Whether the loaded model supports tool/function calling."""
@@ -417,28 +537,74 @@ class LlamaCppClient(GenAIClient):
     def supports_toggleable_thinking(self) -> bool:
         return self._supports_reasoning
 
-    def list_models(self) -> list[str]:
-        """Return available model IDs from the llama.cpp server."""
+    def _fetch_models_data(self) -> list[dict[str, Any]]:
+        """Return the raw /v1/models entries, or an empty list if unreachable."""
         base_url = self.provider or (
             self.genai_config.base_url.rstrip("/")
             if self.genai_config.base_url
             else None
         )
+
         if base_url is None:
             return []
+
         try:
             response = self._get(f"{base_url}/v1/models", timeout=10)
             response.raise_for_status()
-            models = []
-            for m in response.json().get("data", []):
-                models.append(m.get("id", "unknown"))
-                for alias in m.get("aliases", []):
-                    models.append(alias)
-            return sorted(models)
+            data = response.json().get("data", [])
         except Exception as e:
             logger.warning("Failed to list llama.cpp models: %s", e)
             return []
 
+        return data if isinstance(data, list) else []
+
+    def list_models(self) -> list[str]:
+        """Return available model IDs from the llama.cpp server."""
+        models = []
+
+        for m in self._fetch_models_data():
+            models.append(m.get("id", "unknown"))
+
+            for alias in m.get("aliases", []):
+                models.append(alias)
+
+        return sorted(models)
+
+    def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
+        """Report input modalities for every model the server serves.
+
+        Since ggml-org/llama.cpp#22952 each /v1/models entry carries
+        architecture.input_modalities, so a single request describes every
+        model rather than just the configured one. That is what lets the UI
+        answer "can the model I just picked transcribe" before the config is
+        saved and a client for it exists.
+
+        Models whose entry predates that field are omitted rather than reported
+        as incapable, so an older server falls back to the /props probe instead
+        of silently losing capabilities it actually has.
+        """
+        capabilities: dict[str, dict[str, bool]] = {}
+
+        for model in self._fetch_models_data():
+            architecture = model.get("architecture") or {}
+            modalities = architecture.get("input_modalities")
+
+            if not isinstance(modalities, list) or not modalities:
+                continue
+
+            flags = {
+                "supports_vision": "image" in modalities,
+                "supports_transcription": "audio" in modalities,
+            }
+
+            names = [model.get("id"), *(model.get("aliases") or [])]
+
+            for name in names:
+                if isinstance(name, str) and name:
+                    capabilities[name] = flags
+
+        return capabilities
+
     def get_context_size(self) -> int:
         """Get the context window size for llama.cpp.
 

frigate/genai/plugins/openai.py

@@ -15,6 +15,12 @@ from frigate.genai.utils import interleave_images
 
 logger = logging.getLogger(__name__)
 
+# gpt-transcribe replaced the singular `language` field with a `languages` array
+# and rejects a request that sends both. Older transcription models
+# (gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1) still take the singular
+# form. https://developers.openai.com/api/docs/guides/speech-to-text
+_LANGUAGES_ARRAY_MODEL_PREFIX = "gpt-transcribe"
+
 
 def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None:
     """Build a stats dict from an OpenAI-compatible usage object."""
@@ -134,6 +140,51 @@ class OpenAIClient(GenAIClient):
             logger.warning("OpenAI returned an error: %s", str(e))
             return None
 
+    @property
+    def supports_transcription(self) -> bool:
+        """OpenAI exposes /v1/audio/transcriptions for its speech models."""
+        return True
+
+    def transcribe(
+        self,
+        audio: bytes,
+        language: str | None = None,
+        mime_type: str = "audio/wav",
+    ) -> str | None:
+        """Transcribe audio via the OpenAI audio transcriptions endpoint."""
+        try:
+            # runtime_options are chat-completion parameters; the transcriptions
+            # endpoint rejects unknown fields, so they are deliberately not splatted
+            # in here the way _send() does.
+            request_params: dict[str, Any] = {
+                "model": self.genai_config.model,
+                "file": ("audio.wav", audio, mime_type),
+                "response_format": "text",
+                "timeout": self.timeout,
+            }
+
+            if language:
+                if (
+                    self.genai_config.model.strip()
+                    .lower()
+                    .startswith(_LANGUAGES_ARRAY_MODEL_PREFIX)
+                ):
+                    # not a typed parameter on the SDK method, so it has to ride
+                    # along in extra_body
+                    request_params["extra_body"] = {"languages": [language]}
+                else:
+                    request_params["language"] = language
+
+            result = self.provider.audio.transcriptions.create(**request_params)
+        except (TimeoutException, Exception) as e:
+            logger.warning("OpenAI returned an error: %s", str(e))
+            return None
+
+        # response_format="text" yields a bare string, but some compatible
+        # servers still return the object form
+        text = result if isinstance(result, str) else getattr(result, "text", None)
+        return text.strip() if text else None
+
     def list_models(self) -> list[str]:
         """Return available model IDs from the OpenAI-compatible API."""
         try: