Support using GenAI for audio transcription (#24396)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s

* Add support for running transcription with GenAI

* Improve audio joining

* Fix GenAI model capability reporting

* Support language correctly

* Migrate existing users to keep english selected

* Fix models

* Fix tests

* Fix accepted null model

* Handle slwo providers
This commit is contained in:
Nicolas Mowen
2026-09-17 16:34:47 -05:00
committed by GitHub
parent eccd10cd94
commit 334073967b
35 changed files with 2563 additions and 278 deletions
+31 -1
View File
@@ -56,6 +56,7 @@ from .camera.timestamp import TimestampStyleConfig
from .camera_group import CameraGroupConfig
from .classification import (
AudioTranscriptionConfig,
AudioTranscriptionModelEnum,
ClassificationConfig,
FaceRecognitionConfig,
LicensePlateRecognitionConfig,
@@ -884,7 +885,7 @@ class FrigateConfig(FrigateBaseModel):
# set notifications state
self.notifications.enabled_in_config = self.notifications.enabled
# validate genai: each role (chat, descriptions, embeddings) at most once
# validate genai: each role (chat, descriptions, embeddings, transcribe) at most once
role_to_name: dict[GenAIRoleEnum, str] = {}
for name, genai_cfg in self.genai.items():
for role in genai_cfg.roles:
@@ -1245,6 +1246,35 @@ class FrigateConfig(FrigateBaseModel):
for model in self.models:
model.create_colormap(colored_labels)
# validate audio_transcription.model when it is a GenAI provider name.
# this runs here rather than beside the semantic_search check because the
# global->camera merge above is what resolves camera-level enablement.
transcription_active = self.audio_transcription.enabled or any(
camera.audio_transcription.enabled for camera in self.cameras.values()
)
if (
transcription_active
and isinstance(self.audio_transcription.model, str)
and not isinstance(
self.audio_transcription.model, AudioTranscriptionModelEnum
)
):
if self.audio_transcription.model not in self.genai:
raise ValueError(
f"audio_transcription.model '{self.audio_transcription.model}' is not a "
"valid GenAI config key. Must match a key in genai config."
)
if (
GenAIRoleEnum.transcribe
not in self.genai[self.audio_transcription.model].roles
):
raise ValueError(
f"GenAI provider '{self.audio_transcription.model}' must have "
"'transcribe' in its roles for audio transcription."
)
# Check audio transcription and audio detection requirements
if self.audio_transcription.enabled:
# If audio transcription is enabled globally, at least one camera must have audio detection enabled