mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-10-09 16:22:48 +03:00
Support using GenAI for audio transcription (#24396)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
* Add support for running transcription with GenAI * Improve audio joining * Fix GenAI model capability reporting * Support language correctly * Migrate existing users to keep english selected * Fix models * Fix tests * Fix accepted null model * Handle slwo providers
This commit is contained in:
@@ -56,6 +56,7 @@ from .camera.timestamp import TimestampStyleConfig
|
||||
from .camera_group import CameraGroupConfig
|
||||
from .classification import (
|
||||
AudioTranscriptionConfig,
|
||||
AudioTranscriptionModelEnum,
|
||||
ClassificationConfig,
|
||||
FaceRecognitionConfig,
|
||||
LicensePlateRecognitionConfig,
|
||||
@@ -884,7 +885,7 @@ class FrigateConfig(FrigateBaseModel):
|
||||
# set notifications state
|
||||
self.notifications.enabled_in_config = self.notifications.enabled
|
||||
|
||||
# validate genai: each role (chat, descriptions, embeddings) at most once
|
||||
# validate genai: each role (chat, descriptions, embeddings, transcribe) at most once
|
||||
role_to_name: dict[GenAIRoleEnum, str] = {}
|
||||
for name, genai_cfg in self.genai.items():
|
||||
for role in genai_cfg.roles:
|
||||
@@ -1245,6 +1246,35 @@ class FrigateConfig(FrigateBaseModel):
|
||||
for model in self.models:
|
||||
model.create_colormap(colored_labels)
|
||||
|
||||
# validate audio_transcription.model when it is a GenAI provider name.
|
||||
# this runs here rather than beside the semantic_search check because the
|
||||
# global->camera merge above is what resolves camera-level enablement.
|
||||
transcription_active = self.audio_transcription.enabled or any(
|
||||
camera.audio_transcription.enabled for camera in self.cameras.values()
|
||||
)
|
||||
|
||||
if (
|
||||
transcription_active
|
||||
and isinstance(self.audio_transcription.model, str)
|
||||
and not isinstance(
|
||||
self.audio_transcription.model, AudioTranscriptionModelEnum
|
||||
)
|
||||
):
|
||||
if self.audio_transcription.model not in self.genai:
|
||||
raise ValueError(
|
||||
f"audio_transcription.model '{self.audio_transcription.model}' is not a "
|
||||
"valid GenAI config key. Must match a key in genai config."
|
||||
)
|
||||
|
||||
if (
|
||||
GenAIRoleEnum.transcribe
|
||||
not in self.genai[self.audio_transcription.model].roles
|
||||
):
|
||||
raise ValueError(
|
||||
f"GenAI provider '{self.audio_transcription.model}' must have "
|
||||
"'transcribe' in its roles for audio transcription."
|
||||
)
|
||||
|
||||
# Check audio transcription and audio detection requirements
|
||||
if self.audio_transcription.enabled:
|
||||
# If audio transcription is enabled globally, at least one camera must have audio detection enabled
|
||||
|
||||
Reference in New Issue
Block a user