Support using GenAI for audio transcription (#24396)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s

* Add support for running transcription with GenAI

* Improve audio joining

* Fix GenAI model capability reporting

* Support language correctly

* Migrate existing users to keep english selected

* Fix models

* Fix tests

* Fix accepted null model

* Handle slwo providers
This commit is contained in:
Nicolas Mowen
2026-09-17 16:34:47 -05:00
committed by GitHub
parent eccd10cd94
commit 334073967b
35 changed files with 2563 additions and 278 deletions
@@ -0,0 +1,286 @@
"""Config validation and the historical path for the audio_transcription GenAI backend."""
import unittest
from copy import deepcopy
from unittest.mock import MagicMock, patch
from pydantic import ValidationError
from frigate.config import FrigateConfig
from frigate.config.camera.genai import GenAIConfig, GenAIRoleEnum
from frigate.config.classification import AudioTranscriptionModelEnum
from frigate.const import UPDATE_EVENT_DESCRIPTION
from frigate.data_processing.post.audio_transcription import (
AudioTranscriptionPostProcessor,
)
from frigate.data_processing.types import PostProcessDataEnum
class TestAudioTranscriptionGenAIConfig(unittest.TestCase):
def setUp(self):
self.base = {
"mqtt": {"host": "mqtt"},
"cameras": {
"back": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "audio"],
}
]
},
"detect": {"height": 1080, "width": 1920, "fps": 5},
"audio": {"enabled": True},
}
},
}
def _config(self, **overrides) -> dict:
config = deepcopy(self.base)
config.update(deepcopy(overrides))
return config
def _provider(self, roles: list[str]) -> dict:
return {
"whisper_cloud": {
"provider": "openai",
"model": "gpt-4o-transcribe",
"api_key": "k",
"roles": roles,
}
}
def test_default_model_is_whisper_enum(self):
config = FrigateConfig(**self._config())
self.assertEqual(
config.audio_transcription.model, AudioTranscriptionModelEnum.whisper
)
def test_whisper_string_coerces_to_enum(self):
config = FrigateConfig(
**self._config(audio_transcription={"enabled": True, "model": "whisper"})
)
self.assertIsInstance(
config.audio_transcription.model, AudioTranscriptionModelEnum
)
def test_provider_name_stays_a_string(self):
config = FrigateConfig(
**self._config(
genai=self._provider(["transcribe"]),
audio_transcription={"enabled": True, "model": "whisper_cloud"},
)
)
self.assertNotIsInstance(
config.audio_transcription.model, AudioTranscriptionModelEnum
)
self.assertEqual(config.audio_transcription.model, "whisper_cloud")
def test_unspecified_model_falls_back_to_whisper(self):
"""An empty value must not read as a GenAI provider that resolves to no client."""
for value in (None, "", " "):
with self.subTest(repr(value)):
config = FrigateConfig(
**self._config(
audio_transcription={"enabled": True, "model": value}
)
)
self.assertIs(
config.audio_transcription.model,
AudioTranscriptionModelEnum.whisper,
)
def test_missing_genai_key_raises(self):
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(
**self._config(
audio_transcription={"enabled": True, "model": "nope"},
)
)
self.assertIn("is not a valid GenAI config key", str(ctx.exception))
def test_provider_without_role_raises(self):
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(
**self._config(
genai=self._provider(["descriptions"]),
audio_transcription={"enabled": True, "model": "whisper_cloud"},
)
)
self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
def test_global_off_camera_on_still_validates(self):
"""Global-off/camera-on is a supported deployment and must not skip the check."""
config = self._config(
genai=self._provider(["descriptions"]),
audio_transcription={"enabled": False, "model": "whisper_cloud"},
)
config["cameras"]["back"]["audio_transcription"] = {"enabled": True}
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(**config)
self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
def test_disabled_transcription_skips_validation(self):
config = FrigateConfig(
**self._config(audio_transcription={"enabled": False, "model": "nope"})
)
self.assertEqual(config.audio_transcription.model, "nope")
def test_camera_level_model_is_rejected(self):
config = self._config()
config["cameras"]["back"]["audio_transcription"] = {
"enabled": True,
"model": "whisper",
}
with self.assertRaises(ValidationError):
FrigateConfig(**config)
def test_default_roles_do_not_include_transcribe(self):
"""Backward compatibility: existing providers must not silently claim it."""
genai = GenAIConfig(provider="openai", model="gpt-4o")
self.assertNotIn(GenAIRoleEnum.transcribe, genai.roles)
def test_two_providers_claiming_transcribe_raises(self):
genai = self._provider(["transcribe"])
genai["other"] = {
"provider": "gemini",
"model": "gemini-2.0-flash",
"api_key": "k",
"roles": ["transcribe"],
}
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(
**self._config(
genai=genai,
audio_transcription={"enabled": True, "model": "whisper_cloud"},
)
)
self.assertIn("each role must have", str(ctx.exception))
def test_transcribe_rejected_on_provider_without_audio_input(self):
with self.assertRaises(ValidationError) as ctx:
GenAIConfig(provider="ollama", model="llava", roles=["transcribe"])
self.assertIn("does not support audio input", str(ctx.exception))
class TestAudioTranscriptionPostProcessorGenAI(unittest.TestCase):
"""The recorded-speech path must reach the provider and skip the local model."""
def setUp(self):
self.config = FrigateConfig(
**{
"mqtt": {"host": "mqtt"},
"genai": {
"whisper_cloud": {
"provider": "openai",
"model": "gpt-4o-transcribe",
"api_key": "k",
"roles": ["transcribe"],
}
},
"audio_transcription": {
"enabled": True,
"model": "whisper_cloud",
"language": "en",
},
"cameras": {
"back": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "audio"],
}
]
},
"detect": {"height": 1080, "width": 1920, "fps": 5},
"audio": {"enabled": True},
}
},
}
)
self.client = MagicMock()
self.client.transcribe.return_value = "recorded speech"
self.manager = MagicMock()
self.manager.transcribe_client = self.client
self.requestor = MagicMock()
self.processor = AudioTranscriptionPostProcessor(
self.config,
self.requestor,
MagicMock(),
MagicMock(),
self.manager,
)
def _process(self):
self.processor.process_data(
{
"event_id": "1234.5-abc",
"camera": "back",
"event": {
"id": "1234.5-abc",
"camera": "back",
"start_time": 100.0,
"end_time": 110.0,
"data": {},
},
},
PostProcessDataEnum.tracked_object,
)
def test_local_recognizer_is_never_built(self):
self.assertTrue(self.processor._use_genai)
self.assertIsNone(self.processor.recognizer)
def test_audio_bytes_and_language_reach_the_client(self):
with patch(
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
return_value=b"RIFF....WAVE",
):
self._process()
self.client.transcribe.assert_called_once()
self.assertEqual(self.client.transcribe.call_args.args[0], b"RIFF....WAVE")
self.assertEqual(self.client.transcribe.call_args.kwargs["language"], "en")
def test_transcript_is_published_as_the_description(self):
with patch(
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
return_value=b"RIFF....WAVE",
):
self._process()
topics = [call.args[0] for call in self.requestor.send_data.call_args_list]
self.assertIn(UPDATE_EVENT_DESCRIPTION, topics)
payload = next(
call.args[1]
for call in self.requestor.send_data.call_args_list
if call.args[0] == UPDATE_EVENT_DESCRIPTION
)
self.assertEqual(payload["description"], "recorded speech")
self.assertEqual(payload["id"], "1234.5-abc")
def test_missing_client_publishes_nothing(self):
self.manager.transcribe_client = None
with patch(
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
return_value=b"RIFF....WAVE",
):
self._process()
self.requestor.send_data.assert_not_called()
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,288 @@
"""Live GenAI transcription: sliding overlapped windows and their lifecycle."""
import io
import threading
import unittest
import wave
from unittest.mock import MagicMock, patch
import numpy as np
from frigate.config import FrigateConfig
from frigate.const import AUDIO_DURATION, AUDIO_SAMPLE_RATE
from frigate.data_processing.real_time.audio_transcription import (
GENAI_WINDOW_CHUNKS,
AudioTranscriptionRealTimeProcessor,
)
CHUNK_SAMPLES = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE))
def _chunk(amplitude: int) -> np.ndarray:
"""One audio-detector-sized chunk of int16 samples at a constant amplitude."""
return np.full(CHUNK_SAMPLES, amplitude, dtype=np.int16)
class TestLiveGenAITranscription(unittest.TestCase):
def setUp(self):
self.config = FrigateConfig(
**{
"mqtt": {"host": "mqtt"},
"genai": {
"whisper_cloud": {
"provider": "openai",
"model": "gpt-4o-transcribe",
"api_key": "k",
"roles": ["transcribe"],
}
},
"audio_transcription": {
"enabled": True,
"model": "whisper_cloud",
"language": "en",
},
"cameras": {
"back": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "audio"],
}
]
},
"detect": {"height": 1080, "width": 1920, "fps": 5},
"audio": {"enabled": True},
}
},
}
)
self.client = MagicMock()
self.client.transcribe.return_value = "hello"
self.manager = MagicMock()
self.manager.transcribe_client = self.client
self.requestor = MagicMock()
self.processor = AudioTranscriptionRealTimeProcessor(
config=self.config,
camera_config=self.config.cameras["back"],
requestor=self.requestor,
model_runner=None,
metrics=MagicMock(),
stop_event=threading.Event(),
genai_manager=self.manager,
)
def _feed(self, chunk: np.ndarray):
return (
self.processor._AudioTranscriptionRealTimeProcessor__process_audio_stream(
chunk
)
)
def _sent_wav(self, call_index: int) -> wave.Wave_read:
payload = self.client.transcribe.call_args_list[call_index].args[0]
return wave.open(io.BytesIO(payload), "rb")
def test_uses_genai_path(self):
self.assertTrue(self.processor._use_genai)
def test_first_chunk_does_not_transcribe(self):
self.assertIsNone(self._feed(_chunk(4000)))
self.client.transcribe.assert_not_called()
def test_full_window_transcribes_two_chunks(self):
self._feed(_chunk(4000))
result = self._feed(_chunk(4000))
self.assertEqual(result, ("hello", False))
self.client.transcribe.assert_called_once()
self.assertEqual(
self.client.transcribe.call_args.kwargs["language"],
"en",
)
with self._sent_wav(0) as wav:
self.assertEqual(wav.getnchannels(), 1)
self.assertEqual(wav.getsampwidth(), 2)
self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
def test_window_slides_with_overlap(self):
"""The third chunk's window is chunks 2+3, not 3 alone and not 1+2+3."""
self.client.transcribe.side_effect = ["one two", "two three"]
self._feed(_chunk(1000))
self._feed(_chunk(2000))
result = self._feed(_chunk(3000))
self.assertEqual(self.client.transcribe.call_count, 2)
with self._sent_wav(1) as wav:
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
self.assertEqual(samples[0], 2000)
self.assertEqual(samples[-1], 3000)
# the shared "two" appears once
self.assertEqual(result, ("one two three", False))
def test_silent_window_is_not_uploaded(self):
self._feed(_chunk(0))
self.assertIsNone(self._feed(_chunk(0)))
self.client.transcribe.assert_not_called()
def test_silent_window_ends_a_pending_utterance(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
# the gate covers the whole window, so it takes GENAI_WINDOW_CHUNKS
# silent chunks to push the last speech out of it
self._feed(_chunk(0))
self.assertEqual(self._feed(_chunk(0)), ("hello", True))
def test_empty_transcript_ends_a_pending_utterance(self):
self.client.transcribe.side_effect = ["hello", ""]
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
def test_empty_transcript_with_nothing_pending_returns_none(self):
self.client.transcribe.return_value = ""
self._feed(_chunk(4000))
self.assertIsNone(self._feed(_chunk(4000)))
def test_reset_clears_committed_text_and_window(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.processor.reset()
self.assertEqual(self.processor._genai_committed, "")
self.assertEqual(len(self.processor._genai_window), 0)
# a fresh window is required again before the next request
self.client.transcribe.reset_mock()
self._feed(_chunk(4000))
self.client.transcribe.assert_not_called()
def test_check_unload_model_clears_once_then_is_idempotent(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.processor.check_unload_model()
self.requestor.send_data.assert_called_once_with("back/audio/transcription", "")
self.assertEqual(self.processor._genai_committed, "")
self.assertEqual(len(self.processor._genai_window), 0)
self.processor.check_unload_model()
self.requestor.send_data.assert_called_once()
def test_build_recognizer_never_loads_a_local_model(self):
with patch(
"frigate.data_processing.real_time.audio_transcription.FasterWhisperASR"
) as whisper:
self.processor._AudioTranscriptionRealTimeProcessor__build_recognizer()
whisper.assert_not_called()
self.assertIsNone(self.processor.stream)
def test_clear_audio_recognizer_request_only_resets(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
with patch.object(
self.processor,
"_AudioTranscriptionRealTimeProcessor__build_recognizer",
) as build:
result = self.processor.handle_request("clear_audio_recognizer", {})
build.assert_not_called()
self.assertTrue(result["success"])
self.assertEqual(self.processor._genai_committed, "")
def test_missing_client_logs_and_returns_none(self):
self.manager.transcribe_client = None
self.assertIsNone(self._feed(_chunk(4000)))
self.assertIsNone(self._feed(_chunk(4000)))
def test_dropped_audio_discards_the_buffered_window(self):
"""A gap in the stream must not be spliced into a single window.
Dropping a queued chunk leaves the next one non-adjacent to what is
buffered, so concatenating them would hand the provider audio with a
hole in it and break the 50% overlap the stitcher relies on.
"""
self._feed(_chunk(4000))
self.assertEqual(len(self.processor._genai_window), 1)
# the producer discards a chunk while the consumer is blocked
self.processor._audio_dropped.set()
self._feed(_chunk(5000))
# the buffered chunk was discarded, so this one starts a fresh window
self.assertEqual(len(self.processor._genai_window), 1)
self.client.transcribe.assert_not_called()
# and the window that does go out holds only contiguous audio
self._feed(_chunk(5000))
self.client.transcribe.assert_called_once()
with self._sent_wav(0) as wav:
samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
self.assertTrue((samples == 5000).all(), "window spliced across the gap")
def test_dropped_audio_ends_a_pending_utterance(self):
"""Committed text cannot be stitched across missing speech."""
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.assertEqual(self.processor._genai_committed, "hello")
self.processor._audio_dropped.set()
self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
self.assertEqual(len(self.processor._genai_window), 0)
def test_drop_flag_is_consumed_once(self):
self.processor._audio_dropped.set()
self._feed(_chunk(4000))
self.assertFalse(self.processor._audio_dropped.is_set())
def test_full_queue_flags_a_drop(self):
for i in range(self.processor.audio_queue.maxsize + 1):
self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
self.assertTrue(self.processor._audio_dropped.is_set())
def test_queue_is_bounded_and_drops_oldest(self):
maxsize = self.processor.audio_queue.maxsize
self.assertGreater(maxsize, 0)
for i in range(maxsize + 5):
self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
self.assertEqual(self.processor.audio_queue.qsize(), maxsize)
# the newest chunk survived, the oldest did not
remaining = []
while not self.processor.audio_queue.empty():
remaining.append(self.processor.audio_queue.get_nowait()[1][0])
self.assertEqual(remaining[-1], maxsize + 5)
self.assertNotIn(1, remaining)
if __name__ == "__main__":
unittest.main()
+176
View File
@@ -0,0 +1,176 @@
"""Tests for the WAV helpers and transcript stitcher in frigate.util.audio."""
import io
import struct
import unittest
import wave
import numpy as np
from frigate.const import AUDIO_SAMPLE_RATE
from frigate.util.audio import fix_wav_header, pcm16_to_wav, stitch_transcripts
def _wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
buffer = io.BytesIO()
with wave.open(buffer, "wb") as out:
out.setnchannels(1)
out.setsampwidth(2)
out.setframerate(sample_rate)
out.writeframes(samples.tobytes())
return buffer.getvalue()
class TestPcm16ToWav(unittest.TestCase):
def test_round_trips_through_wave(self):
samples = np.arange(-1000, 1000, dtype=np.int16)
with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
self.assertEqual(wav.getnchannels(), 1)
self.assertEqual(wav.getsampwidth(), 2)
self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
self.assertEqual(wav.getnframes(), samples.size)
decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
np.testing.assert_array_equal(decoded, samples)
def test_casts_non_int16_input(self):
samples = np.array([0.0, 100.0, -100.0], dtype=np.float32)
with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
np.testing.assert_array_equal(decoded, np.array([0, 100, -100], np.int16))
def test_honors_sample_rate(self):
with wave.open(
io.BytesIO(pcm16_to_wav(np.zeros(4, np.int16), 8000)), "rb"
) as w:
self.assertEqual(w.getframerate(), 8000)
class TestFixWavHeader(unittest.TestCase):
def test_rewrites_placeholder_sizes(self):
samples = np.arange(64, dtype=np.int16)
data = bytearray(_wav(samples))
# ffmpeg piping to non-seekable stdout leaves both sizes unpatched
struct.pack_into("<I", data, 4, 0xFFFFFFFF)
data_offset = data.index(b"data")
struct.pack_into("<I", data, data_offset + 4, 0xFFFFFFFF)
fixed = fix_wav_header(bytes(data))
self.assertEqual(struct.unpack_from("<I", fixed, 4)[0], len(fixed) - 8)
self.assertEqual(
struct.unpack_from("<I", fixed, data_offset + 4)[0],
len(fixed) - (data_offset + 8),
)
with wave.open(io.BytesIO(fixed), "rb") as wav:
self.assertEqual(wav.getnframes(), samples.size)
def test_leaves_a_well_formed_header_alone(self):
data = _wav(np.arange(32, dtype=np.int16))
self.assertEqual(fix_wav_header(data), data)
def test_non_riff_payload_passes_through(self):
self.assertEqual(fix_wav_header(b"not a wav"), b"not a wav")
self.assertEqual(fix_wav_header(b""), b"")
class TestStitchTranscripts(unittest.TestCase):
def test_table(self):
cases = [
# (committed, incoming, expected, description)
(
"the quick brown",
"brown fox jumps",
"the quick brown fox jumps",
"one word",
),
(
"and then the quick brown",
"the quick brown fox",
"and then the quick brown fox",
"multi word",
),
(
"hello there",
"general kenobi",
"hello there general kenobi",
"no overlap",
),
("the quick brown fox", "brown fox", "the quick brown fox", "contained"),
("", "first words", "first words", "empty committed"),
("already here", "", "already here", "empty incoming"),
(
" spaced out ",
"out again",
"spaced out again",
"whitespace normalized",
),
]
for committed, incoming, expected, description in cases:
with self.subTest(description):
self.assertEqual(stitch_transcripts(committed, incoming), expected)
def test_overlap_found_mid_window(self):
"""The shared run is rarely at the start of the new window.
The provider re-transcribes the overlapping audio independently and
often renders its first word differently, so anchoring the match to the
start of the incoming window duplicates the whole phrase.
"""
self.assertEqual(
stitch_transcripts(
"this is just gonna be a fun time", "It's gonna be a fun time."
),
"this is just gonna be a fun time",
)
def test_overlap_longer_than_five_words(self):
"""The cap is bounded by window duration, not by the old 5-word n-gram."""
self.assertEqual(
stitch_transcripts(
"well anyway one two three four five six",
"one two three four five six seven",
),
"well anyway one two three four five six seven",
)
def test_repeated_phrase_keeps_its_second_utterance(self):
"""Preferring the earliest match is what protects a real repeat."""
self.assertEqual(
stitch_transcripts("a b c fun time", "fun time fun time"),
"a b c fun time fun time",
)
def test_window_wholly_repeating_the_tail_is_dropped(self):
"""The accepted trade-off: an entirely redundant window adds nothing."""
self.assertEqual(stitch_transcripts("go go go", "go go go"), "go go go")
def test_revises_a_mistranscribed_tail(self):
"""A wrong last word would otherwise block every alignment.
Those words came from the newest audio, which the next window re-covers,
so replacing them is better than duplicating the phrase behind them.
"""
self.assertEqual(
stitch_transcripts("Yeah. this is Jessica.", "This is just gonna be fun."),
"Yeah. this is just gonna be fun.",
)
def test_revision_needs_more_than_one_shared_word(self):
"""A revision deletes published text, so it takes real evidence."""
self.assertEqual(
stitch_transcripts("the cat sat on a mat", "a dog barked"),
"the cat sat on a mat a dog barked",
)
if __name__ == "__main__":
unittest.main()
+266
View File
@@ -581,5 +581,271 @@ class TestLlamaCppProvider(unittest.TestCase):
self.assertEqual(client.get_context_size(), 32768)
# ---------------------------------------------------------------------------
# transcribe role
# ---------------------------------------------------------------------------
WAV_BYTES = b"RIFF$\x00\x00\x00WAVEfmt "
class TestOpenAITranscribe(unittest.TestCase):
def _client(self):
return _make_client(
"openai",
model="gpt-4o-transcribe",
api_key="k",
base_url="http://localhost:9999/v1",
runtime_options={"temperature": 0.7},
)
def test_supports_transcription(self):
self.assertTrue(self._client().supports_transcription)
def test_passes_file_tuple_and_language(self):
client = self._client()
create = MagicMock(return_value=" hello there ")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "hello there")
kwargs = create.call_args.kwargs
self.assertEqual(kwargs["model"], "gpt-4o-transcribe")
self.assertEqual(kwargs["file"], ("audio.wav", WAV_BYTES, "audio/wav"))
self.assertEqual(kwargs["language"], "en")
self.assertEqual(kwargs["response_format"], "text")
def test_does_not_forward_runtime_options(self):
"""runtime_options are chat parameters; /audio/transcriptions rejects them."""
client = self._client()
create = MagicMock(return_value="hi")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
client.transcribe(WAV_BYTES)
self.assertNotIn("temperature", create.call_args.kwargs)
def test_gpt_transcribe_uses_languages_array(self):
"""gpt-transcribe replaced `language` with a `languages` array."""
client = _make_client("openai", model="gpt-transcribe", api_key="k")
create = MagicMock(return_value="hi")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
client.transcribe(WAV_BYTES, language="en")
kwargs = create.call_args.kwargs
self.assertEqual(kwargs["extra_body"], {"languages": ["en"]})
# sending both fields is rejected by the API
self.assertNotIn("language", kwargs)
def test_older_models_use_singular_language(self):
for model in ("gpt-4o-transcribe", "whisper-1"):
with self.subTest(model):
client = _make_client("openai", model=model, api_key="k")
create = MagicMock(return_value="hi")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
client.transcribe(WAV_BYTES, language="en")
kwargs = create.call_args.kwargs
self.assertEqual(kwargs["language"], "en")
self.assertNotIn("extra_body", kwargs)
def test_object_response_form(self):
client = self._client()
create = MagicMock(return_value=SimpleNamespace(text="hi"))
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertEqual(client.transcribe(WAV_BYTES), "hi")
def test_error_returns_none(self):
client = self._client()
create = MagicMock(side_effect=RuntimeError("boom"))
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertIsNone(client.transcribe(WAV_BYTES))
class TestAzureOpenAITranscribe(unittest.TestCase):
def _client(self):
return _make_client(
"azure_openai",
model="my-deployment",
api_key="k",
base_url="https://example.openai.azure.com/?api-version=2024-06-01",
)
def test_routes_through_azure_client(self):
from openai import AzureOpenAI
client = self._client()
self.assertIsInstance(client.provider, AzureOpenAI)
self.assertTrue(client.supports_transcription)
def test_transcribe_inherited(self):
client = self._client()
create = MagicMock(return_value="azure text")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertEqual(client.transcribe(WAV_BYTES, language="fr"), "azure text")
self.assertEqual(create.call_args.kwargs["model"], "my-deployment")
class TestGeminiTranscribe(unittest.TestCase):
def _client(self):
return _make_client("gemini", model="gemini-2.0-flash", api_key="k")
def test_supports_transcription(self):
self.assertTrue(self._client().supports_transcription)
def test_sends_audio_part(self):
client = self._client()
generate = MagicMock(return_value=SimpleNamespace(text=" spoken words "))
client.provider = SimpleNamespace(
models=SimpleNamespace(generate_content=generate)
)
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "spoken words")
contents = generate.call_args.kwargs["contents"]
audio_parts = [
p for p in contents if getattr(p, "inline_data", None) is not None
]
self.assertEqual(len(audio_parts), 1)
self.assertEqual(audio_parts[0].inline_data.mime_type, "audio/wav")
self.assertEqual(audio_parts[0].inline_data.data, WAV_BYTES)
def test_oversized_payload_is_skipped(self):
from frigate.genai.plugins.gemini import GEMINI_MAX_INLINE_BYTES
client = self._client()
generate = MagicMock()
client.provider = SimpleNamespace(
models=SimpleNamespace(generate_content=generate)
)
self.assertIsNone(client.transcribe(b"\x00" * (GEMINI_MAX_INLINE_BYTES + 1)))
generate.assert_not_called()
class TestLlamaCppTranscribe(unittest.TestCase):
def _client(self, supports_audio: bool):
cfg = GenAIConfig(
provider="llamacpp",
model="m",
base_url="http://localhost:9999",
)
info = {
"context_size": 4096,
"supports_vision": False,
"supports_audio": supports_audio,
"supports_tools": False,
"supports_reasoning": False,
"media_marker": "<__media__>",
}
cls = PROVIDERS[GenAIProviderEnum.llamacpp]
with patch.object(cls, "_get_model_info", return_value=info):
return cls(cfg, timeout=5)
def test_supports_transcription_tracks_supports_audio(self):
self.assertTrue(self._client(True).supports_transcription)
self.assertFalse(self._client(False).supports_transcription)
@staticmethod
def _transcriptions_response(text: str = " transcript "):
response = MagicMock()
response.status_code = 200
response.json.return_value = {"text": text}
return response
@staticmethod
def _chat_response(content: str = " fallback transcript "):
response = MagicMock()
response.status_code = 200
response.json.return_value = {"choices": [{"message": {"content": content}}]}
return response
def test_posts_multipart_to_transcriptions(self):
client = self._client(True)
with patch.object(
client, "_post", return_value=self._transcriptions_response()
) as post:
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "transcript")
self.assertTrue(post.call_args.args[0].endswith("/v1/audio/transcriptions"))
self.assertEqual(
post.call_args.kwargs["files"]["file"],
("audio.wav", WAV_BYTES, "audio/wav"),
)
self.assertEqual(post.call_args.kwargs["data"]["language"], "en")
def test_omits_language_when_not_set(self):
"""An unset language is what lets the model detect one itself."""
client = self._client(True)
with patch.object(
client, "_post", return_value=self._transcriptions_response()
) as post:
client.transcribe(WAV_BYTES)
self.assertNotIn("language", post.call_args.kwargs["data"])
def test_falls_back_to_chat_completions_on_404(self):
"""Servers predating llama.cpp#21863 have no transcriptions route."""
client = self._client(True)
missing = MagicMock()
missing.status_code = 404
with patch.object(
client, "_post", side_effect=[missing, self._chat_response()]
) as post:
self.assertEqual(
client.transcribe(WAV_BYTES, language="en"), "fallback transcript"
)
urls = [call.args[0] for call in post.call_args_list]
self.assertTrue(urls[0].endswith("/v1/audio/transcriptions"))
self.assertTrue(urls[1].endswith("/v1/chat/completions"))
payload = post.call_args_list[1].kwargs["json"]
content = payload["messages"][0]["content"]
audio_parts = [p for p in content if p["type"] == "input_audio"]
self.assertEqual(len(audio_parts), 1)
self.assertEqual(audio_parts[0]["input_audio"]["format"], "wav")
self.assertEqual(
base64.b64decode(audio_parts[0]["input_audio"]["data"]), WAV_BYTES
)
def test_audio_unsupported_returns_none(self):
client = self._client(False)
with patch.object(client, "_post") as post:
self.assertIsNone(client.transcribe(WAV_BYTES))
post.assert_not_called()
class TestBaseClientTranscribe(unittest.TestCase):
"""Providers that don't implement the role must be inert, not broken."""
def test_ollama_reports_and_returns_nothing(self):
client = _make_client("ollama", model="llava", base_url="http://localhost:9999")
self.assertFalse(client.supports_transcription)
self.assertIsNone(client.transcribe(WAV_BYTES, language="en"))
if __name__ == "__main__":
unittest.main()