mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-17 19:13:02 +03:00
Support using GenAI for audio transcription (#24396)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
* Add support for running transcription with GenAI * Improve audio joining * Fix GenAI model capability reporting * Support language correctly * Migrate existing users to keep english selected * Fix models * Fix tests * Fix accepted null model * Handle slwo providers
This commit is contained in:
@@ -0,0 +1,286 @@
|
||||
"""Config validation and the historical path for the audio_transcription GenAI backend."""
|
||||
|
||||
import unittest
|
||||
from copy import deepcopy
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.camera.genai import GenAIConfig, GenAIRoleEnum
|
||||
from frigate.config.classification import AudioTranscriptionModelEnum
|
||||
from frigate.const import UPDATE_EVENT_DESCRIPTION
|
||||
from frigate.data_processing.post.audio_transcription import (
|
||||
AudioTranscriptionPostProcessor,
|
||||
)
|
||||
from frigate.data_processing.types import PostProcessDataEnum
|
||||
|
||||
|
||||
class TestAudioTranscriptionGenAIConfig(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.base = {
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"cameras": {
|
||||
"back": {
|
||||
"ffmpeg": {
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video",
|
||||
"roles": ["detect", "audio"],
|
||||
}
|
||||
]
|
||||
},
|
||||
"detect": {"height": 1080, "width": 1920, "fps": 5},
|
||||
"audio": {"enabled": True},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def _config(self, **overrides) -> dict:
|
||||
config = deepcopy(self.base)
|
||||
config.update(deepcopy(overrides))
|
||||
return config
|
||||
|
||||
def _provider(self, roles: list[str]) -> dict:
|
||||
return {
|
||||
"whisper_cloud": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-4o-transcribe",
|
||||
"api_key": "k",
|
||||
"roles": roles,
|
||||
}
|
||||
}
|
||||
|
||||
def test_default_model_is_whisper_enum(self):
|
||||
config = FrigateConfig(**self._config())
|
||||
self.assertEqual(
|
||||
config.audio_transcription.model, AudioTranscriptionModelEnum.whisper
|
||||
)
|
||||
|
||||
def test_whisper_string_coerces_to_enum(self):
|
||||
config = FrigateConfig(
|
||||
**self._config(audio_transcription={"enabled": True, "model": "whisper"})
|
||||
)
|
||||
self.assertIsInstance(
|
||||
config.audio_transcription.model, AudioTranscriptionModelEnum
|
||||
)
|
||||
|
||||
def test_provider_name_stays_a_string(self):
|
||||
config = FrigateConfig(
|
||||
**self._config(
|
||||
genai=self._provider(["transcribe"]),
|
||||
audio_transcription={"enabled": True, "model": "whisper_cloud"},
|
||||
)
|
||||
)
|
||||
self.assertNotIsInstance(
|
||||
config.audio_transcription.model, AudioTranscriptionModelEnum
|
||||
)
|
||||
self.assertEqual(config.audio_transcription.model, "whisper_cloud")
|
||||
|
||||
def test_unspecified_model_falls_back_to_whisper(self):
|
||||
"""An empty value must not read as a GenAI provider that resolves to no client."""
|
||||
for value in (None, "", " "):
|
||||
with self.subTest(repr(value)):
|
||||
config = FrigateConfig(
|
||||
**self._config(
|
||||
audio_transcription={"enabled": True, "model": value}
|
||||
)
|
||||
)
|
||||
self.assertIs(
|
||||
config.audio_transcription.model,
|
||||
AudioTranscriptionModelEnum.whisper,
|
||||
)
|
||||
|
||||
def test_missing_genai_key_raises(self):
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
FrigateConfig(
|
||||
**self._config(
|
||||
audio_transcription={"enabled": True, "model": "nope"},
|
||||
)
|
||||
)
|
||||
self.assertIn("is not a valid GenAI config key", str(ctx.exception))
|
||||
|
||||
def test_provider_without_role_raises(self):
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
FrigateConfig(
|
||||
**self._config(
|
||||
genai=self._provider(["descriptions"]),
|
||||
audio_transcription={"enabled": True, "model": "whisper_cloud"},
|
||||
)
|
||||
)
|
||||
self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
|
||||
|
||||
def test_global_off_camera_on_still_validates(self):
|
||||
"""Global-off/camera-on is a supported deployment and must not skip the check."""
|
||||
config = self._config(
|
||||
genai=self._provider(["descriptions"]),
|
||||
audio_transcription={"enabled": False, "model": "whisper_cloud"},
|
||||
)
|
||||
config["cameras"]["back"]["audio_transcription"] = {"enabled": True}
|
||||
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
FrigateConfig(**config)
|
||||
|
||||
self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
|
||||
|
||||
def test_disabled_transcription_skips_validation(self):
|
||||
config = FrigateConfig(
|
||||
**self._config(audio_transcription={"enabled": False, "model": "nope"})
|
||||
)
|
||||
self.assertEqual(config.audio_transcription.model, "nope")
|
||||
|
||||
def test_camera_level_model_is_rejected(self):
|
||||
config = self._config()
|
||||
config["cameras"]["back"]["audio_transcription"] = {
|
||||
"enabled": True,
|
||||
"model": "whisper",
|
||||
}
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
FrigateConfig(**config)
|
||||
|
||||
def test_default_roles_do_not_include_transcribe(self):
|
||||
"""Backward compatibility: existing providers must not silently claim it."""
|
||||
genai = GenAIConfig(provider="openai", model="gpt-4o")
|
||||
self.assertNotIn(GenAIRoleEnum.transcribe, genai.roles)
|
||||
|
||||
def test_two_providers_claiming_transcribe_raises(self):
|
||||
genai = self._provider(["transcribe"])
|
||||
genai["other"] = {
|
||||
"provider": "gemini",
|
||||
"model": "gemini-2.0-flash",
|
||||
"api_key": "k",
|
||||
"roles": ["transcribe"],
|
||||
}
|
||||
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
FrigateConfig(
|
||||
**self._config(
|
||||
genai=genai,
|
||||
audio_transcription={"enabled": True, "model": "whisper_cloud"},
|
||||
)
|
||||
)
|
||||
|
||||
self.assertIn("each role must have", str(ctx.exception))
|
||||
|
||||
def test_transcribe_rejected_on_provider_without_audio_input(self):
|
||||
with self.assertRaises(ValidationError) as ctx:
|
||||
GenAIConfig(provider="ollama", model="llava", roles=["transcribe"])
|
||||
|
||||
self.assertIn("does not support audio input", str(ctx.exception))
|
||||
|
||||
|
||||
class TestAudioTranscriptionPostProcessorGenAI(unittest.TestCase):
|
||||
"""The recorded-speech path must reach the provider and skip the local model."""
|
||||
|
||||
def setUp(self):
|
||||
self.config = FrigateConfig(
|
||||
**{
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"genai": {
|
||||
"whisper_cloud": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-4o-transcribe",
|
||||
"api_key": "k",
|
||||
"roles": ["transcribe"],
|
||||
}
|
||||
},
|
||||
"audio_transcription": {
|
||||
"enabled": True,
|
||||
"model": "whisper_cloud",
|
||||
"language": "en",
|
||||
},
|
||||
"cameras": {
|
||||
"back": {
|
||||
"ffmpeg": {
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video",
|
||||
"roles": ["detect", "audio"],
|
||||
}
|
||||
]
|
||||
},
|
||||
"detect": {"height": 1080, "width": 1920, "fps": 5},
|
||||
"audio": {"enabled": True},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.client = MagicMock()
|
||||
self.client.transcribe.return_value = "recorded speech"
|
||||
self.manager = MagicMock()
|
||||
self.manager.transcribe_client = self.client
|
||||
self.requestor = MagicMock()
|
||||
|
||||
self.processor = AudioTranscriptionPostProcessor(
|
||||
self.config,
|
||||
self.requestor,
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
self.manager,
|
||||
)
|
||||
|
||||
def _process(self):
|
||||
self.processor.process_data(
|
||||
{
|
||||
"event_id": "1234.5-abc",
|
||||
"camera": "back",
|
||||
"event": {
|
||||
"id": "1234.5-abc",
|
||||
"camera": "back",
|
||||
"start_time": 100.0,
|
||||
"end_time": 110.0,
|
||||
"data": {},
|
||||
},
|
||||
},
|
||||
PostProcessDataEnum.tracked_object,
|
||||
)
|
||||
|
||||
def test_local_recognizer_is_never_built(self):
|
||||
self.assertTrue(self.processor._use_genai)
|
||||
self.assertIsNone(self.processor.recognizer)
|
||||
|
||||
def test_audio_bytes_and_language_reach_the_client(self):
|
||||
with patch(
|
||||
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
|
||||
return_value=b"RIFF....WAVE",
|
||||
):
|
||||
self._process()
|
||||
|
||||
self.client.transcribe.assert_called_once()
|
||||
self.assertEqual(self.client.transcribe.call_args.args[0], b"RIFF....WAVE")
|
||||
self.assertEqual(self.client.transcribe.call_args.kwargs["language"], "en")
|
||||
|
||||
def test_transcript_is_published_as_the_description(self):
|
||||
with patch(
|
||||
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
|
||||
return_value=b"RIFF....WAVE",
|
||||
):
|
||||
self._process()
|
||||
|
||||
topics = [call.args[0] for call in self.requestor.send_data.call_args_list]
|
||||
self.assertIn(UPDATE_EVENT_DESCRIPTION, topics)
|
||||
|
||||
payload = next(
|
||||
call.args[1]
|
||||
for call in self.requestor.send_data.call_args_list
|
||||
if call.args[0] == UPDATE_EVENT_DESCRIPTION
|
||||
)
|
||||
self.assertEqual(payload["description"], "recorded speech")
|
||||
self.assertEqual(payload["id"], "1234.5-abc")
|
||||
|
||||
def test_missing_client_publishes_nothing(self):
|
||||
self.manager.transcribe_client = None
|
||||
|
||||
with patch(
|
||||
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
|
||||
return_value=b"RIFF....WAVE",
|
||||
):
|
||||
self._process()
|
||||
|
||||
self.requestor.send_data.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Live GenAI transcription: sliding overlapped windows and their lifecycle."""
|
||||
|
||||
import io
|
||||
import threading
|
||||
import unittest
|
||||
import wave
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.const import AUDIO_DURATION, AUDIO_SAMPLE_RATE
|
||||
from frigate.data_processing.real_time.audio_transcription import (
|
||||
GENAI_WINDOW_CHUNKS,
|
||||
AudioTranscriptionRealTimeProcessor,
|
||||
)
|
||||
|
||||
CHUNK_SAMPLES = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE))
|
||||
|
||||
|
||||
def _chunk(amplitude: int) -> np.ndarray:
|
||||
"""One audio-detector-sized chunk of int16 samples at a constant amplitude."""
|
||||
return np.full(CHUNK_SAMPLES, amplitude, dtype=np.int16)
|
||||
|
||||
|
||||
class TestLiveGenAITranscription(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.config = FrigateConfig(
|
||||
**{
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"genai": {
|
||||
"whisper_cloud": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-4o-transcribe",
|
||||
"api_key": "k",
|
||||
"roles": ["transcribe"],
|
||||
}
|
||||
},
|
||||
"audio_transcription": {
|
||||
"enabled": True,
|
||||
"model": "whisper_cloud",
|
||||
"language": "en",
|
||||
},
|
||||
"cameras": {
|
||||
"back": {
|
||||
"ffmpeg": {
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video",
|
||||
"roles": ["detect", "audio"],
|
||||
}
|
||||
]
|
||||
},
|
||||
"detect": {"height": 1080, "width": 1920, "fps": 5},
|
||||
"audio": {"enabled": True},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.client = MagicMock()
|
||||
self.client.transcribe.return_value = "hello"
|
||||
self.manager = MagicMock()
|
||||
self.manager.transcribe_client = self.client
|
||||
self.requestor = MagicMock()
|
||||
|
||||
self.processor = AudioTranscriptionRealTimeProcessor(
|
||||
config=self.config,
|
||||
camera_config=self.config.cameras["back"],
|
||||
requestor=self.requestor,
|
||||
model_runner=None,
|
||||
metrics=MagicMock(),
|
||||
stop_event=threading.Event(),
|
||||
genai_manager=self.manager,
|
||||
)
|
||||
|
||||
def _feed(self, chunk: np.ndarray):
|
||||
return (
|
||||
self.processor._AudioTranscriptionRealTimeProcessor__process_audio_stream(
|
||||
chunk
|
||||
)
|
||||
)
|
||||
|
||||
def _sent_wav(self, call_index: int) -> wave.Wave_read:
|
||||
payload = self.client.transcribe.call_args_list[call_index].args[0]
|
||||
return wave.open(io.BytesIO(payload), "rb")
|
||||
|
||||
def test_uses_genai_path(self):
|
||||
self.assertTrue(self.processor._use_genai)
|
||||
|
||||
def test_first_chunk_does_not_transcribe(self):
|
||||
self.assertIsNone(self._feed(_chunk(4000)))
|
||||
self.client.transcribe.assert_not_called()
|
||||
|
||||
def test_full_window_transcribes_two_chunks(self):
|
||||
self._feed(_chunk(4000))
|
||||
result = self._feed(_chunk(4000))
|
||||
|
||||
self.assertEqual(result, ("hello", False))
|
||||
self.client.transcribe.assert_called_once()
|
||||
self.assertEqual(
|
||||
self.client.transcribe.call_args.kwargs["language"],
|
||||
"en",
|
||||
)
|
||||
|
||||
with self._sent_wav(0) as wav:
|
||||
self.assertEqual(wav.getnchannels(), 1)
|
||||
self.assertEqual(wav.getsampwidth(), 2)
|
||||
self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
|
||||
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
|
||||
|
||||
def test_window_slides_with_overlap(self):
|
||||
"""The third chunk's window is chunks 2+3, not 3 alone and not 1+2+3."""
|
||||
self.client.transcribe.side_effect = ["one two", "two three"]
|
||||
|
||||
self._feed(_chunk(1000))
|
||||
self._feed(_chunk(2000))
|
||||
result = self._feed(_chunk(3000))
|
||||
|
||||
self.assertEqual(self.client.transcribe.call_count, 2)
|
||||
|
||||
with self._sent_wav(1) as wav:
|
||||
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
|
||||
samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
|
||||
|
||||
self.assertEqual(samples[0], 2000)
|
||||
self.assertEqual(samples[-1], 3000)
|
||||
|
||||
# the shared "two" appears once
|
||||
self.assertEqual(result, ("one two three", False))
|
||||
|
||||
def test_silent_window_is_not_uploaded(self):
|
||||
self._feed(_chunk(0))
|
||||
self.assertIsNone(self._feed(_chunk(0)))
|
||||
self.client.transcribe.assert_not_called()
|
||||
|
||||
def test_silent_window_ends_a_pending_utterance(self):
|
||||
self._feed(_chunk(4000))
|
||||
self._feed(_chunk(4000))
|
||||
|
||||
# the gate covers the whole window, so it takes GENAI_WINDOW_CHUNKS
|
||||
# silent chunks to push the last speech out of it
|
||||
self._feed(_chunk(0))
|
||||
|
||||
self.assertEqual(self._feed(_chunk(0)), ("hello", True))
|
||||
|
||||
def test_empty_transcript_ends_a_pending_utterance(self):
|
||||
self.client.transcribe.side_effect = ["hello", ""]
|
||||
|
||||
self._feed(_chunk(4000))
|
||||
self._feed(_chunk(4000))
|
||||
|
||||
self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
|
||||
|
||||
def test_empty_transcript_with_nothing_pending_returns_none(self):
|
||||
self.client.transcribe.return_value = ""
|
||||
|
||||
self._feed(_chunk(4000))
|
||||
self.assertIsNone(self._feed(_chunk(4000)))
|
||||
|
||||
def test_reset_clears_committed_text_and_window(self):
|
||||
self._feed(_chunk(4000))
|
||||
self._feed(_chunk(4000))
|
||||
|
||||
self.processor.reset()
|
||||
|
||||
self.assertEqual(self.processor._genai_committed, "")
|
||||
self.assertEqual(len(self.processor._genai_window), 0)
|
||||
|
||||
# a fresh window is required again before the next request
|
||||
self.client.transcribe.reset_mock()
|
||||
self._feed(_chunk(4000))
|
||||
self.client.transcribe.assert_not_called()
|
||||
|
||||
def test_check_unload_model_clears_once_then_is_idempotent(self):
|
||||
self._feed(_chunk(4000))
|
||||
self._feed(_chunk(4000))
|
||||
|
||||
self.processor.check_unload_model()
|
||||
|
||||
self.requestor.send_data.assert_called_once_with("back/audio/transcription", "")
|
||||
self.assertEqual(self.processor._genai_committed, "")
|
||||
self.assertEqual(len(self.processor._genai_window), 0)
|
||||
|
||||
self.processor.check_unload_model()
|
||||
self.requestor.send_data.assert_called_once()
|
||||
|
||||
def test_build_recognizer_never_loads_a_local_model(self):
|
||||
with patch(
|
||||
"frigate.data_processing.real_time.audio_transcription.FasterWhisperASR"
|
||||
) as whisper:
|
||||
self.processor._AudioTranscriptionRealTimeProcessor__build_recognizer()
|
||||
|
||||
whisper.assert_not_called()
|
||||
self.assertIsNone(self.processor.stream)
|
||||
|
||||
def test_clear_audio_recognizer_request_only_resets(self):
|
||||
self._feed(_chunk(4000))
|
||||
self._feed(_chunk(4000))
|
||||
|
||||
with patch.object(
|
||||
self.processor,
|
||||
"_AudioTranscriptionRealTimeProcessor__build_recognizer",
|
||||
) as build:
|
||||
result = self.processor.handle_request("clear_audio_recognizer", {})
|
||||
|
||||
build.assert_not_called()
|
||||
self.assertTrue(result["success"])
|
||||
self.assertEqual(self.processor._genai_committed, "")
|
||||
|
||||
def test_missing_client_logs_and_returns_none(self):
|
||||
self.manager.transcribe_client = None
|
||||
|
||||
self.assertIsNone(self._feed(_chunk(4000)))
|
||||
self.assertIsNone(self._feed(_chunk(4000)))
|
||||
|
||||
def test_dropped_audio_discards_the_buffered_window(self):
|
||||
"""A gap in the stream must not be spliced into a single window.
|
||||
|
||||
Dropping a queued chunk leaves the next one non-adjacent to what is
|
||||
buffered, so concatenating them would hand the provider audio with a
|
||||
hole in it and break the 50% overlap the stitcher relies on.
|
||||
"""
|
||||
self._feed(_chunk(4000))
|
||||
self.assertEqual(len(self.processor._genai_window), 1)
|
||||
|
||||
# the producer discards a chunk while the consumer is blocked
|
||||
self.processor._audio_dropped.set()
|
||||
|
||||
self._feed(_chunk(5000))
|
||||
|
||||
# the buffered chunk was discarded, so this one starts a fresh window
|
||||
self.assertEqual(len(self.processor._genai_window), 1)
|
||||
self.client.transcribe.assert_not_called()
|
||||
|
||||
# and the window that does go out holds only contiguous audio
|
||||
self._feed(_chunk(5000))
|
||||
self.client.transcribe.assert_called_once()
|
||||
|
||||
with self._sent_wav(0) as wav:
|
||||
samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
|
||||
|
||||
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
|
||||
self.assertTrue((samples == 5000).all(), "window spliced across the gap")
|
||||
|
||||
def test_dropped_audio_ends_a_pending_utterance(self):
|
||||
"""Committed text cannot be stitched across missing speech."""
|
||||
self._feed(_chunk(4000))
|
||||
self._feed(_chunk(4000))
|
||||
self.assertEqual(self.processor._genai_committed, "hello")
|
||||
|
||||
self.processor._audio_dropped.set()
|
||||
|
||||
self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
|
||||
self.assertEqual(len(self.processor._genai_window), 0)
|
||||
|
||||
def test_drop_flag_is_consumed_once(self):
|
||||
self.processor._audio_dropped.set()
|
||||
self._feed(_chunk(4000))
|
||||
|
||||
self.assertFalse(self.processor._audio_dropped.is_set())
|
||||
|
||||
def test_full_queue_flags_a_drop(self):
|
||||
for i in range(self.processor.audio_queue.maxsize + 1):
|
||||
self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
|
||||
|
||||
self.assertTrue(self.processor._audio_dropped.is_set())
|
||||
|
||||
def test_queue_is_bounded_and_drops_oldest(self):
|
||||
maxsize = self.processor.audio_queue.maxsize
|
||||
self.assertGreater(maxsize, 0)
|
||||
|
||||
for i in range(maxsize + 5):
|
||||
self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
|
||||
|
||||
self.assertEqual(self.processor.audio_queue.qsize(), maxsize)
|
||||
|
||||
# the newest chunk survived, the oldest did not
|
||||
remaining = []
|
||||
while not self.processor.audio_queue.empty():
|
||||
remaining.append(self.processor.audio_queue.get_nowait()[1][0])
|
||||
|
||||
self.assertEqual(remaining[-1], maxsize + 5)
|
||||
self.assertNotIn(1, remaining)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Tests for the WAV helpers and transcript stitcher in frigate.util.audio."""
|
||||
|
||||
import io
|
||||
import struct
|
||||
import unittest
|
||||
import wave
|
||||
|
||||
import numpy as np
|
||||
|
||||
from frigate.const import AUDIO_SAMPLE_RATE
|
||||
from frigate.util.audio import fix_wav_header, pcm16_to_wav, stitch_transcripts
|
||||
|
||||
|
||||
def _wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
|
||||
with wave.open(buffer, "wb") as out:
|
||||
out.setnchannels(1)
|
||||
out.setsampwidth(2)
|
||||
out.setframerate(sample_rate)
|
||||
out.writeframes(samples.tobytes())
|
||||
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
class TestPcm16ToWav(unittest.TestCase):
|
||||
def test_round_trips_through_wave(self):
|
||||
samples = np.arange(-1000, 1000, dtype=np.int16)
|
||||
|
||||
with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
|
||||
self.assertEqual(wav.getnchannels(), 1)
|
||||
self.assertEqual(wav.getsampwidth(), 2)
|
||||
self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
|
||||
self.assertEqual(wav.getnframes(), samples.size)
|
||||
decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
|
||||
|
||||
np.testing.assert_array_equal(decoded, samples)
|
||||
|
||||
def test_casts_non_int16_input(self):
|
||||
samples = np.array([0.0, 100.0, -100.0], dtype=np.float32)
|
||||
|
||||
with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
|
||||
decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
|
||||
|
||||
np.testing.assert_array_equal(decoded, np.array([0, 100, -100], np.int16))
|
||||
|
||||
def test_honors_sample_rate(self):
|
||||
with wave.open(
|
||||
io.BytesIO(pcm16_to_wav(np.zeros(4, np.int16), 8000)), "rb"
|
||||
) as w:
|
||||
self.assertEqual(w.getframerate(), 8000)
|
||||
|
||||
|
||||
class TestFixWavHeader(unittest.TestCase):
|
||||
def test_rewrites_placeholder_sizes(self):
|
||||
samples = np.arange(64, dtype=np.int16)
|
||||
data = bytearray(_wav(samples))
|
||||
|
||||
# ffmpeg piping to non-seekable stdout leaves both sizes unpatched
|
||||
struct.pack_into("<I", data, 4, 0xFFFFFFFF)
|
||||
data_offset = data.index(b"data")
|
||||
struct.pack_into("<I", data, data_offset + 4, 0xFFFFFFFF)
|
||||
|
||||
fixed = fix_wav_header(bytes(data))
|
||||
|
||||
self.assertEqual(struct.unpack_from("<I", fixed, 4)[0], len(fixed) - 8)
|
||||
self.assertEqual(
|
||||
struct.unpack_from("<I", fixed, data_offset + 4)[0],
|
||||
len(fixed) - (data_offset + 8),
|
||||
)
|
||||
|
||||
with wave.open(io.BytesIO(fixed), "rb") as wav:
|
||||
self.assertEqual(wav.getnframes(), samples.size)
|
||||
|
||||
def test_leaves_a_well_formed_header_alone(self):
|
||||
data = _wav(np.arange(32, dtype=np.int16))
|
||||
self.assertEqual(fix_wav_header(data), data)
|
||||
|
||||
def test_non_riff_payload_passes_through(self):
|
||||
self.assertEqual(fix_wav_header(b"not a wav"), b"not a wav")
|
||||
self.assertEqual(fix_wav_header(b""), b"")
|
||||
|
||||
|
||||
class TestStitchTranscripts(unittest.TestCase):
|
||||
def test_table(self):
|
||||
cases = [
|
||||
# (committed, incoming, expected, description)
|
||||
(
|
||||
"the quick brown",
|
||||
"brown fox jumps",
|
||||
"the quick brown fox jumps",
|
||||
"one word",
|
||||
),
|
||||
(
|
||||
"and then the quick brown",
|
||||
"the quick brown fox",
|
||||
"and then the quick brown fox",
|
||||
"multi word",
|
||||
),
|
||||
(
|
||||
"hello there",
|
||||
"general kenobi",
|
||||
"hello there general kenobi",
|
||||
"no overlap",
|
||||
),
|
||||
("the quick brown fox", "brown fox", "the quick brown fox", "contained"),
|
||||
("", "first words", "first words", "empty committed"),
|
||||
("already here", "", "already here", "empty incoming"),
|
||||
(
|
||||
" spaced out ",
|
||||
"out again",
|
||||
"spaced out again",
|
||||
"whitespace normalized",
|
||||
),
|
||||
]
|
||||
|
||||
for committed, incoming, expected, description in cases:
|
||||
with self.subTest(description):
|
||||
self.assertEqual(stitch_transcripts(committed, incoming), expected)
|
||||
|
||||
def test_overlap_found_mid_window(self):
|
||||
"""The shared run is rarely at the start of the new window.
|
||||
|
||||
The provider re-transcribes the overlapping audio independently and
|
||||
often renders its first word differently, so anchoring the match to the
|
||||
start of the incoming window duplicates the whole phrase.
|
||||
"""
|
||||
self.assertEqual(
|
||||
stitch_transcripts(
|
||||
"this is just gonna be a fun time", "It's gonna be a fun time."
|
||||
),
|
||||
"this is just gonna be a fun time",
|
||||
)
|
||||
|
||||
def test_overlap_longer_than_five_words(self):
|
||||
"""The cap is bounded by window duration, not by the old 5-word n-gram."""
|
||||
self.assertEqual(
|
||||
stitch_transcripts(
|
||||
"well anyway one two three four five six",
|
||||
"one two three four five six seven",
|
||||
),
|
||||
"well anyway one two three four five six seven",
|
||||
)
|
||||
|
||||
def test_repeated_phrase_keeps_its_second_utterance(self):
|
||||
"""Preferring the earliest match is what protects a real repeat."""
|
||||
self.assertEqual(
|
||||
stitch_transcripts("a b c fun time", "fun time fun time"),
|
||||
"a b c fun time fun time",
|
||||
)
|
||||
|
||||
def test_window_wholly_repeating_the_tail_is_dropped(self):
|
||||
"""The accepted trade-off: an entirely redundant window adds nothing."""
|
||||
self.assertEqual(stitch_transcripts("go go go", "go go go"), "go go go")
|
||||
|
||||
def test_revises_a_mistranscribed_tail(self):
|
||||
"""A wrong last word would otherwise block every alignment.
|
||||
|
||||
Those words came from the newest audio, which the next window re-covers,
|
||||
so replacing them is better than duplicating the phrase behind them.
|
||||
"""
|
||||
self.assertEqual(
|
||||
stitch_transcripts("Yeah. this is Jessica.", "This is just gonna be fun."),
|
||||
"Yeah. this is just gonna be fun.",
|
||||
)
|
||||
|
||||
def test_revision_needs_more_than_one_shared_word(self):
|
||||
"""A revision deletes published text, so it takes real evidence."""
|
||||
self.assertEqual(
|
||||
stitch_transcripts("the cat sat on a mat", "a dog barked"),
|
||||
"the cat sat on a mat a dog barked",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -581,5 +581,271 @@ class TestLlamaCppProvider(unittest.TestCase):
|
||||
self.assertEqual(client.get_context_size(), 32768)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# transcribe role
|
||||
# ---------------------------------------------------------------------------
|
||||
WAV_BYTES = b"RIFF$\x00\x00\x00WAVEfmt "
|
||||
|
||||
|
||||
class TestOpenAITranscribe(unittest.TestCase):
|
||||
def _client(self):
|
||||
return _make_client(
|
||||
"openai",
|
||||
model="gpt-4o-transcribe",
|
||||
api_key="k",
|
||||
base_url="http://localhost:9999/v1",
|
||||
runtime_options={"temperature": 0.7},
|
||||
)
|
||||
|
||||
def test_supports_transcription(self):
|
||||
self.assertTrue(self._client().supports_transcription)
|
||||
|
||||
def test_passes_file_tuple_and_language(self):
|
||||
client = self._client()
|
||||
create = MagicMock(return_value=" hello there ")
|
||||
client.provider = SimpleNamespace(
|
||||
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
|
||||
)
|
||||
|
||||
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "hello there")
|
||||
|
||||
kwargs = create.call_args.kwargs
|
||||
self.assertEqual(kwargs["model"], "gpt-4o-transcribe")
|
||||
self.assertEqual(kwargs["file"], ("audio.wav", WAV_BYTES, "audio/wav"))
|
||||
self.assertEqual(kwargs["language"], "en")
|
||||
self.assertEqual(kwargs["response_format"], "text")
|
||||
|
||||
def test_does_not_forward_runtime_options(self):
|
||||
"""runtime_options are chat parameters; /audio/transcriptions rejects them."""
|
||||
client = self._client()
|
||||
create = MagicMock(return_value="hi")
|
||||
client.provider = SimpleNamespace(
|
||||
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
|
||||
)
|
||||
|
||||
client.transcribe(WAV_BYTES)
|
||||
|
||||
self.assertNotIn("temperature", create.call_args.kwargs)
|
||||
|
||||
def test_gpt_transcribe_uses_languages_array(self):
|
||||
"""gpt-transcribe replaced `language` with a `languages` array."""
|
||||
client = _make_client("openai", model="gpt-transcribe", api_key="k")
|
||||
create = MagicMock(return_value="hi")
|
||||
client.provider = SimpleNamespace(
|
||||
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
|
||||
)
|
||||
|
||||
client.transcribe(WAV_BYTES, language="en")
|
||||
|
||||
kwargs = create.call_args.kwargs
|
||||
self.assertEqual(kwargs["extra_body"], {"languages": ["en"]})
|
||||
# sending both fields is rejected by the API
|
||||
self.assertNotIn("language", kwargs)
|
||||
|
||||
def test_older_models_use_singular_language(self):
|
||||
for model in ("gpt-4o-transcribe", "whisper-1"):
|
||||
with self.subTest(model):
|
||||
client = _make_client("openai", model=model, api_key="k")
|
||||
create = MagicMock(return_value="hi")
|
||||
client.provider = SimpleNamespace(
|
||||
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
|
||||
)
|
||||
|
||||
client.transcribe(WAV_BYTES, language="en")
|
||||
|
||||
kwargs = create.call_args.kwargs
|
||||
self.assertEqual(kwargs["language"], "en")
|
||||
self.assertNotIn("extra_body", kwargs)
|
||||
|
||||
def test_object_response_form(self):
|
||||
client = self._client()
|
||||
create = MagicMock(return_value=SimpleNamespace(text="hi"))
|
||||
client.provider = SimpleNamespace(
|
||||
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
|
||||
)
|
||||
|
||||
self.assertEqual(client.transcribe(WAV_BYTES), "hi")
|
||||
|
||||
def test_error_returns_none(self):
|
||||
client = self._client()
|
||||
create = MagicMock(side_effect=RuntimeError("boom"))
|
||||
client.provider = SimpleNamespace(
|
||||
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
|
||||
)
|
||||
|
||||
self.assertIsNone(client.transcribe(WAV_BYTES))
|
||||
|
||||
|
||||
class TestAzureOpenAITranscribe(unittest.TestCase):
|
||||
def _client(self):
|
||||
return _make_client(
|
||||
"azure_openai",
|
||||
model="my-deployment",
|
||||
api_key="k",
|
||||
base_url="https://example.openai.azure.com/?api-version=2024-06-01",
|
||||
)
|
||||
|
||||
def test_routes_through_azure_client(self):
|
||||
from openai import AzureOpenAI
|
||||
|
||||
client = self._client()
|
||||
self.assertIsInstance(client.provider, AzureOpenAI)
|
||||
self.assertTrue(client.supports_transcription)
|
||||
|
||||
def test_transcribe_inherited(self):
|
||||
client = self._client()
|
||||
create = MagicMock(return_value="azure text")
|
||||
client.provider = SimpleNamespace(
|
||||
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
|
||||
)
|
||||
|
||||
self.assertEqual(client.transcribe(WAV_BYTES, language="fr"), "azure text")
|
||||
self.assertEqual(create.call_args.kwargs["model"], "my-deployment")
|
||||
|
||||
|
||||
class TestGeminiTranscribe(unittest.TestCase):
|
||||
def _client(self):
|
||||
return _make_client("gemini", model="gemini-2.0-flash", api_key="k")
|
||||
|
||||
def test_supports_transcription(self):
|
||||
self.assertTrue(self._client().supports_transcription)
|
||||
|
||||
def test_sends_audio_part(self):
|
||||
client = self._client()
|
||||
generate = MagicMock(return_value=SimpleNamespace(text=" spoken words "))
|
||||
client.provider = SimpleNamespace(
|
||||
models=SimpleNamespace(generate_content=generate)
|
||||
)
|
||||
|
||||
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "spoken words")
|
||||
|
||||
contents = generate.call_args.kwargs["contents"]
|
||||
audio_parts = [
|
||||
p for p in contents if getattr(p, "inline_data", None) is not None
|
||||
]
|
||||
self.assertEqual(len(audio_parts), 1)
|
||||
self.assertEqual(audio_parts[0].inline_data.mime_type, "audio/wav")
|
||||
self.assertEqual(audio_parts[0].inline_data.data, WAV_BYTES)
|
||||
|
||||
def test_oversized_payload_is_skipped(self):
|
||||
from frigate.genai.plugins.gemini import GEMINI_MAX_INLINE_BYTES
|
||||
|
||||
client = self._client()
|
||||
generate = MagicMock()
|
||||
client.provider = SimpleNamespace(
|
||||
models=SimpleNamespace(generate_content=generate)
|
||||
)
|
||||
|
||||
self.assertIsNone(client.transcribe(b"\x00" * (GEMINI_MAX_INLINE_BYTES + 1)))
|
||||
generate.assert_not_called()
|
||||
|
||||
|
||||
class TestLlamaCppTranscribe(unittest.TestCase):
|
||||
def _client(self, supports_audio: bool):
|
||||
cfg = GenAIConfig(
|
||||
provider="llamacpp",
|
||||
model="m",
|
||||
base_url="http://localhost:9999",
|
||||
)
|
||||
info = {
|
||||
"context_size": 4096,
|
||||
"supports_vision": False,
|
||||
"supports_audio": supports_audio,
|
||||
"supports_tools": False,
|
||||
"supports_reasoning": False,
|
||||
"media_marker": "<__media__>",
|
||||
}
|
||||
cls = PROVIDERS[GenAIProviderEnum.llamacpp]
|
||||
with patch.object(cls, "_get_model_info", return_value=info):
|
||||
return cls(cfg, timeout=5)
|
||||
|
||||
def test_supports_transcription_tracks_supports_audio(self):
|
||||
self.assertTrue(self._client(True).supports_transcription)
|
||||
self.assertFalse(self._client(False).supports_transcription)
|
||||
|
||||
@staticmethod
|
||||
def _transcriptions_response(text: str = " transcript "):
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.json.return_value = {"text": text}
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _chat_response(content: str = " fallback transcript "):
|
||||
response = MagicMock()
|
||||
response.status_code = 200
|
||||
response.json.return_value = {"choices": [{"message": {"content": content}}]}
|
||||
return response
|
||||
|
||||
def test_posts_multipart_to_transcriptions(self):
|
||||
client = self._client(True)
|
||||
|
||||
with patch.object(
|
||||
client, "_post", return_value=self._transcriptions_response()
|
||||
) as post:
|
||||
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "transcript")
|
||||
|
||||
self.assertTrue(post.call_args.args[0].endswith("/v1/audio/transcriptions"))
|
||||
self.assertEqual(
|
||||
post.call_args.kwargs["files"]["file"],
|
||||
("audio.wav", WAV_BYTES, "audio/wav"),
|
||||
)
|
||||
self.assertEqual(post.call_args.kwargs["data"]["language"], "en")
|
||||
|
||||
def test_omits_language_when_not_set(self):
|
||||
"""An unset language is what lets the model detect one itself."""
|
||||
client = self._client(True)
|
||||
|
||||
with patch.object(
|
||||
client, "_post", return_value=self._transcriptions_response()
|
||||
) as post:
|
||||
client.transcribe(WAV_BYTES)
|
||||
|
||||
self.assertNotIn("language", post.call_args.kwargs["data"])
|
||||
|
||||
def test_falls_back_to_chat_completions_on_404(self):
|
||||
"""Servers predating llama.cpp#21863 have no transcriptions route."""
|
||||
client = self._client(True)
|
||||
missing = MagicMock()
|
||||
missing.status_code = 404
|
||||
|
||||
with patch.object(
|
||||
client, "_post", side_effect=[missing, self._chat_response()]
|
||||
) as post:
|
||||
self.assertEqual(
|
||||
client.transcribe(WAV_BYTES, language="en"), "fallback transcript"
|
||||
)
|
||||
|
||||
urls = [call.args[0] for call in post.call_args_list]
|
||||
self.assertTrue(urls[0].endswith("/v1/audio/transcriptions"))
|
||||
self.assertTrue(urls[1].endswith("/v1/chat/completions"))
|
||||
|
||||
payload = post.call_args_list[1].kwargs["json"]
|
||||
content = payload["messages"][0]["content"]
|
||||
audio_parts = [p for p in content if p["type"] == "input_audio"]
|
||||
self.assertEqual(len(audio_parts), 1)
|
||||
self.assertEqual(audio_parts[0]["input_audio"]["format"], "wav")
|
||||
self.assertEqual(
|
||||
base64.b64decode(audio_parts[0]["input_audio"]["data"]), WAV_BYTES
|
||||
)
|
||||
|
||||
def test_audio_unsupported_returns_none(self):
|
||||
client = self._client(False)
|
||||
|
||||
with patch.object(client, "_post") as post:
|
||||
self.assertIsNone(client.transcribe(WAV_BYTES))
|
||||
|
||||
post.assert_not_called()
|
||||
|
||||
|
||||
class TestBaseClientTranscribe(unittest.TestCase):
|
||||
"""Providers that don't implement the role must be inert, not broken."""
|
||||
|
||||
def test_ollama_reports_and_returns_nothing(self):
|
||||
client = _make_client("ollama", model="llava", base_url="http://localhost:9999")
|
||||
self.assertFalse(client.supports_transcription)
|
||||
self.assertIsNone(client.transcribe(WAV_BYTES, language="en"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user