frigate/frigate/events/audio.py

"""Handle creating audio events."""

import datetime
import logging
import subprocess
import threading
import time
from multiprocessing.managers import DictProxy
from multiprocessing.synchronize import Event as MpEvent
from typing import Any

import numpy as np

from frigate.camera import CameraMetrics
from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import CameraConfig, CameraInput, FrigateConfig
from frigate.config.camera.ffmpeg import CameraFfmpegConfig
from frigate.config.camera.updater import (
    CameraConfigUpdateEnum,
    CameraConfigUpdateSubscriber,
)
from frigate.config.classification import AudioTranscriptionModelEnum
from frigate.const import (
    AUDIO_DURATION,
    AUDIO_FORMAT,
    AUDIO_MAX_BIT_RANGE,
    AUDIO_MIN_CONFIDENCE,
    AUDIO_SAMPLE_RATE,
    EXPIRE_AUDIO_ACTIVITY,
    PROCESS_PRIORITY_HIGH,
    UPDATE_AUDIO_ACTIVITY,
)
from frigate.data_processing.common.audio_transcription.model import (
    AudioTranscriptionModelRunner,
)
from frigate.data_processing.real_time.audio_transcription import (
    AudioTranscriptionRealTimeProcessor,
)
from frigate.data_processing.types import DataProcessorMetrics
from frigate.ffmpeg_presets import parse_preset_input
from frigate.log import LogPipe, suppress_stderr_during
from frigate.util.builtin import get_ffmpeg_arg_list, load_labels
from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg
from frigate.util.process import FrigateProcess

try:
    from tflite_runtime.interpreter import Interpreter
except ModuleNotFoundError:
    from ai_edge_litert.interpreter import Interpreter


logger = logging.getLogger(__name__)


def get_ffmpeg_command(ffmpeg: CameraFfmpegConfig) -> list[str]:
    ffmpeg_input: CameraInput = [i for i in ffmpeg.inputs if "audio" in i.roles][0]
    input_args = get_ffmpeg_arg_list(ffmpeg.global_args) + (
        parse_preset_input(ffmpeg_input.input_args, 1)
        or get_ffmpeg_arg_list(ffmpeg_input.input_args)
        or parse_preset_input(ffmpeg.input_args, 1)
        or get_ffmpeg_arg_list(ffmpeg.input_args)
    )
    return (
        [ffmpeg.ffmpeg_path, "-vn", "-threads", "1"]
        + input_args
        + ["-i"]
        + [ffmpeg_input.path]
        + [
            "-threads",
            "1",
            "-f",
            f"{AUDIO_FORMAT}",
            "-ar",
            f"{AUDIO_SAMPLE_RATE}",
            "-ac",
            "1",
            "-y",
            "pipe:",
        ]
    )


class AudioProcessor(FrigateProcess):
    name = "frigate.audio_manager"

    def __init__(
        self,
        config: FrigateConfig,
        camera_metrics: DictProxy,
        embeddings_metrics: DataProcessorMetrics,
        stop_event: MpEvent,
    ):
        super().__init__(
            stop_event, PROCESS_PRIORITY_HIGH, name="frigate.audio_manager", daemon=True
        )

        self.camera_metrics = camera_metrics
        self.embeddings_metrics = embeddings_metrics
        self.config = config

    def spawn_if_needed(self, camera: CameraConfig) -> None:
        """Start an audio maintainer for the camera once everything it needs
        has arrived. Returning early leaves the camera for the next poll."""
        name = camera.name
        if name is None or name in self.audio_threads:
            return
        if not camera.enabled or not camera.audio.enabled:
            return
        # ffmpeg update may not have arrived yet
        if not any("audio" in i.roles for i in camera.ffmpeg.inputs):
            return
        # the camera maintainer creates metrics on its own poll of the same
        # add update and may not have gotten there yet
        metrics = self.camera_metrics.get(name)
        if metrics is None:
            return
        thread = AudioEventMaintainer(
            camera,
            self.config,
            metrics,
            self.embeddings_metrics,
            self.transcription_model_runner,
            self.stop_event,  # type: ignore[arg-type]
            self.genai_manager,
        )
        self.audio_threads[name] = thread
        thread.start()
        self.logger.info(f"Audio maintainer started for {name}")

    def __stop_audio_thread(self, camera: str) -> None:
        thread = self.audio_threads.pop(camera, None)
        if thread is None:
            return

        thread.stop()
        thread.join(10)
        if thread.is_alive():
            self.logger.warning(f"Audio maintainer thread for {camera} is still alive")
        else:
            self.logger.info(f"Audio maintainer stopped for {camera}")

    def run(self) -> None:
        self.pre_run_setup(self.config.logger)
        self.audio_threads: dict[str, AudioEventMaintainer] = {}

        threading.current_thread().name = "process:audio_manager"

        self.transcription_model_runner: AudioTranscriptionModelRunner | None = None
        self.genai_manager: Any = None

        if any(
            c.enabled_in_config and c.audio_transcription.enabled
            for c in self.config.cameras.values()
        ):
            if isinstance(
                self.config.audio_transcription.model, AudioTranscriptionModelEnum
            ):
                # AudioTranscriptionModelRunner.__init__ unconditionally fetches
                # sherpa-onnx or whisper weights, so only build it on the local path
                self.transcription_model_runner = AudioTranscriptionModelRunner(
                    self.config.audio_transcription.device or "AUTO",
                    self.config.audio_transcription.model_size,
                )
            else:
                # imported here rather than at module scope: frigate.genai pulls in
                # numpy, the provider SDKs, frigate.models, and the prompt builders,
                # and this process runs at PROCESS_PRIORITY_HIGH. built after the
                # fork because SDK clients hold sockets and TLS state that must not
                # cross it; clients themselves stay lazy behind the role property.
                from frigate.genai.manager import GenAIClientManager

                self.genai_manager = GenAIClientManager(self.config)

        config_subscriber = CameraConfigUpdateSubscriber(
            self.config,
            self.config.cameras,
            [
                CameraConfigUpdateEnum.add,
                CameraConfigUpdateEnum.audio,
                CameraConfigUpdateEnum.ffmpeg,
                CameraConfigUpdateEnum.remove,
            ],
        )

        for camera in self.config.cameras.values():
            self.spawn_if_needed(camera)

        self.logger.info(f"Audio processor started (pid: {self.pid})")

        # poll for newly added/removed cameras or cameras flipped to
        # audio.enabled at runtime
        while not self.stop_event.wait(timeout=1.0):
            updated_topics = config_subscriber.check_for_updates()

            # stop maintainers for removed cameras so their ffmpeg process is
            # torn down
            for removed_camera in updated_topics.get(
                CameraConfigUpdateEnum.remove.name, []
            ):
                self.__stop_audio_thread(removed_camera)

            for camera in self.config.cameras.values():
                self.spawn_if_needed(camera)

        config_subscriber.stop()

        for thread in self.audio_threads.values():
            thread.join(1)
            if thread.is_alive():
                self.logger.info(f"Waiting for thread {thread.name:s} to exit")
                thread.join(10)

        for thread in self.audio_threads.values():
            if thread.is_alive():
                self.logger.warning(f"Thread {thread.name} is still alive")

        self.logger.info("Exiting audio processor")


class AudioEventMaintainer(threading.Thread):
    def __init__(
        self,
        camera: CameraConfig,
        config: FrigateConfig,
        metrics: CameraMetrics,
        embeddings_metrics: DataProcessorMetrics,
        audio_transcription_model_runner: AudioTranscriptionModelRunner | None,
        stop_event: threading.Event,
        genai_manager: Any = None,
    ) -> None:
        super().__init__(name=f"{camera.name}_audio_event_processor")

        self.config = config
        self.camera_config = camera
        # hold the metrics object rather than indexing the manager dict per
        # chunk, which costs an IPC round trip and breaks once the camera
        # maintainer pops the entry on removal
        self.metrics = metrics
        self.embeddings_metrics = embeddings_metrics
        self.stop_event = stop_event
        # per-camera stop signal so a single maintainer can be torn down at
        # runtime (e.g. on camera removal) without stopping the whole process
        self.camera_stop_event = threading.Event()
        self.detector = AudioTfl(
            stop_event,
            self.camera_config.audio.num_threads,
            self.camera_config.audio.labelmap,
        )
        self.shape = (int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE)),)
        self.chunk_size = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE * 2))
        self.logger = logging.getLogger(f"audio.{self.camera_config.name}")
        self.ffmpeg_cmd = get_ffmpeg_command(self.camera_config.ffmpeg)
        self.logpipe = LogPipe(f"ffmpeg.{self.camera_config.name}.audio")
        self.audio_listener: subprocess.Popen[Any] | None = None
        self.audio_transcription_model_runner = audio_transcription_model_runner
        self.genai_manager = genai_manager
        self.transcription_processor = None
        self.transcription_thread = None

        # create communication for audio detections
        self.requestor = InterProcessRequestor()
        self.config_subscriber = CameraConfigUpdateSubscriber(
            None,
            {str(self.camera_config.name): self.camera_config},
            [
                CameraConfigUpdateEnum.audio,
                CameraConfigUpdateEnum.enabled,
                CameraConfigUpdateEnum.audio_transcription,
            ],
        )
        self.detection_publisher = DetectionPublisher(DetectionTypeEnum.audio.value)

        if self.camera_config.audio_transcription.enabled and (
            self.audio_transcription_model_runner is not None
            or self.genai_manager is not None
        ):
            # init the transcription processor for this camera
            self.transcription_processor = AudioTranscriptionRealTimeProcessor(
                config=self.config,
                camera_config=self.camera_config,
                requestor=self.requestor,
                model_runner=self.audio_transcription_model_runner,
                metrics=self.embeddings_metrics,
                stop_event=self.stop_event,
                genai_manager=self.genai_manager,
            )

            self.transcription_thread = threading.Thread(
                target=self.transcription_processor.run,
                name=f"{self.camera_config.name}_transcription_processor",
                daemon=True,
            )
            self.transcription_thread.start()

        self.was_enabled = camera.enabled
        self.was_audio_enabled = camera.audio.enabled

    def detect_audio(self, audio: np.ndarray) -> None:
        if (
            not self.camera_config.audio.enabled
            or self.stop_event.is_set()
            or self.camera_stop_event.is_set()
        ):
            return

        audio_as_float: np.ndarray = audio.astype(np.float32)
        rms, dBFS = self.calculate_audio_levels(audio_as_float)

        self.metrics.audio_rms.value = rms
        self.metrics.audio_dBFS.value = dBFS

        audio_detections: list[tuple[str, float]] = []

        # only run audio detection when volume is above min_volume
        if rms >= self.camera_config.audio.min_volume:
            # create waveform relative to max range and look for detections
            waveform = (audio / AUDIO_MAX_BIT_RANGE).astype(np.float32)
            model_detections = self.detector.detect(waveform)

            for label, score, _ in model_detections:
                self.logger.debug(
                    f"{self.camera_config.name} heard {label} with a score of {score}"
                )

                if label not in self.camera_config.audio.listen:
                    continue

                if score > dict(
                    (self.camera_config.audio.filters or {}).get(label, {})
                ).get("threshold", 0.8):
                    audio_detections.append((label, score))

            # send audio detection data
            self.detection_publisher.publish(
                (
                    self.camera_config.name,
                    datetime.datetime.now().timestamp(),
                    dBFS,
                    [label for label, _ in audio_detections],
                )
            )

        # send audio activity update
        self.requestor.send_data(
            UPDATE_AUDIO_ACTIVITY,
            {self.camera_config.name: {"detections": audio_detections}},
        )

        # run audio transcription
        if self.transcription_processor is not None:
            if self.camera_config.audio_transcription.live_enabled:
                # process audio until we've reached the endpoint
                self.transcription_processor.process_audio(
                    {
                        "id": f"{self.camera_config.name}_audio",
                        "camera": self.camera_config.name,
                    },
                    audio,
                )
            else:
                self.transcription_processor.check_unload_model()

    def calculate_audio_levels(self, audio_as_float: np.ndarray) -> tuple[float, float]:
        # Calculate RMS (Root-Mean-Square) which represents the average signal amplitude
        # Note: np.float32 isn't serializable, we must use np.float64 to publish the message
        rms = np.sqrt(np.mean(np.absolute(np.square(audio_as_float))))

        # Transform RMS to dBFS (decibels relative to full scale)
        if rms > 0:
            dBFS = 20 * np.log10(np.abs(rms) / AUDIO_MAX_BIT_RANGE)
        else:
            dBFS = 0

        self.requestor.send_data(f"{self.camera_config.name}/audio/dBFS", float(dBFS))
        self.requestor.send_data(f"{self.camera_config.name}/audio/rms", float(rms))

        return float(rms), float(dBFS)

    def start_or_restart_ffmpeg(self) -> None:
        self.audio_listener = start_or_restart_ffmpeg(
            self.ffmpeg_cmd,
            self.logger,
            self.logpipe,
            self.chunk_size,
            self.audio_listener,
        )
        self.requestor.send_data(f"{self.camera_config.name}/status/audio", "online")

    def read_audio(self) -> None:
        def log_and_restart() -> None:
            if self.stop_event.is_set():
                return

            time.sleep(self.camera_config.ffmpeg.retry_interval)
            self.start_or_restart_ffmpeg()

        if self.audio_listener is None or self.audio_listener.stdout is None:
            log_and_restart()
            return

        try:
            chunk = self.audio_listener.stdout.read(self.chunk_size)

            if not chunk:
                if self.audio_listener.poll() is not None:
                    self.requestor.send_data(
                        f"{self.camera_config.name}/status/audio", "offline"
                    )
                    self.logger.error("ffmpeg process is not running, restarting...")
                    log_and_restart()
                    return

                return

            audio = np.frombuffer(chunk, dtype=np.int16)
            self.detect_audio(audio)
        except Exception as e:
            self.logger.error(f"Error reading audio data from ffmpeg process: {e}")
            log_and_restart()

    def stop(self) -> None:
        """Signal this maintainer to exit its run loop and clean up."""
        self.camera_stop_event.set()

    def run(self) -> None:
        if self.camera_config.enabled:
            self.start_or_restart_ffmpeg()

        while not self.stop_event.is_set() and not self.camera_stop_event.is_set():
            # check if there is an updated config
            updated_topics = self.config_subscriber.check_for_updates()

            if CameraConfigUpdateEnum.audio.name in updated_topics:
                self.detector.update_labelmap(self.camera_config.audio.labelmap)

            enabled = self.camera_config.enabled
            if enabled != self.was_enabled:
                if enabled:
                    self.logger.debug(
                        f"Enabling audio detections for {self.camera_config.name}"
                    )
                    self.start_or_restart_ffmpeg()
                else:
                    self.requestor.send_data(
                        f"{self.camera_config.name}/status/audio", "disabled"
                    )
                    self.logger.debug(
                        f"Disabling audio detections for {self.camera_config.name}, ending events"
                    )
                    self.requestor.send_data(
                        EXPIRE_AUDIO_ACTIVITY, self.camera_config.name
                    )

                    if self.audio_listener:
                        stop_ffmpeg(self.audio_listener, self.logger)

                    self.audio_listener = None
                self.was_enabled = enabled
                continue

            if not enabled:
                time.sleep(0.1)
                continue

            audio_enabled = self.camera_config.audio.enabled
            if audio_enabled != self.was_audio_enabled:
                if not audio_enabled:
                    self.logger.debug(
                        f"Disabling audio detections for {self.camera_config.name}, ending events"
                    )
                    self.requestor.send_data(
                        EXPIRE_AUDIO_ACTIVITY, self.camera_config.name
                    )
                self.was_audio_enabled = audio_enabled

            self.read_audio()

        if self.audio_listener:
            stop_ffmpeg(self.audio_listener, self.logger)
        if self.transcription_thread:
            self.transcription_thread.join(timeout=2)
            if self.transcription_thread.is_alive():
                self.logger.warning(
                    f"Audio transcription thread {self.transcription_thread.name} is still alive"
                )
        self.logpipe.close()
        self.requestor.stop()
        self.config_subscriber.stop()
        self.detection_publisher.stop()


class AudioTfl:
    def __init__(
        self,
        stop_event: threading.Event,
        num_threads: int = 2,
        labelmap: dict[int, str] | None = None,
    ) -> None:
        self.stop_event = stop_event
        self.num_threads = num_threads
        self._default_labels = load_labels("/audio-labelmap.txt", prefill=521)
        self.labels: dict[int, str] = {}
        self.update_labelmap(labelmap or {})
        # Suppress TFLite delegate creation messages that bypass Python logging
        with suppress_stderr_during("tflite_interpreter_init"):
            self.interpreter = Interpreter(
                model_path="/cpu_audio_model.tflite",
                num_threads=self.num_threads,
            )
            self.interpreter.allocate_tensors()

        self.tensor_input_details = self.interpreter.get_input_details()
        self.tensor_output_details = self.interpreter.get_output_details()

    def update_labelmap(self, labelmap: dict[int, str]) -> None:
        """Merge configured label overrides into the default audio labelmap."""
        self.labels = {**self._default_labels, **labelmap}

    def _detect_raw(self, tensor_input: np.ndarray) -> np.ndarray:
        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input)
        self.interpreter.invoke()
        detections = np.zeros((20, 6), np.float32)

        res = self.interpreter.get_tensor(self.tensor_output_details[0]["index"])[0]
        non_zero_indices = res > 0
        class_ids = np.argpartition(-res, 20)[:20]
        class_ids = class_ids[np.argsort(-res[class_ids])]
        class_ids = class_ids[non_zero_indices[class_ids]]
        scores = res[class_ids]
        boxes = np.full((scores.shape[0], 4), -1, np.float32)
        count = len(scores)

        for i in range(count):
            if scores[i] < AUDIO_MIN_CONFIDENCE or i == 20:
                break
            detections[i] = [
                class_ids[i],
                float(scores[i]),
                boxes[i][0],
                boxes[i][1],
                boxes[i][2],
                boxes[i][3],
            ]

        return detections

    def detect(
        self, tensor_input: np.ndarray, threshold: float = AUDIO_MIN_CONFIDENCE
    ) -> list[tuple[str, float, tuple[float, float, float, float]]]:
        detections: list[tuple[str, float, tuple[float, float, float, float]]] = []

        if self.stop_event.is_set():
            return detections

        raw_detections = self._detect_raw(tensor_input)

        detected_labels: set[str] = set()

        for d in raw_detections:
            if d[1] < threshold:
                break
            label = self.labels[int(d[0])]
            if label in detected_labels:
                continue
            detected_labels.add(label)
            detections.append((label, float(d[1]), (d[2], d[3], d[4], d[5])))
        return detections