Add audio labelmap grouping (#24004)

Allow audio classes to be grouped under a shared configured label.

Keep audio overrides separate from object labels and retain only the highest-scoring grouped detection.

Refs #23967

frigate/config/camera/audio.py

@@ -41,6 +41,11 @@ class AudioConfig(FrigateBaseModel):
         title="Listen types",
         description="List of audio event types to detect (for example: bark, fire_alarm, speech, yell).",
     )
+    labelmap: dict[int, str] = Field(
+        default_factory=dict,
+        title="Audio labelmap customization",
+        description="Overrides or remapping entries to merge into the standard audio labelmap.",
+    )
     filters: dict[str, AudioFilterConfig] | None = Field(
         None,
         title="Audio filters",

frigate/events/audio.py

@@ -210,7 +210,11 @@ class AudioEventMaintainer(threading.Thread):
         # per-camera stop signal so a single maintainer can be torn down at
         # runtime (e.g. on camera removal) without stopping the whole process
         self.camera_stop_event = threading.Event()
-        self.detector = AudioTfl(stop_event, self.camera_config.audio.num_threads)
+        self.detector = AudioTfl(
+            stop_event,
+            self.camera_config.audio.num_threads,
+            self.camera_config.audio.labelmap,
+        )
         self.shape = (int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE)),)
         self.chunk_size = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE * 2))
         self.logger = logging.getLogger(f"audio.{self.camera_config.name}")
@@ -392,7 +396,10 @@ class AudioEventMaintainer(threading.Thread):
 
         while not self.stop_event.is_set() and not self.camera_stop_event.is_set():
             # check if there is an updated config
-            self.config_subscriber.check_for_updates()
+            updated_topics = self.config_subscriber.check_for_updates()
+
+            if CameraConfigUpdateEnum.audio.name in updated_topics:
+                self.detector.update_labelmap(self.camera_config.audio.labelmap)
 
             enabled = self.camera_config.enabled
             if enabled != self.was_enabled:
@@ -451,10 +458,17 @@ class AudioEventMaintainer(threading.Thread):
 
 
 class AudioTfl:
-    def __init__(self, stop_event: threading.Event, num_threads: int = 2) -> None:
+    def __init__(
+        self,
+        stop_event: threading.Event,
+        num_threads: int = 2,
+        labelmap: dict[int, str] | None = None,
+    ) -> None:
         self.stop_event = stop_event
         self.num_threads = num_threads
-        self.labels = load_labels("/audio-labelmap.txt", prefill=521)
+        self._default_labels = load_labels("/audio-labelmap.txt", prefill=521)
+        self.labels: dict[int, str] = {}
+        self.update_labelmap(labelmap or {})
         # Suppress TFLite delegate creation messages that bypass Python logging
         with suppress_stderr_during("tflite_interpreter_init"):
             self.interpreter = Interpreter(
@@ -466,6 +480,10 @@ class AudioTfl:
         self.tensor_input_details = self.interpreter.get_input_details()
         self.tensor_output_details = self.interpreter.get_output_details()
 
+    def update_labelmap(self, labelmap: dict[int, str]) -> None:
+        """Merge configured label overrides into the default audio labelmap."""
+        self.labels = {**self._default_labels, **labelmap}
+
     def _detect_raw(self, tensor_input: np.ndarray) -> np.ndarray:
         self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input)
         self.interpreter.invoke()
@@ -504,10 +522,14 @@ class AudioTfl:
 
         raw_detections = self._detect_raw(tensor_input)
 
+        detected_labels: set[str] = set()
+
         for d in raw_detections:
             if d[1] < threshold:
                 break
-            detections.append(
-                (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5]))
-            )
+            label = self.labels[int(d[0])]
+            if label in detected_labels:
+                continue
+            detected_labels.add(label)
+            detections.append((label, float(d[1]), (d[2], d[3], d[4], d[5])))
         return detections

frigate/test/test_audio.py

@@ -0,0 +1,75 @@
+"""Tests for audio label mapping."""
+
+import threading
+import unittest
+from unittest.mock import Mock
+
+import numpy as np
+
+from frigate.events.audio import AudioTfl
+
+
+class TestAudioTfl(unittest.TestCase):
+    def setUp(self):
+        self.detector = AudioTfl.__new__(AudioTfl)
+        self.detector.stop_event = threading.Event()
+        self.detector._default_labels = {
+            69: "dog",
+            70: "bark",
+            75: "whimper_dog",
+            117: "dogs",
+        }
+
+    def test_update_labelmap_replaces_and_resets_overrides(self):
+        self.detector.update_labelmap({69: "dogs", 70: "dogs"})
+        assert self.detector.labels == {
+            69: "dogs",
+            70: "dogs",
+            75: "whimper_dog",
+            117: "dogs",
+        }
+
+        self.detector.update_labelmap({75: "whimper"})
+        assert self.detector.labels == {
+            69: "dog",
+            70: "bark",
+            75: "whimper",
+            117: "dogs",
+        }
+
+    def test_detect_returns_highest_scoring_detection_for_grouped_label(self):
+        self.detector.update_labelmap({69: "dogs", 70: "dogs", 75: "dogs"})
+        self.detector._detect_raw = Mock(
+            return_value=np.array(
+                [
+                    [117, 0.95, -1, -1, -1, -1],
+                    [70, 0.9, -1, -1, -1, -1],
+                    [69, 0.8, -1, -1, -1, -1],
+                    [75, 0.7, -1, -1, -1, -1],
+                ],
+                dtype=np.float32,
+            )
+        )
+
+        detections = self.detector.detect(np.array([], dtype=np.float32))
+
+        assert len(detections) == 1
+        assert detections[0][0] == "dogs"
+        self.assertAlmostEqual(detections[0][1], 0.95)
+
+    def test_each_dog_audio_label_maps_to_grouped_label(self):
+        self.detector.update_labelmap({69: "dogs", 70: "dogs", 75: "dogs"})
+
+        for class_id in (69, 70, 75):
+            with self.subTest(class_id=class_id):
+                self.detector._detect_raw = Mock(
+                    return_value=np.array(
+                        [[class_id, 0.9, -1, -1, -1, -1]], dtype=np.float32
+                    )
+                )
+
+                detections = self.detector.detect(np.array([], dtype=np.float32))
+
+                assert len(detections) == 1
+                assert detections[0][0] == "dogs"
+                self.assertAlmostEqual(detections[0][1], 0.9)

frigate/test/test_config.py

@@ -1154,6 +1154,28 @@ class TestConfig(unittest.TestCase):
         frigate_config = FrigateConfig(**config)
         assert frigate_config.model.merged_labelmap[7] == "truck"
 
+    def test_audio_labelmap_inheritance_is_separate_from_model_labelmap(self):
+        config = deep_merge(
+            {
+                "audio": {"labelmap": {69: "dogs", 70: "dogs"}},
+                "cameras": {
+                    "back": {
+                        "audio": {"labelmap": {75: "dogs"}},
+                    }
+                },
+            },
+            self.minimal,
+        )
+
+        frigate_config = FrigateConfig(**config)
+
+        assert frigate_config.cameras["back"].audio.labelmap == {
+            69: "dogs",
+            70: "dogs",
+            75: "dogs",
+        }
+        assert frigate_config.model.merged_labelmap[69] != "dogs"
+
     def test_default_labelmap_empty(self):
         config = {
             "mqtt": {"host": "mqtt"},