From 87a4d1d15b201e0d6f618f6f94afd6bbc0a9e1f3 Mon Sep 17 00:00:00 2001
From: Nick Mowen <nickmowen213@gmail.com>
Date: Sat, 17 Jun 2023 17:14:43 -0600
Subject: [PATCH] Add audio detection process keypoints

---
 frigate/app.py                         | 40 ++++++++++++++++++++++--
 frigate/detectors/plugins/audio_tfl.py |  2 +-
 frigate/events/audio.py                | 43 ++++++++++++++++++++++++++
 3 files changed, 81 insertions(+), 4 deletions(-)

diff --git a/frigate/app.py b/frigate/app.py
index d37372538..fec751939 100644
--- a/frigate/app.py
+++ b/frigate/app.py
@@ -30,6 +30,8 @@ from frigate.const import (
     MODEL_CACHE_DIR,
     RECORD_DIR,
 )
+from frigate.detectors.plugins.audio_tfl import AudioDetectorConfig
+from frigate.events.audio import AudioDetectProcess
 from frigate.events.cleanup import EventCleanup
 from frigate.events.external import ExternalEventProcessor
 from frigate.events.maintainer import EventProcessor
@@ -60,6 +62,7 @@ class FrigateApp:
         self.detectors: dict[str, ObjectDetectProcess] = {}
         self.detection_out_events: dict[str, MpEvent] = {}
         self.detection_shms: list[mp.shared_memory.SharedMemory] = []
+        self.audio_queue: Queue = mp.Queue()
         self.log_queue: Queue = mp.Queue()
         self.plus_api = PlusApi()
         self.camera_metrics: dict[str, CameraMetricsTypes] = {}
@@ -287,6 +290,8 @@ class FrigateApp:
         )
 
     def start_detectors(self) -> None:
+        audio_enabled = any(c.audio.enabled for c in self.config.cameras.items())
+
         for name in self.config.cameras.keys():
             self.detection_out_events[name] = mp.Event()
 
@@ -315,6 +320,33 @@ class FrigateApp:
             self.detection_shms.append(shm_in)
             self.detection_shms.append(shm_out)
 
+
+        if audio_enabled:
+            try:
+                shm_in_audio = mp.shared_memory.SharedMemory(
+                    name=f"{name}-audio",
+                    create=True,
+                    size=int(
+                        round(
+                            self.config.audio_model.duration
+                            * self.config.audio_model.sample_rate
+                        )
+                    )
+                    * 4,  # stored as float32, so 4 bytes per sample
+                )
+            except FileExistsError:
+                shm_in_audio = mp.shared_memory.SharedMemory(name=f"{name}-audio")
+
+            try:
+                shm_out_audio = mp.shared_memory.SharedMemory(
+                    name=f"out-{name}-audio", create=True, size=20 * 6 * 4
+                )
+            except FileExistsError:
+                shm_out_audio = mp.shared_memory.SharedMemory(
+                    name=f"out-{name}-audio"
+                )
+
+
         for name, detector_config in self.config.detectors.items():
             self.detectors[name] = ObjectDetectProcess(
                 name,
@@ -323,10 +355,12 @@ class FrigateApp:
                 detector_config,
             )
 
-        if any(c.audio.enabled for c in self.config.cameras.items()):
-            self.detectors[AUDIO_DETECTOR] = ObjectDetectProcess(
+        if audio_enabled:
+            self.detectors[AUDIO_DETECTOR] = AudioDetectProcess(
                 AUDIO_DETECTOR,
-                
+                self.audio_queue,
+                self.detection_out_events,
+                AudioDetectorConfig
             )
 
     def start_detected_frames_processor(self) -> None:
diff --git a/frigate/detectors/plugins/audio_tfl.py b/frigate/detectors/plugins/audio_tfl.py
index cc9252417..3f24b7790 100644
--- a/frigate/detectors/plugins/audio_tfl.py
+++ b/frigate/detectors/plugins/audio_tfl.py
@@ -22,7 +22,7 @@ class AudioDetectorConfig(BaseDetectorConfig):
     type: Literal[DETECTOR_KEY]
 
 
-class CpuTfl(DetectionApi):
+class AudioTfl(DetectionApi):
     type_key = DETECTOR_KEY
 
     def __init__(self, detector_config: AudioDetectorConfig):
diff --git a/frigate/events/audio.py b/frigate/events/audio.py
index 13975edac..43dac978e 100644
--- a/frigate/events/audio.py
+++ b/frigate/events/audio.py
@@ -36,6 +36,49 @@ def listen_to_audio(config: FrigateConfig, event_queue: mp.Queue) -> None:
             AudioEventMaintainer(camera, stop_event)
 
 
+class AudioDetectProcess:
+    def __init__(
+        self,
+        name,
+        detection_queue,
+        out_events,
+    ):
+        self.name = name
+        self.out_events = out_events
+        self.detection_queue = detection_queue
+        self.detect_process = None
+        self.start_or_restart()
+
+    def stop(self):
+        # if the process has already exited on its own, just return
+        if self.detect_process and self.detect_process.exitcode:
+            return
+        self.detect_process.terminate()
+        logging.info("Waiting for detection process to exit gracefully...")
+        self.detect_process.join(timeout=30)
+        if self.detect_process.exitcode is None:
+            logging.info("Detection process didnt exit. Force killing...")
+            self.detect_process.kill()
+            self.detect_process.join()
+        logging.info("Detection process has exited...")
+
+    def start_or_restart(self):
+        self.detection_start.value = 0.0
+        if (self.detect_process is not None) and self.detect_process.is_alive():
+            self.stop()
+        self.detect_process = mp.Process(
+            target=run_detector,
+            name=f"detector:{self.name}",
+            args=(
+                self.name,
+                self.detection_queue,
+                self.out_events,
+            ),
+        )
+        self.detect_process.daemon = True
+        self.detect_process.start()
+
+
 class AudioEventMaintainer(threading.Thread):
     def __init__(self, camera: CameraConfig, stop_event: mp.Event) -> None:
         threading.Thread.__init__(self)