From 734eb07a23ac078f2bcfdf72f6facbc5f819d1bb Mon Sep 17 00:00:00 2001 From: Nick Mowen Date: Sun, 18 Jun 2023 17:11:11 -0600 Subject: [PATCH] Save event to db --- frigate/audio.py | 126 ----------------------------------- frigate/events/audio.py | 30 +++++++-- frigate/events/maintainer.py | 34 +++++++++- 3 files changed, 58 insertions(+), 132 deletions(-) delete mode 100644 frigate/audio.py diff --git a/frigate/audio.py b/frigate/audio.py deleted file mode 100644 index ebefcd5ce..000000000 --- a/frigate/audio.py +++ /dev/null @@ -1,126 +0,0 @@ -import datetime -import logging -import multiprocessing as mp -import queue -import random -import signal -import string -import threading - -import numpy as np -from setproctitle import setproctitle - -from frigate.config import CameraConfig, AudioModelConfig -from frigate.object_detection import RemoteObjectDetector -from frigate.util import listen, SharedMemoryFrameManager - - -logger = logging.getLogger(__name__) - - -def capture_audio( - name: str, - model_config: AudioModelConfig, - process_info, -): - stop_event = mp.Event() - - def receiveSignal(signalNumber, frame): - stop_event.set() - - signal.signal(signal.SIGTERM, receiveSignal) - signal.signal(signal.SIGINT, receiveSignal) - - threading.current_thread().name = f"capture:{name}" - setproctitle(f"frigate.capture:{name}") - listen() - - chunk_size = int(round(model_config.duration * model_config.sample_rate * 2)) - - key = f"{name}-audio" - - audio_queue = process_info["audio_queue"] - frame_manager = SharedMemoryFrameManager() - current_frame = mp.Value("d", 0.0) - - pipe = open(f"/tmp/{key}", "rb") - - while not stop_event.is_set(): - current_frame.value = datetime.datetime.now().timestamp() - frame_name = f"{key}{current_frame.value}" - frame_buffer = frame_manager.create(frame_name, chunk_size) - - try: - frame_buffer[:] = pipe.read(chunk_size) - except Exception as e: - continue - - # if the queue is full, skip this frame - if audio_queue.full(): - frame_manager.delete(frame_name) - continue - - # close the frame - frame_manager.close(frame_name) - - # add to the queue - audio_queue.put(current_frame.value) - - -def process_audio( - name: str, - camera_config: CameraConfig, - model_config: AudioModelConfig, - labelmap, - detection_queue: mp.Queue, - result_connection, - process_info, -): - stop_event = mp.Event() - - def receiveSignal(signalNumber, frame): - stop_event.set() - - signal.signal(signal.SIGTERM, receiveSignal) - signal.signal(signal.SIGINT, receiveSignal) - - threading.current_thread().name = f"process:{name}" - setproctitle(f"frigate.process:{name}") - listen() - - shape = (int(round(model_config.duration * model_config.sample_rate)),) - - key = f"{name}-audio" - - audio_queue = process_info["audio_queue"] - frame_manager = SharedMemoryFrameManager() - - detector = RemoteObjectDetector( - key, labelmap, detection_queue, result_connection, model_config - ) - - while not stop_event.is_set(): - try: - frame_time = audio_queue.get(True, 10) - except queue.Empty: - continue - - audio = frame_manager.get(f"{key}{frame_time}", shape, dtype=np.int16) - - if audio is None: - logger.info(f"{key}: audio {frame_time} is not in memory store.") - continue - - waveform = (audio / 32768.0).astype(np.float32) - model_detections = detector.detect(waveform) - - for label, score, _ in model_detections: - if label not in camera_config.objects.track: - continue - filters = camera_config.objects.filters.get(label) - if filters: - if score < filters.min_score: - continue - logger.info(f"{label}: {score}") - - frame_manager.close(f"{key}{frame_time}") diff --git a/frigate/events/audio.py b/frigate/events/audio.py index 68eda9558..36890c762 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -1,10 +1,13 @@ """Handle creating audio events.""" +import datetime import logging import multiprocessing as mp import numpy as np import os +import random import signal +import string import subprocess as sp import threading from types import FrameType @@ -20,6 +23,7 @@ from frigate.const import ( AUDIO_SAMPLE_RATE, CACHE_DIR, ) +from frigate.events.maintainer import EventTypeEnum from frigate.ffmpeg_presets import parse_preset_input from frigate.object_detection import load_labels from frigate.util import get_ffmpeg_arg_list, listen @@ -51,7 +55,7 @@ def listen_to_audio(config: FrigateConfig, event_queue: mp.Queue) -> None: for camera in config.cameras.values(): if camera.enabled and camera.audio.enabled: - AudioEventMaintainer(camera, stop_event).start() + AudioEventMaintainer(camera, event_queue, stop_event).start() class AudioTfl: @@ -110,10 +114,12 @@ class AudioTfl: class AudioEventMaintainer(threading.Thread): - def __init__(self, camera: CameraConfig, stop_event: mp.Event) -> None: + def __init__(self, camera: CameraConfig, event_queue: mp.Queue, stop_event: mp.Event) -> None: threading.Thread.__init__(self) self.name = f"{camera.name}_audio_event_processor" self.config = camera + self.queue = event_queue + self.detections: dict[dict[str, any]] = {} self.stop_event = stop_event self.detector = AudioTfl() self.shape = (int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE)),) @@ -140,8 +146,24 @@ class AudioEventMaintainer(threading.Thread): if label not in self.config.audio.listen: continue - logger.error(f"Detected audio: {label} with score {score}") - # TODO handle valid detect + self.handle_detection(label, score) + + def handle_detection(self, label: str, score: float) -> None: + if self.detections[label] is not None: + self.detections[label]["last_detection"] = datetime.datetime.now().timestamp() + else: + now = datetime.datetime.now().timestamp() + rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) + event_id = f"{now}-{rand_id}" + self.detections[label] = { + "id": event_id, + "label": label, + "camera": self.config.name, + "start_time": now, + "last_detection": now, + } + self.queue.put((EventTypeEnum.audio, "start", self.config.name, self.detections[label])) + def init_ffmpeg(self) -> None: try: diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index 264ab1142..20124376a 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) class EventTypeEnum(str, Enum): api = "api" - # audio = "audio" + audio = "audio" tracked_object = "tracked_object" @@ -90,6 +90,8 @@ class EventProcessor(threading.Thread): continue self.handle_object_detection(event_type, camera, event_data) + elif source_type == EventTypeEnum.audio: + self.handle_audio_detection(event_type, camera, event_data) elif source_type == EventTypeEnum.api: self.handle_external_detection(event_type, event_data) @@ -214,7 +216,35 @@ class EventProcessor(threading.Thread): del self.events_in_process[event_data["id"]] self.event_processed_queue.put((event_data["id"], camera)) - def handle_external_detection(self, type: str, event_data: Event): + def handle_audio_detection(self, type: str, event_data: Event) -> None: + if type == "new": + event = { + Event.id: event_data["id"], + Event.label: event_data["label"], + Event.camera: event_data["camera"], + Event.start_time: event_data["start_time"], + Event.thumbnail: None, + Event.has_clip: True, + Event.has_snapshot: True, + Event.zones: [], + Event.data: {}, + } + elif type == "end": + event = { + Event.id: event_data["id"], + Event.end_time: event_data["end_time"], + } + + ( + Event.insert(event) + .on_conflict( + conflict_target=[Event.id], + update=event, + ) + .execute() + ) + + def handle_external_detection(self, type: str, event_data: Event) -> None: if type == "new": event = { Event.id: event_data["id"],