From 38782fe7183299930e8cabe3cc81a986ba4c770a Mon Sep 17 00:00:00 2001 From: JP Verdejo Date: Mon, 26 Jun 2023 19:33:11 +0000 Subject: [PATCH] Expose dBFS when doing audio analysis --- frigate/app.py | 1 + frigate/events/audio.py | 15 ++++++++++++++- frigate/http.py | 35 +++++++++++++++++++++++++++++++++++ frigate/test/test_http.py | 10 ++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/frigate/app.py b/frigate/app.py index ccfbd4696..6c36ea98e 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -303,6 +303,7 @@ class FrigateApp: self.onvif_controller, self.external_event_processor, self.plus_api, + self.dispatcher, ) def init_onvif(self) -> None: diff --git a/frigate/events/audio.py b/frigate/events/audio.py index 4f40334d4..53ec23690 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -168,9 +168,22 @@ class AudioEventMaintainer(threading.Thread): if not self.feature_metrics[self.config.name]["audio_enabled"].value: return - waveform = (audio / AUDIO_MAX_BIT_RANGE).astype(np.float32) + audio_as_float = audio.astype(np.float32) + waveform = audio_as_float / AUDIO_MAX_BIT_RANGE model_detections = self.detector.detect(waveform) + # Calculate RMS (Root-Mean-Square) which represents the average signal amplitude + # Note: np.float32 isn't serializable, we must use np.float64 to publish the message + rms = np.sqrt(np.mean(np.absolute(audio_as_float**2))).astype(np.float64) + + # Transform RMS to dBFS (decibels relative to full scale) + dBFS = 20 * np.log10(np.abs(rms) / AUDIO_MAX_BIT_RANGE) + + requests.post( + f"http://127.0.0.1:5000/api/{self.config.name}/metadata", + json={"dBFS": dBFS, "rms": rms}, + ) + for label, score, _ in model_detections: if label not in self.config.audio.listen: continue diff --git a/frigate/http.py b/frigate/http.py index f3632a0cf..f6859a210 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -28,6 +28,7 @@ from peewee import DoesNotExist, SqliteDatabase, fn, operator from playhouse.shortcuts import model_to_dict from tzlocal import get_localzone_name +from frigate.comms.dispatcher import Dispatcher from frigate.config import FrigateConfig from frigate.const import CLIPS_DIR, MAX_SEGMENT_DURATION, RECORD_DIR from frigate.events.external import ExternalEventProcessor @@ -61,6 +62,7 @@ def create_app( onvif: OnvifController, external_processor: ExternalEventProcessor, plus_api: PlusApi, + dispatcher: Dispatcher, ): app = Flask(__name__) @@ -81,6 +83,7 @@ def create_app( app.onvif = onvif app.external_processor = external_processor app.plus_api = plus_api + app.dispatcher = dispatcher app.camera_error_image = None app.hwaccel_errors = [] @@ -1423,6 +1426,38 @@ def recording_clip(camera_name, start_ts, end_ts): return response +@bp.route("//metadata", methods=["POST"]) +def create_metadata_message(camera_name): + if not camera_name or not current_app.frigate_config.cameras.get(camera_name): + return jsonify( + {"success": False, "message": f"{camera_name} is not a valid camera."}, 404 + ) + + request_json = request.get_json(silent=True) + if request_json == {}: + return jsonify( + {"success": False, "message": "Metadata json cannot be empty."}, 404 + ) + + try: + current_app.dispatcher.publish( + "metadata", json.dumps(request_json), retain=False + ) + except Exception as e: + logger.error(f"The error is {e}") + return jsonify( + {"success": False, "message": f"An unknown error occurred: {e}"}, 404 + ) + + return jsonify( + { + "success": True, + "message": "Successfully published metadata message.", + }, + 200, + ) + + @bp.route("/vod//start//end/") @bp.route("/vod//start//end/") def vod_ts(camera_name, start_ts, end_ts): diff --git a/frigate/test/test_http.py b/frigate/test/test_http.py index 3557eccd3..f48505784 100644 --- a/frigate/test/test_http.py +++ b/frigate/test/test_http.py @@ -121,6 +121,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" id2 = "7890.random" @@ -157,6 +158,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" @@ -178,6 +180,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" bad_id = "654321.other" @@ -198,6 +201,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" @@ -220,6 +224,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" @@ -246,6 +251,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" sub_label = "sub" @@ -281,6 +287,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" sub_label = "sub" @@ -306,6 +313,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) with app.test_client() as client: @@ -323,6 +331,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) id = "123456.random" @@ -343,6 +352,7 @@ class TestHttp(unittest.TestCase): None, None, PlusApi(), + None, ) mock_stats.return_value = self.test_stats