From 61c62d468564b94b4cf6fac3e3014db4cdf89a07 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Wed, 6 Oct 2021 06:11:36 -0500 Subject: [PATCH 01/20] version tick --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d11f9359a..fb1a1447f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ default_target: amd64_frigate COMMIT_HASH := $(shell git log -1 --pretty=format:"%h"|tail -1) version: - echo "VERSION='0.9.4-$(COMMIT_HASH)'" > frigate/version.py + echo "VERSION='0.10.0-$(COMMIT_HASH)'" > frigate/version.py web: docker build --tag frigate-web --file docker/Dockerfile.web web/ From b63c56d8106d477682b301b03e823347a7747601 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 23 Oct 2021 16:18:13 -0500 Subject: [PATCH 02/20] only save recordings when an event is in progress --- frigate/events.py | 40 +++- frigate/http.py | 7 +- frigate/object_processing.py | 5 + frigate/record.py | 175 ++++++++++++------ migrations/005_make_end_time_nullable.py | 43 +++++ web/src/routes/Event.jsx | 6 +- web/src/routes/Events/components/tableRow.jsx | 4 +- 7 files changed, 215 insertions(+), 65 deletions(-) create mode 100644 migrations/005_make_end_time_nullable.py diff --git a/frigate/events.py b/frigate/events.py index 86fd2082b..2b7110dc7 100644 --- a/frigate/events.py +++ b/frigate/events.py @@ -30,6 +30,11 @@ class EventProcessor(threading.Thread): self.stop_event = stop_event def run(self): + # set an end_time on events without an end_time on startup + Event.update(end_time=Event.start_time + 30).where( + Event.end_time == None + ).execute() + while not self.stop_event.is_set(): try: event_type, camera, event_data = self.event_queue.get(timeout=10) @@ -38,14 +43,35 @@ class EventProcessor(threading.Thread): logger.debug(f"Event received: {event_type} {camera} {event_data['id']}") + event_config: EventsConfig = self.config.cameras[camera].record.events + if event_type == "start": self.events_in_process[event_data["id"]] = event_data - if event_type == "end": - event_config: EventsConfig = self.config.cameras[camera].record.events - + elif event_type == "update": + self.events_in_process[event_data["id"]] = event_data + # TODO: this will generate a lot of db activity possibly if event_data["has_clip"] or event_data["has_snapshot"]: - Event.create( + Event.replace( + id=event_data["id"], + label=event_data["label"], + camera=camera, + start_time=event_data["start_time"] - event_config.pre_capture, + end_time=None, + top_score=event_data["top_score"], + false_positive=event_data["false_positive"], + zones=list(event_data["entered_zones"]), + thumbnail=event_data["thumbnail"], + region=event_data["region"], + box=event_data["box"], + area=event_data["area"], + has_clip=event_data["has_clip"], + has_snapshot=event_data["has_snapshot"], + ).execute() + + elif event_type == "end": + if event_data["has_clip"] or event_data["has_snapshot"]: + Event.replace( id=event_data["id"], label=event_data["label"], camera=camera, @@ -60,11 +86,15 @@ class EventProcessor(threading.Thread): area=event_data["area"], has_clip=event_data["has_clip"], has_snapshot=event_data["has_snapshot"], - ) + ).execute() del self.events_in_process[event_data["id"]] self.event_processed_queue.put((event_data["id"], camera)) + # set an end_time on events without an end_time before exiting + Event.update(end_time=datetime.datetime.now().timestamp()).where( + Event.end_time == None + ).execute() logger.info(f"Exiting event processor...") diff --git a/frigate/http.py b/frigate/http.py index 1ef35dd92..32dcdb130 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -190,7 +190,7 @@ def event_snapshot(id): download = request.args.get("download", type=bool) jpg_bytes = None try: - event = Event.get(Event.id == id) + event = Event.get(Event.id == id, Event.end_time != None) if not event.has_snapshot: return "Snapshot not available", 404 # read snapshot from disk @@ -697,7 +697,10 @@ def vod_event(id): clip_path = os.path.join(CLIPS_DIR, f"{event.camera}-{id}.mp4") if not os.path.isfile(clip_path): - return vod_ts(event.camera, event.start_time, event.end_time) + end_ts = ( + datetime.now().timestamp() if event.end_time is None else event.end_time + ) + return vod_ts(event.camera, event.start_time, end_ts) duration = int((event.end_time - event.start_time) * 1000) return jsonify( diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 2597893d7..9a8ad8cc6 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -603,6 +603,8 @@ class TrackedObjectProcessor(threading.Thread): self.event_queue.put(("start", camera, obj.to_dict())) def update(camera, obj: TrackedObject, current_frame_time): + obj.has_snapshot = self.should_save_snapshot(camera, obj) + obj.has_clip = self.should_retain_recording(camera, obj) after = obj.to_dict() message = { "before": obj.previous, @@ -613,6 +615,9 @@ class TrackedObjectProcessor(threading.Thread): f"{self.topic_prefix}/events", json.dumps(message), retain=False ) obj.previous = after + self.event_queue.put( + ("update", camera, obj.to_dict(include_thumbnail=True)) + ) def end(camera, obj: TrackedObject, current_frame_time): # populate has_snapshot diff --git a/frigate/record.py b/frigate/record.py index b0fd64798..e2427e729 100644 --- a/frigate/record.py +++ b/frigate/record.py @@ -7,6 +7,7 @@ import shutil import string import subprocess as sp import threading +from collections import defaultdict from pathlib import Path import psutil @@ -45,7 +46,7 @@ class RecordingMaintainer(threading.Thread): self.stop_event = stop_event def move_files(self): - recordings = [ + cache_files = [ d for d in os.listdir(CACHE_DIR) if os.path.isfile(os.path.join(CACHE_DIR, d)) @@ -66,7 +67,9 @@ class RecordingMaintainer(threading.Thread): except: continue - for f in recordings: + # group recordings by camera + grouped_recordings = defaultdict(list) + for f in cache_files: # Skip files currently in use if f in files_in_use: continue @@ -76,58 +79,124 @@ class RecordingMaintainer(threading.Thread): camera, date = basename.rsplit("-", maxsplit=1) start_time = datetime.datetime.strptime(date, "%Y%m%d%H%M%S") - # Just delete files if recordings are turned off - if ( - not camera in self.config.cameras - or not self.config.cameras[camera].record.enabled - ): - Path(cache_path).unlink(missing_ok=True) - continue - - ffprobe_cmd = [ - "ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - f"{cache_path}", - ] - p = sp.run(ffprobe_cmd, capture_output=True) - if p.returncode == 0: - duration = float(p.stdout.decode().strip()) - end_time = start_time + datetime.timedelta(seconds=duration) - else: - logger.warning(f"Discarding a corrupt recording segment: {f}") - Path(cache_path).unlink(missing_ok=True) - continue - - directory = os.path.join( - RECORD_DIR, start_time.strftime("%Y-%m/%d/%H"), camera + grouped_recordings[camera].append( + { + "cache_path": cache_path, + "start_time": start_time, + } ) - if not os.path.exists(directory): - os.makedirs(directory) - - file_name = f"{start_time.strftime('%M.%S.mp4')}" - file_path = os.path.join(directory, file_name) - - # copy then delete is required when recordings are stored on some network drives - shutil.copyfile(cache_path, file_path) - os.remove(cache_path) - - rand_id = "".join( - random.choices(string.ascii_lowercase + string.digits, k=6) - ) - Recordings.create( - id=f"{start_time.timestamp()}-{rand_id}", - camera=camera, - path=file_path, - start_time=start_time.timestamp(), - end_time=end_time.timestamp(), - duration=duration, + for camera, recordings in grouped_recordings.items(): + # get all events with the end time after the start of the oldest cache file + # or with end_time None + events: Event = ( + Event.select() + .where( + Event.camera == camera, + (Event.end_time == None) + | (Event.end_time >= recordings[0]["start_time"]), + Event.has_clip, + ) + .order_by(Event.start_time) ) + for r in recordings: + cache_path = r["cache_path"] + start_time = r["start_time"] + + # Just delete files if recordings are turned off + if ( + not camera in self.config.cameras + or not self.config.cameras[camera].record.enabled + ): + Path(cache_path).unlink(missing_ok=True) + continue + + ffprobe_cmd = [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + f"{cache_path}", + ] + p = sp.run(ffprobe_cmd, capture_output=True) + if p.returncode == 0: + duration = float(p.stdout.decode().strip()) + end_time = start_time + datetime.timedelta(seconds=duration) + else: + logger.warning(f"Discarding a corrupt recording segment: {f}") + Path(cache_path).unlink(missing_ok=True) + continue + + # if cached file's start_time is earlier than the retain_days for the camera + if start_time <= ( + ( + datetime.datetime.now() + - datetime.timedelta( + days=self.config.cameras[camera].record.retain_days + ) + ) + ): + # if the cached segment overlaps with the events: + overlaps = False + for event in events: + # if the event starts in the future, stop checking events + # and let this recording segment expire + if event.start_time > end_time.timestamp(): + overlaps = False + break + + # if the event is in progress or ends after the recording starts, keep it + # and stop looking at events + if event.end_time is None or event.end_time >= start_time: + overlaps = True + break + + if overlaps: + # move from cache to recordings immediately + self.store_segment( + camera, + start_time, + end_time, + duration, + cache_path, + ) + # else retain_days includes this segment + else: + self.store_segment( + camera, start_time, end_time, duration, cache_path + ) + + if len(recordings) > 2: + # delete all cached files past the most recent 2 + to_remove = sorted(recordings, key=lambda i: i["start_time"])[:-2] + for f in to_remove: + Path(cache_path).unlink(missing_ok=True) + + def store_segment(self, camera, start_time, end_time, duration, cache_path): + directory = os.path.join(RECORD_DIR, start_time.strftime("%Y-%m/%d/%H"), camera) + + if not os.path.exists(directory): + os.makedirs(directory) + + file_name = f"{start_time.strftime('%M.%S.mp4')}" + file_path = os.path.join(directory, file_name) + + # copy then delete is required when recordings are stored on some network drives + shutil.copyfile(cache_path, file_path) + os.remove(cache_path) + + rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) + Recordings.create( + id=f"{start_time.timestamp()}-{rand_id}", + camera=camera, + path=file_path, + start_time=start_time.timestamp(), + end_time=end_time.timestamp(), + duration=duration, + ) def run(self): # Check for new files every 5 seconds @@ -231,9 +300,9 @@ class RecordingCleanup(threading.Thread): keep = False break - # if the event ends after the recording starts, keep it + # if the event is in progress or ends after the recording starts, keep it # and stop looking at events - if event.end_time >= recording.start_time: + if event.end_time is None or event.end_time >= recording.start_time: keep = True break diff --git a/migrations/005_make_end_time_nullable.py b/migrations/005_make_end_time_nullable.py new file mode 100644 index 000000000..5c8cf6505 --- /dev/null +++ b/migrations/005_make_end_time_nullable.py @@ -0,0 +1,43 @@ +"""Peewee migrations -- 004_add_bbox_region_area.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import datetime as dt +import peewee as pw +from playhouse.sqlite_ext import * +from decimal import ROUND_HALF_EVEN +from frigate.models import Event + +try: + import playhouse.postgres_ext as pw_pext +except ImportError: + pass + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + migrator.drop_not_null(Event, "end_time") + + +def rollback(migrator, database, fake=False, **kwargs): + pass diff --git a/web/src/routes/Event.jsx b/web/src/routes/Event.jsx index d69a882e0..4a371a9d0 100644 --- a/web/src/routes/Event.jsx +++ b/web/src/routes/Event.jsx @@ -99,7 +99,7 @@ export default function Event({ eventId, close, scrollRef }) { } const startime = new Date(data.start_time * 1000); - const endtime = new Date(data.end_time * 1000); + const endtime = data.end_time ? new Date(data.end_time * 1000) : null; return (
@@ -155,7 +155,7 @@ export default function Event({ eventId, close, scrollRef }) { Timeframe - {startime.toLocaleString()} – {endtime.toLocaleString()} + {startime.toLocaleString()}{endtime === null ? ` – ${endtime.toLocaleString()}`:''} @@ -186,7 +186,7 @@ export default function Event({ eventId, close, scrollRef }) { }, ], poster: data.has_snapshot - ? `${apiHost}/clips/${data.camera}-${eventId}.jpg` + ? `${apiHost}/api/events/${eventId}/snapshot.jpg` : `data:image/jpeg;base64,${data.thumbnail}`, }} seekOptions={{ forward: 10, back: 5 }} diff --git a/web/src/routes/Events/components/tableRow.jsx b/web/src/routes/Events/components/tableRow.jsx index 262f3408a..f358153b2 100644 --- a/web/src/routes/Events/components/tableRow.jsx +++ b/web/src/routes/Events/components/tableRow.jsx @@ -42,7 +42,7 @@ const EventsRow = memo( ); const start = new Date(parseInt(startTime * 1000, 10)); - const end = new Date(parseInt(endTime * 1000, 10)); + const end = endTime ? new Date(parseInt(endTime * 1000, 10)) : null; return ( @@ -102,7 +102,7 @@ const EventsRow = memo( {start.toLocaleDateString()} {start.toLocaleTimeString()} - {end.toLocaleTimeString()} + {end === null ? 'In progress' : end.toLocaleTimeString()} {viewEvent === id ? ( From 8f101ccca81daf4ce01cd2c3ddbf44cac80310a6 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 30 Oct 2021 07:24:26 -0500 Subject: [PATCH 03/20] improve box merging and keep tracking --- frigate/test/test_reduce_boxes.py | 27 ++++++++++++++ frigate/util.py | 2 +- frigate/video.py | 58 ++++++++++++++----------------- 3 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 frigate/test/test_reduce_boxes.py diff --git a/frigate/test/test_reduce_boxes.py b/frigate/test/test_reduce_boxes.py new file mode 100644 index 000000000..db3803162 --- /dev/null +++ b/frigate/test/test_reduce_boxes.py @@ -0,0 +1,27 @@ +import cv2 +import numpy as np +from unittest import TestCase, main +from frigate.video import box_overlaps, reduce_boxes + + +class TestBoxOverlaps(TestCase): + def test_overlap(self): + assert box_overlaps((100, 100, 200, 200), (50, 50, 150, 150)) + + def test_overlap_2(self): + assert box_overlaps((50, 50, 150, 150), (100, 100, 200, 200)) + + def test_no_overlap(self): + assert not box_overlaps((100, 100, 200, 200), (250, 250, 350, 350)) + + +class TestReduceBoxes(TestCase): + def test_cluster(self): + clusters = reduce_boxes( + [(144, 290, 221, 459), (225, 178, 426, 341), (343, 105, 584, 250)] + ) + assert len(clusters) == 2 + + +if __name__ == "__main__": + main(verbosity=2) diff --git a/frigate/util.py b/frigate/util.py index 24a75b775..36848f0da 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -191,7 +191,7 @@ def draw_box_with_label( def calculate_region(frame_shape, xmin, ymin, xmax, ymax, multiplier=2): # size is the longest edge and divisible by 4 - size = int(max(xmax - xmin, ymax - ymin) // 4 * 4 * multiplier) + size = int((max(xmax - xmin, ymax - ymin) * multiplier) // 4 * 4) # dont go any smaller than 300 if size < 300: size = 300 diff --git a/frigate/video.py b/frigate/video.py index 3108db4a3..2e46bc590 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -379,26 +379,37 @@ def track_camera( logger.info(f"{name}: exiting subprocess") +def box_overlaps(b1, b2): + if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]: + return False + return True + + def reduce_boxes(boxes): - if len(boxes) == 0: - return [] - reduced_boxes = cv2.groupRectangles( - [list(b) for b in itertools.chain(boxes, boxes)], 1, 0.2 - )[0] - return [tuple(b) for b in reduced_boxes] + clusters = [] + + for box in boxes: + matched = 0 + for cluster in clusters: + if box_overlaps(box, cluster): + matched = 1 + cluster[0] = min(cluster[0], box[0]) + cluster[1] = min(cluster[1], box[1]) + cluster[2] = max(cluster[2], box[2]) + cluster[3] = max(cluster[3], box[3]) + + if not matched: + clusters.append(list(box)) + + return [tuple(c) for c in clusters] -# modified from https://stackoverflow.com/a/40795835 def intersects_any(box_a, boxes): for box in boxes: - if ( - box_a[2] < box[0] - or box_a[0] > box[2] - or box_a[1] > box[3] - or box_a[3] < box[1] - ): + if box_overlaps(box_a, box): continue return True + return False def detect( @@ -489,9 +500,7 @@ def process_frames( # only get the tracked object boxes that intersect with motion tracked_object_boxes = [ - obj["box"] - for obj in object_tracker.tracked_objects.values() - if intersects_any(obj["box"], motion_boxes) + obj["box"] for obj in object_tracker.tracked_objects.values() ] # combine motion boxes with known locations of existing objects @@ -503,15 +512,6 @@ def process_frames( for a in combined_boxes ] - # combine overlapping regions - combined_regions = reduce_boxes(regions) - - # re-compute regions - regions = [ - calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.0) - for a in combined_regions - ] - # resize regions and detect detections = [] for region in regions: @@ -582,14 +582,8 @@ def process_frames( if refining: refine_count += 1 - # Limit to the detections overlapping with motion areas - # to avoid picking up stationary background objects - detections_with_motion = [ - d for d in detections if intersects_any(d[2], motion_boxes) - ] - # now that we have refined our detections, we need to track objects - object_tracker.match_and_update(frame_time, detections_with_motion) + object_tracker.match_and_update(frame_time, detections) # add to the queue if not full if detected_objects_queue.full(): From d17bd74c9ac86311a3e52be5e145932de35d78ff Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 30 Oct 2021 14:01:31 -0500 Subject: [PATCH 04/20] reduce detection rate for stationary objects --- frigate/objects.py | 10 +++++++++- frigate/video.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/frigate/objects.py b/frigate/objects.py index 4da7f12d4..dc2b1b55e 100644 --- a/frigate/objects.py +++ b/frigate/objects.py @@ -13,7 +13,7 @@ import numpy as np from scipy.spatial import distance as dist from frigate.config import DetectConfig -from frigate.util import draw_box_with_label +from frigate.util import intersection_over_union class ObjectTracker: @@ -27,6 +27,7 @@ class ObjectTracker: id = f"{obj['frame_time']}-{rand_id}" obj["id"] = id obj["start_time"] = obj["frame_time"] + obj["motionless_count"] = 0 self.tracked_objects[id] = obj self.disappeared[id] = 0 @@ -36,6 +37,13 @@ class ObjectTracker: def update(self, id, new_obj): self.disappeared[id] = 0 + if ( + intersection_over_union(self.tracked_objects[id]["box"], new_obj["box"]) + > 0.9 + ): + self.tracked_objects[id]["motionless_count"] += 1 + else: + self.tracked_objects[id]["motionless_count"] = 0 self.tracked_objects[id].update(new_obj) def match_and_update(self, frame_time, new_objects): diff --git a/frigate/video.py b/frigate/video.py index 2e46bc590..0d6bf201e 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -498,9 +498,21 @@ def process_frames( # look for motion motion_boxes = motion_detector.detect(frame) - # only get the tracked object boxes that intersect with motion + # get stationary object ids + # check every 10th frame for stationary objects + stationary_object_ids = [ + obj["id"] + for obj in object_tracker.tracked_objects.values() + if obj["motionless_count"] >= 10 + and obj["motionless_count"] % 10 != 0 + and object_tracker.disappeared[obj["id"]] == 0 + ] + + # get tracked object boxes that aren't stationary tracked_object_boxes = [ - obj["box"] for obj in object_tracker.tracked_objects.values() + obj["box"] + for obj in object_tracker.tracked_objects.values() + if not obj["id"] in stationary_object_ids ] # combine motion boxes with known locations of existing objects @@ -513,7 +525,18 @@ def process_frames( ] # resize regions and detect - detections = [] + # seed with stationary objects + detections = [ + ( + obj["label"], + obj["score"], + obj["box"], + obj["area"], + obj["region"], + ) + for obj in object_tracker.tracked_objects.values() + if obj["id"] in stationary_object_ids + ] for region in regions: detections.extend( detect( From 24cc63d6d395b523a5f09f13aca585f2926dc50f Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 31 Oct 2021 11:12:44 -0500 Subject: [PATCH 05/20] drop high overlap detections --- frigate/video.py | 49 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/frigate/video.py b/frigate/video.py index 0d6bf201e..9c4ba9e61 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -3,16 +3,16 @@ import itertools import logging import multiprocessing as mp import queue -import subprocess as sp import signal +import subprocess as sp import threading import time from collections import defaultdict -from setproctitle import setproctitle from typing import Dict, List -from cv2 import cv2 import numpy as np +from cv2 import cv2 +from setproctitle import setproctitle from frigate.config import CameraConfig from frigate.edgetpu import RemoteObjectDetector @@ -23,8 +23,11 @@ from frigate.util import ( EventsPerSecond, FrameManager, SharedMemoryFrameManager, + area, calculate_region, clipped, + intersection, + intersection_over_union, listen, yuv_region_2_rgb, ) @@ -605,8 +608,46 @@ def process_frames( if refining: refine_count += 1 + ## drop detections that overlap too much + consolidated_detections = [] + # group by name + detected_object_groups = defaultdict(lambda: []) + for detection in detections: + detected_object_groups[detection[0]].append(detection) + + # loop over detections grouped by label + for group in detected_object_groups.values(): + # if the group only has 1 item, skip + if len(group) == 1: + consolidated_detections.append(group[0]) + continue + + # sort smallest to largest by area + sorted_by_area = sorted(group, key=lambda g: g[3]) + + for current_detection_idx in range(0, len(sorted_by_area)): + current_detection = sorted_by_area[current_detection_idx][2] + overlap = 0 + for to_check_idx in range( + min(current_detection_idx + 1, len(sorted_by_area)), + len(sorted_by_area), + ): + to_check = sorted_by_area[to_check_idx][2] + # if 90% of smaller detection is inside of another detection, consolidate + if ( + area(intersection(current_detection, to_check)) + / area(current_detection) + > 0.9 + ): + overlap = 1 + break + if overlap == 0: + consolidated_detections.append( + sorted_by_area[current_detection_idx] + ) + # now that we have refined our detections, we need to track objects - object_tracker.match_and_update(frame_time, detections) + object_tracker.match_and_update(frame_time, consolidated_detections) # add to the queue if not full if detected_objects_queue.full(): From 5bec438f9c91be4268581401206110db36fb385e Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 31 Oct 2021 11:48:49 -0500 Subject: [PATCH 06/20] config option for stationary detection interval --- docs/docs/configuration/index.md | 2 ++ docs/docs/configuration/record.md | 2 +- frigate/config.py | 8 ++++++++ frigate/video.py | 9 ++++++--- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/docs/configuration/index.md b/docs/docs/configuration/index.md index 9461ec6bd..89eb9ee4a 100644 --- a/docs/docs/configuration/index.md +++ b/docs/docs/configuration/index.md @@ -159,6 +159,8 @@ detect: enabled: True # Optional: Number of frames without a detection before frigate considers an object to be gone. (default: 5x the frame rate) max_disappeared: 25 + # Optional: Frequency for running detection on stationary objects (default: 10x the frame rate) + stationary_interval: 50 # Optional: Object configuration # NOTE: Can be overridden at the camera level diff --git a/docs/docs/configuration/record.md b/docs/docs/configuration/record.md index 23189d1af..d59530e07 100644 --- a/docs/docs/configuration/record.md +++ b/docs/docs/configuration/record.md @@ -22,4 +22,4 @@ record: This configuration will retain recording segments that overlap with events for 10 days. Because multiple events can reference the same recording segments, this avoids storing duplicate footage for overlapping events and reduces overall storage needs. -When `retain_days` is set to `0`, events will have up to `max_seconds` (defaults to 5 minutes) of recordings retained. Increasing `retain_days` to `1` will allow events to exceed the `max_seconds` limitation of up to 1 day. +When `retain_days` is set to `0`, segments will be deleted from the cache if no events are in progress diff --git a/frigate/config.py b/frigate/config.py index 6ff88dc00..e3db81a68 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -162,6 +162,9 @@ class DetectConfig(FrigateBaseModel): max_disappeared: Optional[int] = Field( title="Maximum number of frames the object can dissapear before detection ends." ) + stationary_interval: Optional[int] = Field( + title="Frame interval for checking stationary objects." + ) class FilterConfig(FrigateBaseModel): @@ -745,6 +748,11 @@ class FrigateConfig(FrigateBaseModel): if camera_config.detect.max_disappeared is None: camera_config.detect.max_disappeared = max_disappeared + # Default stationary_interval configuration + stationary_interval = camera_config.detect.fps * 10 + if camera_config.detect.stationary_interval is None: + camera_config.detect.stationary_interval = stationary_interval + # FFMPEG input substitution for input in camera_config.ffmpeg.inputs: input.path = input.path.format(**FRIGATE_ENV_VARS) diff --git a/frigate/video.py b/frigate/video.py index 9c4ba9e61..8a66f9f6c 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -14,7 +14,7 @@ import numpy as np from cv2 import cv2 from setproctitle import setproctitle -from frigate.config import CameraConfig +from frigate.config import CameraConfig, DetectConfig from frigate.edgetpu import RemoteObjectDetector from frigate.log import LogPipe from frigate.motion import MotionDetector @@ -367,6 +367,7 @@ def track_camera( frame_queue, frame_shape, model_shape, + config.detect, frame_manager, motion_detector, object_detector, @@ -448,6 +449,7 @@ def process_frames( frame_queue: mp.Queue, frame_shape, model_shape, + detect_config: DetectConfig, frame_manager: FrameManager, motion_detector: MotionDetector, object_detector: RemoteObjectDetector, @@ -502,12 +504,13 @@ def process_frames( motion_boxes = motion_detector.detect(frame) # get stationary object ids - # check every 10th frame for stationary objects + # check every Nth frame for stationary objects + # disappeared objects are not stationary stationary_object_ids = [ obj["id"] for obj in object_tracker.tracked_objects.values() if obj["motionless_count"] >= 10 - and obj["motionless_count"] % 10 != 0 + and obj["motionless_count"] % detect_config.stationary_interval != 0 and object_tracker.disappeared[obj["id"]] == 0 ] From 58117e2a3e3df0574477e0fe1bc667e77bffc3f5 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Thu, 4 Nov 2021 09:25:17 -0500 Subject: [PATCH 07/20] check for overlapping motion boxes --- frigate/video.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frigate/video.py b/frigate/video.py index 8a66f9f6c..774b6fc91 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -411,8 +411,7 @@ def reduce_boxes(boxes): def intersects_any(box_a, boxes): for box in boxes: if box_overlaps(box_a, box): - continue - return True + return True return False @@ -506,12 +505,18 @@ def process_frames( # get stationary object ids # check every Nth frame for stationary objects # disappeared objects are not stationary + # also check for overlapping motion boxes stationary_object_ids = [ obj["id"] for obj in object_tracker.tracked_objects.values() + # if there hasn't been motion for 10 frames if obj["motionless_count"] >= 10 + # and it isn't due for a periodic check and obj["motionless_count"] % detect_config.stationary_interval != 0 + # and it hasn't disappeared and object_tracker.disappeared[obj["id"]] == 0 + # and it doesn't overlap with any current motion boxes + and not intersects_any(obj["box"], motion_boxes) ] # get tracked object boxes that aren't stationary From 9dc6c423b7f70ee0fb90aa35a58ce86ef9d48cc3 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Thu, 4 Nov 2021 11:01:12 -0500 Subject: [PATCH 08/20] improve contrast --- frigate/motion.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frigate/motion.py b/frigate/motion.py index 3b9eb20bb..84f05b9e1 100644 --- a/frigate/motion.py +++ b/frigate/motion.py @@ -36,7 +36,13 @@ class MotionDetector: interpolation=cv2.INTER_LINEAR, ) - # TODO: can I improve the contrast of the grayscale image here? + # Improve contrast + minval = np.percentile(resized_frame, 5) + maxval = np.percentile(resized_frame, 95) + resized_frame = np.clip(resized_frame, minval, maxval) + resized_frame = (((resized_frame - minval) / (maxval - minval)) * 255).astype( + np.uint8 + ) # convert to grayscale # resized_frame = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY) From 57dcb29f8b1987e33a6a90a135c37bfc67ff7aeb Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Thu, 4 Nov 2021 11:57:26 -0500 Subject: [PATCH 09/20] consolidate regions --- frigate/video.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frigate/video.py b/frigate/video.py index 774b6fc91..7f3659144 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -11,7 +11,7 @@ from collections import defaultdict from typing import Dict, List import numpy as np -from cv2 import cv2 +from cv2 import cv2, reduce from setproctitle import setproctitle from frigate.config import CameraConfig, DetectConfig @@ -389,13 +389,13 @@ def box_overlaps(b1, b2): return True -def reduce_boxes(boxes): +def reduce_boxes(boxes, iou_threshold=0.0): clusters = [] for box in boxes: matched = 0 for cluster in clusters: - if box_overlaps(box, cluster): + if intersection_over_union(box, cluster) > iou_threshold: matched = 1 cluster[0] = min(cluster[0], box[0]) cluster[1] = min(cluster[1], box[1]) @@ -535,6 +535,12 @@ def process_frames( for a in combined_boxes ] + # consolidate regions with heavy overlap + regions = [ + calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.0) + for a in reduce_boxes(regions, 0.4) + ] + # resize regions and detect # seed with stationary objects detections = [ From ff667b019a5659c89fc73e2209ab1cc24b1bdc2e Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Thu, 4 Nov 2021 11:58:45 -0500 Subject: [PATCH 10/20] remove min frame height of 180 and increase contour area --- frigate/config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frigate/config.py b/frigate/config.py index e3db81a68..a5eda7b9c 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -120,13 +120,11 @@ class RuntimeMotionConfig(MotionConfig): frame_shape = config.get("frame_shape", (1, 1)) if "frame_height" not in config: - config["frame_height"] = max(frame_shape[0] // 6, 180) + config["frame_height"] = frame_shape[0] // 6 if "contour_area" not in config: frame_width = frame_shape[1] * config["frame_height"] / frame_shape[0] - config["contour_area"] = ( - config["frame_height"] * frame_width * 0.00173611111 - ) + config["contour_area"] = config["frame_height"] * frame_width * 0.004 mask = config.get("mask", "") config["raw_mask"] = mask From d5541756310b19d18b5870ca13db20d934d821be Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 7 Nov 2021 13:16:38 -0600 Subject: [PATCH 11/20] no longer make motion settings dynamic --- docs/docs/configuration/index.md | 20 ++++++++------ frigate/config.py | 11 ++------ frigate/motion.py | 46 ++++++++++++++++++++++++++------ 3 files changed, 52 insertions(+), 25 deletions(-) diff --git a/docs/docs/configuration/index.md b/docs/docs/configuration/index.md index 89eb9ee4a..03b71b3ef 100644 --- a/docs/docs/configuration/index.md +++ b/docs/docs/configuration/index.md @@ -194,10 +194,14 @@ motion: # Increasing this value will make motion detection less sensitive and decreasing it will make motion detection more sensitive. # The value should be between 1 and 255. threshold: 25 - # Optional: Minimum size in pixels in the resized motion image that counts as motion (default: ~0.17% of the motion frame area) - # Increasing this value will prevent smaller areas of motion from being detected. Decreasing will make motion detection more sensitive to smaller - # moving objects. - contour_area: 100 + # Optional: Minimum size in pixels in the resized motion image that counts as motion (default: 30) + # Increasing this value will prevent smaller areas of motion from being detected. Decreasing will + # make motion detection more sensitive to smaller moving objects. + # As a rule of thumb: + # - 15 - high sensitivity + # - 30 - medium sensitivity + # - 50 - low sensitivity + contour_area: 30 # Optional: Alpha value passed to cv2.accumulateWeighted when averaging the motion delta across multiple frames (default: shown below) # Higher values mean the current frame impacts the delta a lot, and a single raindrop may register as motion. # Too low and a fast moving person wont be detected as motion. @@ -207,10 +211,10 @@ motion: # Low values will cause things like moving shadows to be detected as motion for longer. # https://www.geeksforgeeks.org/background-subtraction-in-an-image-using-concept-of-running-average/ frame_alpha: 0.2 - # Optional: Height of the resized motion frame (default: 1/6th of the original frame height, but no less than 180) - # This operates as an efficient blur alternative. Higher values will result in more granular motion detection at the expense of higher CPU usage. - # Lower values result in less CPU, but small changes may not register as motion. - frame_height: 180 + # Optional: Height of the resized motion frame (default: 80) + # This operates as an efficient blur alternative. Higher values will result in more granular motion detection at the expense + # of higher CPU usage. Lower values result in less CPU, but small changes may not register as motion. + frame_height: 50 # Optional: motion mask # NOTE: see docs for more detailed info on creating masks mask: 0,900,1080,900,1080,1920,0,1920 diff --git a/frigate/config.py b/frigate/config.py index a5eda7b9c..e911f76cd 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -103,10 +103,10 @@ class MotionConfig(FrigateBaseModel): ge=1, le=255, ) - contour_area: Optional[int] = Field(title="Contour Area") + contour_area: Optional[int] = Field(default=30, title="Contour Area") delta_alpha: float = Field(default=0.2, title="Delta Alpha") frame_alpha: float = Field(default=0.2, title="Frame Alpha") - frame_height: Optional[int] = Field(title="Frame Height") + frame_height: Optional[int] = Field(default=50, title="Frame Height") mask: Union[str, List[str]] = Field( default="", title="Coordinates polygon for the motion mask." ) @@ -119,13 +119,6 @@ class RuntimeMotionConfig(MotionConfig): def __init__(self, **config): frame_shape = config.get("frame_shape", (1, 1)) - if "frame_height" not in config: - config["frame_height"] = frame_shape[0] // 6 - - if "contour_area" not in config: - frame_width = frame_shape[1] * config["frame_height"] / frame_shape[0] - config["contour_area"] = config["frame_height"] * frame_width * 0.004 - mask = config.get("mask", "") config["raw_mask"] = mask diff --git a/frigate/motion.py b/frigate/motion.py index 84f05b9e1..9d6b9a6ca 100644 --- a/frigate/motion.py +++ b/frigate/motion.py @@ -23,6 +23,7 @@ class MotionDetector: interpolation=cv2.INTER_LINEAR, ) self.mask = np.where(resized_mask == [0]) + self.save_images = False def detect(self, frame): motion_boxes = [] @@ -37,16 +38,13 @@ class MotionDetector: ) # Improve contrast - minval = np.percentile(resized_frame, 5) - maxval = np.percentile(resized_frame, 95) + minval = np.percentile(resized_frame, 4) + maxval = np.percentile(resized_frame, 96) resized_frame = np.clip(resized_frame, minval, maxval) resized_frame = (((resized_frame - minval) / (maxval - minval)) * 255).astype( np.uint8 ) - # convert to grayscale - # resized_frame = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY) - # mask frame resized_frame[self.mask] = [255] @@ -55,6 +53,8 @@ class MotionDetector: if self.frame_counter < 30: self.frame_counter += 1 else: + if self.save_images: + self.frame_counter += 1 # compare to average frameDelta = cv2.absdiff(resized_frame, cv2.convertScaleAbs(self.avg_frame)) @@ -64,7 +64,6 @@ class MotionDetector: cv2.accumulateWeighted(frameDelta, self.avg_delta, self.config.delta_alpha) # compute the threshold image for the current frame - # TODO: threshold current_thresh = cv2.threshold( frameDelta, self.config.threshold, 255, cv2.THRESH_BINARY )[1] @@ -81,8 +80,10 @@ class MotionDetector: # dilate the thresholded image to fill in holes, then find contours # on thresholded image - thresh = cv2.dilate(thresh, None, iterations=2) - cnts = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + thresh_dilated = cv2.dilate(thresh, None, iterations=2) + cnts = cv2.findContours( + thresh_dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE + ) cnts = imutils.grab_contours(cnts) # loop over the contours @@ -100,6 +101,35 @@ class MotionDetector: ) ) + if self.save_images: + thresh_dilated = cv2.cvtColor(thresh_dilated, cv2.COLOR_GRAY2BGR) + # print("--------") + # print(self.frame_counter) + for c in cnts: + contour_area = cv2.contourArea(c) + # print(contour_area) + if contour_area > self.config.contour_area: + x, y, w, h = cv2.boundingRect(c) + cv2.rectangle( + thresh_dilated, + (x, y), + (x + w, y + h), + (0, 0, 255), + 2, + ) + # print("--------") + image_row_1 = cv2.hconcat( + [ + cv2.cvtColor(frameDelta, cv2.COLOR_GRAY2BGR), + cv2.cvtColor(avg_delta_image, cv2.COLOR_GRAY2BGR), + ] + ) + image_row_2 = cv2.hconcat( + [cv2.cvtColor(thresh, cv2.COLOR_GRAY2BGR), thresh_dilated] + ) + combined_image = cv2.vconcat([image_row_1, image_row_2]) + cv2.imwrite(f"motion/motion-{self.frame_counter}.jpg", combined_image) + if len(motion_boxes) > 0: self.motion_frame_count += 1 if self.motion_frame_count >= 10: From bd7755fdd36638f49daf7baa8035cd77543b2758 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 7 Nov 2021 14:55:09 -0600 Subject: [PATCH 12/20] revamp process clip --- .gitignore | 1 + frigate/process_clip.py => process_clip.py | 125 +++++++++++++-------- 2 files changed, 79 insertions(+), 47 deletions(-) rename frigate/process_clip.py => process_clip.py (71%) diff --git a/.gitignore b/.gitignore index 2e509347e..8de9cbdf9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ models *.mp4 *.ts *.db +*.csv frigate/version.py web/build web/node_modules diff --git a/frigate/process_clip.py b/process_clip.py similarity index 71% rename from frigate/process_clip.py rename to process_clip.py index ee9240338..9a2ea30e2 100644 --- a/frigate/process_clip.py +++ b/process_clip.py @@ -1,23 +1,26 @@ -import datetime +import sys +from typing_extensions import runtime + +sys.path.append("/lab/frigate") + import json import logging import multiprocessing as mp import os import subprocess as sp import sys -from unittest import TestCase, main import click +import csv import cv2 import numpy as np -from frigate.config import FRIGATE_CONFIG_SCHEMA, FrigateConfig +from frigate.config import FrigateConfig from frigate.edgetpu import LocalObjectDetector from frigate.motion import MotionDetector from frigate.object_processing import CameraState from frigate.objects import ObjectTracker from frigate.util import ( - DictFrameManager, EventsPerSecond, SharedMemoryFrameManager, draw_box_with_label, @@ -96,20 +99,22 @@ class ProcessClip: ffmpeg_process.wait() ffmpeg_process.communicate() - def process_frames(self, objects_to_track=["person"], object_filters={}): + def process_frames( + self, object_detector, objects_to_track=["person"], object_filters={} + ): mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8) mask[:] = 255 - motion_detector = MotionDetector( - self.frame_shape, mask, self.camera_config.motion - ) + motion_detector = MotionDetector(self.frame_shape, self.camera_config.motion) + motion_detector.save_images = False - object_detector = LocalObjectDetector(labels="/labelmap.txt") object_tracker = ObjectTracker(self.camera_config.detect) process_info = { "process_fps": mp.Value("d", 0.0), "detection_fps": mp.Value("d", 0.0), "detection_frame": mp.Value("d", 0.0), } + + detection_enabled = mp.Value("d", 1) stop_event = mp.Event() model_shape = (self.config.model.height, self.config.model.width) @@ -118,6 +123,7 @@ class ProcessClip: self.frame_queue, self.frame_shape, model_shape, + self.camera_config.detect, self.frame_manager, motion_detector, object_detector, @@ -126,25 +132,16 @@ class ProcessClip: process_info, objects_to_track, object_filters, - mask, + detection_enabled, stop_event, exit_on_empty=True, ) - def top_object(self, debug_path=None): - obj_detected = False - top_computed_score = 0.0 - - def handle_event(name, obj, frame_time): - nonlocal obj_detected - nonlocal top_computed_score - if obj.computed_score > top_computed_score: - top_computed_score = obj.computed_score - if not obj.false_positive: - obj_detected = True - - self.camera_state.on("new", handle_event) - self.camera_state.on("update", handle_event) + def stats(self, debug_path=None): + total_regions = 0 + total_motion_boxes = 0 + object_ids = set() + total_frames = 0 while not self.detected_objects_queue.empty(): ( @@ -154,7 +151,8 @@ class ProcessClip: motion_boxes, regions, ) = self.detected_objects_queue.get() - if not debug_path is None: + + if debug_path: self.save_debug_frame( debug_path, frame_time, current_tracked_objects.values() ) @@ -162,10 +160,22 @@ class ProcessClip: self.camera_state.update( frame_time, current_tracked_objects, motion_boxes, regions ) + total_regions += len(regions) + total_motion_boxes += len(motion_boxes) + for id, obj in self.camera_state.tracked_objects.items(): + if not obj.false_positive: + object_ids.add(id) - self.frame_manager.delete(self.camera_state.previous_frame_id) + total_frames += 1 - return {"object_detected": obj_detected, "top_score": top_computed_score} + self.frame_manager.delete(self.camera_state.previous_frame_id) + + return { + "total_regions": total_regions, + "total_motion_boxes": total_motion_boxes, + "true_positive_objects": len(object_ids), + "total_frames": total_frames, + } def save_debug_frame(self, debug_path, frame_time, tracked_objects): current_frame = cv2.cvtColor( @@ -178,7 +188,6 @@ class ProcessClip: for obj in tracked_objects: thickness = 2 color = (0, 0, 175) - if obj["frame_time"] != frame_time: thickness = 1 color = (255, 0, 0) @@ -221,10 +230,9 @@ class ProcessClip: @click.command() @click.option("-p", "--path", required=True, help="Path to clip or directory to test.") @click.option("-l", "--label", default="person", help="Label name to detect.") -@click.option("-t", "--threshold", default=0.85, help="Threshold value for objects.") -@click.option("-s", "--scores", default=None, help="File to save csv of top scores") +@click.option("-o", "--output", default=None, help="File to save csv of data") @click.option("--debug-path", default=None, help="Path to output frames for debugging.") -def process(path, label, threshold, scores, debug_path): +def process(path, label, output, debug_path): clips = [] if os.path.isdir(path): files = os.listdir(path) @@ -235,51 +243,74 @@ def process(path, label, threshold, scores, debug_path): json_config = { "mqtt": {"host": "mqtt"}, + "detectors": {"coral": {"type": "edgetpu", "device": "usb"}}, "cameras": { "camera": { "ffmpeg": { "inputs": [ { "path": "path.mp4", - "global_args": "", - "input_args": "", + "global_args": "-hide_banner", + "input_args": "-loglevel info", "roles": ["detect"], } ] }, - "height": 1920, - "width": 1080, + "rtmp": {"enabled": False}, + "record": {"enabled": False}, } }, } + object_detector = LocalObjectDetector(labels="/labelmap.txt") + results = [] for c in clips: logger.info(c) frame_shape = get_frame_shape(c) - json_config["cameras"]["camera"]["height"] = frame_shape[0] - json_config["cameras"]["camera"]["width"] = frame_shape[1] json_config["cameras"]["camera"]["ffmpeg"]["inputs"][0]["path"] = c - config = FrigateConfig(config=FRIGATE_CONFIG_SCHEMA(json_config)) + frigate_config = FrigateConfig(**json_config) + runtime_config = frigate_config.runtime_config - process_clip = ProcessClip(c, frame_shape, config) + process_clip = ProcessClip(c, frame_shape, runtime_config) process_clip.load_frames() - process_clip.process_frames(objects_to_track=[label]) + process_clip.process_frames(object_detector, objects_to_track=[label]) - results.append((c, process_clip.top_object(debug_path))) + results.append((c, process_clip.stats(debug_path))) - if not scores is None: - with open(scores, "w") as writer: - for result in results: - writer.write(f"{result[0]},{result[1]['top_score']}\n") - - positive_count = sum(1 for result in results if result[1]["object_detected"]) + positive_count = sum( + 1 for result in results if result[1]["true_positive_objects"] > 0 + ) print( f"Objects were detected in {positive_count}/{len(results)}({positive_count/len(results)*100:.2f}%) clip(s)." ) + if output: + # now we will open a file for writing + data_file = open(output, "w") + + # create the csv writer object + csv_writer = csv.writer(data_file) + + # Counter variable used for writing + # headers to the CSV file + count = 0 + + for result in results: + if count == 0: + + # Writing headers of CSV file + header = ["file"] + list(result[1].keys()) + csv_writer.writerow(header) + count += 1 + + # Writing data of CSV file + csv_writer.writerow([result[0]] + list(result[1].values())) + + data_file.close() + if __name__ == "__main__": process() From 8f43a2d1091eccbbbca93209cf3f53609ee570f3 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Mon, 8 Nov 2021 07:29:01 -0600 Subject: [PATCH 13/20] use resolution of clip --- process_clip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/process_clip.py b/process_clip.py index 9a2ea30e2..3220f6a47 100644 --- a/process_clip.py +++ b/process_clip.py @@ -269,6 +269,10 @@ def process(path, label, output, debug_path): logger.info(c) frame_shape = get_frame_shape(c) + json_config["cameras"]["camera"]["detect"] = { + "height": frame_shape[0], + "width": frame_shape[1], + } json_config["cameras"]["camera"]["ffmpeg"]["inputs"][0]["path"] = c frigate_config = FrigateConfig(**json_config) From 4422e86907ae9d95bf9e88c06662c0cb8f5212f3 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Mon, 8 Nov 2021 07:29:20 -0600 Subject: [PATCH 14/20] clarify shm in docs --- docs/docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/installation.md b/docs/docs/installation.md index f138a54e5..e18b7c18d 100644 --- a/docs/docs/installation.md +++ b/docs/docs/installation.md @@ -78,7 +78,7 @@ Frigate utilizes shared memory to store frames during processing. The default `s The default shm-size of 64m is fine for setups with 2 or less 1080p cameras. If frigate is exiting with "Bus error" messages, it is likely because you have too many high resolution cameras and you need to specify a higher shm size. -You can calculate the necessary shm-size for each camera with the following formula: +You can calculate the necessary shm-size for each camera with the following formula using the resolution specified for detect: ``` (width * height * 1.5 * 9 + 270480)/1048576 = From 1c1c28d0e5a9625d3592348704c0cc2dc3d1ea5a Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Mon, 8 Nov 2021 07:32:29 -0600 Subject: [PATCH 15/20] create ffmpeg commands on startup --- frigate/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frigate/config.py b/frigate/config.py index e911f76cd..8126ecfcc 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -489,6 +489,7 @@ class CameraConfig(FrigateBaseModel): timestamp_style: TimestampStyleConfig = Field( default_factory=TimestampStyleConfig, title="Timestamp style configuration." ) + _ffmpeg_cmds: List[Dict[str, List[str]]] = PrivateAttr() def __init__(self, **config): # Set zone colors @@ -505,6 +506,8 @@ class CameraConfig(FrigateBaseModel): super().__init__(**config) + self._ffmpeg_cmds = self._create_ffmpeg_cmds() + @property def frame_shape(self) -> Tuple[int, int]: return self.detect.height, self.detect.width @@ -515,6 +518,9 @@ class CameraConfig(FrigateBaseModel): @property def ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: + return self._ffmpeg_cmds + + def _create_ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: ffmpeg_cmds = [] for ffmpeg_input in self.ffmpeg.inputs: ffmpeg_cmd = self._get_ffmpeg_cmd(ffmpeg_input) From cdd3000315e108be55809d996a85db8759a1f13a Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Mon, 8 Nov 2021 18:20:47 -0600 Subject: [PATCH 16/20] fix ffmpeg config for env vars --- frigate/app.py | 3 +++ frigate/config.py | 8 +++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index c9812e05f..5833d5fe9 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -71,6 +71,9 @@ class FrigateApp: self.config = user_config.runtime_config for camera_name in self.config.cameras.keys(): + # generage the ffmpeg commands + self.config.cameras[camera_name].create_ffmpeg_cmds() + # create camera_metrics self.camera_metrics[camera_name] = { "camera_fps": mp.Value("d", 0.0), diff --git a/frigate/config.py b/frigate/config.py index 8126ecfcc..aee716a84 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -12,7 +12,7 @@ import yaml from pydantic import BaseModel, Extra, Field, validator from pydantic.fields import PrivateAttr -from frigate.const import BASE_DIR, CACHE_DIR, RECORD_DIR +from frigate.const import BASE_DIR, CACHE_DIR from frigate.edgetpu import load_labels from frigate.util import create_mask, deep_merge @@ -506,8 +506,6 @@ class CameraConfig(FrigateBaseModel): super().__init__(**config) - self._ffmpeg_cmds = self._create_ffmpeg_cmds() - @property def frame_shape(self) -> Tuple[int, int]: return self.detect.height, self.detect.width @@ -520,7 +518,7 @@ class CameraConfig(FrigateBaseModel): def ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: return self._ffmpeg_cmds - def _create_ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: + def create_ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: ffmpeg_cmds = [] for ffmpeg_input in self.ffmpeg.inputs: ffmpeg_cmd = self._get_ffmpeg_cmd(ffmpeg_input) @@ -528,7 +526,7 @@ class CameraConfig(FrigateBaseModel): continue ffmpeg_cmds.append({"roles": ffmpeg_input.roles, "cmd": ffmpeg_cmd}) - return ffmpeg_cmds + self._ffmpeg_cmds = ffmpeg_cmds def _get_ffmpeg_cmd(self, ffmpeg_input: CameraInput): ffmpeg_output_args = [] From c7faef8faa9b725284415f2a66fc9152d0e274a4 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Mon, 8 Nov 2021 19:05:39 -0600 Subject: [PATCH 17/20] don't modify ffmpeg_cmd object --- frigate/config.py | 2 +- frigate/http.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frigate/config.py b/frigate/config.py index aee716a84..0ed749d72 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -518,7 +518,7 @@ class CameraConfig(FrigateBaseModel): def ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: return self._ffmpeg_cmds - def create_ffmpeg_cmds(self) -> List[Dict[str, List[str]]]: + def create_ffmpeg_cmds(self): ffmpeg_cmds = [] for ffmpeg_input in self.ffmpeg.inputs: ffmpeg_cmd = self._get_ffmpeg_cmd(ffmpeg_input) diff --git a/frigate/http.py b/frigate/http.py index 32dcdb130..69c06c41d 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -1,6 +1,7 @@ import base64 from collections import OrderedDict from datetime import datetime, timedelta +import copy import json import glob import logging @@ -321,7 +322,7 @@ def config(): # add in the ffmpeg_cmds for camera_name, camera in current_app.frigate_config.cameras.items(): camera_dict = config["cameras"][camera_name] - camera_dict["ffmpeg_cmds"] = camera.ffmpeg_cmds + camera_dict["ffmpeg_cmds"] = copy.deepcopy(camera.ffmpeg_cmds) for cmd in camera_dict["ffmpeg_cmds"]: cmd["cmd"] = " ".join(cmd["cmd"]) From 3541f966e3d60b544b49654cf7443d251f1bf4b5 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Tue, 9 Nov 2021 07:05:21 -0600 Subject: [PATCH 18/20] error handling for the recording maintainer --- frigate/record.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/frigate/record.py b/frigate/record.py index e2427e729..1035ea529 100644 --- a/frigate/record.py +++ b/frigate/record.py @@ -185,25 +185,38 @@ class RecordingMaintainer(threading.Thread): file_path = os.path.join(directory, file_name) # copy then delete is required when recordings are stored on some network drives - shutil.copyfile(cache_path, file_path) - os.remove(cache_path) + try: + shutil.copyfile(cache_path, file_path) + os.remove(cache_path) - rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) - Recordings.create( - id=f"{start_time.timestamp()}-{rand_id}", - camera=camera, - path=file_path, - start_time=start_time.timestamp(), - end_time=end_time.timestamp(), - duration=duration, - ) + rand_id = "".join( + random.choices(string.ascii_lowercase + string.digits, k=6) + ) + Recordings.create( + id=f"{start_time.timestamp()}-{rand_id}", + camera=camera, + path=file_path, + start_time=start_time.timestamp(), + end_time=end_time.timestamp(), + duration=duration, + ) + except Exception as e: + logger.error(f"Unable to store recording segment {cache_path}") + Path(cache_path).unlink(missing_ok=True) + logger.error(e) def run(self): # Check for new files every 5 seconds wait_time = 5 while not self.stop_event.wait(wait_time): run_start = datetime.datetime.now().timestamp() - self.move_files() + try: + self.move_files() + except Exception as e: + logger.error( + "Error occurred when attempting to maintain recording cache" + ) + logger.error(e) wait_time = max(0, 5 - (datetime.datetime.now().timestamp() - run_start)) logger.info(f"Exiting recording maintenance...") From 51fb532e1a4417c3868161bafa3e6963090505ad Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Tue, 9 Nov 2021 07:40:23 -0600 Subject: [PATCH 19/20] set retain when setting switches from frontend --- web/src/api/__tests__/mqtt.test.jsx | 12 ++++++------ web/src/api/mqtt.jsx | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/web/src/api/__tests__/mqtt.test.jsx b/web/src/api/__tests__/mqtt.test.jsx index 31b539522..164776407 100644 --- a/web/src/api/__tests__/mqtt.test.jsx +++ b/web/src/api/__tests__/mqtt.test.jsx @@ -121,12 +121,12 @@ describe('MqttProvider', () => { ); await screen.findByTestId('data'); - expect(screen.getByTestId('front/detect/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"ON"}'); - expect(screen.getByTestId('front/recordings/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF"}'); - expect(screen.getByTestId('front/snapshots/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"ON"}'); - expect(screen.getByTestId('side/detect/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF"}'); - expect(screen.getByTestId('side/recordings/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF"}'); - expect(screen.getByTestId('side/snapshots/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF"}'); + expect(screen.getByTestId('front/detect/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"ON","retain":true}'); + expect(screen.getByTestId('front/recordings/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF","retain":true}'); + expect(screen.getByTestId('front/snapshots/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"ON","retain":true}'); + expect(screen.getByTestId('side/detect/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF","retain":true}'); + expect(screen.getByTestId('side/recordings/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF","retain":true}'); + expect(screen.getByTestId('side/snapshots/state')).toHaveTextContent('{"lastUpdate":123456,"payload":"OFF","retain":true}'); }); }); diff --git a/web/src/api/mqtt.jsx b/web/src/api/mqtt.jsx index 5d7639450..7505702d6 100644 --- a/web/src/api/mqtt.jsx +++ b/web/src/api/mqtt.jsx @@ -42,9 +42,9 @@ export function MqttProvider({ useEffect(() => { Object.keys(config.cameras).forEach((camera) => { const { name, record, detect, snapshots } = config.cameras[camera]; - dispatch({ topic: `${name}/recordings/state`, payload: record.enabled ? 'ON' : 'OFF' }); - dispatch({ topic: `${name}/detect/state`, payload: detect.enabled ? 'ON' : 'OFF' }); - dispatch({ topic: `${name}/snapshots/state`, payload: snapshots.enabled ? 'ON' : 'OFF' }); + dispatch({ topic: `${name}/recordings/state`, payload: record.enabled ? 'ON' : 'OFF', retain: true }); + dispatch({ topic: `${name}/detect/state`, payload: detect.enabled ? 'ON' : 'OFF', retain: true }); + dispatch({ topic: `${name}/snapshots/state`, payload: snapshots.enabled ? 'ON' : 'OFF', retain: true }); }); }, [config]); From 14c74e4361975c6c5916bee82b99d97fcae40a25 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Wed, 10 Nov 2021 21:12:41 -0600 Subject: [PATCH 20/20] more robust cache management --- frigate/record.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/frigate/record.py b/frigate/record.py index 1035ea529..7da57dae6 100644 --- a/frigate/record.py +++ b/frigate/record.py @@ -1,4 +1,5 @@ import datetime +import time import itertools import logging import os @@ -86,6 +87,20 @@ class RecordingMaintainer(threading.Thread): } ) + # delete all cached files past the most recent 2 + for camera in grouped_recordings.keys(): + if len(grouped_recordings[camera]) > 2: + logger.warning( + "Proactively cleaning cache. Your recordings disk may be too slow." + ) + sorted_recordings = sorted( + grouped_recordings[camera], key=lambda i: i["start_time"] + ) + to_remove = sorted_recordings[:-2] + for f in to_remove: + Path(f["cache_path"]).unlink(missing_ok=True) + grouped_recordings[camera] = sorted_recordings[-2:] + for camera, recordings in grouped_recordings.items(): # get all events with the end time after the start of the oldest cache file # or with end_time None @@ -169,12 +184,6 @@ class RecordingMaintainer(threading.Thread): camera, start_time, end_time, duration, cache_path ) - if len(recordings) > 2: - # delete all cached files past the most recent 2 - to_remove = sorted(recordings, key=lambda i: i["start_time"])[:-2] - for f in to_remove: - Path(cache_path).unlink(missing_ok=True) - def store_segment(self, camera, start_time, end_time, duration, cache_path): directory = os.path.join(RECORD_DIR, start_time.strftime("%Y-%m/%d/%H"), camera) @@ -184,9 +193,13 @@ class RecordingMaintainer(threading.Thread): file_name = f"{start_time.strftime('%M.%S.mp4')}" file_path = os.path.join(directory, file_name) - # copy then delete is required when recordings are stored on some network drives try: + start_frame = datetime.datetime.now().timestamp() + # copy then delete is required when recordings are stored on some network drives shutil.copyfile(cache_path, file_path) + logger.debug( + f"Copied {file_path} in {datetime.datetime.now().timestamp()-start_frame} seconds." + ) os.remove(cache_path) rand_id = "".join( @@ -362,6 +375,9 @@ class RecordingCleanup(threading.Thread): oldest_timestamp = p.stat().st_mtime - 1 except DoesNotExist: oldest_timestamp = datetime.datetime.now().timestamp() + except FileNotFoundError: + logger.warning(f"Unable to find file from recordings database: {p}") + oldest_timestamp = datetime.datetime.now().timestamp() logger.debug(f"Oldest recording in the db: {oldest_timestamp}") process = sp.run(