From 7c1568fcb941ad17256ab73438cb997b23fb7f33 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 18 Jun 2023 11:56:41 -0500 Subject: [PATCH 1/7] use clahe for contrast improvement (#6835) * use clahe for contrast improvement * update tests --- frigate/config.py | 6 +++--- frigate/motion/improved_motion.py | 3 ++- frigate/test/test_config.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frigate/config.py b/frigate/config.py index 9b434ca1e..2d9bb102a 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -187,7 +187,7 @@ class RecordConfig(FrigateBaseModel): class MotionConfig(FrigateBaseModel): threshold: int = Field( - default=40, + default=30, title="Motion detection threshold (1-255).", ge=1, le=255, @@ -196,10 +196,10 @@ class MotionConfig(FrigateBaseModel): default=0.8, title="Lightning detection threshold (0.3-1.0).", ge=0.3, le=1.0 ) improve_contrast: bool = Field(default=True, title="Improve Contrast") - contour_area: Optional[int] = Field(default=15, title="Contour Area") + contour_area: Optional[int] = Field(default=10, title="Contour Area") delta_alpha: float = Field(default=0.2, title="Delta Alpha") frame_alpha: float = Field(default=0.02, title="Frame Alpha") - frame_height: Optional[int] = Field(default=50, title="Frame Height") + frame_height: Optional[int] = Field(default=100, title="Frame Height") mask: Union[str, List[str]] = Field( default="", title="Coordinates polygon for the motion mask." ) diff --git a/frigate/motion/improved_motion.py b/frigate/motion/improved_motion.py index 882cd22ee..0aa259940 100644 --- a/frigate/motion/improved_motion.py +++ b/frigate/motion/improved_motion.py @@ -38,6 +38,7 @@ class ImprovedMotionDetector(MotionDetector): self.improve_contrast = improve_contrast self.threshold = threshold self.contour_area = contour_area + self.clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) def detect(self, frame): motion_boxes = [] @@ -55,7 +56,7 @@ class ImprovedMotionDetector(MotionDetector): # Improve contrast if self.improve_contrast.value: - resized_frame = cv2.equalizeHist(resized_frame) + resized_frame = self.clahe.apply(resized_frame) # mask frame resized_frame[self.mask] = [255] diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index a9e665706..8e767f77f 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -730,7 +730,7 @@ class TestConfig(unittest.TestCase): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config() - assert runtime_config.cameras["back"].motion.frame_height == 50 + assert runtime_config.cameras["back"].motion.frame_height == 100 def test_motion_contour_area_dynamic(self): config = { @@ -758,7 +758,7 @@ class TestConfig(unittest.TestCase): assert config == frigate_config.dict(exclude_unset=True) runtime_config = frigate_config.runtime_config() - assert round(runtime_config.cameras["back"].motion.contour_area) == 15 + assert round(runtime_config.cameras["back"].motion.contour_area) == 10 def test_merge_labelmap(self): config = { From 9e531b0b5bf34634dc6c44e7eca7e5eaf5a52a25 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Wed, 21 Jun 2023 08:38:51 -0500 Subject: [PATCH 2/7] reduce grid size for contrast improvement (#6870) --- benchmark_motion.py | 106 +++++++++++++++--------------- docs/docs/configuration/index.md | 12 ++-- frigate/config.py | 4 +- frigate/motion/improved_motion.py | 28 +++++++- 4 files changed, 87 insertions(+), 63 deletions(-) diff --git a/benchmark_motion.py b/benchmark_motion.py index a9ecc56b2..167770280 100644 --- a/benchmark_motion.py +++ b/benchmark_motion.py @@ -1,14 +1,13 @@ import datetime import multiprocessing as mp import os -from statistics import mean import cv2 import numpy as np from frigate.config import MotionConfig -from frigate.motion.frigate_motion import FrigateMotionDetector from frigate.motion.improved_motion import ImprovedMotionDetector +from frigate.util import create_mask # get info on the video # cap = cv2.VideoCapture("debug/front_cam_2023_05_23_08_41__2023_05_23_08_43.mp4") @@ -20,84 +19,85 @@ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = cap.get(cv2.CAP_PROP_FPS) frame_shape = (height, width, 3) +mask = create_mask( + (height, width), + [], +) + # create the motion config -motion_config = MotionConfig() -motion_config.mask = np.zeros((height, width), np.uint8) -motion_config.mask[:] = 255 -motion_config.improve_contrast = 1 -motion_config.frame_alpha = 0.02 -motion_config.threshold = 40 -motion_config.contour_area = 15 +motion_config_1 = MotionConfig() +motion_config_1.mask = np.zeros((height, width), np.uint8) +motion_config_1.mask[:] = mask +# motion_config_1.improve_contrast = 1 +# motion_config_1.frame_height = 150 +# motion_config_1.frame_alpha = 0.02 +# motion_config_1.threshold = 30 +# motion_config_1.contour_area = 10 + +motion_config_2 = MotionConfig() +motion_config_2.mask = np.zeros((height, width), np.uint8) +motion_config_2.mask[:] = mask +# motion_config_2.improve_contrast = 1 +# motion_config_2.frame_height = 150 +# motion_config_2.frame_alpha = 0.01 +# motion_config_2.threshold = 20 +# motion_config.contour_area = 10 save_images = True -# create motion detectors -frigate_motion_detector = FrigateMotionDetector( +improved_motion_detector_1 = ImprovedMotionDetector( frame_shape=frame_shape, - config=motion_config, + config=motion_config_1, fps=fps, - improve_contrast=mp.Value("i", motion_config.improve_contrast), - threshold=mp.Value("i", motion_config.threshold), - contour_area=mp.Value("i", motion_config.contour_area), + improve_contrast=mp.Value("i", motion_config_1.improve_contrast), + threshold=mp.Value("i", motion_config_1.threshold), + contour_area=mp.Value("i", motion_config_1.contour_area), + name="default", + clipLimit=2.0, + tileGridSize=(8, 8), ) -frigate_motion_detector.save_images = save_images +improved_motion_detector_1.save_images = save_images -improved_motion_detector = ImprovedMotionDetector( +improved_motion_detector_2 = ImprovedMotionDetector( frame_shape=frame_shape, - config=motion_config, + config=motion_config_2, fps=fps, - improve_contrast=mp.Value("i", motion_config.improve_contrast), - threshold=mp.Value("i", motion_config.threshold), - contour_area=mp.Value("i", motion_config.contour_area), + improve_contrast=mp.Value("i", motion_config_2.improve_contrast), + threshold=mp.Value("i", motion_config_2.threshold), + contour_area=mp.Value("i", motion_config_2.contour_area), + name="compare", ) -improved_motion_detector.save_images = save_images +improved_motion_detector_2.save_images = save_images # read and process frames -frame_times = {"frigate": [], "improved": []} ret, frame = cap.read() frame_counter = 1 while ret: yuv_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420) start_frame = datetime.datetime.now().timestamp() - frigate_motion_detector.detect(yuv_frame) - frame_times["frigate"].append(datetime.datetime.now().timestamp() - start_frame) + improved_motion_detector_1.detect(yuv_frame) start_frame = datetime.datetime.now().timestamp() - improved_motion_detector.detect(yuv_frame) - frame_times["improved"].append(datetime.datetime.now().timestamp() - start_frame) + improved_motion_detector_2.detect(yuv_frame) - frigate_frame = f"debug/frames/frigate-{frame_counter}.jpg" - improved_frame = f"debug/frames/improved-{frame_counter}.jpg" - if os.path.exists(frigate_frame) and os.path.exists(improved_frame): - image_row_1 = cv2.hconcat( - [ - cv2.imread(frigate_frame), - cv2.imread(improved_frame), - ] - ) - - image_row_2 = cv2.resize( - frame, - dsize=( - frigate_motion_detector.motion_frame_size[1] * 2, - frigate_motion_detector.motion_frame_size[0] * 2, - ), - interpolation=cv2.INTER_LINEAR, - ) + default_frame = f"debug/frames/default-{frame_counter}.jpg" + compare_frame = f"debug/frames/compare-{frame_counter}.jpg" + if os.path.exists(default_frame) and os.path.exists(compare_frame): + images = [ + cv2.imread(default_frame), + cv2.imread(compare_frame), + ] cv2.imwrite( f"debug/frames/all-{frame_counter}.jpg", - cv2.vconcat([image_row_1, image_row_2]), + cv2.vconcat(images) + if frame_shape[0] > frame_shape[1] + else cv2.hconcat(images), ) - os.unlink(frigate_frame) - os.unlink(improved_frame) + os.unlink(default_frame) + os.unlink(compare_frame) frame_counter += 1 ret, frame = cap.read() cap.release() - -print("Frigate Motion Detector") -print(f"Average frame processing time: {mean(frame_times['frigate'])*1000:.2f}ms") -print("Improved Motion Detector") -print(f"Average frame processing time: {mean(frame_times['improved'])*1000:.2f}ms") diff --git a/docs/docs/configuration/index.md b/docs/docs/configuration/index.md index 4d8cc4f17..ac65a1018 100644 --- a/docs/docs/configuration/index.md +++ b/docs/docs/configuration/index.md @@ -230,7 +230,7 @@ detect: # especially when using separate streams for detect and record. # Use this setting to make the timeline bounding boxes more closely align # with the recording. The value can be positive or negative. - # TIP: Imagine there is an event clip with a person walking from left to right. + # TIP: Imagine there is an event clip with a person walking from left to right. # If the event timeline bounding box is consistently to the left of the person # then the value should be decreased. Similarly, if a person is walking from # left to right and the bounding box is consistently ahead of the person @@ -275,7 +275,7 @@ motion: # Optional: The threshold passed to cv2.threshold to determine if a pixel is different enough to be counted as motion. (default: shown below) # Increasing this value will make motion detection less sensitive and decreasing it will make motion detection more sensitive. # The value should be between 1 and 255. - threshold: 40 + threshold: 20 # Optional: The percentage of the image used to detect lightning or other substantial changes where motion detection # needs to recalibrate. (default: shown below) # Increasing this value will make motion detection more likely to consider lightning or ir mode changes as valid motion. @@ -286,19 +286,19 @@ motion: # Increasing this value will prevent smaller areas of motion from being detected. Decreasing will # make motion detection more sensitive to smaller moving objects. # As a rule of thumb: - # - 15 - high sensitivity + # - 10 - high sensitivity # - 30 - medium sensitivity # - 50 - low sensitivity - contour_area: 15 + contour_area: 10 # Optional: Alpha value passed to cv2.accumulateWeighted when averaging frames to determine the background (default: shown below) # Higher values mean the current frame impacts the average a lot, and a new object will be averaged into the background faster. # Low values will cause things like moving shadows to be detected as motion for longer. # https://www.geeksforgeeks.org/background-subtraction-in-an-image-using-concept-of-running-average/ - frame_alpha: 0.02 + frame_alpha: 0.01 # Optional: Height of the resized motion frame (default: 50) # Higher values will result in more granular motion detection at the expense of higher CPU usage. # Lower values result in less CPU, but small changes may not register as motion. - frame_height: 50 + frame_height: 100 # Optional: motion mask # NOTE: see docs for more detailed info on creating masks mask: 0,900,1080,900,1080,1920,0,1920 diff --git a/frigate/config.py b/frigate/config.py index 2d9bb102a..b71ba1907 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -187,7 +187,7 @@ class RecordConfig(FrigateBaseModel): class MotionConfig(FrigateBaseModel): threshold: int = Field( - default=30, + default=20, title="Motion detection threshold (1-255).", ge=1, le=255, @@ -198,7 +198,7 @@ class MotionConfig(FrigateBaseModel): improve_contrast: bool = Field(default=True, title="Improve Contrast") contour_area: Optional[int] = Field(default=10, title="Contour Area") delta_alpha: float = Field(default=0.2, title="Delta Alpha") - frame_alpha: float = Field(default=0.02, title="Frame Alpha") + frame_alpha: float = Field(default=0.01, title="Frame Alpha") frame_height: Optional[int] = Field(default=100, title="Frame Height") mask: Union[str, List[str]] = Field( default="", title="Coordinates polygon for the motion mask." diff --git a/frigate/motion/improved_motion.py b/frigate/motion/improved_motion.py index 0aa259940..525854b82 100644 --- a/frigate/motion/improved_motion.py +++ b/frigate/motion/improved_motion.py @@ -15,7 +15,11 @@ class ImprovedMotionDetector(MotionDetector): improve_contrast, threshold, contour_area, + clipLimit=2.0, + tileGridSize=(2, 2), + name="improved", ): + self.name = name self.config = config self.frame_shape = frame_shape self.resize_factor = frame_shape[0] / config.frame_height @@ -38,7 +42,7 @@ class ImprovedMotionDetector(MotionDetector): self.improve_contrast = improve_contrast self.threshold = threshold self.contour_area = contour_area - self.clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + self.clahe = cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize) def detect(self, frame): motion_boxes = [] @@ -52,12 +56,21 @@ class ImprovedMotionDetector(MotionDetector): interpolation=cv2.INTER_LINEAR, ) + if self.save_images: + resized_saved = resized_frame.copy() + resized_frame = cv2.GaussianBlur(resized_frame, (3, 3), cv2.BORDER_DEFAULT) + if self.save_images: + blurred_saved = resized_frame.copy() + # Improve contrast if self.improve_contrast.value: resized_frame = self.clahe.apply(resized_frame) + if self.save_images: + contrasted_saved = resized_frame.copy() + # mask frame resized_frame[self.mask] = [255] @@ -119,8 +132,19 @@ class ImprovedMotionDetector(MotionDetector): (0, 0, 255), 2, ) + frames = [ + cv2.cvtColor(resized_saved, cv2.COLOR_GRAY2BGR), + cv2.cvtColor(blurred_saved, cv2.COLOR_GRAY2BGR), + cv2.cvtColor(contrasted_saved, cv2.COLOR_GRAY2BGR), + cv2.cvtColor(frameDelta, cv2.COLOR_GRAY2BGR), + cv2.cvtColor(thresh, cv2.COLOR_GRAY2BGR), + thresh_dilated, + ] cv2.imwrite( - f"debug/frames/improved-{self.frame_counter}.jpg", thresh_dilated + f"debug/frames/{self.name}-{self.frame_counter}.jpg", + cv2.hconcat(frames) + if self.frame_shape[0] > self.frame_shape[1] + else cv2.vconcat(frames), ) if len(motion_boxes) > 0: From e640981cc4fca80405376d4453325fa0622f64d1 Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Wed, 28 Jun 2023 13:39:39 +0300 Subject: [PATCH 3/7] Performance: multiprocessing improvement (#6936) * Add faster-fifo dependency for improved performance * isort --- Dockerfile | 4 +++- frigate/app.py | 2 +- frigate/events/external.py | 2 +- frigate/events/maintainer.py | 3 ++- frigate/log.py | 2 +- frigate/timeline.py | 3 ++- frigate/types.py | 3 ++- requirements-wheels.txt | 1 + 8 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0c244f901..058bb6ad7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -127,7 +127,9 @@ RUN apt-get -qq update \ libtbb2 libtbb-dev libdc1394-22-dev libopenexr-dev \ libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \ # scipy dependencies - gcc gfortran libopenblas-dev liblapack-dev && \ + gcc gfortran libopenblas-dev liblapack-dev \ + # faster-fifo dependencies + g++ cython3 && \ rm -rf /var/lib/apt/lists/* RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ diff --git a/frigate/app.py b/frigate/app.py index 9d85f461e..8d3972343 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -6,12 +6,12 @@ import shutil import signal import sys import traceback -from multiprocessing.queues import Queue from multiprocessing.synchronize import Event as MpEvent from types import FrameType from typing import Optional import psutil +from faster_fifo import Queue from peewee_migrate import Router from playhouse.sqlite_ext import SqliteExtDatabase from playhouse.sqliteq import SqliteQueueDatabase diff --git a/frigate/events/external.py b/frigate/events/external.py index 5422de260..910aee35f 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -6,10 +6,10 @@ import logging import os import random import string -from multiprocessing.queues import Queue from typing import Optional import cv2 +from faster_fifo import Queue from frigate.config import CameraConfig, FrigateConfig from frigate.const import CLIPS_DIR diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index 264ab1142..28fb4646b 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -3,10 +3,11 @@ import logging import queue import threading from enum import Enum -from multiprocessing.queues import Queue from multiprocessing.synchronize import Event as MpEvent from typing import Dict +from faster_fifo import Queue + from frigate.config import EventsConfig, FrigateConfig from frigate.models import Event from frigate.types import CameraMetricsTypes diff --git a/frigate/log.py b/frigate/log.py index 5dbf4eed0..ac51fc3da 100644 --- a/frigate/log.py +++ b/frigate/log.py @@ -7,10 +7,10 @@ import signal import threading from collections import deque from logging import handlers -from multiprocessing.queues import Queue from types import FrameType from typing import Deque, Optional +from faster_fifo import Queue from setproctitle import setproctitle from frigate.util import clean_camera_user_pass diff --git a/frigate/timeline.py b/frigate/timeline.py index 9ca617ba9..6cfcbe928 100644 --- a/frigate/timeline.py +++ b/frigate/timeline.py @@ -3,9 +3,10 @@ import logging import queue import threading -from multiprocessing.queues import Queue from multiprocessing.synchronize import Event as MpEvent +from faster_fifo import Queue + from frigate.config import FrigateConfig from frigate.events.maintainer import EventTypeEnum from frigate.models import Timeline diff --git a/frigate/types.py b/frigate/types.py index 8c3e54654..23751d499 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -1,8 +1,9 @@ from multiprocessing.context import Process -from multiprocessing.queues import Queue from multiprocessing.sharedctypes import Synchronized from typing import Optional, TypedDict +from faster_fifo import Queue + from frigate.object_detection import ObjectDetectProcess diff --git a/requirements-wheels.txt b/requirements-wheels.txt index f02317e41..19bd1077c 100644 --- a/requirements-wheels.txt +++ b/requirements-wheels.txt @@ -1,5 +1,6 @@ click == 8.1.* Flask == 2.3.* +faster-fifo == 1.4.* imutils == 0.5.* matplotlib == 3.7.* mypy == 0.942 From 3d40ed5d474305ea66c515e619900d2a22e472bc Mon Sep 17 00:00:00 2001 From: spacebares <57186372+spacebares@users.noreply.github.com> Date: Wed, 28 Jun 2023 06:45:54 -0400 Subject: [PATCH 4/7] fix tooltip not showing if too far left (#6909) --- web/src/components/Tooltip.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/components/Tooltip.jsx b/web/src/components/Tooltip.jsx index ba56f20b8..58f723ac5 100644 --- a/web/src/components/Tooltip.jsx +++ b/web/src/components/Tooltip.jsx @@ -29,7 +29,7 @@ export default function Tooltip({ relativeTo, text }) { let newLeft = left - Math.round(tipWidth / 2); // too far right if (newLeft + tipWidth + TIP_SPACE > windowWidth - window.scrollX) { - newLeft = left - tipWidth - TIP_SPACE; + newLeft = Math.max(0, left - tipWidth - TIP_SPACE); newTop = top - Math.round(tipHeight / 2); } // too far left From ece070fee1359ea00cbe52a4b39a2308d34801e8 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 28 Jun 2023 04:51:53 -0600 Subject: [PATCH 5/7] Load labels dynamically for event filters (#6896) * Load labels dynamically to include custom events and audio, do not include attribute labels * Formatting * Fix sorting * Also filter tracked object list on camera page * isort * Don't fail before load --- frigate/const.py | 10 ++++++++++ frigate/http.py | 18 ++++++++++++++++++ frigate/video.py | 14 +++----------- web/src/routes/Camera.jsx | 7 +++++-- web/src/routes/Events.jsx | 10 +++------- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/frigate/const.py b/frigate/const.py index 9b7e177f2..28ce766e4 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -12,6 +12,16 @@ PLUS_ENV_VAR = "PLUS_API_KEY" PLUS_API_HOST = "https://api.frigate.video" BTBN_PATH = "/usr/lib/btbn-ffmpeg" +# Attributes + +ATTRIBUTE_LABEL_MAP = { + "person": ["face", "amazon"], + "car": ["ups", "fedex", "amazon", "license_plate"], +} +ALL_ATTRIBUTE_LABELS = [ + item for sublist in ATTRIBUTE_LABEL_MAP.values() for item in sublist +] + # Regex Consts REGEX_CAMERA_NAME = r"^[a-zA-Z0-9_-]+$" diff --git a/frigate/http.py b/frigate/http.py index b4813c1f2..44706f2c2 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -410,6 +410,24 @@ def set_sub_label(id): ) +@bp.route("/labels") +def get_labels(): + camera = request.args.get("camera", type=str, default="") + + try: + if camera: + events = Event.select(Event.label).where(Event.camera == camera).distinct() + else: + events = Event.select(Event.label).distinct() + except Exception as e: + return jsonify( + {"success": False, "message": f"Failed to get labels: {e}"}, "404" + ) + + labels = sorted([e.label for e in events]) + return jsonify(labels) + + @bp.route("/sub_labels") def get_sub_labels(): split_joined = request.args.get("split_joined", type=int) diff --git a/frigate/video.py b/frigate/video.py index c02ad15c4..f60bce1d9 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -15,7 +15,7 @@ import numpy as np from setproctitle import setproctitle from frigate.config import CameraConfig, DetectConfig -from frigate.const import CACHE_DIR +from frigate.const import ALL_ATTRIBUTE_LABELS, ATTRIBUTE_LABEL_MAP, CACHE_DIR from frigate.detectors.detector_config import PixelFormatEnum from frigate.log import LogPipe from frigate.motion import MotionDetector @@ -723,14 +723,6 @@ def process_frames( stop_event, exit_on_empty: bool = False, ): - # attribute labels are not tracked and are not assigned regions - attribute_label_map = { - "person": ["face", "amazon"], - "car": ["ups", "fedex", "amazon", "license_plate"], - } - all_attribute_labels = [ - item for sublist in attribute_label_map.values() for item in sublist - ] fps = process_info["process_fps"] detection_fps = process_info["detection_fps"] current_frame_time = process_info["detection_frame"] @@ -906,7 +898,7 @@ def process_frames( tracked_detections = [ d for d in consolidated_detections - if d[0] not in all_attribute_labels + if d[0] not in ALL_ATTRIBUTE_LABELS ] # now that we have refined our detections, we need to track objects object_tracker.match_and_update(frame_time, tracked_detections) @@ -916,7 +908,7 @@ def process_frames( # group the attribute detections based on what label they apply to attribute_detections = {} - for label, attribute_labels in attribute_label_map.items(): + for label, attribute_labels in ATTRIBUTE_LABEL_MAP.items(): attribute_detections[label] = [ d for d in consolidated_detections if d[0] in attribute_labels ] diff --git a/web/src/routes/Camera.jsx b/web/src/routes/Camera.jsx index 7f5e3ca77..11ae2cdea 100644 --- a/web/src/routes/Camera.jsx +++ b/web/src/routes/Camera.jsx @@ -22,6 +22,7 @@ const emptyObject = Object.freeze({}); export default function Camera({ camera }) { const { data: config } = useSWR('config'); + const { data: trackedLabels } = useSWR(['labels', { camera }]); const apiHost = useApiHost(); const [showSettings, setShowSettings] = useState(false); const [viewMode, setViewMode] = useState('live'); @@ -121,7 +122,9 @@ export default function Camera({ camera }) {
@@ -203,7 +206,7 @@ export default function Camera({ camera }) {
Tracked objects
- {cameraConfig.objects.track.map((objectType) => ( + {(trackedLabels || []).map((objectType) => ( self.indexOf(value) === i), 'None', ], - labels: Object.values(config?.cameras || {}) - .reduce((memo, camera) => { - memo = memo.concat(camera?.objects?.track || []); - return memo; - }, config?.objects?.track || []) - .filter((value, i, self) => self.indexOf(value) === i), + labels: Object.values(allLabels || {}), sub_labels: (allSubLabels || []).length > 0 ? [...Object.values(allSubLabels), 'None'] : [], }), - [config, allSubLabels] + [config, allLabels, allSubLabels] ); const onSave = async (e, eventId, save) => { From ee4a1336558eb3711ee42158a138d54f5fe8c712 Mon Sep 17 00:00:00 2001 From: Cody Cutrer Date: Wed, 28 Jun 2023 04:53:28 -0600 Subject: [PATCH 6/7] actually keep track of skipped frames (#6889) * actually keep track of skipped frames fixes #6863 * fix black errors --- frigate/video.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/frigate/video.py b/frigate/video.py index f60bce1d9..4f315885c 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -172,7 +172,7 @@ def capture_frames( skipped_eps.start() while True: fps.value = frame_rate.eps() - skipped_eps.eps() + skipped_fps.value = skipped_eps.eps() current_frame.value = datetime.datetime.now().timestamp() frame_name = f"{camera_name}{current_frame.value}" @@ -215,6 +215,7 @@ class CameraWatchdog(threading.Thread): config: CameraConfig, frame_queue, camera_fps, + skipped_fps, ffmpeg_pid, stop_event, ): @@ -227,6 +228,7 @@ class CameraWatchdog(threading.Thread): self.logpipe = LogPipe(f"ffmpeg.{self.camera_name}.detect") self.ffmpeg_other_processes: list[dict[str, any]] = [] self.camera_fps = camera_fps + self.skipped_fps = skipped_fps self.ffmpeg_pid = ffmpeg_pid self.frame_queue = frame_queue self.frame_shape = self.config.frame_shape_yuv @@ -346,6 +348,7 @@ class CameraWatchdog(threading.Thread): self.frame_shape, self.frame_queue, self.camera_fps, + self.skipped_fps, self.stop_event, ) self.capture_thread.start() @@ -376,7 +379,14 @@ class CameraWatchdog(threading.Thread): class CameraCapture(threading.Thread): def __init__( - self, camera_name, ffmpeg_process, frame_shape, frame_queue, fps, stop_event + self, + camera_name, + ffmpeg_process, + frame_shape, + frame_queue, + fps, + skipped_fps, + stop_event, ): threading.Thread.__init__(self) self.name = f"capture:{camera_name}" @@ -385,14 +395,13 @@ class CameraCapture(threading.Thread): self.frame_queue = frame_queue self.fps = fps self.stop_event = stop_event - self.skipped_fps = EventsPerSecond() + self.skipped_fps = skipped_fps self.frame_manager = SharedMemoryFrameManager() self.ffmpeg_process = ffmpeg_process self.current_frame = mp.Value("d", 0.0) self.last_frame = 0 def run(self): - self.skipped_fps.start() capture_frames( self.ffmpeg_process, self.camera_name, @@ -424,6 +433,7 @@ def capture_camera(name, config: CameraConfig, process_info): config, frame_queue, process_info["camera_fps"], + process_info["skipped_fps"], process_info["ffmpeg_pid"], stop_event, ) From ef14a43930636ec9d87efda9a6a5b527f96a513c Mon Sep 17 00:00:00 2001 From: Cody Cutrer Date: Wed, 28 Jun 2023 04:55:53 -0600 Subject: [PATCH 7/7] optimize frame-per-second calculations (#6887) * optimize frame-per-second calculations FPS is never calculated over differing periods for even given counter. By only storing timestamps within the period that is used, we avoid having to iterate 1000 entries every time it's re-calculated (multiple times per second). 1000 entries would normally only be needed if something is running at 100fps. A more common speed - anywhere from 5 to 30fps, only needs 50 to 300 entries. This isn't a huge improvement in the grand scheme, but when motion detection is disabled, it takes nearly as much time in a flamegraph as actually transferring the video frame. * fix python checks --- frigate/util.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/frigate/util.py b/frigate/util.py index aa19e99fa..f535a9572 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -650,34 +650,42 @@ def restart_frigate(): class EventsPerSecond: - def __init__(self, max_events=1000): + def __init__(self, max_events=1000, last_n_seconds=10): self._start = None self._max_events = max_events + self._last_n_seconds = last_n_seconds self._timestamps = [] def start(self): self._start = datetime.datetime.now().timestamp() def update(self): + now = datetime.datetime.now().timestamp() if self._start is None: - self.start() - self._timestamps.append(datetime.datetime.now().timestamp()) + self._start = now + self._timestamps.append(now) # truncate the list when it goes 100 over the max_size if len(self._timestamps) > self._max_events + 100: self._timestamps = self._timestamps[(1 - self._max_events) :] + self.expire_timestamps(now) - def eps(self, last_n_seconds=10): - if self._start is None: - self.start() - # compute the (approximate) events in the last n seconds + def eps(self): now = datetime.datetime.now().timestamp() - seconds = min(now - self._start, last_n_seconds) + if self._start is None: + self._start = now + # compute the (approximate) events in the last n seconds + self.expire_timestamps(now) + seconds = min(now - self._start, self._last_n_seconds) # avoid divide by zero if seconds == 0: seconds = 1 - return ( - len([t for t in self._timestamps if t > (now - last_n_seconds)]) / seconds - ) + return len(self._timestamps) / seconds + + # remove aged out timestamps + def expire_timestamps(self, now): + threshold = now - self._last_n_seconds + while self._timestamps and self._timestamps[0] < threshold: + del self._timestamps[0] def print_stack(sig, frame):