From 4469507e5bebc40508c7504bd0139a0566cdc805 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 15 Oct 2023 14:31:56 -0400 Subject: [PATCH 01/26] dont set has_clip to false unless the event is older (#8179) --- frigate/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frigate/http.py b/frigate/http.py index 64ba51ea0..695ba8473 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -1766,9 +1766,10 @@ def vod_event(id): datetime.now().timestamp() if event.end_time is None else event.end_time ) vod_response = vod_ts(event.camera, event.start_time, end_ts) - # If the recordings are not found, set has_clip to false + # If the recordings are not found and the event started more than 5 minutes ago, set has_clip to false if ( - type(vod_response) == tuple + event.start_time < datetime.now().timestamp() - 300 + and type(vod_response) == tuple and len(vod_response) == 2 and vod_response[1] == 404 ): From cac37e484db618d0d2e3aa82abf06eef213c28db Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Mon, 16 Oct 2023 14:42:24 +0300 Subject: [PATCH 02/26] Upd: go2rtc v1.8.1 (#8166) * go2rtc v1.8.0 * 1.8.1 --- docker/main/Dockerfile | 2 +- docs/docs/configuration/advanced.md | 2 +- docs/docs/configuration/camera_specific.md | 2 +- docs/docs/configuration/index.md | 2 +- docs/docs/configuration/live.md | 2 +- docs/docs/configuration/restream.md | 4 ++-- docs/docs/guides/configuring_go2rtc.md | 4 ++-- docs/sidebars.js | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index c7fd149fd..aee067b25 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -33,7 +33,7 @@ RUN --mount=type=tmpfs,target=/tmp --mount=type=tmpfs,target=/var/cache/apt \ FROM scratch AS go2rtc ARG TARGETARCH WORKDIR /rootfs/usr/local/go2rtc/bin -ADD --link --chmod=755 "https://github.com/AlexxIT/go2rtc/releases/download/v1.7.1/go2rtc_linux_${TARGETARCH}" go2rtc +ADD --link --chmod=755 "https://github.com/AlexxIT/go2rtc/releases/download/v1.8.1/go2rtc_linux_${TARGETARCH}" go2rtc #### diff --git a/docs/docs/configuration/advanced.md b/docs/docs/configuration/advanced.md index e32795946..d652b3526 100644 --- a/docs/docs/configuration/advanced.md +++ b/docs/docs/configuration/advanced.md @@ -120,7 +120,7 @@ NOTE: The folder that is mapped from the host needs to be the folder that contai ## Custom go2rtc version -Frigate currently includes go2rtc v1.7.1, there may be certain cases where you want to run a different version of go2rtc. +Frigate currently includes go2rtc v1.8.1, there may be certain cases where you want to run a different version of go2rtc. To do this: diff --git a/docs/docs/configuration/camera_specific.md b/docs/docs/configuration/camera_specific.md index 270b12f70..2567b2c81 100644 --- a/docs/docs/configuration/camera_specific.md +++ b/docs/docs/configuration/camera_specific.md @@ -140,7 +140,7 @@ go2rtc: - rtspx://192.168.1.1:7441/abcdefghijk ``` -[See the go2rtc docs for more information](https://github.com/AlexxIT/go2rtc/tree/v1.7.1#source-rtsp) +[See the go2rtc docs for more information](https://github.com/AlexxIT/go2rtc/tree/v1.8.1#source-rtsp) In the Unifi 2.0 update Unifi Protect Cameras had a change in audio sample rate which causes issues for ffmpeg. The input rate needs to be set for record and rtmp if used directly with unifi protect. diff --git a/docs/docs/configuration/index.md b/docs/docs/configuration/index.md index db59cd62d..f659b5339 100644 --- a/docs/docs/configuration/index.md +++ b/docs/docs/configuration/index.md @@ -436,7 +436,7 @@ rtmp: enabled: False # Optional: Restream configuration -# Uses https://github.com/AlexxIT/go2rtc (v1.7.1) +# Uses https://github.com/AlexxIT/go2rtc (v1.8.1) go2rtc: # Optional: jsmpeg stream configuration for WebUI diff --git a/docs/docs/configuration/live.md b/docs/docs/configuration/live.md index f061d65ff..452daa68c 100644 --- a/docs/docs/configuration/live.md +++ b/docs/docs/configuration/live.md @@ -115,4 +115,4 @@ services: ::: -See [go2rtc WebRTC docs](https://github.com/AlexxIT/go2rtc/tree/v1.7.1#module-webrtc) for more information about this. +See [go2rtc WebRTC docs](https://github.com/AlexxIT/go2rtc/tree/v1.8.1#module-webrtc) for more information about this. diff --git a/docs/docs/configuration/restream.md b/docs/docs/configuration/restream.md index 405ffa7d4..319884ff0 100644 --- a/docs/docs/configuration/restream.md +++ b/docs/docs/configuration/restream.md @@ -7,7 +7,7 @@ title: Restream Frigate can restream your video feed as an RTSP feed for other applications such as Home Assistant to utilize it at `rtsp://:8554/`. Port 8554 must be open. [This allows you to use a video feed for detection in Frigate and Home Assistant live view at the same time without having to make two separate connections to the camera](#reduce-connections-to-camera). The video feed is copied from the original video feed directly to avoid re-encoding. This feed does not include any annotation by Frigate. -Frigate uses [go2rtc](https://github.com/AlexxIT/go2rtc/tree/v1.7.1) to provide its restream and MSE/WebRTC capabilities. The go2rtc config is hosted at the `go2rtc` in the config, see [go2rtc docs](https://github.com/AlexxIT/go2rtc/tree/v1.7.1#configuration) for more advanced configurations and features. +Frigate uses [go2rtc](https://github.com/AlexxIT/go2rtc/tree/v1.8.1) to provide its restream and MSE/WebRTC capabilities. The go2rtc config is hosted at the `go2rtc` in the config, see [go2rtc docs](https://github.com/AlexxIT/go2rtc/tree/v1.8.1#configuration) for more advanced configurations and features. :::note @@ -138,7 +138,7 @@ cameras: ## Advanced Restream Configurations -The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.7.1#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below: +The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.8.1#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below: NOTE: The output will need to be passed with two curly braces `{{output}}` diff --git a/docs/docs/guides/configuring_go2rtc.md b/docs/docs/guides/configuring_go2rtc.md index 8c9a492af..37ea0f819 100644 --- a/docs/docs/guides/configuring_go2rtc.md +++ b/docs/docs/guides/configuring_go2rtc.md @@ -11,7 +11,7 @@ Use of the bundled go2rtc is optional. You can still configure FFmpeg to connect # Setup a go2rtc stream -First, you will want to configure go2rtc to connect to your camera stream by adding the stream you want to use for live view in your Frigate config file. If you set the stream name under go2rtc to match the name of your camera, it will automatically be mapped and you will get additional live view options for the camera. Avoid changing any other parts of your config at this step. Note that go2rtc supports [many different stream types](https://github.com/AlexxIT/go2rtc/tree/v1.7.1#module-streams), not just rtsp. +First, you will want to configure go2rtc to connect to your camera stream by adding the stream you want to use for live view in your Frigate config file. If you set the stream name under go2rtc to match the name of your camera, it will automatically be mapped and you will get additional live view options for the camera. Avoid changing any other parts of your config at this step. Note that go2rtc supports [many different stream types](https://github.com/AlexxIT/go2rtc/tree/v1.8.1#module-streams), not just rtsp. ```yaml go2rtc: @@ -24,7 +24,7 @@ The easiest live view to get working is MSE. After adding this to the config, re ### What if my video doesn't play? -If you are unable to see your video feed, first check the go2rtc logs in the Frigate UI under Logs in the sidebar. If go2rtc is having difficulty connecting to your camera, you should see some error messages in the log. If you do not see any errors, then the video codec of the stream may not be supported in your browser. If your camera stream is set to H265, try switching to H264. You can see more information about [video codec compatibility](https://github.com/AlexxIT/go2rtc/tree/v1.7.1#codecs-madness) in the go2rtc documentation. If you are not able to switch your camera settings from H265 to H264 or your stream is a different format such as MJPEG, you can use go2rtc to re-encode the video using the [FFmpeg parameters](https://github.com/AlexxIT/go2rtc/tree/v1.7.1#source-ffmpeg). It supports rotating and resizing video feeds and hardware acceleration. Keep in mind that transcoding video from one format to another is a resource intensive task and you may be better off using the built-in jsmpeg view. Here is an example of a config that will re-encode the stream to H264 without hardware acceleration: +If you are unable to see your video feed, first check the go2rtc logs in the Frigate UI under Logs in the sidebar. If go2rtc is having difficulty connecting to your camera, you should see some error messages in the log. If you do not see any errors, then the video codec of the stream may not be supported in your browser. If your camera stream is set to H265, try switching to H264. You can see more information about [video codec compatibility](https://github.com/AlexxIT/go2rtc/tree/v1.8.1#codecs-madness) in the go2rtc documentation. If you are not able to switch your camera settings from H265 to H264 or your stream is a different format such as MJPEG, you can use go2rtc to re-encode the video using the [FFmpeg parameters](https://github.com/AlexxIT/go2rtc/tree/v1.8.1#source-ffmpeg). It supports rotating and resizing video feeds and hardware acceleration. Keep in mind that transcoding video from one format to another is a resource intensive task and you may be better off using the built-in jsmpeg view. Here is an example of a config that will re-encode the stream to H264 without hardware acceleration: ```yaml go2rtc: diff --git a/docs/sidebars.js b/docs/sidebars.js index 079af88f7..c814e6e6a 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -21,7 +21,7 @@ module.exports = { { type: "link", label: "Go2RTC Configuration Reference", - href: "https://github.com/AlexxIT/go2rtc/tree/v1.7.1#configuration", + href: "https://github.com/AlexxIT/go2rtc/tree/v1.8.1#configuration", }, ], Detectors: [ From 1bdfc380c39a3d058cbdbadd556410cf9025722a Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 17 Oct 2023 05:37:07 -0600 Subject: [PATCH 03/26] Delete timeline items along with event (#8192) --- frigate/http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frigate/http.py b/frigate/http.py index 695ba8473..48e9661b4 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -516,6 +516,7 @@ def delete_event(id): media.unlink(missing_ok=True) event.delete_instance() + Timeline.delete().where(Timeline.source_id == id).execute() return make_response( jsonify({"success": True, "message": "Event " + id + " deleted"}), 200 ) From c7b2c6b95da84866c2384e7a12daf508c49f890d Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 17 Oct 2023 05:37:40 -0600 Subject: [PATCH 04/26] Pin all hwaccel deps (#8191) --- docker/main/install_deps.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/docker/main/install_deps.sh b/docker/main/install_deps.sh index a42b439ca..18d8976c1 100755 --- a/docker/main/install_deps.sh +++ b/docker/main/install_deps.sh @@ -55,24 +55,16 @@ fi # arch specific packages if [[ "${TARGETARCH}" == "amd64" ]]; then - # use debian bookworm for AMD hwaccel packages + # use debian bookworm for hwaccel packages echo 'deb https://deb.debian.org/debian bookworm main contrib' >/etc/apt/sources.list.d/debian-bookworm.list apt-get -qq update - apt-get -qq install --no-install-recommends --no-install-suggests -y \ - mesa-va-drivers radeontop - rm -f /etc/apt/sources.list.d/debian-bookworm.list - - # Use debian testing repo only for intel hwaccel packages - echo 'deb http://deb.debian.org/debian testing main non-free' >/etc/apt/sources.list.d/debian-testing.list - apt-get -qq update - # intel-opencl-icd specifically for GPU support in OpenVino apt-get -qq install --no-install-recommends --no-install-suggests -y \ intel-opencl-icd \ - libva-drm2 intel-media-va-driver-non-free i965-va-driver libmfx1 intel-gpu-tools + mesa-va-drivers radeontop libva-drm2 intel-media-va-driver-non-free i965-va-driver libmfx1 intel-gpu-tools # something about this dependency requires it to be installed in a separate call rather than in the line above apt-get -qq install --no-install-recommends --no-install-suggests -y \ i965-va-driver-shaders - rm -f /etc/apt/sources.list.d/debian-testing.list + rm -f /etc/apt/sources.list.d/debian-bookworm.list fi if [[ "${TARGETARCH}" == "arm64" ]]; then From efbc094bbc2d42aecc6777221112bd6ea03e998e Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 17 Oct 2023 19:18:06 -0600 Subject: [PATCH 05/26] Fixes for ongoing events (#8208) * Refresh ongoing and standard events * Collapse ongoing when props are set * Fix --- web/src/routes/Events.jsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/web/src/routes/Events.jsx b/web/src/routes/Events.jsx index 55ab44f95..df3416cc8 100644 --- a/web/src/routes/Events.jsx +++ b/web/src/routes/Events.jsx @@ -94,7 +94,7 @@ export default function Events({ path, ...props }) { showDeleteFavorite: false, }); - const [showInProgress, setShowInProgress] = useState(true); + const [showInProgress, setShowInProgress] = useState((props.event || props.cameras || props.labels) == null); const eventsFetcher = useCallback( (path, params) => { @@ -121,8 +121,12 @@ export default function Events({ path, ...props }) { [searchParams] ); - const { data: ongoingEvents } = useSWR(['events', { in_progress: 1, include_thumbnails: 0 }]); - const { data: eventPages, mutate, size, setSize, isValidating } = useSWRInfinite(getKey, eventsFetcher); + const { data: ongoingEvents, mutate: refreshOngoingEvents } = useSWR(['events', { in_progress: 1, include_thumbnails: 0 }]); + const { data: eventPages, mutate: refreshEvents, size, setSize, isValidating } = useSWRInfinite(getKey, eventsFetcher); + const mutate = () => { + refreshEvents(); + refreshOngoingEvents(); + } const { data: allLabels } = useSWR(['labels']); const { data: allSubLabels } = useSWR(['sub_labels', { split_joined: 1 }]); From 126aed2798f201ddc7d113abc4c95e1a41be79b3 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 17 Oct 2023 19:18:50 -0600 Subject: [PATCH 06/26] Include non-free in hwaccel deps types (#8203) --- docker/main/install_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/main/install_deps.sh b/docker/main/install_deps.sh index 18d8976c1..43fff479b 100755 --- a/docker/main/install_deps.sh +++ b/docker/main/install_deps.sh @@ -56,7 +56,7 @@ fi # arch specific packages if [[ "${TARGETARCH}" == "amd64" ]]; then # use debian bookworm for hwaccel packages - echo 'deb https://deb.debian.org/debian bookworm main contrib' >/etc/apt/sources.list.d/debian-bookworm.list + echo 'deb https://deb.debian.org/debian bookworm main contrib non-free' >/etc/apt/sources.list.d/debian-bookworm.list apt-get -qq update apt-get -qq install --no-install-recommends --no-install-suggests -y \ intel-opencl-icd \ From cd35481e92f206119874c5e0acabaf9dca12de8d Mon Sep 17 00:00:00 2001 From: winstona Date: Wed, 18 Oct 2023 04:52:48 -0700 Subject: [PATCH 07/26] Fix recording events intermittently missing (#8162) * fix queues not emptying fully by changing gets to a blocking call with short timeout * add extra error/warning messages when there's a possibility of missing recording segments --- frigate/record/maintainer.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 9aa60585d..c2264d63f 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -410,7 +410,8 @@ class RecordingMaintainer(threading.Thread): wait_time = 0.0 while not self.stop_event.wait(wait_time): run_start = datetime.datetime.now().timestamp() - + stale_frame_count = 0 + stale_frame_count_threshold = 10 # empty the object recordings info queue while True: try: @@ -420,7 +421,10 @@ class RecordingMaintainer(threading.Thread): current_tracked_objects, motion_boxes, regions, - ) = self.object_recordings_info_queue.get(False) + ) = self.object_recordings_info_queue.get(True, timeout=0.1) + + if frame_time < run_start - stale_frame_count_threshold: + stale_frame_count += 1 if self.process_info[camera]["record_enabled"].value: self.object_recordings_info[camera].append( @@ -432,17 +436,32 @@ class RecordingMaintainer(threading.Thread): ) ) except queue.Empty: + q_size = self.object_recordings_info_queue.qsize() + if q_size > 5: + logger.warning( + f"object_recordings_info loop queue not empty ({q_size}) - recording segments may be missing" + ) break + if stale_frame_count > 0: + logger.error( + f"Found {stale_frame_count} old frames, segments from recordings may be missing" + ) + # empty the audio recordings info queue if audio is enabled if self.audio_recordings_info_queue: + stale_frame_count = 0 + while True: try: ( camera, frame_time, dBFS, - ) = self.audio_recordings_info_queue.get(False) + ) = self.audio_recordings_info_queue.get(True, timeout=0.1) + + if frame_time < run_start - stale_frame_count_threshold: + stale_frame_count += 1 if self.process_info[camera]["record_enabled"].value: self.audio_recordings_info[camera].append( @@ -452,8 +471,18 @@ class RecordingMaintainer(threading.Thread): ) ) except queue.Empty: + q_size = self.audio_recordings_info_queue.qsize() + if q_size > 5: + logger.warning( + f"object_recordings_info loop audio queue not empty ({q_size}) - recording segments may be missing" + ) break + if stale_frame_count > 0: + logger.error( + f"Found {stale_frame_count} old audio frames, segments from recordings may be missing" + ) + try: asyncio.run(self.move_files()) except Exception as e: From 282cbf8f4070473b1a63be84492ca40245004b4d Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 18 Oct 2023 17:17:53 -0600 Subject: [PATCH 08/26] Add FAQ item for cameras with bad sub streams (#8224) --- docs/docs/troubleshooting/faqs.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/docs/troubleshooting/faqs.md b/docs/docs/troubleshooting/faqs.md index da334d26e..ca64974fa 100644 --- a/docs/docs/troubleshooting/faqs.md +++ b/docs/docs/troubleshooting/faqs.md @@ -23,6 +23,17 @@ Ensure your cameras send h264 encoded video, or [transcode them](/configuration/ You can open `chrome://media-internals/` in another tab and then try to playback, the media internals page will give information about why playback is failing. +### What do I do if my cameras sub stream is not good enough? + +Frigate generally [recommends cameras with configurable sub streams](/frigate/hardware.md). However, if your camera does not have a sub stream that a suitable resolution, the main stream can be resized. + +To do this efficiently the following setup is required: +1. A GPU or iGPU must be available to do the scaling. +2. [ffmpeg presets for hwaccel](/configuration/hardware_acceleration.md) must be used +3. Set the desired detection resolution for `detect -> width` and `detect -> height`. + +When this is done correctly, the GPU will do the decoding and scaling which will result in a small increase in CPU usage but with better results. + ### My mjpeg stream or snapshots look green and crazy This almost always means that the width/height defined for your camera are not correct. Double check the resolution with VLC or another player. Also make sure you don't have the width and height values backwards. From 98200b7ddaaf2d25f32792321a9a2ee32214dfb3 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 18 Oct 2023 17:18:22 -0600 Subject: [PATCH 09/26] Fix recording segment management (#8220) * Fix timing error * Downgrade logs --- frigate/record/maintainer.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index c2264d63f..ae1e3b04d 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -406,6 +406,7 @@ class RecordingMaintainer(threading.Thread): return None def run(self) -> None: + camera_count = len(self.config.cameras.keys()) # Check for new files every 5 seconds wait_time = 0.0 while not self.stop_event.wait(wait_time): @@ -421,7 +422,7 @@ class RecordingMaintainer(threading.Thread): current_tracked_objects, motion_boxes, regions, - ) = self.object_recordings_info_queue.get(True, timeout=0.1) + ) = self.object_recordings_info_queue.get(True, timeout=0.01) if frame_time < run_start - stale_frame_count_threshold: stale_frame_count += 1 @@ -437,15 +438,15 @@ class RecordingMaintainer(threading.Thread): ) except queue.Empty: q_size = self.object_recordings_info_queue.qsize() - if q_size > 5: - logger.warning( - f"object_recordings_info loop queue not empty ({q_size}) - recording segments may be missing" + if q_size > camera_count: + logger.debug( + f"object_recordings_info loop queue not empty ({q_size})." ) break if stale_frame_count > 0: - logger.error( - f"Found {stale_frame_count} old frames, segments from recordings may be missing" + logger.warning( + f"Found {stale_frame_count} old frames, segments from recordings may be missing." ) # empty the audio recordings info queue if audio is enabled @@ -458,7 +459,7 @@ class RecordingMaintainer(threading.Thread): camera, frame_time, dBFS, - ) = self.audio_recordings_info_queue.get(True, timeout=0.1) + ) = self.audio_recordings_info_queue.get(True, timeout=0.01) if frame_time < run_start - stale_frame_count_threshold: stale_frame_count += 1 @@ -472,9 +473,9 @@ class RecordingMaintainer(threading.Thread): ) except queue.Empty: q_size = self.audio_recordings_info_queue.qsize() - if q_size > 5: - logger.warning( - f"object_recordings_info loop audio queue not empty ({q_size}) - recording segments may be missing" + if q_size > camera_count: + logger.debug( + f"object_recordings_info loop audio queue not empty ({q_size})." ) break From 91f7d67c5eb2637676e28cb0e8eedee141d2669c Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 18 Oct 2023 17:21:52 -0600 Subject: [PATCH 10/26] Smarter Regions (#8194) * Smarter Regions * Formatting * Cleanup * Fix motion region checking logic * Add database table and migration for regions * Update region grid on startup * Revert init delay change * Fix mypy * Move object related functions to util * Remove unused * Fix tests * Remove log * Update the region daily at 2 * Fix logic * Formatting * Initialize grid before starting processing frames * Move back to creating grid in main process * Formatting * Fixes * Formating * Fix region check * Accept all but true * Use regions grid for startup scan * Add clarifying comment * Fix new grid requests * Add tests * Delete stale region grids from DB --- frigate/app.py | 21 +- frigate/comms/dispatcher.py | 8 +- frigate/const.py | 1 + frigate/models.py | 6 + frigate/test/test_reduce_boxes.py | 2 +- frigate/test/test_video.py | 36 +- frigate/track/norfair_tracker.py | 7 +- frigate/util/builtin.py | 8 + frigate/util/object.py | 481 +++++++++++++++++++++++++ frigate/video.py | 355 ++++-------------- migrations/019_create_regions_table.py | 35 ++ 11 files changed, 678 insertions(+), 282 deletions(-) create mode 100644 frigate/util/object.py create mode 100644 migrations/019_create_regions_table.py diff --git a/frigate/app.py b/frigate/app.py index a2e300526..1b807dd5b 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -36,7 +36,7 @@ from frigate.events.external import ExternalEventProcessor from frigate.events.maintainer import EventProcessor from frigate.http import create_app from frigate.log import log_process, root_configurer -from frigate.models import Event, Recordings, RecordingsToDelete, Timeline +from frigate.models import Event, Recordings, RecordingsToDelete, Regions, Timeline from frigate.object_detection import ObjectDetectProcess from frigate.object_processing import TrackedObjectProcessor from frigate.output import output_frames @@ -49,6 +49,7 @@ from frigate.stats import StatsEmitter, stats_init from frigate.storage import StorageMaintainer from frigate.timeline import TimelineProcessor from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes +from frigate.util.object import get_camera_regions_grid from frigate.version import VERSION from frigate.video import capture_camera, track_camera from frigate.watchdog import FrigateWatchdog @@ -69,6 +70,7 @@ class FrigateApp: self.feature_metrics: dict[str, FeatureMetricsTypes] = {} self.ptz_metrics: dict[str, PTZMetricsTypes] = {} self.processes: dict[str, int] = {} + self.region_grids: dict[str, list[list[dict[str, int]]]] = {} def set_environment_vars(self) -> None: for key, value in self.config.environment_vars.items(): @@ -161,6 +163,7 @@ class FrigateApp: # issue https://github.com/python/typeshed/issues/8799 # from mypy 0.981 onwards "frame_queue": mp.Queue(maxsize=2), + "region_grid_queue": mp.Queue(maxsize=1), "capture_process": None, "process": None, "audio_rms": mp.Value("d", 0.0), # type: ignore[typeddict-item] @@ -327,7 +330,7 @@ class FrigateApp: 60, 10 * len([c for c in self.config.cameras.values() if c.enabled]) ), ) - models = [Event, Recordings, RecordingsToDelete, Timeline] + models = [Event, Recordings, RecordingsToDelete, Regions, Timeline] self.db.bind(models) def init_stats(self) -> None: @@ -452,6 +455,17 @@ class FrigateApp: output_processor.start() logger.info(f"Output process started: {output_processor.pid}") + def init_historical_regions(self) -> None: + # delete region grids for removed or renamed cameras + cameras = list(self.config.cameras.keys()) + Regions.delete().where(~(Regions.camera << cameras)).execute() + + # create or update region grids for each camera + for camera in self.config.cameras.values(): + self.region_grids[camera.name] = get_camera_regions_grid( + camera.name, camera.detect + ) + def start_camera_processors(self) -> None: for name, config in self.config.cameras.items(): if not self.config.cameras[name].enabled: @@ -469,8 +483,10 @@ class FrigateApp: self.detection_queue, self.detection_out_events[name], self.detected_frames_queue, + self.inter_process_queue, self.camera_metrics[name], self.ptz_metrics[name], + self.region_grids[name], ), ) camera_process.daemon = True @@ -611,6 +627,7 @@ class FrigateApp: self.start_detectors() self.start_video_output_processor() self.start_ptz_autotracker() + self.init_historical_regions() self.start_detected_frames_processor() self.start_camera_processors() self.start_camera_capture_processes() diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index e97095c8a..f3886a331 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -5,10 +5,11 @@ from abc import ABC, abstractmethod from typing import Any, Callable from frigate.config import FrigateConfig -from frigate.const import INSERT_MANY_RECORDINGS +from frigate.const import INSERT_MANY_RECORDINGS, REQUEST_REGION_GRID from frigate.models import Recordings from frigate.ptz.onvif import OnvifCommandEnum, OnvifController from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes +from frigate.util.object import get_camera_regions_grid from frigate.util.services import restart_frigate logger = logging.getLogger(__name__) @@ -90,6 +91,11 @@ class Dispatcher: restart_frigate() elif topic == INSERT_MANY_RECORDINGS: Recordings.insert_many(payload).execute() + elif topic == REQUEST_REGION_GRID: + camera = payload + self.camera_metrics[camera]["region_grid_queue"].put( + get_camera_regions_grid(camera, self.config.cameras[camera].detect) + ) else: self.publish(topic, payload, retain=False) diff --git a/frigate/const.py b/frigate/const.py index c6912471b..56d0f4517 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -51,3 +51,4 @@ MAX_PLAYLIST_SECONDS = 7200 # support 2 hour segments for a single playlist to # Internal Comms Topics INSERT_MANY_RECORDINGS = "insert_many_recordings" +REQUEST_REGION_GRID = "request_region_grid" diff --git a/frigate/models.py b/frigate/models.py index b29ae91dc..65cbfbaac 100644 --- a/frigate/models.py +++ b/frigate/models.py @@ -57,6 +57,12 @@ class Timeline(Model): # type: ignore[misc] data = JSONField() # ex: tracked object id, region, box, etc. +class Regions(Model): # type: ignore[misc] + camera = CharField(null=False, primary_key=True, max_length=20) + grid = JSONField() # json blob of grid + last_update = DateTimeField() + + class Recordings(Model): # type: ignore[misc] id = CharField(null=False, primary_key=True, max_length=30) camera = CharField(index=True, max_length=20) diff --git a/frigate/test/test_reduce_boxes.py b/frigate/test/test_reduce_boxes.py index d26fcd40c..5ac913dfe 100644 --- a/frigate/test/test_reduce_boxes.py +++ b/frigate/test/test_reduce_boxes.py @@ -1,6 +1,6 @@ from unittest import TestCase, main -from frigate.video import box_overlaps, reduce_boxes +from frigate.util.object import box_overlaps, reduce_boxes class TestBoxOverlaps(TestCase): diff --git a/frigate/test/test_video.py b/frigate/test/test_video.py index 99736f658..9fd46a877 100644 --- a/frigate/test/test_video.py +++ b/frigate/test/test_video.py @@ -6,10 +6,11 @@ from norfair.drawing.color import Palette from norfair.drawing.drawer import Drawer from frigate.util.image import intersection -from frigate.video import ( +from frigate.util.object import ( get_cluster_boundary, get_cluster_candidates, get_cluster_region, + get_region_from_grid, ) @@ -190,3 +191,36 @@ class TestObjectBoundingBoxes(unittest.TestCase): assert intersection(box_a, box_b) == None assert intersection(box_b, box_c) == (899, 128, 985, 151) + + +class TestRegionGrid(unittest.TestCase): + def setUp(self) -> None: + pass + + def test_region_in_range(self): + """Test that region is kept at minimal size when within std dev.""" + frame_shape = (720, 1280) + box = [450, 450, 550, 550] + region_grid = [ + [], + [], + [], + [{}, {}, {}, {}, {}, {"sizes": [0.25], "mean": 0.26, "std_dev": 0.01}], + ] + + region = get_region_from_grid(frame_shape, box, 320, region_grid) + assert region[2] - region[0] == 320 + + def test_region_out_of_range(self): + """Test that region is upsized when outside of std dev.""" + frame_shape = (720, 1280) + box = [450, 450, 550, 550] + region_grid = [ + [], + [], + [], + [{}, {}, {}, {}, {}, {"sizes": [0.5], "mean": 0.5, "std_dev": 0.1}], + ] + + region = get_region_from_grid(frame_shape, box, 320, region_grid) + assert region[2] - region[0] > 320 diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py index ff63ba563..42a2fde2f 100644 --- a/frigate/track/norfair_tracker.py +++ b/frigate/track/norfair_tracker.py @@ -77,7 +77,7 @@ class NorfairTracker(ObjectTracker): self.tracker = Tracker( distance_function=frigate_distance, distance_threshold=2.5, - initialization_delay=config.detect.fps / 2, + initialization_delay=0, hit_counter_max=self.max_disappeared, ) if self.ptz_autotracker_enabled.value: @@ -106,6 +106,11 @@ class NorfairTracker(ObjectTracker): "ymax": self.detect_config.height, } + # start object with a hit count of `fps` to avoid quick detection -> loss + next( + (o for o in self.tracker.tracked_objects if o.global_id == track_id) + ).hit_counter = self.camera_config.detect.fps + def deregister(self, id, track_id): del self.tracked_objects[id] del self.disappeared[id] diff --git a/frigate/util/builtin.py b/frigate/util/builtin.py index 5cb01e1c5..2a9b7053a 100644 --- a/frigate/util/builtin.py +++ b/frigate/util/builtin.py @@ -14,6 +14,7 @@ import numpy as np import pytz import yaml from ruamel.yaml import YAML +from tzlocal import get_localzone from frigate.const import REGEX_HTTP_CAMERA_USER_PASS, REGEX_RTSP_CAMERA_USER_PASS @@ -262,3 +263,10 @@ def find_by_key(dictionary, target_key): if result is not None: return result return None + + +def get_tomorrow_at_2() -> datetime.datetime: + tomorrow = datetime.datetime.now(get_localzone()) + datetime.timedelta(days=1) + return tomorrow.replace(hour=2, minute=0, second=0).astimezone( + datetime.timezone.utc + ) diff --git a/frigate/util/object.py b/frigate/util/object.py new file mode 100644 index 000000000..3fa98df59 --- /dev/null +++ b/frigate/util/object.py @@ -0,0 +1,481 @@ +"""Utils for reading and writing object detection data.""" + +import datetime +import logging +import math + +import cv2 +import numpy as np +from peewee import DoesNotExist + +from frigate.config import DetectConfig, ModelConfig +from frigate.detectors.detector_config import PixelFormatEnum +from frigate.models import Event, Regions, Timeline +from frigate.util.image import ( + area, + calculate_region, + intersection, + intersection_over_union, + yuv_region_2_bgr, + yuv_region_2_rgb, + yuv_region_2_yuv, +) + +logger = logging.getLogger(__name__) + +GRID_SIZE = 8 + + +def get_camera_regions_grid( + name: str, detect: DetectConfig +) -> list[list[dict[str, any]]]: + """Build a grid of expected region sizes for a camera.""" + # get grid from db if available + try: + regions: Regions = Regions.select().where(Regions.camera == name).get() + grid = regions.grid + last_update = regions.last_update + except DoesNotExist: + grid = [] + for x in range(GRID_SIZE): + row = [] + for y in range(GRID_SIZE): + row.append({"sizes": []}) + grid.append(row) + last_update = 0 + + # get events for timeline entries + events = ( + Event.select(Event.id) + .where(Event.camera == name) + .where((Event.false_positive == None) | (Event.false_positive == False)) + .where(Event.start_time > last_update) + ) + valid_event_ids = [e["id"] for e in events.dicts()] + logger.debug(f"Found {len(valid_event_ids)} new events for {name}") + + # no new events, return as is + if not valid_event_ids: + return grid + + new_update = datetime.datetime.now().timestamp() + timeline = ( + Timeline.select( + *[ + Timeline.camera, + Timeline.source, + Timeline.data, + ] + ) + .where(Timeline.source_id << valid_event_ids) + .limit(10000) + .dicts() + ) + + logger.debug(f"Found {len(timeline)} new entries for {name}") + + width = detect.width + height = detect.height + + for t in timeline: + if t.get("source") != "tracked_object": + continue + + box = t["data"]["box"] + + # calculate centroid position + x = box[0] + (box[2] / 2) + y = box[1] + (box[3] / 2) + + x_pos = int(x * GRID_SIZE) + y_pos = int(y * GRID_SIZE) + + calculated_region = calculate_region( + (height, width), + box[0] * width, + box[1] * height, + (box[0] + box[2]) * width, + (box[1] + box[3]) * height, + 320, + 1.35, + ) + # save width of region to grid as relative + grid[x_pos][y_pos]["sizes"].append( + (calculated_region[2] - calculated_region[0]) / width + ) + + for x in range(GRID_SIZE): + for y in range(GRID_SIZE): + cell = grid[x][y] + + if len(cell["sizes"]) == 0: + continue + + std_dev = np.std(cell["sizes"]) + mean = np.mean(cell["sizes"]) + logger.debug(f"std dev: {std_dev} mean: {mean}") + cell["x"] = x + cell["y"] = y + cell["std_dev"] = std_dev + cell["mean"] = mean + + # update db with new grid + region = { + Regions.camera: name, + Regions.grid: grid, + Regions.last_update: new_update, + } + ( + Regions.insert(region) + .on_conflict( + conflict_target=[Regions.camera], + update=region, + ) + .execute() + ) + + return grid + + +def get_cluster_region_from_grid(frame_shape, min_region, cluster, boxes, region_grid): + min_x = frame_shape[1] + min_y = frame_shape[0] + max_x = 0 + max_y = 0 + for b in cluster: + min_x = min(boxes[b][0], min_x) + min_y = min(boxes[b][1], min_y) + max_x = max(boxes[b][2], max_x) + max_y = max(boxes[b][3], max_y) + return get_region_from_grid( + frame_shape, [min_x, min_y, max_x, max_y], min_region, region_grid + ) + + +def get_region_from_grid( + frame_shape: tuple[int], + cluster: list[int], + min_region: int, + region_grid: list[list[dict[str, any]]], +) -> list[int]: + """Get a region for a box based on the region grid.""" + box = calculate_region( + frame_shape, cluster[0], cluster[1], cluster[2], cluster[3], min_region + ) + centroid = ( + box[0] + (min(frame_shape[1], box[2]) - box[0]) / 2, + box[1] + (min(frame_shape[0], box[3]) - box[1]) / 2, + ) + grid_x = int(centroid[0] / frame_shape[1] * GRID_SIZE) + grid_y = int(centroid[1] / frame_shape[0] * GRID_SIZE) + + cell = region_grid[grid_x][grid_y] + + # if there is no known data, get standard region for motion box + if not cell or not cell["sizes"]: + return calculate_region(frame_shape, box[0], box[1], box[2], box[3], min_region) + + # convert the calculated region size to relative + calc_size = (box[2] - box[0]) / frame_shape[1] + + # if region is within expected size, don't resize + if ( + (cell["mean"] - cell["std_dev"]) + <= calc_size + <= (cell["mean"] + cell["std_dev"]) + ): + return box + # TODO not sure how to handle case where cluster is larger than expected region + elif calc_size > (cell["mean"] + cell["std_dev"]): + return box + + size = cell["mean"] * frame_shape[1] + + # get region based on grid size + return calculate_region( + frame_shape, + max(0, centroid[0] - size / 2), + max(0, centroid[1] - size / 2), + min(frame_shape[1], centroid[0] + size / 2), + min(frame_shape[0], centroid[1] + size / 2), + min_region, + ) + + +def is_object_filtered(obj, objects_to_track, object_filters): + object_name = obj[0] + object_score = obj[1] + object_box = obj[2] + object_area = obj[3] + object_ratio = obj[4] + + if object_name not in objects_to_track: + return True + + if object_name in object_filters: + obj_settings = object_filters[object_name] + + # if the min area is larger than the + # detected object, don't add it to detected objects + if obj_settings.min_area > object_area: + return True + + # if the detected object is larger than the + # max area, don't add it to detected objects + if obj_settings.max_area < object_area: + return True + + # if the score is lower than the min_score, skip + if obj_settings.min_score > object_score: + return True + + # if the object is not proportionally wide enough + if obj_settings.min_ratio > object_ratio: + return True + + # if the object is proportionally too wide + if obj_settings.max_ratio < object_ratio: + return True + + if obj_settings.mask is not None: + # compute the coordinates of the object and make sure + # the location isn't outside the bounds of the image (can happen from rounding) + object_xmin = object_box[0] + object_xmax = object_box[2] + object_ymax = object_box[3] + y_location = min(int(object_ymax), len(obj_settings.mask) - 1) + x_location = min( + int((object_xmax + object_xmin) / 2.0), + len(obj_settings.mask[0]) - 1, + ) + + # if the object is in a masked location, don't add it to detected objects + if obj_settings.mask[y_location][x_location] == 0: + return True + + return False + + +def get_min_region_size(model_config: ModelConfig) -> int: + """Get the min region size.""" + return max(model_config.height, model_config.width) + + +def create_tensor_input(frame, model_config: ModelConfig, region): + if model_config.input_pixel_format == PixelFormatEnum.rgb: + cropped_frame = yuv_region_2_rgb(frame, region) + elif model_config.input_pixel_format == PixelFormatEnum.bgr: + cropped_frame = yuv_region_2_bgr(frame, region) + else: + cropped_frame = yuv_region_2_yuv(frame, region) + + # Resize if needed + if cropped_frame.shape != (model_config.height, model_config.width, 3): + cropped_frame = cv2.resize( + cropped_frame, + dsize=(model_config.width, model_config.height), + interpolation=cv2.INTER_LINEAR, + ) + + # Expand dimensions since the model expects images to have shape: [1, height, width, 3] + return np.expand_dims(cropped_frame, axis=0) + + +def box_overlaps(b1, b2): + if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]: + return False + return True + + +def box_inside(b1, b2): + # check if b2 is inside b1 + if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]: + return True + return False + + +def reduce_boxes(boxes, iou_threshold=0.0): + clusters = [] + + for box in boxes: + matched = 0 + for cluster in clusters: + if intersection_over_union(box, cluster) > iou_threshold: + matched = 1 + cluster[0] = min(cluster[0], box[0]) + cluster[1] = min(cluster[1], box[1]) + cluster[2] = max(cluster[2], box[2]) + cluster[3] = max(cluster[3], box[3]) + + if not matched: + clusters.append(list(box)) + + return [tuple(c) for c in clusters] + + +def intersects_any(box_a, boxes): + for box in boxes: + if box_overlaps(box_a, box): + return True + return False + + +def inside_any(box_a, boxes): + for box in boxes: + # check if box_a is inside of box + if box_inside(box, box_a): + return True + return False + + +def get_cluster_boundary(box, min_region): + # compute the max region size for the current box (box is 10% of region) + box_width = box[2] - box[0] + box_height = box[3] - box[1] + max_region_area = abs(box_width * box_height) / 0.1 + max_region_size = max(min_region, int(math.sqrt(max_region_area))) + + centroid = (box_width / 2 + box[0], box_height / 2 + box[1]) + + max_x_dist = int(max_region_size - box_width / 2 * 1.1) + max_y_dist = int(max_region_size - box_height / 2 * 1.1) + + return [ + int(centroid[0] - max_x_dist), + int(centroid[1] - max_y_dist), + int(centroid[0] + max_x_dist), + int(centroid[1] + max_y_dist), + ] + + +def get_cluster_candidates(frame_shape, min_region, boxes): + # and create a cluster of other boxes using it's max region size + # only include boxes where the region is an appropriate(except the region could possibly be smaller?) + # size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset + # determined by the max_region size minus half the box + 20% + # TODO: see if we can do this with numpy + cluster_candidates = [] + used_boxes = [] + # loop over each box + for current_index, b in enumerate(boxes): + if current_index in used_boxes: + continue + cluster = [current_index] + used_boxes.append(current_index) + cluster_boundary = get_cluster_boundary(b, min_region) + # find all other boxes that fit inside the boundary + for compare_index, compare_box in enumerate(boxes): + if compare_index in used_boxes: + continue + + # if the box is not inside the potential cluster area, cluster them + if not box_inside(cluster_boundary, compare_box): + continue + + # get the region if you were to add this box to the cluster + potential_cluster = cluster + [compare_index] + cluster_region = get_cluster_region( + frame_shape, min_region, potential_cluster, boxes + ) + # if region could be smaller and either box would be too small + # for the resulting region, dont cluster + should_cluster = True + if (cluster_region[2] - cluster_region[0]) > min_region: + for b in potential_cluster: + box = boxes[b] + # boxes should be more than 5% of the area of the region + if area(box) / area(cluster_region) < 0.05: + should_cluster = False + break + + if should_cluster: + cluster.append(compare_index) + used_boxes.append(compare_index) + cluster_candidates.append(cluster) + + # return the unique clusters only + unique = {tuple(sorted(c)) for c in cluster_candidates} + return [list(tup) for tup in unique] + + +def get_cluster_region(frame_shape, min_region, cluster, boxes): + min_x = frame_shape[1] + min_y = frame_shape[0] + max_x = 0 + max_y = 0 + for b in cluster: + min_x = min(boxes[b][0], min_x) + min_y = min(boxes[b][1], min_y) + max_x = max(boxes[b][2], max_x) + max_y = max(boxes[b][3], max_y) + return calculate_region( + frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2 + ) + + +def get_consolidated_object_detections(detected_object_groups): + """Drop detections that overlap too much""" + consolidated_detections = [] + for group in detected_object_groups.values(): + # if the group only has 1 item, skip + if len(group) == 1: + consolidated_detections.append(group[0]) + continue + + # sort smallest to largest by area + sorted_by_area = sorted(group, key=lambda g: g[3]) + + for current_detection_idx in range(0, len(sorted_by_area)): + current_detection = sorted_by_area[current_detection_idx][2] + overlap = 0 + for to_check_idx in range( + min(current_detection_idx + 1, len(sorted_by_area)), + len(sorted_by_area), + ): + to_check = sorted_by_area[to_check_idx][2] + intersect_box = intersection(current_detection, to_check) + # if 90% of smaller detection is inside of another detection, consolidate + if ( + intersect_box is not None + and area(intersect_box) / area(current_detection) > 0.9 + ): + overlap = 1 + break + if overlap == 0: + consolidated_detections.append(sorted_by_area[current_detection_idx]) + + return consolidated_detections + + +def get_startup_regions( + frame_shape: tuple[int], + region_min_size: int, + region_grid: list[list[dict[str, any]]], +) -> list[list[int]]: + """Get a list of regions to run on startup.""" + # return 8 most popular regions for the camera + all_cells = np.concatenate(region_grid).flat + startup_cells = sorted(all_cells, key=lambda c: len(c["sizes"]), reverse=True)[0:8] + regions = [] + + for cell in startup_cells: + # rest of the cells are empty + if not cell["sizes"]: + break + + x = frame_shape[1] / GRID_SIZE * (0.5 + cell["x"]) + y = frame_shape[0] / GRID_SIZE * (0.5 + cell["y"]) + size = cell["mean"] * frame_shape[1] + regions.append( + calculate_region( + frame_shape, + x - size / 2, + y - size / 2, + x + size / 2, + y + size / 2, + region_min_size, + multiplier=1, + ) + ) + + return regions diff --git a/frigate/video.py b/frigate/video.py index 47e65811d..b961cc8a6 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -1,6 +1,5 @@ import datetime import logging -import math import multiprocessing as mp import os import queue @@ -15,8 +14,12 @@ import numpy as np from setproctitle import setproctitle from frigate.config import CameraConfig, DetectConfig, ModelConfig -from frigate.const import ALL_ATTRIBUTE_LABELS, ATTRIBUTE_LABEL_MAP, CACHE_DIR -from frigate.detectors.detector_config import PixelFormatEnum +from frigate.const import ( + ALL_ATTRIBUTE_LABELS, + ATTRIBUTE_LABEL_MAP, + CACHE_DIR, + REQUEST_REGION_GRID, +) from frigate.log import LogPipe from frigate.motion import MotionDetector from frigate.motion.improved_motion import ImprovedMotionDetector @@ -24,103 +27,30 @@ from frigate.object_detection import RemoteObjectDetector from frigate.track import ObjectTracker from frigate.track.norfair_tracker import NorfairTracker from frigate.types import PTZMetricsTypes -from frigate.util.builtin import EventsPerSecond +from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_2 from frigate.util.image import ( FrameManager, SharedMemoryFrameManager, - area, - calculate_region, draw_box_with_label, - intersection, - intersection_over_union, - yuv_region_2_bgr, - yuv_region_2_rgb, - yuv_region_2_yuv, +) +from frigate.util.object import ( + box_inside, + create_tensor_input, + get_cluster_candidates, + get_cluster_region, + get_cluster_region_from_grid, + get_consolidated_object_detections, + get_min_region_size, + get_startup_regions, + inside_any, + intersects_any, + is_object_filtered, ) from frigate.util.services import listen logger = logging.getLogger(__name__) -def filtered(obj, objects_to_track, object_filters): - object_name = obj[0] - object_score = obj[1] - object_box = obj[2] - object_area = obj[3] - object_ratio = obj[4] - - if object_name not in objects_to_track: - return True - - if object_name in object_filters: - obj_settings = object_filters[object_name] - - # if the min area is larger than the - # detected object, don't add it to detected objects - if obj_settings.min_area > object_area: - return True - - # if the detected object is larger than the - # max area, don't add it to detected objects - if obj_settings.max_area < object_area: - return True - - # if the score is lower than the min_score, skip - if obj_settings.min_score > object_score: - return True - - # if the object is not proportionally wide enough - if obj_settings.min_ratio > object_ratio: - return True - - # if the object is proportionally too wide - if obj_settings.max_ratio < object_ratio: - return True - - if obj_settings.mask is not None: - # compute the coordinates of the object and make sure - # the location isn't outside the bounds of the image (can happen from rounding) - object_xmin = object_box[0] - object_xmax = object_box[2] - object_ymax = object_box[3] - y_location = min(int(object_ymax), len(obj_settings.mask) - 1) - x_location = min( - int((object_xmax + object_xmin) / 2.0), - len(obj_settings.mask[0]) - 1, - ) - - # if the object is in a masked location, don't add it to detected objects - if obj_settings.mask[y_location][x_location] == 0: - return True - - return False - - -def get_min_region_size(model_config: ModelConfig) -> int: - """Get the min region size.""" - return max(model_config.height, model_config.width) - - -def create_tensor_input(frame, model_config: ModelConfig, region): - if model_config.input_pixel_format == PixelFormatEnum.rgb: - cropped_frame = yuv_region_2_rgb(frame, region) - elif model_config.input_pixel_format == PixelFormatEnum.bgr: - cropped_frame = yuv_region_2_bgr(frame, region) - else: - cropped_frame = yuv_region_2_yuv(frame, region) - - # Resize if needed - if cropped_frame.shape != (model_config.height, model_config.width, 3): - cropped_frame = cv2.resize( - cropped_frame, - dsize=(model_config.width, model_config.height), - interpolation=cv2.INTER_LINEAR, - ) - - # Expand dimensions since the model expects images to have shape: [1, height, width, 3] - return np.expand_dims(cropped_frame, axis=0) - - def stop_ffmpeg(ffmpeg_process, logger): logger.info("Terminating the existing ffmpeg process...") ffmpeg_process.terminate() @@ -455,8 +385,10 @@ def track_camera( detection_queue, result_connection, detected_objects_queue, + inter_process_queue, process_info, ptz_metrics, + region_grid, ): stop_event = mp.Event() @@ -471,6 +403,7 @@ def track_camera( listen() frame_queue = process_info["frame_queue"] + region_grid_queue = process_info["region_grid_queue"] detection_enabled = process_info["detection_enabled"] motion_enabled = process_info["motion_enabled"] improve_contrast_enabled = process_info["improve_contrast_enabled"] @@ -499,7 +432,9 @@ def track_camera( process_frames( name, + inter_process_queue, frame_queue, + region_grid_queue, frame_shape, model_config, config.detect, @@ -515,50 +450,12 @@ def track_camera( motion_enabled, stop_event, ptz_metrics, + region_grid, ) logger.info(f"{name}: exiting subprocess") -def box_overlaps(b1, b2): - if b1[2] < b2[0] or b1[0] > b2[2] or b1[1] > b2[3] or b1[3] < b2[1]: - return False - return True - - -def box_inside(b1, b2): - # check if b2 is inside b1 - if b2[0] >= b1[0] and b2[1] >= b1[1] and b2[2] <= b1[2] and b2[3] <= b1[3]: - return True - return False - - -def reduce_boxes(boxes, iou_threshold=0.0): - clusters = [] - - for box in boxes: - matched = 0 - for cluster in clusters: - if intersection_over_union(box, cluster) > iou_threshold: - matched = 1 - cluster[0] = min(cluster[0], box[0]) - cluster[1] = min(cluster[1], box[1]) - cluster[2] = max(cluster[2], box[2]) - cluster[3] = max(cluster[3], box[3]) - - if not matched: - clusters.append(list(box)) - - return [tuple(c) for c in clusters] - - -def intersects_any(box_a, boxes): - for box in boxes: - if box_overlaps(box_a, box): - return True - return False - - def detect( detect_config: DetectConfig, object_detector, @@ -597,134 +494,17 @@ def detect( region, ) # apply object filters - if filtered(det, objects_to_track, object_filters): + if is_object_filtered(det, objects_to_track, object_filters): continue detections.append(det) return detections -def get_cluster_boundary(box, min_region): - # compute the max region size for the current box (box is 10% of region) - box_width = box[2] - box[0] - box_height = box[3] - box[1] - max_region_area = abs(box_width * box_height) / 0.1 - max_region_size = max(min_region, int(math.sqrt(max_region_area))) - - centroid = (box_width / 2 + box[0], box_height / 2 + box[1]) - - max_x_dist = int(max_region_size - box_width / 2 * 1.1) - max_y_dist = int(max_region_size - box_height / 2 * 1.1) - - return [ - int(centroid[0] - max_x_dist), - int(centroid[1] - max_y_dist), - int(centroid[0] + max_x_dist), - int(centroid[1] + max_y_dist), - ] - - -def get_cluster_candidates(frame_shape, min_region, boxes): - # and create a cluster of other boxes using it's max region size - # only include boxes where the region is an appropriate(except the region could possibly be smaller?) - # size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset - # determined by the max_region size minus half the box + 20% - # TODO: see if we can do this with numpy - cluster_candidates = [] - used_boxes = [] - # loop over each box - for current_index, b in enumerate(boxes): - if current_index in used_boxes: - continue - cluster = [current_index] - used_boxes.append(current_index) - cluster_boundary = get_cluster_boundary(b, min_region) - # find all other boxes that fit inside the boundary - for compare_index, compare_box in enumerate(boxes): - if compare_index in used_boxes: - continue - - # if the box is not inside the potential cluster area, cluster them - if not box_inside(cluster_boundary, compare_box): - continue - - # get the region if you were to add this box to the cluster - potential_cluster = cluster + [compare_index] - cluster_region = get_cluster_region( - frame_shape, min_region, potential_cluster, boxes - ) - # if region could be smaller and either box would be too small - # for the resulting region, dont cluster - should_cluster = True - if (cluster_region[2] - cluster_region[0]) > min_region: - for b in potential_cluster: - box = boxes[b] - # boxes should be more than 5% of the area of the region - if area(box) / area(cluster_region) < 0.05: - should_cluster = False - break - - if should_cluster: - cluster.append(compare_index) - used_boxes.append(compare_index) - cluster_candidates.append(cluster) - - # return the unique clusters only - unique = {tuple(sorted(c)) for c in cluster_candidates} - return [list(tup) for tup in unique] - - -def get_cluster_region(frame_shape, min_region, cluster, boxes): - min_x = frame_shape[1] - min_y = frame_shape[0] - max_x = 0 - max_y = 0 - for b in cluster: - min_x = min(boxes[b][0], min_x) - min_y = min(boxes[b][1], min_y) - max_x = max(boxes[b][2], max_x) - max_y = max(boxes[b][3], max_y) - return calculate_region( - frame_shape, min_x, min_y, max_x, max_y, min_region, multiplier=1.2 - ) - - -def get_consolidated_object_detections(detected_object_groups): - """Drop detections that overlap too much""" - consolidated_detections = [] - for group in detected_object_groups.values(): - # if the group only has 1 item, skip - if len(group) == 1: - consolidated_detections.append(group[0]) - continue - - # sort smallest to largest by area - sorted_by_area = sorted(group, key=lambda g: g[3]) - - for current_detection_idx in range(0, len(sorted_by_area)): - current_detection = sorted_by_area[current_detection_idx][2] - overlap = 0 - for to_check_idx in range( - min(current_detection_idx + 1, len(sorted_by_area)), - len(sorted_by_area), - ): - to_check = sorted_by_area[to_check_idx][2] - intersect_box = intersection(current_detection, to_check) - # if 90% of smaller detection is inside of another detection, consolidate - if ( - intersect_box is not None - and area(intersect_box) / area(current_detection) > 0.9 - ): - overlap = 1 - break - if overlap == 0: - consolidated_detections.append(sorted_by_area[current_detection_idx]) - - return consolidated_detections - - def process_frames( camera_name: str, + inter_process_queue: mp.Queue, frame_queue: mp.Queue, + region_grid_queue: mp.Queue, frame_shape, model_config: ModelConfig, detect_config: DetectConfig, @@ -740,20 +520,35 @@ def process_frames( motion_enabled: mp.Value, stop_event, ptz_metrics: PTZMetricsTypes, + region_grid, exit_on_empty: bool = False, ): fps = process_info["process_fps"] detection_fps = process_info["detection_fps"] current_frame_time = process_info["detection_frame"] + next_region_update = get_tomorrow_at_2() fps_tracker = EventsPerSecond() fps_tracker.start() - startup_scan_counter = 0 + startup_scan = True region_min_size = get_min_region_size(model_config) while not stop_event.is_set(): + if ( + datetime.datetime.now().astimezone(datetime.timezone.utc) + > next_region_update + ): + inter_process_queue.put((REQUEST_REGION_GRID, camera_name)) + + try: + region_grid = region_grid_queue.get(True, 10) + except queue.Empty: + logger.error(f"Unable to get updated region grid for {camera_name}") + + next_region_update = get_tomorrow_at_2() + try: if exit_on_empty: frame_time = frame_queue.get(False) @@ -815,40 +610,48 @@ def process_frames( if obj["id"] not in stationary_object_ids ] - combined_boxes = tracked_object_boxes - # only add in the motion boxes when not calibrating - if not motion_detector.is_calibrating(): - combined_boxes += motion_boxes - - cluster_candidates = get_cluster_candidates( - frame_shape, region_min_size, combined_boxes - ) - + # get consolidated regions for tracked objects regions = [ get_cluster_region( - frame_shape, region_min_size, candidate, combined_boxes + frame_shape, region_min_size, candidate, tracked_object_boxes + ) + for candidate in get_cluster_candidates( + frame_shape, region_min_size, tracked_object_boxes ) - for candidate in cluster_candidates ] - # if starting up, get the next startup scan region - if startup_scan_counter < 9: - ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3) - ymax = int(frame_shape[0] / 3 + ymin) - xmin = int(frame_shape[1] / 3 * startup_scan_counter / 3) - xmax = int(frame_shape[1] / 3 + xmin) - regions.append( - calculate_region( + # only add in the motion boxes when not calibrating + if not motion_detector.is_calibrating(): + # find motion boxes that are not inside tracked object regions + standalone_motion_boxes = [ + b for b in motion_boxes if not inside_any(b, regions) + ] + + if standalone_motion_boxes: + motion_clusters = get_cluster_candidates( frame_shape, - xmin, - ymin, - xmax, - ymax, region_min_size, - multiplier=1.2, + standalone_motion_boxes, ) - ) - startup_scan_counter += 1 + motion_regions = [ + get_cluster_region_from_grid( + frame_shape, + region_min_size, + candidate, + standalone_motion_boxes, + region_grid, + ) + for candidate in motion_clusters + ] + regions += motion_regions + + # if starting up, get the next startup scan region + if startup_scan: + for region in get_startup_regions( + frame_shape, region_min_size, region_grid + ): + regions.append(region) + startup_scan = False # resize regions and detect # seed with stationary objects diff --git a/migrations/019_create_regions_table.py b/migrations/019_create_regions_table.py new file mode 100644 index 000000000..e1492581b --- /dev/null +++ b/migrations/019_create_regions_table.py @@ -0,0 +1,35 @@ +"""Peewee migrations -- 019_create_regions_table.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + migrator.sql( + 'CREATE TABLE IF NOT EXISTS "regions" ("camera" VARCHAR(20) NOT NULL PRIMARY KEY, "last_update" DATETIME NOT NULL, "grid" JSON)' + ) + + +def rollback(migrator, database, fake=False, **kwargs): + pass From 8f349a6365e232b02e1034c4075c1b80ab5c7f0b Mon Sep 17 00:00:00 2001 From: Sergey Krashevich Date: Fri, 20 Oct 2023 01:14:06 +0300 Subject: [PATCH 11/26] use sum() instead of len() to count only enabled cameras (#8232) --- frigate/record/maintainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index ae1e3b04d..3cda65c29 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -406,7 +406,7 @@ class RecordingMaintainer(threading.Thread): return None def run(self) -> None: - camera_count = len(self.config.cameras.keys()) + camera_count = sum(camera.enabled for camera in self.config.cameras.values()) # Check for new files every 5 seconds wait_time = 0.0 while not self.stop_event.wait(wait_time): From 12487b3b602db263f0ad9cb199e49b438f9d7293 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 19 Oct 2023 16:14:33 -0600 Subject: [PATCH 12/26] Sync stationary object checks (#8238) * Sync stationary object checks for all objects on a camera * Formatting --- frigate/video.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/frigate/video.py b/frigate/video.py index b961cc8a6..a6d52dfc7 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -532,6 +532,7 @@ def process_frames( fps_tracker.start() startup_scan = True + stationary_frame_counter = 0 region_min_size = get_min_region_size(model_config) @@ -585,23 +586,24 @@ def process_frames( # check every Nth frame for stationary objects # disappeared objects are not stationary # also check for overlapping motion boxes - stationary_object_ids = [ - obj["id"] - for obj in object_tracker.tracked_objects.values() - # if it has exceeded the stationary threshold - if obj["motionless_count"] >= detect_config.stationary.threshold - # and it isn't due for a periodic check - and ( - detect_config.stationary.interval == 0 - or obj["motionless_count"] % detect_config.stationary.interval != 0 - ) - # and it hasn't disappeared - and object_tracker.disappeared[obj["id"]] == 0 - # and it doesn't overlap with any current motion boxes when not calibrating - and not intersects_any( - obj["box"], [] if motion_detector.is_calibrating() else motion_boxes - ) - ] + if stationary_frame_counter == detect_config.stationary.interval: + stationary_frame_counter = 0 + stationary_object_ids = [] + else: + stationary_frame_counter += 1 + stationary_object_ids = [ + obj["id"] + for obj in object_tracker.tracked_objects.values() + # if it has exceeded the stationary threshold + if obj["motionless_count"] >= detect_config.stationary.threshold + # and it hasn't disappeared + and object_tracker.disappeared[obj["id"]] == 0 + # and it doesn't overlap with any current motion boxes when not calibrating + and not intersects_any( + obj["box"], + [] if motion_detector.is_calibrating() else motion_boxes, + ) + ] # get tracked object boxes that aren't stationary tracked_object_boxes = [ From facd557f8cf912dc97abc1241e4178e79d6ff1d9 Mon Sep 17 00:00:00 2001 From: tpjanssen <25168870+tpjanssen@users.noreply.github.com> Date: Fri, 20 Oct 2023 00:15:47 +0200 Subject: [PATCH 13/26] Change camera stats to be more structured (#8151) * Change camera stats to be more structured * Update stats.py * Update stats.py * Update System.jsx Front end also breaks due to moved camera stats --- frigate/stats.py | 3 ++- web/src/routes/System.jsx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frigate/stats.py b/frigate/stats.py index 8f8d03ed4..083926996 100644 --- a/frigate/stats.py +++ b/frigate/stats.py @@ -248,6 +248,7 @@ def stats_snapshot( total_detection_fps = 0 + stats["cameras"] = {} for name, camera_stats in camera_metrics.items(): total_detection_fps += camera_stats["detection_fps"].value pid = camera_stats["process"].pid if camera_stats["process"] else None @@ -259,7 +260,7 @@ def stats_snapshot( if camera_stats["capture_process"] else None ) - stats[name] = { + stats["cameras"][name] = { "camera_fps": round(camera_stats["camera_fps"].value, 2), "process_fps": round(camera_stats["process_fps"].value, 2), "skipped_fps": round(camera_stats["skipped_fps"].value, 2), diff --git a/web/src/routes/System.jsx b/web/src/routes/System.jsx index 0074763fa..5ad3fd215 100644 --- a/web/src/routes/System.jsx +++ b/web/src/routes/System.jsx @@ -32,7 +32,7 @@ export default function System() { service = {}, detection_fps: _, processes, - ...cameras + cameras, } = stats || initialStats || emptyObject; const detectorNames = Object.keys(detectors || emptyObject); From b4d5a3ef14b576b62a2f9cb18be5ea114d18d67c Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 20 Oct 2023 17:20:38 -0600 Subject: [PATCH 14/26] Fix dangling webrtc connections (#8251) * fix dangling webrtc connections * Make more efficient * Close pc as well --- web/src/components/WebRtcPlayer.jsx | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/web/src/components/WebRtcPlayer.jsx b/web/src/components/WebRtcPlayer.jsx index a454c67d3..b5760afed 100644 --- a/web/src/components/WebRtcPlayer.jsx +++ b/web/src/components/WebRtcPlayer.jsx @@ -1,10 +1,11 @@ import { h } from 'preact'; import { baseUrl } from '../api/baseUrl'; import { useCallback, useEffect } from 'preact/hooks'; +import { useMemo } from 'react'; export default function WebRtcPlayer({ camera, width, height }) { const url = `${baseUrl.replace(/^http/, 'ws')}live/webrtc/api/ws?src=${camera}`; - + const ws = useMemo(() => new WebSocket(url), [url]) const PeerConnection = useCallback(async (media) => { const pc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }], @@ -60,7 +61,6 @@ export default function WebRtcPlayer({ camera, width, height }) { const connect = useCallback(async () => { const pc = await PeerConnection('video+audio'); - const ws = new WebSocket(url); ws.addEventListener('open', () => { pc.addEventListener('icecandidate', (ev) => { @@ -85,11 +85,19 @@ export default function WebRtcPlayer({ camera, width, height }) { pc.setRemoteDescription({ type: 'answer', sdp: msg.value }); } }); - }, [PeerConnection, url]); + + ws.addEventListener('close', () => { + pc.close(); + }) + }, [PeerConnection, ws]); useEffect(() => { connect(); - }, [connect]); + + return () => { + ws.close(); + } + }, [connect, ws]); return (
From a3c0e30502f880cb78800a5f4eb2e8c2de30f290 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 20 Oct 2023 17:21:34 -0600 Subject: [PATCH 15/26] Use existing bounding box for region when object is stationary (#8248) --- frigate/video.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frigate/video.py b/frigate/video.py index a6d52dfc7..aa85a89df 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -607,7 +607,12 @@ def process_frames( # get tracked object boxes that aren't stationary tracked_object_boxes = [ - obj["estimate"] + ( + # use existing object box for stationary objects + obj["estimate"] + if obj["motionless_count"] < detect_config.stationary.threshold + else obj["box"] + ) for obj in object_tracker.tracked_objects.values() if obj["id"] not in stationary_object_ids ] From 0c2f3a97022b269ea752ac289fe5be8cd41d7b06 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 20 Oct 2023 17:22:38 -0600 Subject: [PATCH 16/26] Adjust motion calibration to be more dynamic (#8250) * Adjust motion calibration to be more dynamic * isort --- frigate/motion/improved_motion.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frigate/motion/improved_motion.py b/frigate/motion/improved_motion.py index b9d72da29..603d8fda4 100644 --- a/frigate/motion/improved_motion.py +++ b/frigate/motion/improved_motion.py @@ -1,3 +1,5 @@ +import logging + import cv2 import imutils import numpy as np @@ -6,6 +8,8 @@ from scipy.ndimage import gaussian_filter from frigate.config import MotionConfig from frigate.motion import MotionDetector +logger = logging.getLogger(__name__) + class ImprovedMotionDetector(MotionDetector): def __init__( @@ -138,8 +142,8 @@ class ImprovedMotionDetector(MotionDetector): self.motion_frame_size[0] * self.motion_frame_size[1] ) - # once the motion drops to less than 1% for the first time, assume its calibrated - if pct_motion < 0.01: + # once the motion is less than 5% and the number of contours is < 4, assume its calibrated + if pct_motion < 0.05 and len(motion_boxes) <= 4: self.calibrating = False # if calibrating or the motion contours are > 80% of the image area (lightning, ir, ptz) recalibrate From ee1e1b748c98d274fc7fcaaab3b5877904f7cc8d Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 20 Oct 2023 18:27:47 -0500 Subject: [PATCH 17/26] fix logic error in preset fetch (#8245) --- frigate/ptz/onvif.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frigate/ptz/onvif.py b/frigate/ptz/onvif.py index 0315446e9..5bd245167 100644 --- a/frigate/ptz/onvif.py +++ b/frigate/ptz/onvif.py @@ -176,7 +176,7 @@ class OnvifController: for preset in presets: self.cams[camera_name]["presets"][ - getattr(preset, "Name", f"preset {preset['token']}").lower() + (getattr(preset, "Name") or f"preset {preset['token']}").lower() ] = preset["token"] # get list of supported features From e80b6d9e5bca79037398fb906cd9a23490a380ea Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 20 Oct 2023 17:29:52 -0600 Subject: [PATCH 18/26] Use different consolidation requirement depending on label (#8249) --- frigate/const.py | 7 ++++++- frigate/util/object.py | 14 +++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/frigate/const.py b/frigate/const.py index 56d0f4517..f96665ce2 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -12,7 +12,7 @@ FRIGATE_LOCALHOST = "http://127.0.0.1:5000" PLUS_ENV_VAR = "PLUS_API_KEY" PLUS_API_HOST = "https://api.frigate.video" -# Attributes +# Attribute & Object Consts ATTRIBUTE_LABEL_MAP = { "person": ["face", "amazon"], @@ -21,6 +21,11 @@ ATTRIBUTE_LABEL_MAP = { ALL_ATTRIBUTE_LABELS = [ item for sublist in ATTRIBUTE_LABEL_MAP.values() for item in sublist ] +LABEL_CONSOLIDATION_MAP = { + "car": 0.8, + "face": 0.5, +} +LABEL_CONSOLIDATION_DEFAULT = 0.9 # Audio Consts diff --git a/frigate/util/object.py b/frigate/util/object.py index 3fa98df59..6c7c8f9b9 100644 --- a/frigate/util/object.py +++ b/frigate/util/object.py @@ -9,6 +9,7 @@ import numpy as np from peewee import DoesNotExist from frigate.config import DetectConfig, ModelConfig +from frigate.const import LABEL_CONSOLIDATION_DEFAULT, LABEL_CONSOLIDATION_MAP from frigate.detectors.detector_config import PixelFormatEnum from frigate.models import Event, Regions, Timeline from frigate.util.image import ( @@ -426,18 +427,21 @@ def get_consolidated_object_detections(detected_object_groups): sorted_by_area = sorted(group, key=lambda g: g[3]) for current_detection_idx in range(0, len(sorted_by_area)): - current_detection = sorted_by_area[current_detection_idx][2] + current_detection = sorted_by_area[current_detection_idx] + current_label = current_detection[0] + current_box = current_detection[2] overlap = 0 for to_check_idx in range( min(current_detection_idx + 1, len(sorted_by_area)), len(sorted_by_area), ): to_check = sorted_by_area[to_check_idx][2] - intersect_box = intersection(current_detection, to_check) + intersect_box = intersection(current_box, to_check) # if 90% of smaller detection is inside of another detection, consolidate - if ( - intersect_box is not None - and area(intersect_box) / area(current_detection) > 0.9 + if intersect_box is not None and area(intersect_box) / area( + current_box + ) > LABEL_CONSOLIDATION_MAP.get( + current_label, LABEL_CONSOLIDATION_DEFAULT ): overlap = 1 break From c8b38bdd47974c90857b1661da1e9d9f475fa32e Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 21 Oct 2023 09:08:03 -0400 Subject: [PATCH 19/26] address codeql scan results (#8260) --- frigate/http.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/frigate/http.py b/frigate/http.py index 48e9661b4..42d9c1610 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -261,7 +261,7 @@ def send_to_plus(id): except Exception as ex: logger.exception(ex) return make_response( - jsonify({"success": False, "message": str(ex)}), + jsonify({"success": False, "message": "Error uploading image"}), 400, ) @@ -281,7 +281,7 @@ def send_to_plus(id): except Exception as ex: logger.exception(ex) return make_response( - jsonify({"success": False, "message": str(ex)}), + jsonify({"success": False, "message": "Error uploading annotation"}), 400, ) @@ -352,7 +352,7 @@ def false_positive(id): except Exception as ex: logger.exception(ex) return make_response( - jsonify({"success": False, "message": str(ex)}), + jsonify({"success": False, "message": "Error uploading false positive"}), 400, ) @@ -455,8 +455,9 @@ def get_labels(): else: events = Event.select(Event.label).distinct() except Exception as e: + logger.error(e) return make_response( - jsonify({"success": False, "message": f"Failed to get labels: {e}"}), 404 + jsonify({"success": False, "message": "Failed to get labels"}), 404 ) labels = sorted([e.label for e in events]) @@ -469,9 +470,9 @@ def get_sub_labels(): try: events = Event.select(Event.sub_label).distinct() - except Exception as e: + except Exception: return make_response( - jsonify({"success": False, "message": f"Failed to get sub_labels: {e}"}), + jsonify({"success": False, "message": "Failed to get sub_labels"}), 404, ) @@ -649,7 +650,7 @@ def event_snapshot(id): ) # read snapshot from disk with open( - os.path.join(CLIPS_DIR, f"{event.camera}-{id}.jpg"), "rb" + os.path.join(CLIPS_DIR, f"{event.camera}-{event.id}.jpg"), "rb" ) as image_file: jpg_bytes = image_file.read() except DoesNotExist: @@ -741,7 +742,7 @@ def event_clip(id): jsonify({"success": False, "message": "Clip not available"}), 404 ) - file_name = f"{event.camera}-{id}.mp4" + file_name = f"{event.camera}-{event.id}.mp4" clip_path = os.path.join(CLIPS_DIR, file_name) if not os.path.isfile(clip_path): @@ -994,8 +995,9 @@ def create_event(camera_name, label): frame, ) except Exception as e: + logger.error(e) return make_response( - jsonify({"success": False, "message": f"An unknown error occurred: {e}"}), + jsonify({"success": False, "message": "An unknown error occurred"}), 500, ) @@ -1188,11 +1190,12 @@ def config_set(): with open(config_file, "w") as f: f.write(old_raw_config) f.close() + logger.error(f"\nConfig Error:\n\n{str(traceback.format_exc())}") return make_response( jsonify( { "success": False, - "message": f"\nConfig Error:\n\n{str(traceback.format_exc())}", + "message": "Error parsing config. Check logs for error message.", } ), 400, @@ -1366,7 +1369,10 @@ def latest_frame(camera_name): @bp.route("//recordings//snapshot.png") def get_snapshot_from_recording(camera_name: str, frame_time: str): if camera_name not in current_app.frigate_config.cameras: - return "Camera named {} not found".format(camera_name), 404 + return make_response( + jsonify({"success": False, "message": "Camera not found"}), + 404, + ) frame_time = float(frame_time) recording_query = ( @@ -1592,7 +1598,7 @@ def recording_clip(camera_name, start_ts, end_ts): if clip.end_time > end_ts: playlist_lines.append(f"outpoint {int(end_ts - clip.start_time)}") - file_name = f"clip_{camera_name}_{start_ts}-{end_ts}.mp4" + file_name = secure_filename(f"clip_{camera_name}_{start_ts}-{end_ts}.mp4") path = os.path.join(CACHE_DIR, file_name) if not os.path.exists(path): @@ -1760,7 +1766,7 @@ def vod_event(id): 404, ) - clip_path = os.path.join(CLIPS_DIR, f"{event.camera}-{id}.mp4") + clip_path = os.path.join(CLIPS_DIR, f"{event.camera}-{event.id}.mp4") if not os.path.isfile(clip_path): end_ts = ( @@ -1979,7 +1985,8 @@ def logs(service: str): file.close() return contents, 200 except FileNotFoundError as e: + logger.error(e) return make_response( - jsonify({"success": False, "message": f"Could not find log file: {e}"}), + jsonify({"success": False, "message": "Could not find log file"}), 500, ) From 18545718c1c8c68b30353a176d6d71c13fe447c0 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 21 Oct 2023 09:15:24 -0400 Subject: [PATCH 20/26] refactor and disable access logs for stats and version (#8259) --- .../rootfs/usr/local/nginx/conf/nginx.conf | 51 +++++++++---------- .../rootfs/usr/local/nginx/conf/proxy.conf | 4 ++ 2 files changed, 27 insertions(+), 28 deletions(-) create mode 100644 docker/main/rootfs/usr/local/nginx/conf/proxy.conf diff --git a/docker/main/rootfs/usr/local/nginx/conf/nginx.conf b/docker/main/rootfs/usr/local/nginx/conf/nginx.conf index a857461b5..edf28efea 100644 --- a/docker/main/rootfs/usr/local/nginx/conf/nginx.conf +++ b/docker/main/rootfs/usr/local/nginx/conf/nginx.conf @@ -149,62 +149,57 @@ http { location /ws { proxy_pass http://mqtt_ws/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; + include proxy.conf; } location /live/jsmpeg/ { proxy_pass http://jsmpeg/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; + include proxy.conf; } location /live/mse/ { proxy_pass http://go2rtc/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; + include proxy.conf; } location /live/webrtc/ { proxy_pass http://go2rtc/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; + include proxy.conf; } location ~* /api/go2rtc([/]?.*)$ { proxy_pass http://go2rtc; rewrite ^/api/go2rtc(.*)$ /api$1 break; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; + include proxy.conf; } location ~* /api/.*\.(jpg|jpeg|png)$ { rewrite ^/api/(.*)$ $1 break; proxy_pass http://frigate_api; - proxy_pass_request_headers on; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + include proxy.conf; } location /api/ { add_header Cache-Control "no-store"; expires off; proxy_pass http://frigate_api/; - proxy_pass_request_headers on; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + include proxy.conf; + + location /api/stats { + access_log off; + add_header Cache-Control "no-store"; + expires off; + proxy_pass http://frigate_api/; + include proxy.conf; + } + + location /api/version { + access_log off; + add_header Cache-Control "no-store"; + expires off; + proxy_pass http://frigate_api/; + include proxy.conf; + } } location / { diff --git a/docker/main/rootfs/usr/local/nginx/conf/proxy.conf b/docker/main/rootfs/usr/local/nginx/conf/proxy.conf new file mode 100644 index 000000000..442c78718 --- /dev/null +++ b/docker/main/rootfs/usr/local/nginx/conf/proxy.conf @@ -0,0 +1,4 @@ +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection "Upgrade"; +proxy_set_header Host $host; \ No newline at end of file From 1e71e36056495f9bf92b955b9ea33630320140a7 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 21 Oct 2023 11:40:46 -0400 Subject: [PATCH 21/26] fix route for stats and version (#8263) --- docker/main/rootfs/usr/local/nginx/conf/nginx.conf | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/main/rootfs/usr/local/nginx/conf/nginx.conf b/docker/main/rootfs/usr/local/nginx/conf/nginx.conf index edf28efea..6d0861406 100644 --- a/docker/main/rootfs/usr/local/nginx/conf/nginx.conf +++ b/docker/main/rootfs/usr/local/nginx/conf/nginx.conf @@ -187,17 +187,15 @@ http { location /api/stats { access_log off; - add_header Cache-Control "no-store"; - expires off; - proxy_pass http://frigate_api/; + rewrite ^/api/(.*)$ $1 break; + proxy_pass http://frigate_api; include proxy.conf; } location /api/version { access_log off; - add_header Cache-Control "no-store"; - expires off; - proxy_pass http://frigate_api/; + rewrite ^/api/(.*)$ $1 break; + proxy_pass http://frigate_api; include proxy.conf; } } From e13a1768207e35d03a3dcea109c52386279fdf18 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 21 Oct 2023 11:53:21 -0400 Subject: [PATCH 22/26] Update deps (#8261) * update web deps * update python deps * actions deps --- .github/workflows/pull_request.yml | 2 +- docker/main/requirements-dev.txt | 2 +- docker/main/requirements-wheels.txt | 6 +- web/package-lock.json | 266 +++++++++++++++------------- 4 files changed, 145 insertions(+), 131 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d5a8d049a..10ba723eb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -65,7 +65,7 @@ jobs: - name: Check out the repository uses: actions/checkout@v4 - name: Set up Python ${{ env.DEFAULT_PYTHON }} - uses: actions/setup-python@v4.7.0 + uses: actions/setup-python@v4.7.1 with: python-version: ${{ env.DEFAULT_PYTHON }} - name: Install requirements diff --git a/docker/main/requirements-dev.txt b/docker/main/requirements-dev.txt index 0acd15ae4..9a68e601d 100644 --- a/docker/main/requirements-dev.txt +++ b/docker/main/requirements-dev.txt @@ -1,3 +1,3 @@ -black == 23.3.* +black == 23.10.* isort ruff diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index e01b4a15a..37028d33b 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -2,12 +2,12 @@ click == 8.1.* Flask == 2.3.* imutils == 0.5.* matplotlib == 3.7.* -mypy == 1.4.1 +mypy == 1.6.1 numpy == 1.23.* onvif_zeep == 0.2.12 opencv-python-headless == 4.7.0.* paho-mqtt == 1.6.* -peewee == 3.16.* +peewee == 3.17.* peewee_migrate == 1.12.* psutil == 5.9.* pydantic == 1.10.* @@ -15,7 +15,7 @@ git+https://github.com/fbcotter/py3nvml#egg=py3nvml PyYAML == 6.0.* pytz == 2023.3 ruamel.yaml == 0.17.* -tzlocal == 5.0.* +tzlocal == 5.1 types-PyYAML == 6.0.* requests == 2.31.* types-requests == 2.31.* diff --git a/web/package-lock.json b/web/package-lock.json index 0591ebda9..3a8acbc41 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -978,21 +978,21 @@ } }, "node_modules/@eslint/js": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.51.0.tgz", - "integrity": "sha512-HxjQ8Qn+4SI3/AFv6sOrDB+g6PpUTDwSJiQqOrnneEk8L71161srI9gjzzZvYVbzHiVg/BvcH95+cK/zfIt4pg==", + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.52.0.tgz", + "integrity": "sha512-mjZVbpaeMZludF2fsWLD0Z9gCref1Tk4i9+wddjRvpUNqqcndPkBD09N/Mapey0b3jaXbLm2kICwFv2E64QinA==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.11", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz", - "integrity": "sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA==", + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", + "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^1.2.1", + "@humanwhocodes/object-schema": "^2.0.1", "debug": "^4.1.1", "minimatch": "^3.0.5" }, @@ -1014,9 +1014,9 @@ } }, "node_modules/@humanwhocodes/object-schema": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", - "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", + "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", "dev": true }, "node_modules/@istanbuljs/schema": { @@ -1664,16 +1664,16 @@ } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.5.tgz", - "integrity": "sha512-JhtAwTRhOUcP96D0Y6KYnwig/MRQbOoLGXTON2+LlyB/N35SP9j1boai2zzwXb7ypKELXMx3DVk9UTaEq1vHEw==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.8.0.tgz", + "integrity": "sha512-GosF4238Tkes2SHPQ1i8f6rMtG6zlKwMEB0abqSJ3Npvos+doIlc/ATG+vX1G9coDF3Ex78zM3heXHLyWEwLUw==", "dev": true, "dependencies": { "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.7.5", - "@typescript-eslint/type-utils": "6.7.5", - "@typescript-eslint/utils": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5", + "@typescript-eslint/scope-manager": "6.8.0", + "@typescript-eslint/type-utils": "6.8.0", + "@typescript-eslint/utils": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.4", @@ -1870,15 +1870,15 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.7.5.tgz", - "integrity": "sha512-bIZVSGx2UME/lmhLcjdVc7ePBwn7CLqKarUBL4me1C5feOd663liTGjMBGVcGr+BhnSLeP4SgwdvNnnkbIdkCw==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.8.0.tgz", + "integrity": "sha512-5tNs6Bw0j6BdWuP8Fx+VH4G9fEPDxnVI7yH1IAPkQH5RUtvKwRoqdecAPdQXv4rSOADAaz1LFBZvZG7VbXivSg==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "6.7.5", - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/typescript-estree": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5", + "@typescript-eslint/scope-manager": "6.8.0", + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/typescript-estree": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0", "debug": "^4.3.4" }, "engines": { @@ -1898,13 +1898,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.7.5.tgz", - "integrity": "sha512-GAlk3eQIwWOJeb9F7MKQ6Jbah/vx1zETSDw8likab/eFcqkjSD7BI75SDAeC5N2L0MmConMoPvTsmkrg71+B1A==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.8.0.tgz", + "integrity": "sha512-xe0HNBVwCph7rak+ZHcFD6A+q50SMsFwcmfdjs9Kz4qDh5hWhaPhFjRs/SODEhroBI5Ruyvyz9LfwUJ624O40g==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5" + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0" }, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -1915,13 +1915,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.7.5.tgz", - "integrity": "sha512-Gs0qos5wqxnQrvpYv+pf3XfcRXW6jiAn9zE/K+DlmYf6FcpxeNYN0AIETaPR7rHO4K2UY+D0CIbDP9Ut0U4m1g==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.8.0.tgz", + "integrity": "sha512-RYOJdlkTJIXW7GSldUIHqc/Hkto8E+fZN96dMIFhuTJcQwdRoGN2rEWA8U6oXbLo0qufH7NPElUb+MceHtz54g==", "dev": true, "dependencies": { - "@typescript-eslint/typescript-estree": "6.7.5", - "@typescript-eslint/utils": "6.7.5", + "@typescript-eslint/typescript-estree": "6.8.0", + "@typescript-eslint/utils": "6.8.0", "debug": "^4.3.4", "ts-api-utils": "^1.0.1" }, @@ -1942,9 +1942,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.5.tgz", - "integrity": "sha512-WboQBlOXtdj1tDFPyIthpKrUb+kZf2VroLZhxKa/VlwLlLyqv/PwUNgL30BlTVZV1Wu4Asu2mMYPqarSO4L5ZQ==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.8.0.tgz", + "integrity": "sha512-p5qOxSum7W3k+llc7owEStXlGmSl8FcGvhYt8Vjy7FqEnmkCVlM3P57XQEGj58oqaBWDQXbJDZxwUWMS/EAPNQ==", "dev": true, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -1955,13 +1955,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.5.tgz", - "integrity": "sha512-NhJiJ4KdtwBIxrKl0BqG1Ur+uw7FiOnOThcYx9DpOGJ/Abc9z2xNzLeirCG02Ig3vkvrc2qFLmYSSsaITbKjlg==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.8.0.tgz", + "integrity": "sha512-ISgV0lQ8XgW+mvv5My/+iTUdRmGspducmQcDw5JxznasXNnZn3SKNrTRuMsEXv+V/O+Lw9AGcQCfVaOPCAk/Zg==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5", + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -1997,17 +1997,17 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.7.5.tgz", - "integrity": "sha512-pfRRrH20thJbzPPlPc4j0UNGvH1PjPlhlCMq4Yx7EGjV7lvEeGX0U6MJYe8+SyFutWgSHsdbJ3BXzZccYggezA==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.8.0.tgz", + "integrity": "sha512-dKs1itdE2qFG4jr0dlYLQVppqTE+Itt7GmIf/vX6CSvsW+3ov8PbWauVKyyfNngokhIO9sKZeRGCUo1+N7U98Q==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.7.5", - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/typescript-estree": "6.7.5", + "@typescript-eslint/scope-manager": "6.8.0", + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/typescript-estree": "6.8.0", "semver": "^7.5.4" }, "engines": { @@ -2037,12 +2037,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.5.tgz", - "integrity": "sha512-3MaWdDZtLlsexZzDSdQWsFQ9l9nL8B80Z4fImSpyllFC/KLqWQRdEcB+gGGO+N3Q2uL40EsG66wZLsohPxNXvg==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.8.0.tgz", + "integrity": "sha512-oqAnbA7c+pgOhW2OhGvxm0t1BULX5peQI/rLsNDpGM78EebV3C9IGbX5HNZabuZ6UQrYveCLjKo8Iy/lLlBkkg==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/types": "6.8.0", "eslint-visitor-keys": "^3.4.1" }, "engines": { @@ -2053,6 +2053,12 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true + }, "node_modules/@videojs/http-streaming": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/@videojs/http-streaming/-/http-streaming-3.5.3.tgz", @@ -3677,18 +3683,19 @@ } }, "node_modules/eslint": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.51.0.tgz", - "integrity": "sha512-2WuxRZBrlwnXi+/vFSJyjMqrNjtJqiasMzehF0shoLaW7DzS3/9Yvrmq5JiT66+pNjiX4UBnLDiKHcWAr/OInA==", + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.52.0.tgz", + "integrity": "sha512-zh/JHnaixqHZsolRB/w9/02akBk9EPrOs9JwcTP2ek7yL5bVvXuRariiaAjjoJ5DvuwQ1WAE/HsMz+w17YgBCg==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.2", - "@eslint/js": "8.51.0", - "@humanwhocodes/config-array": "^0.11.11", + "@eslint/js": "8.52.0", + "@humanwhocodes/config-array": "^0.11.13", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -4071,9 +4078,9 @@ } }, "node_modules/eslint-plugin-jest": { - "version": "27.4.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-27.4.2.tgz", - "integrity": "sha512-3Nfvv3wbq2+PZlRTf2oaAWXWwbdBejFRBR2O8tAO67o+P8zno+QGbcDYaAXODlreXVg+9gvWhKKmG2rgfb8GEg==", + "version": "27.4.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-27.4.3.tgz", + "integrity": "sha512-7S6SmmsHsgIm06BAGCAxL+ABd9/IB3MWkz2pudj6Qqor2y1qQpWPfuFU4SG9pWj4xDjF0e+D7Llh5useuSzAZw==", "dev": true, "dependencies": { "@typescript-eslint/utils": "^5.10.0" @@ -9223,9 +9230,9 @@ } }, "node_modules/vite": { - "version": "4.4.11", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz", - "integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.0.tgz", + "integrity": "sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==", "dev": true, "dependencies": { "esbuild": "^0.18.10", @@ -10291,18 +10298,18 @@ } }, "@eslint/js": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.51.0.tgz", - "integrity": "sha512-HxjQ8Qn+4SI3/AFv6sOrDB+g6PpUTDwSJiQqOrnneEk8L71161srI9gjzzZvYVbzHiVg/BvcH95+cK/zfIt4pg==", + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.52.0.tgz", + "integrity": "sha512-mjZVbpaeMZludF2fsWLD0Z9gCref1Tk4i9+wddjRvpUNqqcndPkBD09N/Mapey0b3jaXbLm2kICwFv2E64QinA==", "dev": true }, "@humanwhocodes/config-array": { - "version": "0.11.11", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz", - "integrity": "sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA==", + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", + "integrity": "sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==", "dev": true, "requires": { - "@humanwhocodes/object-schema": "^1.2.1", + "@humanwhocodes/object-schema": "^2.0.1", "debug": "^4.1.1", "minimatch": "^3.0.5" } @@ -10314,9 +10321,9 @@ "dev": true }, "@humanwhocodes/object-schema": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", - "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz", + "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", "dev": true }, "@istanbuljs/schema": { @@ -10830,16 +10837,16 @@ } }, "@typescript-eslint/eslint-plugin": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.5.tgz", - "integrity": "sha512-JhtAwTRhOUcP96D0Y6KYnwig/MRQbOoLGXTON2+LlyB/N35SP9j1boai2zzwXb7ypKELXMx3DVk9UTaEq1vHEw==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.8.0.tgz", + "integrity": "sha512-GosF4238Tkes2SHPQ1i8f6rMtG6zlKwMEB0abqSJ3Npvos+doIlc/ATG+vX1G9coDF3Ex78zM3heXHLyWEwLUw==", "dev": true, "requires": { "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.7.5", - "@typescript-eslint/type-utils": "6.7.5", - "@typescript-eslint/utils": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5", + "@typescript-eslint/scope-manager": "6.8.0", + "@typescript-eslint/type-utils": "6.8.0", + "@typescript-eslint/utils": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.4", @@ -10953,54 +10960,54 @@ } }, "@typescript-eslint/parser": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.7.5.tgz", - "integrity": "sha512-bIZVSGx2UME/lmhLcjdVc7ePBwn7CLqKarUBL4me1C5feOd663liTGjMBGVcGr+BhnSLeP4SgwdvNnnkbIdkCw==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.8.0.tgz", + "integrity": "sha512-5tNs6Bw0j6BdWuP8Fx+VH4G9fEPDxnVI7yH1IAPkQH5RUtvKwRoqdecAPdQXv4rSOADAaz1LFBZvZG7VbXivSg==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "6.7.5", - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/typescript-estree": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5", + "@typescript-eslint/scope-manager": "6.8.0", + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/typescript-estree": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0", "debug": "^4.3.4" } }, "@typescript-eslint/scope-manager": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.7.5.tgz", - "integrity": "sha512-GAlk3eQIwWOJeb9F7MKQ6Jbah/vx1zETSDw8likab/eFcqkjSD7BI75SDAeC5N2L0MmConMoPvTsmkrg71+B1A==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.8.0.tgz", + "integrity": "sha512-xe0HNBVwCph7rak+ZHcFD6A+q50SMsFwcmfdjs9Kz4qDh5hWhaPhFjRs/SODEhroBI5Ruyvyz9LfwUJ624O40g==", "dev": true, "requires": { - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5" + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0" } }, "@typescript-eslint/type-utils": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.7.5.tgz", - "integrity": "sha512-Gs0qos5wqxnQrvpYv+pf3XfcRXW6jiAn9zE/K+DlmYf6FcpxeNYN0AIETaPR7rHO4K2UY+D0CIbDP9Ut0U4m1g==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.8.0.tgz", + "integrity": "sha512-RYOJdlkTJIXW7GSldUIHqc/Hkto8E+fZN96dMIFhuTJcQwdRoGN2rEWA8U6oXbLo0qufH7NPElUb+MceHtz54g==", "dev": true, "requires": { - "@typescript-eslint/typescript-estree": "6.7.5", - "@typescript-eslint/utils": "6.7.5", + "@typescript-eslint/typescript-estree": "6.8.0", + "@typescript-eslint/utils": "6.8.0", "debug": "^4.3.4", "ts-api-utils": "^1.0.1" } }, "@typescript-eslint/types": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.5.tgz", - "integrity": "sha512-WboQBlOXtdj1tDFPyIthpKrUb+kZf2VroLZhxKa/VlwLlLyqv/PwUNgL30BlTVZV1Wu4Asu2mMYPqarSO4L5ZQ==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.8.0.tgz", + "integrity": "sha512-p5qOxSum7W3k+llc7owEStXlGmSl8FcGvhYt8Vjy7FqEnmkCVlM3P57XQEGj58oqaBWDQXbJDZxwUWMS/EAPNQ==", "dev": true }, "@typescript-eslint/typescript-estree": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.5.tgz", - "integrity": "sha512-NhJiJ4KdtwBIxrKl0BqG1Ur+uw7FiOnOThcYx9DpOGJ/Abc9z2xNzLeirCG02Ig3vkvrc2qFLmYSSsaITbKjlg==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.8.0.tgz", + "integrity": "sha512-ISgV0lQ8XgW+mvv5My/+iTUdRmGspducmQcDw5JxznasXNnZn3SKNrTRuMsEXv+V/O+Lw9AGcQCfVaOPCAk/Zg==", "dev": true, "requires": { - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/visitor-keys": "6.7.5", + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/visitor-keys": "6.8.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -11020,17 +11027,17 @@ } }, "@typescript-eslint/utils": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.7.5.tgz", - "integrity": "sha512-pfRRrH20thJbzPPlPc4j0UNGvH1PjPlhlCMq4Yx7EGjV7lvEeGX0U6MJYe8+SyFutWgSHsdbJ3BXzZccYggezA==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.8.0.tgz", + "integrity": "sha512-dKs1itdE2qFG4jr0dlYLQVppqTE+Itt7GmIf/vX6CSvsW+3ov8PbWauVKyyfNngokhIO9sKZeRGCUo1+N7U98Q==", "dev": true, "requires": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.7.5", - "@typescript-eslint/types": "6.7.5", - "@typescript-eslint/typescript-estree": "6.7.5", + "@typescript-eslint/scope-manager": "6.8.0", + "@typescript-eslint/types": "6.8.0", + "@typescript-eslint/typescript-estree": "6.8.0", "semver": "^7.5.4" }, "dependencies": { @@ -11046,15 +11053,21 @@ } }, "@typescript-eslint/visitor-keys": { - "version": "6.7.5", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.5.tgz", - "integrity": "sha512-3MaWdDZtLlsexZzDSdQWsFQ9l9nL8B80Z4fImSpyllFC/KLqWQRdEcB+gGGO+N3Q2uL40EsG66wZLsohPxNXvg==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.8.0.tgz", + "integrity": "sha512-oqAnbA7c+pgOhW2OhGvxm0t1BULX5peQI/rLsNDpGM78EebV3C9IGbX5HNZabuZ6UQrYveCLjKo8Iy/lLlBkkg==", "dev": true, "requires": { - "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/types": "6.8.0", "eslint-visitor-keys": "^3.4.1" } }, + "@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true + }, "@videojs/http-streaming": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/@videojs/http-streaming/-/http-streaming-3.5.3.tgz", @@ -12259,18 +12272,19 @@ "dev": true }, "eslint": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.51.0.tgz", - "integrity": "sha512-2WuxRZBrlwnXi+/vFSJyjMqrNjtJqiasMzehF0shoLaW7DzS3/9Yvrmq5JiT66+pNjiX4UBnLDiKHcWAr/OInA==", + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.52.0.tgz", + "integrity": "sha512-zh/JHnaixqHZsolRB/w9/02akBk9EPrOs9JwcTP2ek7yL5bVvXuRariiaAjjoJ5DvuwQ1WAE/HsMz+w17YgBCg==", "dev": true, "requires": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.2", - "@eslint/js": "8.51.0", - "@humanwhocodes/config-array": "^0.11.11", + "@eslint/js": "8.52.0", + "@humanwhocodes/config-array": "^0.11.13", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -12584,9 +12598,9 @@ } }, "eslint-plugin-jest": { - "version": "27.4.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-27.4.2.tgz", - "integrity": "sha512-3Nfvv3wbq2+PZlRTf2oaAWXWwbdBejFRBR2O8tAO67o+P8zno+QGbcDYaAXODlreXVg+9gvWhKKmG2rgfb8GEg==", + "version": "27.4.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-27.4.3.tgz", + "integrity": "sha512-7S6SmmsHsgIm06BAGCAxL+ABd9/IB3MWkz2pudj6Qqor2y1qQpWPfuFU4SG9pWj4xDjF0e+D7Llh5useuSzAZw==", "dev": true, "requires": { "@typescript-eslint/utils": "^5.10.0" @@ -16240,9 +16254,9 @@ } }, "vite": { - "version": "4.4.11", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz", - "integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.0.tgz", + "integrity": "sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==", "dev": true, "requires": { "esbuild": "^0.18.10", From 9fc93c72a065301a37df770a82d63cbe0c87bdb9 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 21 Oct 2023 11:53:33 -0400 Subject: [PATCH 23/26] more consistent use of iterators in select queries (#8258) --- frigate/events/cleanup.py | 42 ++++++++++++++++++++++-------------- frigate/http.py | 13 +++++++---- frigate/record/cleanup.py | 23 +++++++++++++------- frigate/record/maintainer.py | 2 ++ frigate/storage.py | 41 ++++++++++++++++++++++------------- 5 files changed, 78 insertions(+), 43 deletions(-) diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py index 1f1fb5f76..faba73979 100644 --- a/frigate/events/cleanup.py +++ b/frigate/events/cleanup.py @@ -83,14 +83,19 @@ class EventCleanup(threading.Thread): datetime.datetime.now() - datetime.timedelta(days=expire_days) ).timestamp() # grab all events after specific time - expired_events = Event.select( - Event.id, - Event.camera, - ).where( - Event.camera.not_in(self.camera_keys), - Event.start_time < expire_after, - Event.label == event.label, - Event.retain_indefinitely == False, + expired_events = ( + Event.select( + Event.id, + Event.camera, + ) + .where( + Event.camera.not_in(self.camera_keys), + Event.start_time < expire_after, + Event.label == event.label, + Event.retain_indefinitely == False, + ) + .namedtuples() + .iterator() ) # delete the media from disk for event in expired_events: @@ -136,14 +141,19 @@ class EventCleanup(threading.Thread): datetime.datetime.now() - datetime.timedelta(days=expire_days) ).timestamp() # grab all events after specific time - expired_events = Event.select( - Event.id, - Event.camera, - ).where( - Event.camera == name, - Event.start_time < expire_after, - Event.label == event.label, - Event.retain_indefinitely == False, + expired_events = ( + Event.select( + Event.id, + Event.camera, + ) + .where( + Event.camera == name, + Event.start_time < expire_after, + Event.label == event.label, + Event.retain_indefinitely == False, + ) + .namedtuples() + .iterator() ) # delete the grabbed clips from disk diff --git a/frigate/http.py b/frigate/http.py index 42d9c1610..0061d338f 100644 --- a/frigate/http.py +++ b/frigate/http.py @@ -958,9 +958,10 @@ def events(): .order_by(Event.start_time.desc()) .limit(limit) .dicts() + .iterator() ) - return jsonify([e for e in events]) + return jsonify(list(events)) @bp.route("/events//