From 861ee0485d47d9a1441c9d825a39bdcbff4edd07 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 13 Jun 2021 14:21:20 -0500 Subject: [PATCH 01/12] swith camera view to jsmpeg --- web/src/components/JSMpegPlayer.jsx | 6 ++--- web/src/index.css | 4 ++++ web/src/routes/Camera.jsx | 35 +++++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/web/src/components/JSMpegPlayer.jsx b/web/src/components/JSMpegPlayer.jsx index 323b30771..389d8a8da 100644 --- a/web/src/components/JSMpegPlayer.jsx +++ b/web/src/components/JSMpegPlayer.jsx @@ -5,14 +5,13 @@ import JSMpeg from '@cycjimmy/jsmpeg-player'; export default function JSMpegPlayer({ camera }) { const playerRef = useRef(); - const canvasRef = useRef(); const url = `${baseUrl.replace(/^http/, 'ws')}/live/${camera}` useEffect(() => { const video = new JSMpeg.VideoElement( playerRef.current, url, - {canvas: canvasRef.current}, + {}, {protocols: [], audio: false} ); @@ -22,8 +21,7 @@ export default function JSMpegPlayer({ camera }) { }, [url]); return ( -
- +
); } diff --git a/web/src/index.css b/web/src/index.css index d55a9aabc..1ccb2fad7 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -25,3 +25,7 @@ transform: rotate(360deg); } } + +.jsmpeg canvas { + position: static !important; +} diff --git a/web/src/routes/Camera.jsx b/web/src/routes/Camera.jsx index dab0350e4..fa83255c6 100644 --- a/web/src/routes/Camera.jsx +++ b/web/src/routes/Camera.jsx @@ -1,5 +1,6 @@ -import { h } from 'preact'; +import { h, Fragment } from 'preact'; import AutoUpdatingCameraImage from '../components/AutoUpdatingCameraImage'; +import JSMpegPlayer from '../components/JSMpegPlayer'; import Button from '../components/Button'; import Card from '../components/Card'; import Heading from '../components/Heading'; @@ -16,6 +17,7 @@ export default function Camera({ camera }) { const { data: config } = useConfig(); const apiHost = useApiHost(); const [showSettings, setShowSettings] = useState(false); + const [viewMode, setViewMode] = useState('live'); const cameraConfig = config?.cameras[camera]; const [options, setOptions] = usePersistence(`${camera}-feed`, emptyObject); @@ -79,9 +81,16 @@ export default function Camera({ camera }) {
) : null; - return ( -
- {camera} + let player; + if (viewMode == 'live') { + player = <> +
+ +
+ ; + } + else if (viewMode == 'debug') { + player = <>
@@ -93,6 +102,24 @@ export default function Camera({ camera }) { {showSettings ? 'Hide' : 'Show'} Options {showSettings ? : null} + ; + } + + return ( +
+ {camera} +
+ +
+ + {player}
Tracked objects From 3aa7f753b363081ff2c3348f05741613e030309c Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 13 Jun 2021 14:24:34 -0500 Subject: [PATCH 02/12] lint fixes --- web/src/components/JSMpegPlayer.jsx | 3 +-- web/src/routes/Camera.jsx | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/web/src/components/JSMpegPlayer.jsx b/web/src/components/JSMpegPlayer.jsx index 389d8a8da..7543091ff 100644 --- a/web/src/components/JSMpegPlayer.jsx +++ b/web/src/components/JSMpegPlayer.jsx @@ -21,7 +21,6 @@ export default function JSMpegPlayer({ camera }) { }, [url]); return ( -
-
+
); } diff --git a/web/src/routes/Camera.jsx b/web/src/routes/Camera.jsx index fa83255c6..df09ffc7d 100644 --- a/web/src/routes/Camera.jsx +++ b/web/src/routes/Camera.jsx @@ -82,14 +82,14 @@ export default function Camera({ camera }) { ) : null; let player; - if (viewMode == 'live') { + if (viewMode === 'live') { player = <>
; } - else if (viewMode == 'debug') { + else if (viewMode === 'debug') { player = <>
@@ -110,10 +110,10 @@ export default function Camera({ camera }) { {camera}
From 175c85d69a21ae0ee61e3481ab23352e15960938 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sun, 13 Jun 2021 14:49:13 -0500 Subject: [PATCH 03/12] fix some test errors --- web/src/routes/Camera.jsx | 38 +++++++++++++----------- web/src/routes/__tests__/Camera.test.jsx | 4 +++ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/web/src/routes/Camera.jsx b/web/src/routes/Camera.jsx index df09ffc7d..7f5527df4 100644 --- a/web/src/routes/Camera.jsx +++ b/web/src/routes/Camera.jsx @@ -83,26 +83,30 @@ export default function Camera({ camera }) { let player; if (viewMode === 'live') { - player = <> -
- -
- ; + player = ( + +
+ +
+ + ); } else if (viewMode === 'debug') { - player = <> -
- -
+ player = ( + +
+ +
- - {showSettings ? : null} - ; + + {showSettings ? : null} + + ); } return ( diff --git a/web/src/routes/__tests__/Camera.test.jsx b/web/src/routes/__tests__/Camera.test.jsx index c95a9a1d5..445b45302 100644 --- a/web/src/routes/__tests__/Camera.test.jsx +++ b/web/src/routes/__tests__/Camera.test.jsx @@ -32,7 +32,10 @@ describe('Camera Route', () => { }, mockSetOptions, ]); + render(); + + fireEvent.click(screen.queryByText('Debug')); fireEvent.click(screen.queryByText('Show Options')); expect(screen.queryByTestId('mock-image')).toHaveTextContent( 'bbox=1×tamp=0&zones=1&mask=0&motion=1®ions=0' @@ -47,6 +50,7 @@ describe('Camera Route', () => { render(); + fireEvent.click(screen.queryByText('Debug')); fireEvent.click(screen.queryByText('Show Options')); fireEvent.change(screen.queryByTestId('bbox-input'), { target: { checked: true } }); fireEvent.change(screen.queryByTestId('timestamp-input'), { target: { checked: true } }); From d83ffd89846c3221f7c9dafb61b76001293685bb Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Fri, 18 Jun 2021 21:21:25 -0400 Subject: [PATCH 04/12] fix tests --- web/src/routes/__tests__/Camera.test.jsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/src/routes/__tests__/Camera.test.jsx b/web/src/routes/__tests__/Camera.test.jsx index 445b45302..54f53add3 100644 --- a/web/src/routes/__tests__/Camera.test.jsx +++ b/web/src/routes/__tests__/Camera.test.jsx @@ -3,6 +3,7 @@ import * as AutoUpdatingCameraImage from '../../components/AutoUpdatingCameraIma import * as Api from '../../api'; import * as Context from '../../context'; import Camera from '../Camera'; +import * as JSMpegPlayer from '../../components/JSMpegPlayer'; import { fireEvent, render, screen } from '@testing-library/preact'; describe('Camera Route', () => { @@ -18,6 +19,9 @@ describe('Camera Route', () => { jest.spyOn(AutoUpdatingCameraImage, 'default').mockImplementation(({ searchParams }) => { return
{searchParams.toString()}
; }); + jest.spyOn(JSMpegPlayer, 'default').mockImplementation(() => { + return
; + }); }); test('reads camera feed options from persistence', async () => { @@ -44,6 +48,7 @@ describe('Camera Route', () => { test('updates camera feed options to persistence', async () => { mockUsePersistence + .mockReturnValueOnce([{}, mockSetOptions]) .mockReturnValueOnce([{}, mockSetOptions]) .mockReturnValueOnce([{ bbox: true }, mockSetOptions]) .mockReturnValueOnce([{ bbox: true, timestamp: true }, mockSetOptions]); @@ -56,6 +61,8 @@ describe('Camera Route', () => { fireEvent.change(screen.queryByTestId('timestamp-input'), { target: { checked: true } }); fireEvent.click(screen.queryByText('Hide Options')); + expect(mockUsePersistence).toHaveBeenCalledTimes(4); + expect(mockSetOptions).toHaveBeenCalledTimes(2); expect(mockSetOptions).toHaveBeenCalledWith({ bbox: true, timestamp: true }); expect(screen.queryByTestId('mock-image')).toHaveTextContent('bbox=1×tamp=1'); }); From 9b3a649f17d01c8b43a9a6d64c34817470133c65 Mon Sep 17 00:00:00 2001 From: Sebastian Englbrecht Date: Thu, 17 Jun 2021 18:58:51 +0200 Subject: [PATCH 05/12] Do apt update before upgrade --- docker/Dockerfile.dev | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index ce83ab82a..41a3296c9 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -14,7 +14,8 @@ RUN groupadd --gid $USER_GID $USERNAME \ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ && chmod 0440 /etc/sudoers.d/$USERNAME -RUN apt-get install -y git curl vim htop +RUN apt-get update \ + && apt-get install -y git curl vim htop RUN pip3 install pylint black From b72b66781a716d9f666f592ba5edff4127687f35 Mon Sep 17 00:00:00 2001 From: gpete Date: Sat, 19 Jun 2021 06:11:43 -0600 Subject: [PATCH 06/12] Fixed overwritten argument 'media' (#1026) media variable name is reused and overwritten causing issues with event expiration & clean up. Also changed name in pure_duplicates for consistency. --- frigate/events.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/frigate/events.py b/frigate/events.py index 2325635b5..58ca1986e 100644 --- a/frigate/events.py +++ b/frigate/events.py @@ -281,9 +281,9 @@ class EventCleanup(threading.Thread): self.stop_event = stop_event self.camera_keys = list(self.config.cameras.keys()) - def expire(self, media): + def expire(self, media_type): ## Expire events from unlisted cameras based on the global config - if media == "clips": + if media_type == 'clips': retain_config = self.config.clips.retain file_extension = "mp4" update_params = {"has_clip": False} @@ -314,8 +314,8 @@ class EventCleanup(threading.Thread): # delete the media from disk for event in expired_events: media_name = f"{event.camera}-{event.id}" - media = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") - media.unlink(missing_ok=True) + media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") + media_path.unlink(missing_ok=True) # update the clips attribute for the db entry update_query = Event.update(update_params).where( Event.camera.not_in(self.camera_keys), @@ -326,7 +326,7 @@ class EventCleanup(threading.Thread): ## Expire events from cameras based on the camera config for name, camera in self.config.cameras.items(): - if media == "clips": + if media_type == 'clips': retain_config = camera.clips.retain else: retain_config = camera.snapshots.retain @@ -351,10 +351,8 @@ class EventCleanup(threading.Thread): # delete the grabbed clips from disk for event in expired_events: media_name = f"{event.camera}-{event.id}" - media = Path( - f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}" - ) - media.unlink(missing_ok=True) + media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") + media_path.unlink(missing_ok=True) # update the clips attribute for the db entry update_query = Event.update(update_params).where( Event.camera == name, @@ -385,11 +383,11 @@ class EventCleanup(threading.Thread): logger.debug(f"Removing duplicate: {event.id}") media_name = f"{event.camera}-{event.id}" if event.has_snapshot: - media = Path(f"{os.path.join(CLIPS_DIR, media_name)}.jpg") - media.unlink(missing_ok=True) + media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.jpg") + media_path.unlink(missing_ok=True) if event.has_clip: - media = Path(f"{os.path.join(CLIPS_DIR, media_name)}.mp4") - media.unlink(missing_ok=True) + media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.mp4") + media_path.unlink(missing_ok=True) ( Event.delete() From f110a261b9983c77d71d63c6568f1207c6e18667 Mon Sep 17 00:00:00 2001 From: mrdrup Date: Sat, 19 Jun 2021 13:15:02 +0100 Subject: [PATCH 07/12] Fix 'FileExistsError' shared memory exception (#945) --- frigate/app.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index 1e000d401..a6994d6d9 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -180,14 +180,23 @@ class FrigateApp: model_shape = (self.config.model.height, self.config.model.width) for name in self.config.cameras.keys(): self.detection_out_events[name] = mp.Event() - shm_in = mp.shared_memory.SharedMemory( - name=name, - create=True, - size=self.config.model.height * self.config.model.width * 3, - ) - shm_out = mp.shared_memory.SharedMemory( - name=f"out-{name}", create=True, size=20 * 6 * 4 - ) + + try: + shm_in = mp.shared_memory.SharedMemory( + name=name, + create=True, + size=self.config.model.height*self.config.model.width * 3, + ) + except FileExistsError: + shm_in = mp.shared_memory.SharedMemory(name=name) + + try: + shm_out = mp.shared_memory.SharedMemory( + name=f"out-{name}", create=True, size=20 * 6 * 4 + ) + except FileExistsError: + shm_out = mp.shared_memory.SharedMemory(name=f"out-{name}") + self.detection_shms.append(shm_in) self.detection_shms.append(shm_out) From b134db48b302ed199148682336ce214568c50ae9 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 19 Jun 2021 08:36:03 -0500 Subject: [PATCH 08/12] adding clean_copy to snapshot config --- frigate/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frigate/config.py b/frigate/config.py index 393eecf00..688c98d2f 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -634,6 +634,7 @@ CAMERAS_SCHEMA = vol.Schema( }, vol.Optional("snapshots", default={}): { vol.Optional("enabled", default=False): bool, + vol.Optional("clean_copy", default=True): bool, vol.Optional("timestamp", default=False): bool, vol.Optional("bounding_box", default=False): bool, vol.Optional("crop", default=False): bool, @@ -665,6 +666,7 @@ CAMERAS_SCHEMA = vol.Schema( @dataclasses.dataclass class CameraSnapshotsConfig: enabled: bool + clean_copy: bool timestamp: bool bounding_box: bool crop: bool @@ -676,6 +678,7 @@ class CameraSnapshotsConfig: def build(cls, config, global_config) -> CameraSnapshotsConfig: return CameraSnapshotsConfig( enabled=config["enabled"], + clean_copy=config["clean_copy"], timestamp=config["timestamp"], bounding_box=config["bounding_box"], crop=config["crop"], @@ -689,6 +692,7 @@ class CameraSnapshotsConfig: def to_dict(self) -> Dict[str, Any]: return { "enabled": self.enabled, + "clean_copy": self.clean_copy, "timestamp": self.timestamp, "bounding_box": self.bounding_box, "crop": self.crop, From 3a3b788c6542f363a1ebe5486343bfd10efc80a9 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 19 Jun 2021 08:38:42 -0500 Subject: [PATCH 09/12] save clean snapshot --- frigate/object_processing.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/frigate/object_processing.py b/frigate/object_processing.py index b483940f3..4968f14f1 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -192,6 +192,27 @@ class TrackedObject: ret, jpg = cv2.imencode(".jpg", np.zeros((175, 175, 3), np.uint8)) return jpg.tobytes() + def get_clean_png(self): + if self.thumbnail_data is None: + return None + + try: + best_frame = cv2.cvtColor( + self.frame_cache[self.thumbnail_data["frame_time"]], + cv2.COLOR_YUV2BGR_I420, + ) + except KeyError: + logger.warning( + f"Unable to create clean png because frame {self.thumbnail_data['frame_time']} is not in the cache" + ) + return None + + ret, png = cv2.imencode(".png", best_frame) + if ret: + return png.tobytes() + else: + return None + def get_jpg_bytes( self, timestamp=False, bounding_box=False, crop=False, height=None ): @@ -615,6 +636,23 @@ class TrackedObjectProcessor(threading.Thread): ) as j: j.write(jpg_bytes) event_data["has_snapshot"] = True + + # write clean snapshot if enabled + if snapshot_config.clean_copy: + png_bytes = obj.get_clean_png() + if png_bytes is None: + logger.warning( + f"Unable to save clean snapshot for {obj.obj_data['id']}." + ) + else: + with open( + os.path.join( + CLIPS_DIR, + f"{camera}-{obj.obj_data['id']}-clean.png", + ), + "wb", + ) as p: + p.write(png_bytes) self.event_queue.put(("end", camera, event_data)) def snapshot(camera, obj: TrackedObject, current_frame_time): From fd9c8c1f0d88f1b3bfc7bf9751db13cf789ac816 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 19 Jun 2021 08:40:28 -0500 Subject: [PATCH 10/12] add snapshot time to event data --- docs/docs/usage/mqtt.md | 2 ++ frigate/object_processing.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/docs/docs/usage/mqtt.md b/docs/docs/usage/mqtt.md index d91429b16..27712c124 100644 --- a/docs/docs/usage/mqtt.md +++ b/docs/docs/usage/mqtt.md @@ -37,6 +37,7 @@ Message published for each changed event. The first message is published when th "id": "1607123955.475377-mxklsc", "camera": "front_door", "frame_time": 1607123961.837752, + "snapshot_time": 1607123961.837752, "label": "person", "top_score": 0.958984375, "false_positive": false, @@ -54,6 +55,7 @@ Message published for each changed event. The first message is published when th "id": "1607123955.475377-mxklsc", "camera": "front_door", "frame_time": 1607123962.082975, + "snapshot_time": 1607123961.837752, "label": "person", "top_score": 0.958984375, "false_positive": false, diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 4968f14f1..85d238f24 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -153,10 +153,16 @@ class TrackedObject: return significant_update def to_dict(self, include_thumbnail: bool = False): + snapshot_time = ( + self.thumbnail_data["frame_time"] + if not self.thumbnail_data is None + else 0.0 + ) event = { "id": self.obj_data["id"], "camera": self.camera, "frame_time": self.obj_data["frame_time"], + "snapshot_time": snapshot_time, "label": self.obj_data["label"], "top_score": self.top_score, "false_positive": self.false_positive, From d66f5f6bad2b73a94a1105ff86e5d0c59bdb5161 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 19 Jun 2021 08:44:25 -0500 Subject: [PATCH 11/12] cleanup clean snapshots --- frigate/events.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/frigate/events.py b/frigate/events.py index 58ca1986e..71e9990dc 100644 --- a/frigate/events.py +++ b/frigate/events.py @@ -283,7 +283,7 @@ class EventCleanup(threading.Thread): def expire(self, media_type): ## Expire events from unlisted cameras based on the global config - if media_type == 'clips': + if media_type == "clips": retain_config = self.config.clips.retain file_extension = "mp4" update_params = {"has_clip": False} @@ -314,8 +314,16 @@ class EventCleanup(threading.Thread): # delete the media from disk for event in expired_events: media_name = f"{event.camera}-{event.id}" - media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") + media_path = Path( + f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}" + ) media_path.unlink(missing_ok=True) + if file_extension == "jpg": + media_path = Path( + f"{os.path.join(CLIPS_DIR, media_name)}-clean.png" + ) + media_path.unlink(missing_ok=True) + # update the clips attribute for the db entry update_query = Event.update(update_params).where( Event.camera.not_in(self.camera_keys), @@ -326,7 +334,7 @@ class EventCleanup(threading.Thread): ## Expire events from cameras based on the camera config for name, camera in self.config.cameras.items(): - if media_type == 'clips': + if media_type == "clips": retain_config = camera.clips.retain else: retain_config = camera.snapshots.retain @@ -351,8 +359,15 @@ class EventCleanup(threading.Thread): # delete the grabbed clips from disk for event in expired_events: media_name = f"{event.camera}-{event.id}" - media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}") + media_path = Path( + f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}" + ) media_path.unlink(missing_ok=True) + if file_extension == "jpg": + media_path = Path( + f"{os.path.join(CLIPS_DIR, media_name)}-clean.png" + ) + media_path.unlink(missing_ok=True) # update the clips attribute for the db entry update_query = Event.update(update_params).where( Event.camera == name, From 8e0c2b256ebc2b3ea519d6730fab7e9f6c960368 Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Sat, 19 Jun 2021 08:47:09 -0500 Subject: [PATCH 12/12] update docs --- docs/docs/configuration/cameras.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/docs/configuration/cameras.md b/docs/docs/configuration/cameras.md index faf871f07..707cc830a 100644 --- a/docs/docs/configuration/cameras.md +++ b/docs/docs/configuration/cameras.md @@ -170,6 +170,9 @@ snapshots: # Optional: Enable writing jpg snapshot to /media/frigate/clips (default: shown below) # This value can be set via MQTT and will be updated in startup based on retained value enabled: False + # Optional: Enable writing a clean copy png snapshot to /media/frigate/clips (default: shown below) + # Only works if snapshots are enabled. This image is intended to be used for training purposes. + clean_copy: True # Optional: print a timestamp on the snapshots (default: shown below) timestamp: False # Optional: draw bounding box on the snapshots (default: shown below)