From 5b1b6b5be082fb18ae8ac3b66f81c00ebc8a9493 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 17 Nov 2024 10:25:49 -0700 Subject: [PATCH 1/6] Fix round robin (#15035) * Move camera SHM frame creation to main process * Don't reset frame index * Don't fail if shm exists * Set more types --- frigate/app.py | 8 +++++++- frigate/util/image.py | 14 +++++++++----- frigate/video.py | 20 ++++++++++---------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index 96edfbd15..f56ed1a8b 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -68,7 +68,7 @@ from frigate.stats.util import stats_init from frigate.storage import StorageMaintainer from frigate.timeline import TimelineProcessor from frigate.util.builtin import empty_and_close_queue -from frigate.util.image import UntrackedSharedMemory +from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory from frigate.util.object import get_camera_regions_grid from frigate.version import VERSION from frigate.video import capture_camera, track_camera @@ -426,12 +426,18 @@ class FrigateApp: def start_camera_capture_processes(self) -> None: shm_frame_count = self.shm_frame_count() + frame_manager = SharedMemoryFrameManager() for name, config in self.config.cameras.items(): if not self.config.cameras[name].enabled: logger.info(f"Capture process not started for disabled camera {name}") continue + # pre-create shms + for i in range(shm_frame_count): + frame_size = config.frame_shape_yuv[0] * config.frame_shape_yuv[1] + frame_manager.create(f"{config.name}{i}", frame_size) + capture_process = util.Process( target=capture_camera, name=f"camera_capture:{name}", diff --git a/frigate/util/image.py b/frigate/util/image.py index 4e3161192..7b22c138e 100644 --- a/frigate/util/image.py +++ b/frigate/util/image.py @@ -790,11 +790,15 @@ class SharedMemoryFrameManager(FrameManager): self.shm_store: dict[str, UntrackedSharedMemory] = {} def create(self, name: str, size) -> AnyStr: - shm = UntrackedSharedMemory( - name=name, - create=True, - size=size, - ) + try: + shm = UntrackedSharedMemory( + name=name, + create=True, + size=size, + ) + except FileExistsError: + shm = UntrackedSharedMemory(name=name) + self.shm_store[name] = shm return shm.buf diff --git a/frigate/video.py b/frigate/video.py index 4e7fe660d..5af3e13f4 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -94,7 +94,8 @@ def capture_frames( ffmpeg_process, config: CameraConfig, shm_frame_count: int, - frame_shape, + frame_index: int, + frame_shape: tuple[int, int], frame_manager: FrameManager, frame_queue, fps: mp.Value, @@ -108,12 +109,6 @@ def capture_frames( skipped_eps = EventsPerSecond() skipped_eps.start() - # pre-create shms - for i in range(shm_frame_count): - frame_manager.create(f"{config.name}{i}", frame_size) - - frame_index = 0 - while True: fps.value = frame_rate.eps() skipped_fps.value = skipped_eps.eps() @@ -159,7 +154,7 @@ class CameraWatchdog(threading.Thread): camera_name, config: CameraConfig, shm_frame_count: int, - frame_queue, + frame_queue: mp.Queue, camera_fps, skipped_fps, ffmpeg_pid, @@ -181,6 +176,7 @@ class CameraWatchdog(threading.Thread): self.frame_shape = self.config.frame_shape_yuv self.frame_size = self.frame_shape[0] * self.frame_shape[1] self.fps_overflow_count = 0 + self.frame_index = 0 self.stop_event = stop_event self.sleeptime = self.config.ffmpeg.retry_interval @@ -302,6 +298,7 @@ class CameraWatchdog(threading.Thread): self.capture_thread = CameraCapture( self.config, self.shm_frame_count, + self.frame_index, self.ffmpeg_detect_process, self.frame_shape, self.frame_queue, @@ -342,9 +339,10 @@ class CameraCapture(threading.Thread): self, config: CameraConfig, shm_frame_count: int, + frame_index: int, ffmpeg_process, - frame_shape, - frame_queue, + frame_shape: tuple[int, int], + frame_queue: mp.Queue, fps, skipped_fps, stop_event, @@ -353,6 +351,7 @@ class CameraCapture(threading.Thread): self.name = f"capture:{config.name}" self.config = config self.shm_frame_count = shm_frame_count + self.frame_index = frame_index self.frame_shape = frame_shape self.frame_queue = frame_queue self.fps = fps @@ -368,6 +367,7 @@ class CameraCapture(threading.Thread): self.ffmpeg_process, self.config, self.shm_frame_count, + self.frame_index, self.frame_shape, self.frame_manager, self.frame_queue, From 474c248c9d7501ff7e9677578331d4ca4beff476 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 17 Nov 2024 15:57:58 -0700 Subject: [PATCH 2/6] Cleanup correctly (#15043) --- frigate/app.py | 5 +++-- frigate/video.py | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index f56ed1a8b..6518c1ddf 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -91,6 +91,7 @@ class FrigateApp: self.processes: dict[str, int] = {} self.embeddings: Optional[EmbeddingsContext] = None self.region_grids: dict[str, list[list[dict[str, int]]]] = {} + self.frame_manager = SharedMemoryFrameManager() self.config = config def ensure_dirs(self) -> None: @@ -426,7 +427,6 @@ class FrigateApp: def start_camera_capture_processes(self) -> None: shm_frame_count = self.shm_frame_count() - frame_manager = SharedMemoryFrameManager() for name, config in self.config.cameras.items(): if not self.config.cameras[name].enabled: @@ -436,7 +436,7 @@ class FrigateApp: # pre-create shms for i in range(shm_frame_count): frame_size = config.frame_shape_yuv[0] * config.frame_shape_yuv[1] - frame_manager.create(f"{config.name}{i}", frame_size) + self.frame_manager.create(f"{config.name}{i}", frame_size) capture_process = util.Process( target=capture_camera, @@ -717,6 +717,7 @@ class FrigateApp: self.event_metadata_updater.stop() self.inter_zmq_proxy.stop() + self.frame_manager.cleanup() while len(self.detection_shms) > 0: shm = self.detection_shms.pop() shm.close() diff --git a/frigate/video.py b/frigate/video.py index 5af3e13f4..96b562e8c 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -145,8 +145,6 @@ def capture_frames( frame_index = 0 if frame_index == shm_frame_count - 1 else frame_index + 1 - frame_manager.cleanup() - class CameraWatchdog(threading.Thread): def __init__( From 26c3f9f9148ccfc46a461c6baec000b984827108 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 18 Nov 2024 08:38:58 -0700 Subject: [PATCH 3/6] Fix birdseye (#15051) --- frigate/output/birdseye.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py index cab155b9b..6d6391f14 100644 --- a/frigate/output/birdseye.py +++ b/frigate/output/birdseye.py @@ -388,7 +388,7 @@ class BirdsEyeFrameManager: for cam, cam_data in self.cameras.items() if self.config.cameras[cam].birdseye.enabled and cam_data["last_active_frame"] > 0 - and cam_data["current_frame"] - cam_data["last_active_frame"] + and cam_data["current_frame_time"] - cam_data["last_active_frame"] < self.inactivity_threshold ] ) @@ -405,7 +405,7 @@ class BirdsEyeFrameManager: limited_active_cameras = sorted( active_cameras, key=lambda active_camera: ( - self.cameras[active_camera]["current_frame"] + self.cameras[active_camera]["current_frame_time"] - self.cameras[active_camera]["last_active_frame"] ), ) @@ -517,7 +517,7 @@ class BirdsEyeFrameManager: self.copy_to_position( position[1], position[0], - frame, + self.cameras[position[0]]["current_frame"], ) return True @@ -689,7 +689,8 @@ class BirdsEyeFrameManager: return False # update the last active frame for the camera - self.cameras[camera]["current_frame"] = frame_time + self.cameras[camera]["current_frame"] = frame.copy() + self.cameras[camera]["current_frame_time"] = frame_time if self.camera_active(camera_config.mode, object_count, motion_count): self.cameras[camera]["last_active_frame"] = frame_time @@ -755,7 +756,7 @@ class Birdseye: current_tracked_objects: list[dict[str, any]], motion_boxes: list[list[int]], frame_time: float, - frame, + frame: np.ndarray, ) -> None: # check if there is an updated config while True: From 0b203a3673085da90f4074148b3bc657624f35ec Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 18 Nov 2024 09:14:49 -0700 Subject: [PATCH 4/6] fix writing to birdseye restream buffer (#15052) --- frigate/output/birdseye.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py index 6d6391f14..00f17c8f4 100644 --- a/frigate/output/birdseye.py +++ b/frigate/output/birdseye.py @@ -740,9 +740,10 @@ class Birdseye: ) self.birdseye_manager = BirdsEyeFrameManager(config, stop_event) self.config_subscriber = ConfigSubscriber("config/birdseye/") + self.frame_manager = SharedMemoryFrameManager() if config.birdseye.restream: - self.birdseye_buffer = SharedMemoryFrameManager().create( + self.birdseye_buffer = self.frame_manager.create( "birdseye", self.birdseye_manager.yuv_shape[0] * self.birdseye_manager.yuv_shape[1], ) From 66f71aecf7755771bc9a3c1dd7cf741a6e1b6955 Mon Sep 17 00:00:00 2001 From: Bazyl Ichabod Horsey <31446064+bazylhorsey@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:26:36 -0600 Subject: [PATCH 5/6] fix regex for cookie_name to be general snake case (#14854) * fix regex for cookie_name to be general snake case * Update frigate/config/auth.py Co-authored-by: Blake Blackshear --------- Co-authored-by: Blake Blackshear --- frigate/config/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frigate/config/auth.py b/frigate/config/auth.py index 91a692461..a202fb1af 100644 --- a/frigate/config/auth.py +++ b/frigate/config/auth.py @@ -13,7 +13,7 @@ class AuthConfig(FrigateBaseModel): default=False, title="Reset the admin password on startup" ) cookie_name: str = Field( - default="frigate_token", title="Name for jwt token cookie", pattern=r"^[a-z]_*$" + default="frigate_token", title="Name for jwt token cookie", pattern=r"^[a-z_]+$" ) cookie_secure: bool = Field(default=False, title="Set secure flag on cookie") session_length: int = Field( From 9ae839ad723dcb2b3c89544419a5b96779a44062 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:26:44 -0600 Subject: [PATCH 6/6] Tracked object metadata changes (#15055) * add enum and change topic name * frontend renaming * docs * only display sublabel score if it it exists * remove debug print --- docs/docs/integrations/mqtt.md | 12 ++++++++++++ frigate/comms/dispatcher.py | 12 +++++++++--- frigate/embeddings/maintainer.py | 7 ++++++- frigate/types.py | 4 ++++ web/src/api/ws.tsx | 4 ++-- .../components/overlay/detail/SearchDetailDialog.tsx | 2 +- web/src/pages/Explore.tsx | 8 ++++---- web/src/views/explore/ExploreView.tsx | 6 +++--- 8 files changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index e606d29fc..194821cbd 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -94,6 +94,18 @@ Message published for each changed tracked object. The first message is publishe } ``` +### `frigate/tracked_object_update` + +Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description. + +```json +{ + "type": "description", + "id": "1607123955.475377-mxklsc", + "description": "The car is a red sedan moving away from the camera." +} +``` + ### `frigate/reviews` Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published. diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 1f480fa9c..2bddc97a5 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -22,7 +22,7 @@ from frigate.const import ( ) from frigate.models import Event, Previews, Recordings, ReviewSegment from frigate.ptz.onvif import OnvifCommandEnum, OnvifController -from frigate.types import ModelStatusTypesEnum +from frigate.types import ModelStatusTypesEnum, TrackedObjectUpdateTypesEnum from frigate.util.object import get_camera_regions_grid from frigate.util.services import restart_frigate @@ -137,8 +137,14 @@ class Dispatcher: event.data["description"] = payload["description"] event.save() self.publish( - "event_update", - json.dumps({"id": event.id, "description": event.data["description"]}), + "tracked_object_update", + json.dumps( + { + "type": TrackedObjectUpdateTypesEnum.description, + "id": event.id, + "description": event.data["description"], + } + ), ) def handle_update_model_state(): diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 12c8bac72..dde8f8df4 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -24,6 +24,7 @@ from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION from frigate.events.types import EventTypeEnum from frigate.genai import get_genai_client from frigate.models import Event +from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import serialize from frigate.util.image import SharedMemoryFrameManager, calculate_region @@ -287,7 +288,11 @@ class EmbeddingMaintainer(threading.Thread): # fire and forget description update self.requestor.send_data( UPDATE_EVENT_DESCRIPTION, - {"id": event.id, "description": description}, + { + "type": TrackedObjectUpdateTypesEnum.description, + "id": event.id, + "description": description, + }, ) # Embed the description diff --git a/frigate/types.py b/frigate/types.py index 3e6ad46cc..11ab31238 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -19,3 +19,7 @@ class ModelStatusTypesEnum(str, Enum): downloading = "downloading" downloaded = "downloaded" error = "error" + + +class TrackedObjectUpdateTypesEnum(str, Enum): + description = "description" diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index c7bb74095..9b8924d1b 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -407,9 +407,9 @@ export function useImproveContrast(camera: string): { return { payload: payload as ToggleableSetting, send }; } -export function useEventUpdate(): { payload: string } { +export function useTrackedObjectUpdate(): { payload: string } { const { value: { payload }, - } = useWs("event_update", ""); + } = useWs("tracked_object_update", ""); return useDeepMemo(JSON.parse(payload as string)); } diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 5eca9a934..f7af31606 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -309,7 +309,7 @@ function ObjectDetailsTab({ return undefined; } - if (search.sub_label) { + if (search.sub_label && search.data?.sub_label_score) { return Math.round((search.data?.sub_label_score ?? 0) * 100); } else { return undefined; diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index 711666807..2bf2bb022 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -1,6 +1,6 @@ import { useEmbeddingsReindexProgress, - useEventUpdate, + useTrackedObjectUpdate, useModelState, } from "@/api/ws"; import ActivityIndicator from "@/components/indicators/activity-indicator"; @@ -227,15 +227,15 @@ export default function Explore() { // mutation and revalidation - const eventUpdate = useEventUpdate(); + const trackedObjectUpdate = useTrackedObjectUpdate(); useEffect(() => { - if (eventUpdate) { + if (trackedObjectUpdate) { mutate(); } // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps - }, [eventUpdate]); + }, [trackedObjectUpdate]); // embeddings reindex progress diff --git a/web/src/views/explore/ExploreView.tsx b/web/src/views/explore/ExploreView.tsx index f37c37453..ea9c3cbef 100644 --- a/web/src/views/explore/ExploreView.tsx +++ b/web/src/views/explore/ExploreView.tsx @@ -15,7 +15,7 @@ import { SearchResult } from "@/types/search"; import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator"; import useImageLoaded from "@/hooks/use-image-loaded"; import ActivityIndicator from "@/components/indicators/activity-indicator"; -import { useEventUpdate } from "@/api/ws"; +import { useTrackedObjectUpdate } from "@/api/ws"; import { isEqual } from "lodash"; import TimeAgo from "@/components/dynamic/TimeAgo"; import SearchResultActions from "@/components/menu/SearchResultActions"; @@ -72,13 +72,13 @@ export default function ExploreView({ }, {}); }, [events]); - const eventUpdate = useEventUpdate(); + const trackedObjectUpdate = useTrackedObjectUpdate(); useEffect(() => { mutate(); // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps - }, [eventUpdate]); + }, [trackedObjectUpdate]); // update search detail when results change