From a67ff3843afe5cde45a03bcaa7a374dff67e4e25 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 19 Nov 2024 09:41:16 -0600 Subject: [PATCH 1/3] Update genai docs (#15070) --- docs/docs/configuration/genai.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md index da4b8afd8..67872876b 100644 --- a/docs/docs/configuration/genai.md +++ b/docs/docs/configuration/genai.md @@ -142,6 +142,10 @@ Frigate's thumbnail search excels at identifying specific details about tracked While generating simple descriptions of detected objects is useful, understanding intent provides a deeper layer of insight. Instead of just recognizing "what" is in a scene, Frigate’s default prompts aim to infer "why" it might be there or "what" it could do next. Descriptions tell you what’s happening, but intent gives context. For instance, a person walking toward a door might seem like a visitor, but if they’re moving quickly after hours, you can infer a potential break-in attempt. Detecting a person loitering near a door at night can trigger an alert sooner than simply noting "a person standing by the door," helping you respond based on the situation’s context. +### Using GenAI for notifications + +Frigate provides an [MQTT topic](/integrations/mqtt), `frigate/tracked_object_update`, that is updated with a JSON payload containing `event_id` and `description` when your AI provider returns a description for a tracked object. This description could be used directly in notifications, such as sending alerts to your phone or making audio announcements. If additional details from the tracked object are needed, you can query the [HTTP API](/integrations/api) using the `event_id`, eg: `http://frigate_ip:5000/api/events/`. + ## Custom Prompts Frigate sends multiple frames from the tracked object along with a prompt to your Generative AI provider asking it to generate a description. The default prompt is as follows: @@ -172,7 +176,7 @@ genai: Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire. By default, descriptions will be generated for all tracked objects and all zones. But you can also optionally specify `objects` and `required_zones` to only generate descriptions for certain tracked objects or zones. -Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the thumbnails collected over the object's lifetime to the model. Using a snapshot provides the AI with a higher-resolution image (typically downscaled by the AI itself), but the trade-off is that only a single image is used, which might limit the model's ability to determine object movement or direction. +Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the uncompressed images from the `detect` stream collected over the object's lifetime to the model. Once the object lifecycle ends, only a single compressed and cropped thumbnail is saved with the tracked object. Using a snapshot might be useful when you want to _regenerate_ a tracked object's description as it will provide the AI with a higher-quality image (typically downscaled by the AI itself) than the cropped/compressed thumbnail. Using a snapshot otherwise has a trade-off in that only a single image is sent to your provider, which will limit the model's ability to determine object movement or direction. ```yaml cameras: From 66277fbb6c3085acbb0f669b0c6fc4e9eee3d5b0 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 19 Nov 2024 11:20:04 -0700 Subject: [PATCH 2/3] Fix embeddings (#15072) * Fix embeddings reading frames * Fix event update reading * Formatting * Pin AIO http to fix build failure * Pin starlette --- docker/main/requirements-wheels.txt | 2 ++ frigate/comms/events_updater.py | 2 +- frigate/embeddings/maintainer.py | 9 +++++---- frigate/events/maintainer.py | 2 +- frigate/object_processing.py | 27 +++++++++++++++------------ 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index 795456588..4db88ccd2 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -1,5 +1,7 @@ click == 8.1.* # FastAPI +aiohttp == 3.11.2 +starlette == 0.41.2 starlette-context == 0.3.6 fastapi == 0.115.* uvicorn == 0.30.* diff --git a/frigate/comms/events_updater.py b/frigate/comms/events_updater.py index 7a5772273..98b6ccb7a 100644 --- a/frigate/comms/events_updater.py +++ b/frigate/comms/events_updater.py @@ -14,7 +14,7 @@ class EventUpdatePublisher(Publisher): super().__init__("update") def publish( - self, payload: tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]] + self, payload: tuple[EventTypeEnum, EventStateEnum, str, str, dict[str, any]] ) -> None: super().publish(payload) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index dde8f8df4..d58a7f431 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -114,7 +114,7 @@ class EmbeddingMaintainer(threading.Thread): if update is None: return - source_type, _, camera, data = update + source_type, _, camera, frame_name, data = update if not camera or source_type != EventTypeEnum.tracked_object: return @@ -134,8 +134,9 @@ class EmbeddingMaintainer(threading.Thread): # Create our own thumbnail based on the bounding box and the frame time try: - frame_id = f"{camera}{data['frame_time']}" - yuv_frame = self.frame_manager.get(frame_id, camera_config.frame_shape_yuv) + yuv_frame = self.frame_manager.get( + frame_name, camera_config.frame_shape_yuv + ) if yuv_frame is not None: data["thumbnail"] = self._create_thumbnail(yuv_frame, data["box"]) @@ -147,7 +148,7 @@ class EmbeddingMaintainer(threading.Thread): self.tracked_events[data["id"]].append(data) - self.frame_manager.close(frame_id) + self.frame_manager.close(frame_name) except FileNotFoundError: pass diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index b17bd5d35..3a4209ec3 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -75,7 +75,7 @@ class EventProcessor(threading.Thread): if update == None: continue - source_type, event_type, camera, event_data = update + source_type, event_type, camera, _, event_data = update logger.debug( f"Event received: {source_type} {event_type} {camera} {event_data['id']}" diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 23c84eedd..937c935ba 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -262,7 +262,7 @@ class CameraState: # call event handlers for c in self.callbacks["start"]: - c(self.name, new_obj, frame_time) + c(self.name, new_obj, frame_name) for id in updated_ids: updated_obj = tracked_objects[id] @@ -272,7 +272,7 @@ class CameraState: if autotracker_update or significant_update: for c in self.callbacks["autotrack"]: - c(self.name, updated_obj, frame_time) + c(self.name, updated_obj, frame_name) if thumb_update and current_frame is not None: # ensure this frame is stored in the cache @@ -293,7 +293,7 @@ class CameraState: ) or significant_update: # call event handlers for c in self.callbacks["update"]: - c(self.name, updated_obj, frame_time) + c(self.name, updated_obj, frame_name) updated_obj.last_published = frame_time for id in removed_ids: @@ -302,7 +302,7 @@ class CameraState: if "end_time" not in removed_obj.obj_data: removed_obj.obj_data["end_time"] = frame_time for c in self.callbacks["end"]: - c(self.name, removed_obj, frame_time) + c(self.name, removed_obj, frame_name) # TODO: can i switch to looking this up and only changing when an event ends? # maintain best objects @@ -368,11 +368,11 @@ class CameraState: ): self.best_objects[object_type] = obj for c in self.callbacks["snapshot"]: - c(self.name, self.best_objects[object_type], frame_time) + c(self.name, self.best_objects[object_type], frame_name) else: self.best_objects[object_type] = obj for c in self.callbacks["snapshot"]: - c(self.name, self.best_objects[object_type], frame_time) + c(self.name, self.best_objects[object_type], frame_name) for c in self.callbacks["camera_activity"]: c(self.name, camera_activity) @@ -447,7 +447,7 @@ class CameraState: c(self.name, obj_name, 0) self.active_object_counts[obj_name] = 0 for c in self.callbacks["snapshot"]: - c(self.name, self.best_objects[obj_name], frame_time) + c(self.name, self.best_objects[obj_name], frame_name) # cleanup thumbnail frame cache current_thumb_frames = { @@ -518,17 +518,18 @@ class TrackedObjectProcessor(threading.Thread): self.zone_data = defaultdict(lambda: defaultdict(dict)) self.active_zone_data = defaultdict(lambda: defaultdict(dict)) - def start(camera, obj: TrackedObject, current_frame_time): + def start(camera: str, obj: TrackedObject, frame_name: str): self.event_sender.publish( ( EventTypeEnum.tracked_object, EventStateEnum.start, camera, + frame_name, obj.to_dict(), ) ) - def update(camera, obj: TrackedObject, current_frame_time): + def update(camera: str, obj: TrackedObject, frame_name: str): obj.has_snapshot = self.should_save_snapshot(camera, obj) obj.has_clip = self.should_retain_recording(camera, obj) after = obj.to_dict() @@ -544,14 +545,15 @@ class TrackedObjectProcessor(threading.Thread): EventTypeEnum.tracked_object, EventStateEnum.update, camera, + frame_name, obj.to_dict(include_thumbnail=True), ) ) - def autotrack(camera, obj: TrackedObject, current_frame_time): + def autotrack(camera: str, obj: TrackedObject, frame_name: str): self.ptz_autotracker_thread.ptz_autotracker.autotrack_object(camera, obj) - def end(camera, obj: TrackedObject, current_frame_time): + def end(camera: str, obj: TrackedObject, frame_name: str): # populate has_snapshot obj.has_snapshot = self.should_save_snapshot(camera, obj) obj.has_clip = self.should_retain_recording(camera, obj) @@ -606,11 +608,12 @@ class TrackedObjectProcessor(threading.Thread): EventTypeEnum.tracked_object, EventStateEnum.end, camera, + frame_name, obj.to_dict(include_thumbnail=True), ) ) - def snapshot(camera, obj: TrackedObject, current_frame_time): + def snapshot(camera, obj: TrackedObject, frame_name: str): mqtt_config: MqttConfig = self.config.cameras[camera].mqtt if mqtt_config.enabled and self.should_mqtt_snapshot(camera, obj): jpg_bytes = obj.get_jpg_bytes( From 0df091f38781cec0cba1ca66e0e42a720151c38a Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:33:01 -0600 Subject: [PATCH 3/3] Fix link to api in genai docs (#15075) --- docs/docs/configuration/genai.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md index 67872876b..fac44ed03 100644 --- a/docs/docs/configuration/genai.md +++ b/docs/docs/configuration/genai.md @@ -144,7 +144,7 @@ While generating simple descriptions of detected objects is useful, understandin ### Using GenAI for notifications -Frigate provides an [MQTT topic](/integrations/mqtt), `frigate/tracked_object_update`, that is updated with a JSON payload containing `event_id` and `description` when your AI provider returns a description for a tracked object. This description could be used directly in notifications, such as sending alerts to your phone or making audio announcements. If additional details from the tracked object are needed, you can query the [HTTP API](/integrations/api) using the `event_id`, eg: `http://frigate_ip:5000/api/events/`. +Frigate provides an [MQTT topic](/integrations/mqtt), `frigate/tracked_object_update`, that is updated with a JSON payload containing `event_id` and `description` when your AI provider returns a description for a tracked object. This description could be used directly in notifications, such as sending alerts to your phone or making audio announcements. If additional details from the tracked object are needed, you can query the [HTTP API](/integrations/api/event-events-event-id-get) using the `event_id`, eg: `http://frigate_ip:5000/api/events/`. ## Custom Prompts