Merge remote-tracking branch 'upstream/dev' into unit-test-for-review-controller

docs/docs/integrations/mqtt.md

@@ -94,6 +94,18 @@ Message published for each changed tracked object. The first message is publishe
 }
 ```
 
+### `frigate/tracked_object_update`
+
+Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description.
+
+```json
+{
+  "type": "description",
+  "id": "1607123955.475377-mxklsc",
+  "description": "The car is a red sedan moving away from the camera."
+}
+```
+
 ### `frigate/reviews`
 
 Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published.

frigate/app.py

@@ -68,7 +68,7 @@ from frigate.stats.util import stats_init
 from frigate.storage import StorageMaintainer
 from frigate.timeline import TimelineProcessor
 from frigate.util.builtin import empty_and_close_queue
-from frigate.util.image import UntrackedSharedMemory
+from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory
 from frigate.util.object import get_camera_regions_grid
 from frigate.version import VERSION
 from frigate.video import capture_camera, track_camera
@@ -91,6 +91,7 @@ class FrigateApp:
         self.processes: dict[str, int] = {}
         self.embeddings: Optional[EmbeddingsContext] = None
         self.region_grids: dict[str, list[list[dict[str, int]]]] = {}
+        self.frame_manager = SharedMemoryFrameManager()
         self.config = config
 
     def ensure_dirs(self) -> None:
@@ -432,6 +433,11 @@ class FrigateApp:
                 logger.info(f"Capture process not started for disabled camera {name}")
                 continue
 
+            # pre-create shms
+            for i in range(shm_frame_count):
+                frame_size = config.frame_shape_yuv[0] * config.frame_shape_yuv[1]
+                self.frame_manager.create(f"{config.name}{i}", frame_size)
+
             capture_process = util.Process(
                 target=capture_camera,
                 name=f"camera_capture:{name}",
@@ -711,6 +717,7 @@ class FrigateApp:
         self.event_metadata_updater.stop()
         self.inter_zmq_proxy.stop()
 
+        self.frame_manager.cleanup()
         while len(self.detection_shms) > 0:
             shm = self.detection_shms.pop()
             shm.close()

frigate/comms/dispatcher.py

@@ -22,7 +22,7 @@ from frigate.const import (
 )
 from frigate.models import Event, Previews, Recordings, ReviewSegment
 from frigate.ptz.onvif import OnvifCommandEnum, OnvifController
-from frigate.types import ModelStatusTypesEnum
+from frigate.types import ModelStatusTypesEnum, TrackedObjectUpdateTypesEnum
 from frigate.util.object import get_camera_regions_grid
 from frigate.util.services import restart_frigate
 
@@ -137,8 +137,14 @@ class Dispatcher:
             event.data["description"] = payload["description"]
             event.save()
             self.publish(
-                "event_update",
-                json.dumps({"id": event.id, "description": event.data["description"]}),
+                "tracked_object_update",
+                json.dumps(
+                    {
+                        "type": TrackedObjectUpdateTypesEnum.description,
+                        "id": event.id,
+                        "description": event.data["description"],
+                    }
+                ),
             )
 
         def handle_update_model_state():

frigate/config/auth.py

@@ -13,7 +13,7 @@ class AuthConfig(FrigateBaseModel):
         default=False, title="Reset the admin password on startup"
     )
     cookie_name: str = Field(
-        default="frigate_token", title="Name for jwt token cookie", pattern=r"^[a-z]_*$"
+        default="frigate_token", title="Name for jwt token cookie", pattern=r"^[a-z_]+$"
     )
     cookie_secure: bool = Field(default=False, title="Set secure flag on cookie")
     session_length: int = Field(

frigate/embeddings/maintainer.py

@@ -24,6 +24,7 @@ from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION
 from frigate.events.types import EventTypeEnum
 from frigate.genai import get_genai_client
 from frigate.models import Event
+from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
 from frigate.util.image import SharedMemoryFrameManager, calculate_region
 
@@ -287,7 +288,11 @@ class EmbeddingMaintainer(threading.Thread):
         # fire and forget description update
         self.requestor.send_data(
             UPDATE_EVENT_DESCRIPTION,
-            {"id": event.id, "description": description},
+            {
+                "type": TrackedObjectUpdateTypesEnum.description,
+                "id": event.id,
+                "description": description,
+            },
         )
 
         # Embed the description

frigate/output/birdseye.py

@@ -388,7 +388,7 @@ class BirdsEyeFrameManager:
                 for cam, cam_data in self.cameras.items()
                 if self.config.cameras[cam].birdseye.enabled
                 and cam_data["last_active_frame"] > 0
-                and cam_data["current_frame"] - cam_data["last_active_frame"]
+                and cam_data["current_frame_time"] - cam_data["last_active_frame"]
                 < self.inactivity_threshold
             ]
         )
@@ -405,7 +405,7 @@ class BirdsEyeFrameManager:
                 limited_active_cameras = sorted(
                     active_cameras,
                     key=lambda active_camera: (
-                        self.cameras[active_camera]["current_frame"]
+                        self.cameras[active_camera]["current_frame_time"]
                         - self.cameras[active_camera]["last_active_frame"]
                     ),
                 )
@@ -517,7 +517,7 @@ class BirdsEyeFrameManager:
                 self.copy_to_position(
                     position[1],
                     position[0],
-                    frame,
+                    self.cameras[position[0]]["current_frame"],
                 )
 
         return True
@@ -689,7 +689,8 @@ class BirdsEyeFrameManager:
             return False
 
         # update the last active frame for the camera
-        self.cameras[camera]["current_frame"] = frame_time
+        self.cameras[camera]["current_frame"] = frame.copy()
+        self.cameras[camera]["current_frame_time"] = frame_time
         if self.camera_active(camera_config.mode, object_count, motion_count):
             self.cameras[camera]["last_active_frame"] = frame_time
 
@@ -739,9 +740,10 @@ class Birdseye:
         )
         self.birdseye_manager = BirdsEyeFrameManager(config, stop_event)
         self.config_subscriber = ConfigSubscriber("config/birdseye/")
+        self.frame_manager = SharedMemoryFrameManager()
 
         if config.birdseye.restream:
-            self.birdseye_buffer = SharedMemoryFrameManager().create(
+            self.birdseye_buffer = self.frame_manager.create(
                 "birdseye",
                 self.birdseye_manager.yuv_shape[0] * self.birdseye_manager.yuv_shape[1],
             )
@@ -755,7 +757,7 @@ class Birdseye:
         current_tracked_objects: list[dict[str, any]],
         motion_boxes: list[list[int]],
         frame_time: float,
-        frame,
+        frame: np.ndarray,
     ) -> None:
         # check if there is an updated config
         while True:

frigate/types.py

@@ -19,3 +19,7 @@ class ModelStatusTypesEnum(str, Enum):
     downloading = "downloading"
     downloaded = "downloaded"
     error = "error"
+
+
+class TrackedObjectUpdateTypesEnum(str, Enum):
+    description = "description"

frigate/util/image.py

@@ -790,11 +790,15 @@ class SharedMemoryFrameManager(FrameManager):
         self.shm_store: dict[str, UntrackedSharedMemory] = {}
 
     def create(self, name: str, size) -> AnyStr:
-        shm = UntrackedSharedMemory(
-            name=name,
-            create=True,
-            size=size,
-        )
+        try:
+            shm = UntrackedSharedMemory(
+                name=name,
+                create=True,
+                size=size,
+            )
+        except FileExistsError:
+            shm = UntrackedSharedMemory(name=name)
+
         self.shm_store[name] = shm
         return shm.buf
 

frigate/video.py

@@ -94,7 +94,8 @@ def capture_frames(
     ffmpeg_process,
     config: CameraConfig,
     shm_frame_count: int,
-    frame_shape,
+    frame_index: int,
+    frame_shape: tuple[int, int],
     frame_manager: FrameManager,
     frame_queue,
     fps: mp.Value,
@@ -108,12 +109,6 @@ def capture_frames(
     skipped_eps = EventsPerSecond()
     skipped_eps.start()
 
-    # pre-create shms
-    for i in range(shm_frame_count):
-        frame_manager.create(f"{config.name}{i}", frame_size)
-
-    frame_index = 0
-
     while True:
         fps.value = frame_rate.eps()
         skipped_fps.value = skipped_eps.eps()
@@ -150,8 +145,6 @@ def capture_frames(
 
         frame_index = 0 if frame_index == shm_frame_count - 1 else frame_index + 1
 
-    frame_manager.cleanup()
-
 
 class CameraWatchdog(threading.Thread):
     def __init__(
@@ -159,7 +152,7 @@ class CameraWatchdog(threading.Thread):
         camera_name,
         config: CameraConfig,
         shm_frame_count: int,
-        frame_queue,
+        frame_queue: mp.Queue,
         camera_fps,
         skipped_fps,
         ffmpeg_pid,
@@ -181,6 +174,7 @@ class CameraWatchdog(threading.Thread):
         self.frame_shape = self.config.frame_shape_yuv
         self.frame_size = self.frame_shape[0] * self.frame_shape[1]
         self.fps_overflow_count = 0
+        self.frame_index = 0
         self.stop_event = stop_event
         self.sleeptime = self.config.ffmpeg.retry_interval
 
@@ -302,6 +296,7 @@ class CameraWatchdog(threading.Thread):
         self.capture_thread = CameraCapture(
             self.config,
             self.shm_frame_count,
+            self.frame_index,
             self.ffmpeg_detect_process,
             self.frame_shape,
             self.frame_queue,
@@ -342,9 +337,10 @@ class CameraCapture(threading.Thread):
         self,
         config: CameraConfig,
         shm_frame_count: int,
+        frame_index: int,
         ffmpeg_process,
-        frame_shape,
-        frame_queue,
+        frame_shape: tuple[int, int],
+        frame_queue: mp.Queue,
         fps,
         skipped_fps,
         stop_event,
@@ -353,6 +349,7 @@ class CameraCapture(threading.Thread):
         self.name = f"capture:{config.name}"
         self.config = config
         self.shm_frame_count = shm_frame_count
+        self.frame_index = frame_index
         self.frame_shape = frame_shape
         self.frame_queue = frame_queue
         self.fps = fps
@@ -368,6 +365,7 @@ class CameraCapture(threading.Thread):
             self.ffmpeg_process,
             self.config,
             self.shm_frame_count,
+            self.frame_index,
             self.frame_shape,
             self.frame_manager,
             self.frame_queue,

web/src/api/ws.tsx

@@ -407,9 +407,9 @@ export function useImproveContrast(camera: string): {
   return { payload: payload as ToggleableSetting, send };
 }
 
-export function useEventUpdate(): { payload: string } {
+export function useTrackedObjectUpdate(): { payload: string } {
   const {
     value: { payload },
-  } = useWs("event_update", "");
+  } = useWs("tracked_object_update", "");
   return useDeepMemo(JSON.parse(payload as string));
 }

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -309,7 +309,7 @@ function ObjectDetailsTab({
       return undefined;
     }
 
-    if (search.sub_label) {
+    if (search.sub_label && search.data?.sub_label_score) {
       return Math.round((search.data?.sub_label_score ?? 0) * 100);
     } else {
       return undefined;

web/src/pages/Explore.tsx

@@ -1,6 +1,6 @@
 import {
   useEmbeddingsReindexProgress,
-  useEventUpdate,
+  useTrackedObjectUpdate,
   useModelState,
 } from "@/api/ws";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
@@ -227,15 +227,15 @@ export default function Explore() {
 
   // mutation and revalidation
 
-  const eventUpdate = useEventUpdate();
+  const trackedObjectUpdate = useTrackedObjectUpdate();
 
   useEffect(() => {
-    if (eventUpdate) {
+    if (trackedObjectUpdate) {
       mutate();
     }
     // mutate / revalidate when event description updates come in
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [eventUpdate]);
+  }, [trackedObjectUpdate]);
 
   // embeddings reindex progress
 

web/src/views/explore/ExploreView.tsx

@@ -15,7 +15,7 @@ import { SearchResult } from "@/types/search";
 import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator";
 import useImageLoaded from "@/hooks/use-image-loaded";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
-import { useEventUpdate } from "@/api/ws";
+import { useTrackedObjectUpdate } from "@/api/ws";
 import { isEqual } from "lodash";
 import TimeAgo from "@/components/dynamic/TimeAgo";
 import SearchResultActions from "@/components/menu/SearchResultActions";
@@ -72,13 +72,13 @@ export default function ExploreView({
     }, {});
   }, [events]);
 
-  const eventUpdate = useEventUpdate();
+  const trackedObjectUpdate = useTrackedObjectUpdate();
 
   useEffect(() => {
     mutate();
     // mutate / revalidate when event description updates come in
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [eventUpdate]);
+  }, [trackedObjectUpdate]);
 
   // update search detail when results change