Merge branch 'blakeblackshear:dev' into dev

docker/tensorrt/Dockerfile.arm64

@@ -86,6 +86,9 @@ RUN apt-get -qq update \
     libx264-163 libx265-199 libegl1 \
     && rm -rf /var/lib/apt/lists/*
 
+# Fixes "Error loading shared libs"
+RUN mkdir -p /etc/ld.so.conf.d && echo /usr/lib/ffmpeg/jetson/lib/ > /etc/ld.so.conf.d/ffmpeg.conf
+
 COPY --from=trt-wheels /etc/TENSORRT_VER /etc/TENSORRT_VER
 RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
     --mount=type=bind,from=trt-model-wheels,source=/trt-model-wheels,target=/deps/trt-model-wheels \

frigate/api/auth.py

@@ -136,7 +136,7 @@ def get_jwt_secret() -> str:
             logger.debug("Using jwt secret from .jwt_secret file in config directory.")
             with open(jwt_secret_file) as f:
                 try:
-                    jwt_secret = f.readline()
+                    jwt_secret = f.readline().strip()
                 except Exception:
                     logger.warning(
                         "Unable to read jwt token from .jwt_secret file in config directory. A new jwt token will be created at each startup."
@@ -259,17 +259,24 @@ def auth(request: Request):
         # pass the user header value from the upstream proxy if a mapping is specified
         # or use anonymous if none are specified
         user_header = proxy_config.header_map.user
-        role_header = proxy_config.header_map.get("role", "Remote-Role")
+        role_header = proxy_config.header_map.role
         success_response.headers["remote-user"] = (
             request.headers.get(user_header, default="anonymous")
             if user_header
             else "anonymous"
         )
-        success_response.headers["remote-role"] = (
+        role_header = proxy_config.header_map.role
+        role = (
             request.headers.get(role_header, default="viewer")
             if role_header
             else "viewer"
         )
+
+        # if comma-separated with "admin", use "admin", else "viewer"
+        success_response.headers["remote-role"] = (
+            "admin" if role and "admin" in role else "viewer"
+        )
+
         return success_response
 
     # now apply authentication
@@ -359,14 +366,8 @@ def auth(request: Request):
 @router.get("/profile")
 def profile(request: Request):
     username = request.headers.get("remote-user", "anonymous")
-    if username != "anonymous":
-        try:
-            user = User.get_by_id(username)
-            role = getattr(user, "role", "viewer")
-        except DoesNotExist:
-            role = "viewer"  # Fallback if user deleted
-    else:
-        role = None
+    role = request.headers.get("remote-role", "viewer")
+
     return JSONResponse(content={"username": username, "role": role})
 
 

frigate/api/event.py

@@ -40,6 +40,7 @@ from frigate.api.defs.response.event_response import (
 )
 from frigate.api.defs.response.generic_response import GenericResponse
 from frigate.api.defs.tags import Tags
+from frigate.comms.event_metadata_updater import EventMetadataTypeEnum
 from frigate.const import CLIPS_DIR
 from frigate.embeddings import EmbeddingsContext
 from frigate.events.external import ExternalEventProcessor
@@ -969,27 +970,16 @@ def set_sub_label(
     try:
         event: Event = Event.get(Event.id == event_id)
     except DoesNotExist:
-        if not body.camera:
-            return JSONResponse(
-                content=(
-                    {
-                        "success": False,
-                        "message": "Event "
-                        + event_id
-                        + " not found and camera is not provided.",
-                    }
-                ),
-                status_code=404,
-            )
-
         event = None
 
     if request.app.detected_frames_processor:
-        tracked_obj: TrackedObject = (
-            request.app.detected_frames_processor.camera_states[
-                event.camera if event else body.camera
-            ].tracked_objects.get(event_id)
-        )
+        tracked_obj: TrackedObject = None
+
+        for state in request.app.detected_frames_processor.camera_states.values():
+            tracked_obj = state.tracked_objects.get(event_id)
+
+            if tracked_obj is not None:
+                break
     else:
         tracked_obj = None
 
@@ -1008,23 +998,9 @@ def set_sub_label(
         new_sub_label = None
         new_score = None
 
-    if tracked_obj:
-        tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score)
-
-        # update timeline items
-        Timeline.update(
-            data=Timeline.data.update({"sub_label": (new_sub_label, new_score)})
-        ).where(Timeline.source_id == event_id).execute()
-
-    if event:
-        event.sub_label = new_sub_label
-        data = event.data
-        if new_sub_label is None:
-            data["sub_label_score"] = None
-        elif new_score is not None:
-            data["sub_label_score"] = new_score
-        event.data = data
-        event.save()
+    request.app.event_metadata_updater.publish(
+        EventMetadataTypeEnum.sub_label, (event_id, new_sub_label, new_score)
+    )
 
     return JSONResponse(
         content={
@@ -1105,7 +1081,9 @@ def regenerate_description(
     camera_config = request.app.frigate_config.cameras[event.camera]
 
     if camera_config.genai.enabled:
-        request.app.event_metadata_updater.publish((event.id, params.source))
+        request.app.event_metadata_updater.publish(
+            EventMetadataTypeEnum.regenerate_description, (event.id, params.source)
+        )
 
         return JSONResponse(
             content=(

frigate/app.py

@@ -20,10 +20,7 @@ from frigate.camera import CameraMetrics, PTZMetrics
 from frigate.comms.base_communicator import Communicator
 from frigate.comms.config_updater import ConfigPublisher
 from frigate.comms.dispatcher import Dispatcher
-from frigate.comms.event_metadata_updater import (
-    EventMetadataPublisher,
-    EventMetadataTypeEnum,
-)
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
 from frigate.comms.inter_process import InterProcessCommunicator
 from frigate.comms.mqtt import MqttClient
 from frigate.comms.webpush import WebPushClient
@@ -327,9 +324,7 @@ class FrigateApp:
     def init_inter_process_communicator(self) -> None:
         self.inter_process_communicator = InterProcessCommunicator()
         self.inter_config_updater = ConfigPublisher()
-        self.event_metadata_updater = EventMetadataPublisher(
-            EventMetadataTypeEnum.regenerate_description
-        )
+        self.event_metadata_updater = EventMetadataPublisher()
         self.inter_zmq_proxy = ZmqProxy()
 
     def init_onvif(self) -> None:
@@ -600,6 +595,7 @@ class FrigateApp:
                 User.insert(
                     {
                         User.username: "admin",
+                        User.role: "admin",
                         User.password_hash: password_hash,
                         User.notification_tokens: [],
                     }

frigate/comms/event_metadata_updater.py

@@ -2,9 +2,6 @@
 
 import logging
 from enum import Enum
-from typing import Optional
-
-from frigate.events.types import RegenerateDescriptionEnum
 
 from .zmq_proxy import Publisher, Subscriber
 
@@ -14,6 +11,7 @@ logger = logging.getLogger(__name__)
 class EventMetadataTypeEnum(str, Enum):
     all = ""
     regenerate_description = "regenerate_description"
+    sub_label = "sub_label"
 
 
 class EventMetadataPublisher(Publisher):
@@ -21,12 +19,11 @@ class EventMetadataPublisher(Publisher):
 
     topic_base = "event_metadata/"
 
-    def __init__(self, topic: EventMetadataTypeEnum) -> None:
-        topic = topic.value
-        super().__init__(topic)
+    def __init__(self) -> None:
+        super().__init__()
 
-    def publish(self, payload: tuple[str, RegenerateDescriptionEnum]) -> None:
-        super().publish(payload)
+    def publish(self, topic: EventMetadataTypeEnum, payload: any) -> None:
+        super().publish(payload, topic.value)
 
 
 class EventMetadataSubscriber(Subscriber):
@@ -35,17 +32,14 @@ class EventMetadataSubscriber(Subscriber):
     topic_base = "event_metadata/"
 
     def __init__(self, topic: EventMetadataTypeEnum) -> None:
-        topic = topic.value
-        super().__init__(topic)
+        super().__init__(topic.value)
 
-    def check_for_update(
-        self, timeout: float = 1
-    ) -> Optional[tuple[EventMetadataTypeEnum, str, RegenerateDescriptionEnum]]:
+    def check_for_update(self, timeout: float = 1) -> tuple | None:
         return super().check_for_update(timeout)
 
-    def _return_object(self, topic: str, payload: any) -> any:
+    def _return_object(self, topic: str, payload: tuple) -> tuple:
         if payload is None:
-            return (None, None, None)
+            return (None, None)
+
         topic = EventMetadataTypeEnum[topic[len(self.topic_base) :]]
-        event_id, source = payload
-        return (topic, event_id, RegenerateDescriptionEnum(source))
+        return (topic, payload)

frigate/data_processing/common/license_plate/mixin.py

@@ -8,12 +8,11 @@ from typing import List, Optional, Tuple
 
 import cv2
 import numpy as np
-import requests
 from Levenshtein import distance
 from pyclipper import ET_CLOSEDPOLYGON, JT_ROUND, PyclipperOffset
 from shapely.geometry import Polygon
 
-from frigate.const import FRIGATE_LOCALHOST
+from frigate.comms.event_metadata_updater import EventMetadataTypeEnum
 from frigate.util.image import area
 
 logger = logging.getLogger(__name__)
@@ -34,10 +33,10 @@ class LicensePlateProcessingMixin:
         self.batch_size = 6
 
         # Detection specific parameters
-        self.min_size = 3
+        self.min_size = 8
         self.max_size = 960
-        self.box_thresh = 0.8
-        self.mask_thresh = 0.8
+        self.box_thresh = 0.6
+        self.mask_thresh = 0.6
 
     def _detect(self, image: np.ndarray) -> List[np.ndarray]:
         """
@@ -158,47 +157,40 @@ class LicensePlateProcessingMixin:
             logger.debug("Model runners not loaded")
             return [], [], []
 
-        plate_points = self._detect(image)
-        if len(plate_points) == 0:
-            logger.debug("No points found by OCR detector model")
+        boxes = self._detect(image)
+        if len(boxes) == 0:
+            logger.debug("No boxes found by OCR detector model")
             return [], [], []
 
-        plate_points = self._sort_polygon(list(plate_points))
-        plate_images = [self._crop_license_plate(image, x) for x in plate_points]
-        rotated_images, _ = self._classify(plate_images)
+        boxes = self._sort_boxes(list(boxes))
+        plate_images = [self._crop_license_plate(image, x) for x in boxes]
 
-        # debug rotated and classification result
         if WRITE_DEBUG_IMAGES:
             current_time = int(datetime.datetime.now().timestamp())
             for i, img in enumerate(plate_images):
                 cv2.imwrite(
-                    f"debug/frames/license_plate_rotated_{current_time}_{i + 1}.jpg",
-                    img,
-                )
-            for i, img in enumerate(rotated_images):
-                cv2.imwrite(
-                    f"debug/frames/license_plate_classified_{current_time}_{i + 1}.jpg",
+                    f"debug/frames/license_plate_cropped_{current_time}_{i + 1}.jpg",
                     img,
                 )
 
         # keep track of the index of each image for correct area calc later
-        sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in rotated_images])
+        sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in plate_images])
         reverse_mapping = {
             idx: original_idx for original_idx, idx in enumerate(sorted_indices)
         }
 
-        results, confidences = self._recognize(rotated_images)
+        results, confidences = self._recognize(plate_images)
 
         if results:
-            license_plates = [""] * len(rotated_images)
-            average_confidences = [[0.0]] * len(rotated_images)
-            areas = [0] * len(rotated_images)
+            license_plates = [""] * len(plate_images)
+            average_confidences = [[0.0]] * len(plate_images)
+            areas = [0] * len(plate_images)
 
             # map results back to original image order
             for i, (plate, conf) in enumerate(zip(results, confidences)):
                 original_idx = reverse_mapping[i]
 
-                height, width = rotated_images[original_idx].shape[:2]
+                height, width = plate_images[original_idx].shape[:2]
                 area = height * width
 
                 average_confidence = conf
@@ -206,7 +198,7 @@ class LicensePlateProcessingMixin:
                 # set to True to write each cropped image for debugging
                 if False:
                     save_image = cv2.cvtColor(
-                        rotated_images[original_idx], cv2.COLOR_RGB2BGR
+                        plate_images[original_idx], cv2.COLOR_RGB2BGR
                     )
                     filename = f"debug/frames/plate_{original_idx}_{plate}_{area}.jpg"
                     cv2.imwrite(filename, save_image)
@@ -328,7 +320,7 @@ class LicensePlateProcessingMixin:
             # Use pyclipper to shrink the polygon slightly based on the computed distance.
             offset = PyclipperOffset()
             offset.AddPath(points, JT_ROUND, ET_CLOSEDPOLYGON)
-            points = np.array(offset.Execute(distance * 1.5)).reshape((-1, 1, 2))
+            points = np.array(offset.Execute(distance * 1.75)).reshape((-1, 1, 2))
 
             # get the minimum bounding box around the shrunken polygon.
             box, min_side = self._get_min_boxes(points)
@@ -453,46 +445,64 @@ class LicensePlateProcessingMixin:
         )
 
     @staticmethod
-    def _clockwise_order(point: np.ndarray) -> np.ndarray:
+    def _clockwise_order(pts: np.ndarray) -> np.ndarray:
         """
-        Arrange the points of a polygon in clockwise order based on their angular positions
-        around the polygon's center.
+        Arrange the points of a polygon in order: top-left, top-right, bottom-right, bottom-left.
+        taken from https://github.com/PyImageSearch/imutils/blob/master/imutils/perspective.py
 
         Args:
-            point (np.ndarray): Array of points of the polygon.
+            pts (np.ndarray): Array of points of the polygon.
 
         Returns:
-            np.ndarray: Points ordered in clockwise direction.
+            np.ndarray: Points ordered clockwise starting from top-left.
         """
-        center = point.mean(axis=0)
-        return point[
-            np.argsort(np.arctan2(point[:, 1] - center[1], point[:, 0] - center[0]))
-        ]
+        # Sort the points based on their x-coordinates
+        x_sorted = pts[np.argsort(pts[:, 0]), :]
+
+        # Separate the left-most and right-most points
+        left_most = x_sorted[:2, :]
+        right_most = x_sorted[2:, :]
+
+        # Sort the left-most coordinates by y-coordinates
+        left_most = left_most[np.argsort(left_most[:, 1]), :]
+        (tl, bl) = left_most  # Top-left and bottom-left
+
+        # Use the top-left as an anchor to calculate distances to right points
+        # The further point will be the bottom-right
+        distances = np.sqrt(
+            ((tl[0] - right_most[:, 0]) ** 2) + ((tl[1] - right_most[:, 1]) ** 2)
+        )
+
+        # Sort right points by distance (descending)
+        right_idx = np.argsort(distances)[::-1]
+        (br, tr) = right_most[right_idx, :]  # Bottom-right and top-right
+
+        return np.array([tl, tr, br, bl])
 
     @staticmethod
-    def _sort_polygon(points):
+    def _sort_boxes(boxes):
         """
-        Sort polygons based on their position in the image. If polygons are close in vertical
+        Sort polygons based on their position in the image. If boxes are close in vertical
         position (within 5 pixels), sort them by horizontal position.
 
         Args:
-            points: List of polygons to sort.
+            points: detected text boxes with shape [4, 2]
 
         Returns:
-            List: Sorted list of polygons.
+            List: sorted boxes(array) with shape [4, 2]
         """
-        points.sort(key=lambda x: (x[0][1], x[0][0]))
-        for i in range(len(points) - 1):
+        boxes.sort(key=lambda x: (x[0][1], x[0][0]))
+        for i in range(len(boxes) - 1):
             for j in range(i, -1, -1):
-                if abs(points[j + 1][0][1] - points[j][0][1]) < 5 and (
-                    points[j + 1][0][0] < points[j][0][0]
+                if abs(boxes[j + 1][0][1] - boxes[j][0][1]) < 5 and (
+                    boxes[j + 1][0][0] < boxes[j][0][0]
                 ):
-                    temp = points[j]
-                    points[j] = points[j + 1]
-                    points[j + 1] = temp
+                    temp = boxes[j]
+                    boxes[j] = boxes[j + 1]
+                    boxes[j + 1] = temp
                 else:
                     break
-        return points
+        return boxes
 
     @staticmethod
     def _zero_pad(image: np.ndarray) -> np.ndarray:
@@ -583,9 +593,11 @@ class LicensePlateProcessingMixin:
             for j in range(len(outputs)):
                 label, score = outputs[j]
                 results[indices[i + j]] = [label, score]
-                # make sure we have high confidence if we need to flip a box, this will be rare in lpr
-                if "180" in label and score >= 0.9:
-                    images[indices[i + j]] = cv2.rotate(images[indices[i + j]], 1)
+                # make sure we have high confidence if we need to flip a box
+                if "180" in label and score >= 0.7:
+                    images[indices[i + j]] = cv2.rotate(
+                        images[indices[i + j]], cv2.ROTATE_180
+                    )
 
         return images, results
 
@@ -682,7 +694,7 @@ class LicensePlateProcessingMixin:
         )
         height, width = image.shape[0:2]
         if height * 1.0 / width >= 1.5:
-            image = np.rot90(image, k=3)
+            image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
         return image
 
     def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
@@ -942,9 +954,23 @@ class LicensePlateProcessingMixin:
                 return
 
             license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+
+            # Expand the license_plate_box by 30%
+            box_array = np.array(license_plate_box)
+            expansion = (box_array[2:] - box_array[:2]) * 0.30
+            expanded_box = np.array(
+                [
+                    license_plate_box[0] - expansion[0],
+                    license_plate_box[1] - expansion[1],
+                    license_plate_box[2] + expansion[0],
+                    license_plate_box[3] + expansion[1],
+                ]
+            ).clip(0, [license_plate_frame.shape[1], license_plate_frame.shape[0]] * 2)
+
+            # Crop using the expanded box
             license_plate_frame = license_plate_frame[
-                license_plate_box[1] : license_plate_box[3],
-                license_plate_box[0] : license_plate_box[2],
+                int(expanded_box[1]) : int(expanded_box[3]),
+                int(expanded_box[0]) : int(expanded_box[2]),
             ]
 
         # double the size of the license plate frame for better OCR
@@ -1032,22 +1058,15 @@ class LicensePlateProcessingMixin:
         )
 
         # Send the result to the API
-        resp = requests.post(
-            f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
-            json={
-                "camera": obj_data.get("camera"),
-                "subLabel": sub_label,
-                "subLabelScore": avg_confidence,
-            },
+        self.sub_label_publisher.publish(
+            EventMetadataTypeEnum.sub_label, (id, sub_label, avg_confidence)
         )
-
-        if resp.status_code == 200:
-            self.detected_license_plates[id] = {
-                "plate": top_plate,
-                "char_confidences": top_char_confidences,
-                "area": top_area,
-                "obj_data": obj_data,
-            }
+        self.detected_license_plates[id] = {
+            "plate": top_plate,
+            "char_confidences": top_char_confidences,
+            "area": top_area,
+            "obj_data": obj_data,
+        }
 
     def handle_request(self, topic, request_data) -> dict[str, any] | None:
         return

frigate/data_processing/post/license_plate.py

@@ -8,6 +8,7 @@ import numpy as np
 from peewee import DoesNotExist
 
 from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
 from frigate.config import FrigateConfig
 from frigate.data_processing.common.license_plate.mixin import (
     WRITE_DEBUG_IMAGES,
@@ -30,6 +31,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
     def __init__(
         self,
         config: FrigateConfig,
+        sub_label_publisher: EventMetadataPublisher,
         metrics: DataProcessorMetrics,
         model_runner: LicensePlateModelRunner,
         detected_license_plates: dict[str, dict[str, any]],
@@ -38,6 +40,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
         self.model_runner = model_runner
         self.lpr_config = config.lpr
         self.config = config
+        self.sub_label_publisher = sub_label_publisher
         super().__init__(config, metrics, model_runner)
 
     def process_data(

frigate/data_processing/real_time/bird.py

@@ -5,10 +5,13 @@ import os
 
 import cv2
 import numpy as np
-import requests
 
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
 from frigate.config import FrigateConfig
-from frigate.const import FRIGATE_LOCALHOST, MODEL_CACHE_DIR
+from frigate.const import MODEL_CACHE_DIR
 from frigate.util.object import calculate_region
 
 from ..types import DataProcessorMetrics
@@ -23,9 +26,15 @@ logger = logging.getLogger(__name__)
 
 
 class BirdRealTimeProcessor(RealTimeProcessorApi):
-    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+    ):
         super().__init__(config, metrics)
         self.interpreter: Interpreter = None
+        self.sub_label_publisher = sub_label_publisher
         self.tensor_input_details: dict[str, any] = None
         self.tensor_output_details: dict[str, any] = None
         self.detected_birds: dict[str, float] = {}
@@ -134,17 +143,10 @@ class BirdRealTimeProcessor(RealTimeProcessorApi):
             logger.debug(f"Score {score} is worse than previous score {previous_score}")
             return
 
-        resp = requests.post(
-            f"{FRIGATE_LOCALHOST}/api/events/{obj_data['id']}/sub_label",
-            json={
-                "camera": obj_data.get("camera"),
-                "subLabel": self.labelmap[best_id],
-                "subLabelScore": score,
-            },
+        self.sub_label_publisher.publish(
+            EventMetadataTypeEnum.sub_label, (id, self.labelmap[best_id], score)
         )
-
-        if resp.status_code == 200:
-            self.detected_birds[obj_data["id"]] = score
+        self.detected_birds[obj_data["id"]] = score
 
     def handle_request(self, topic, request_data):
         return None

frigate/data_processing/real_time/face.py

@@ -11,11 +11,14 @@ from typing import Optional
 
 import cv2
 import numpy as np
-import requests
 
 from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
 from frigate.config import FrigateConfig
-from frigate.const import FACE_DIR, FRIGATE_LOCALHOST, MODEL_CACHE_DIR
+from frigate.const import FACE_DIR, MODEL_CACHE_DIR
 from frigate.util.image import area
 
 from ..types import DataProcessorMetrics
@@ -28,9 +31,15 @@ MIN_MATCHING_FACES = 2
 
 
 class FaceRealTimeProcessor(RealTimeProcessorApi):
-    def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+    ):
         super().__init__(config, metrics)
         self.face_config = config.face_recognition
+        self.sub_label_publisher = sub_label_publisher
         self.face_detector: cv2.FaceDetectorYN = None
         self.landmark_detector: cv2.face.FacemarkLBF = None
         self.recognizer: cv2.face.LBPHFaceRecognizer = None
@@ -349,18 +358,10 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
             self.__update_metrics(datetime.datetime.now().timestamp() - start)
             return
 
-        resp = requests.post(
-            f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
-            json={
-                "camera": obj_data.get("camera"),
-                "subLabel": sub_label,
-                "subLabelScore": score,
-            },
+        self.sub_label_publisher.publish(
+            EventMetadataTypeEnum.sub_label, (id, sub_label, score)
         )
-
-        if resp.status_code == 200:
-            self.detected_faces[id] = face_score
-
+        self.detected_faces[id] = face_score
         self.__update_metrics(datetime.datetime.now().timestamp() - start)
 
     def handle_request(self, topic, request_data) -> dict[str, any] | None:

frigate/data_processing/real_time/license_plate.py

@@ -4,6 +4,7 @@ import logging
 
 import numpy as np
 
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
 from frigate.config import FrigateConfig
 from frigate.data_processing.common.license_plate.mixin import (
     LicensePlateProcessingMixin,
@@ -22,6 +23,7 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess
     def __init__(
         self,
         config: FrigateConfig,
+        sub_label_publisher: EventMetadataPublisher,
         metrics: DataProcessorMetrics,
         model_runner: LicensePlateModelRunner,
         detected_license_plates: dict[str, dict[str, any]],
@@ -30,6 +32,7 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess
         self.model_runner = model_runner
         self.lpr_config = config.lpr
         self.config = config
+        self.sub_label_publisher = sub_label_publisher
         super().__init__(config, metrics)
 
     def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):

frigate/embeddings/maintainer.py

@@ -15,6 +15,7 @@ from playhouse.sqliteq import SqliteQueueDatabase
 
 from frigate.comms.embeddings_updater import EmbeddingsRequestEnum, EmbeddingsResponder
 from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
     EventMetadataSubscriber,
     EventMetadataTypeEnum,
 )
@@ -43,7 +44,7 @@ from frigate.data_processing.real_time.license_plate import (
     LicensePlateRealTimeProcessor,
 )
 from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataEnum
-from frigate.events.types import EventTypeEnum
+from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum
 from frigate.genai import get_genai_client
 from frigate.models import Event
 from frigate.types import TrackedObjectUpdateTypesEnum
@@ -89,6 +90,7 @@ class EmbeddingMaintainer(threading.Thread):
 
         self.event_subscriber = EventUpdateSubscriber()
         self.event_end_subscriber = EventEndSubscriber()
+        self.event_metadata_publisher = EventMetadataPublisher()
         self.event_metadata_subscriber = EventMetadataSubscriber(
             EventMetadataTypeEnum.regenerate_description
         )
@@ -108,15 +110,27 @@ class EmbeddingMaintainer(threading.Thread):
         self.realtime_processors: list[RealTimeProcessorApi] = []
 
         if self.config.face_recognition.enabled:
-            self.realtime_processors.append(FaceRealTimeProcessor(self.config, metrics))
+            self.realtime_processors.append(
+                FaceRealTimeProcessor(
+                    self.config, self.event_metadata_publisher, metrics
+                )
+            )
 
         if self.config.classification.bird.enabled:
-            self.realtime_processors.append(BirdRealTimeProcessor(self.config, metrics))
+            self.realtime_processors.append(
+                BirdRealTimeProcessor(
+                    self.config, self.event_metadata_publisher, metrics
+                )
+            )
 
         if self.config.lpr.enabled:
             self.realtime_processors.append(
                 LicensePlateRealTimeProcessor(
-                    self.config, metrics, lpr_model_runner, self.detected_license_plates
+                    self.config,
+                    self.event_metadata_publisher,
+                    metrics,
+                    lpr_model_runner,
+                    self.detected_license_plates,
                 )
             )
 
@@ -126,7 +140,11 @@ class EmbeddingMaintainer(threading.Thread):
         if self.config.lpr.enabled:
             self.post_processors.append(
                 LicensePlatePostProcessor(
-                    self.config, metrics, lpr_model_runner, self.detected_license_plates
+                    self.config,
+                    self.event_metadata_publisher,
+                    metrics,
+                    lpr_model_runner,
+                    self.detected_license_plates,
                 )
             )
 
@@ -150,6 +168,7 @@ class EmbeddingMaintainer(threading.Thread):
         self.event_subscriber.stop()
         self.event_end_subscriber.stop()
         self.recordings_subscriber.stop()
+        self.event_metadata_publisher.stop()
         self.event_metadata_subscriber.stop()
         self.embeddings_responder.stop()
         self.requestor.stop()
@@ -375,15 +394,17 @@ class EmbeddingMaintainer(threading.Thread):
 
     def _process_event_metadata(self):
         # Check for regenerate description requests
-        (topic, event_id, source) = self.event_metadata_subscriber.check_for_update(
-            timeout=0.01
-        )
+        (topic, payload) = self.event_metadata_subscriber.check_for_update(timeout=0.01)
 
         if topic is None:
             return
 
+        event_id, source = payload
+
         if event_id:
-            self.handle_regenerate_description(event_id, source)
+            self.handle_regenerate_description(
+                event_id, RegenerateDescriptionEnum(source)
+            )
 
     def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]:
         """Return jpg thumbnail of a region of the frame."""

frigate/events/audio.py

@@ -351,7 +351,8 @@ class AudioEventMaintainer(threading.Thread):
 
             self.read_audio()
 
-        stop_ffmpeg(self.audio_listener, self.logger)
+        if self.audio_listener:
+            stop_ffmpeg(self.audio_listener, self.logger)
         self.logpipe.close()
         self.requestor.stop()
         self.config_subscriber.stop()

frigate/models.py

@@ -119,7 +119,7 @@ class User(Model):  # type: ignore[misc]
     username = CharField(null=False, primary_key=True, max_length=30)
     role = CharField(
         max_length=20,
-        default="viewer",
+        default="admin",
     )
     password_hash = CharField(null=False, max_length=120)
     notification_tokens = JSONField()

frigate/object_processing.py

@@ -9,10 +9,15 @@ from typing import Callable, Optional
 
 import cv2
 import numpy as np
+from peewee import DoesNotExist
 
 from frigate.comms.config_updater import ConfigSubscriber
 from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
 from frigate.comms.dispatcher import Dispatcher
+from frigate.comms.event_metadata_updater import (
+    EventMetadataSubscriber,
+    EventMetadataTypeEnum,
+)
 from frigate.comms.events_updater import EventEndSubscriber, EventUpdatePublisher
 from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import (
@@ -24,6 +29,7 @@ from frigate.config import (
 )
 from frigate.const import UPDATE_CAMERA_ACTIVITY
 from frigate.events.types import EventStateEnum, EventTypeEnum
+from frigate.models import Event, Timeline
 from frigate.ptz.autotrack import PtzAutoTrackerThread
 from frigate.track.tracked_object import TrackedObject
 from frigate.util.image import (
@@ -446,6 +452,9 @@ class TrackedObjectProcessor(threading.Thread):
         self.detection_publisher = DetectionPublisher(DetectionTypeEnum.video)
         self.event_sender = EventUpdatePublisher()
         self.event_end_subscriber = EventEndSubscriber()
+        self.sub_label_subscriber = EventMetadataSubscriber(
+            EventMetadataTypeEnum.sub_label
+        )
 
         self.camera_activity: dict[str, dict[str, any]] = {}
 
@@ -684,6 +693,46 @@ class TrackedObjectProcessor(threading.Thread):
         """Returns the latest frame time for a given camera."""
         return self.camera_states[camera].current_frame_time
 
+    def set_sub_label(
+        self, event_id: str, sub_label: str | None, score: float | None
+    ) -> None:
+        """Update sub label for given event id."""
+        tracked_obj: TrackedObject = None
+
+        for state in self.camera_states.values():
+            tracked_obj = state.tracked_objects.get(event_id)
+
+            if tracked_obj is not None:
+                break
+
+        try:
+            event: Event = Event.get(Event.id == event_id)
+        except DoesNotExist:
+            event = None
+
+        if not tracked_obj and not event:
+            return
+
+        if tracked_obj:
+            tracked_obj.obj_data["sub_label"] = (sub_label, score)
+
+        if event:
+            event.sub_label = sub_label
+            data = event.data
+            if sub_label is None:
+                data["sub_label_score"] = None
+            elif score is not None:
+                data["sub_label_score"] = score
+            event.data = data
+            event.save()
+
+            # update timeline items
+            Timeline.update(
+                data=Timeline.data.update({"sub_label": (sub_label, score)})
+            ).where(Timeline.source_id == event_id).execute()
+
+        return True
+
     def force_end_all_events(self, camera: str, camera_state: CameraState):
         """Ends all active events on camera when disabling."""
         last_frame_name = camera_state.previous_frame_id
@@ -741,6 +790,18 @@ class TrackedObjectProcessor(threading.Thread):
                 if not current_enabled:
                     continue
 
+            # check for sub label updates
+            while True:
+                (topic, payload) = self.sub_label_subscriber.check_for_update(
+                    timeout=0.1
+                )
+
+                if not topic:
+                    break
+
+                (event_id, sub_label, score) = payload
+                self.set_sub_label(event_id, sub_label, score)
+
             try:
                 (
                     camera,
@@ -799,6 +860,7 @@ class TrackedObjectProcessor(threading.Thread):
         self.detection_publisher.stop()
         self.event_sender.stop()
         self.event_end_subscriber.stop()
+        self.sub_label_subscriber.stop()
         self.config_enabled_subscriber.stop()
 
         logger.info("Exiting object processor...")

frigate/test/test_http.py

@@ -2,6 +2,7 @@ import datetime
 import logging
 import os
 import unittest
+from unittest.mock import Mock
 
 from fastapi.testclient import TestClient
 from peewee_migrate import Router
@@ -10,6 +11,7 @@ from playhouse.sqlite_ext import SqliteExtDatabase
 from playhouse.sqliteq import SqliteQueueDatabase
 
 from frigate.api.fastapi_app import create_fastapi_app
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
 from frigate.config import FrigateConfig
 from frigate.const import BASE_DIR, CACHE_DIR
 from frigate.models import Event, Recordings, Timeline
@@ -243,6 +245,7 @@ class TestHttp(unittest.TestCase):
             assert len(events) == 1
 
     def test_set_delete_sub_label(self):
+        mock_event_updater = Mock(spec=EventMetadataPublisher)
         app = create_fastapi_app(
             FrigateConfig(**self.minimal_config),
             self.db,
@@ -252,11 +255,18 @@ class TestHttp(unittest.TestCase):
             None,
             None,
             None,
-            None,
+            mock_event_updater,
         )
         id = "123456.random"
         sub_label = "sub"
 
+        def update_event(topic, payload):
+            event = Event.get(id=id)
+            event.sub_label = payload[1]
+            event.save()
+
+        mock_event_updater.publish.side_effect = update_event
+
         with TestClient(app) as client:
             _insert_mock_event(id)
             new_sub_label_response = client.post(
@@ -281,6 +291,7 @@ class TestHttp(unittest.TestCase):
             assert event["sub_label"] == None
 
     def test_sub_label_list(self):
+        mock_event_updater = Mock(spec=EventMetadataPublisher)
         app = create_fastapi_app(
             FrigateConfig(**self.minimal_config),
             self.db,
@@ -290,11 +301,18 @@ class TestHttp(unittest.TestCase):
             None,
             None,
             None,
-            None,
+            mock_event_updater,
         )
         id = "123456.random"
         sub_label = "sub"
 
+        def update_event(topic, payload):
+            event = Event.get(id=id)
+            event.sub_label = payload[1]
+            event.save()
+
+        mock_event_updater.publish.side_effect = update_event
+
         with TestClient(app) as client:
             _insert_mock_event(id)
             client.post(

web/src/components/auth/AuthForm.tsx

@@ -87,7 +87,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) {
   return (
     <div className={cn("grid gap-6", className)} {...props}>
       <Form {...form}>
-        <form onSubmit={form.handleSubmit(onSubmit)}>
+        <form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
           <FormField
             name="user"
             render={({ field }) => (

web/src/pages/FaceLibrary.tsx

@@ -19,6 +19,7 @@ import {
   TooltipContent,
   TooltipTrigger,
 } from "@/components/ui/tooltip";
+import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import useOptimisticState from "@/hooks/use-optimistic-state";
 import { cn } from "@/lib/utils";
 import { FrigateConfig } from "@/types/frigateConfig";
@@ -141,6 +142,73 @@ export default function FaceLibrary() {
     [refreshFaces],
   );
 
+  // face multiselect
+
+  const [selectedFaces, setSelectedFaces] = useState<string[]>([]);
+
+  const onClickFace = useCallback(
+    (imageId: string) => {
+      const index = selectedFaces.indexOf(imageId);
+
+      if (index != -1) {
+        if (selectedFaces.length == 1) {
+          setSelectedFaces([]);
+        } else {
+          const copy = [
+            ...selectedFaces.slice(0, index),
+            ...selectedFaces.slice(index + 1),
+          ];
+          setSelectedFaces(copy);
+        }
+      } else {
+        const copy = [...selectedFaces];
+        copy.push(imageId);
+        setSelectedFaces(copy);
+      }
+    },
+    [selectedFaces, setSelectedFaces],
+  );
+
+  const onDelete = useCallback(() => {
+    axios
+      .post(`/faces/train/delete`, { ids: selectedFaces })
+      .then((resp) => {
+        setSelectedFaces([]);
+
+        if (resp.status == 200) {
+          toast.success(`Successfully deleted face.`, {
+            position: "top-center",
+          });
+          refreshFaces();
+        }
+      })
+      .catch((error) => {
+        const errorMessage =
+          error.response?.data?.message ||
+          error.response?.data?.detail ||
+          "Unknown error";
+        toast.error(`Failed to delete: ${errorMessage}`, {
+          position: "top-center",
+        });
+      });
+  }, [selectedFaces, refreshFaces]);
+
+  // keyboard
+
+  useKeyboardListener(["a"], (key, modifiers) => {
+    if (modifiers.repeat || !modifiers.down) {
+      return;
+    }
+
+    switch (key) {
+      case "a":
+        if (modifiers.ctrl) {
+          setSelectedFaces([...trainImages]);
+        }
+        break;
+    }
+  });
+
   if (!config) {
     return <ActivityIndicator />;
   }
@@ -210,16 +278,27 @@ export default function FaceLibrary() {
             <ScrollBar orientation="horizontal" className="h-0" />
           </div>
         </ScrollArea>
-        <div className="flex items-center justify-center gap-2">
-          <Button className="flex gap-2" onClick={() => setAddFace(true)}>
-            <LuScanFace className="size-7 rounded-md p-1 text-secondary-foreground" />
-            Add Face
-          </Button>
-          <Button className="flex gap-2" onClick={() => setUpload(true)}>
-            <LuImagePlus className="size-7 rounded-md p-1 text-secondary-foreground" />
-            Upload Image
-          </Button>
-        </div>
+        {selectedFaces?.length > 0 ? (
+          <div className="flex items-center justify-center gap-2">
+            <Button className="flex gap-2" onClick={() => onDelete()}>
+              <LuTrash2 className="size-7 rounded-md p-1 text-secondary-foreground" />
+              Delete Face Attempts
+            </Button>
+          </div>
+        ) : (
+          <div className="flex items-center justify-center gap-2">
+            <Button className="flex gap-2" onClick={() => setAddFace(true)}>
+              <LuScanFace className="size-7 rounded-md p-1 text-secondary-foreground" />
+              Add Face
+            </Button>
+            {pageToggle != "train" && (
+              <Button className="flex gap-2" onClick={() => setUpload(true)}>
+                <LuImagePlus className="size-7 rounded-md p-1 text-secondary-foreground" />
+                Upload Image
+              </Button>
+            )}
+          </div>
+        )}
       </div>
       {pageToggle &&
         (pageToggle == "train" ? (
@@ -227,6 +306,8 @@ export default function FaceLibrary() {
             config={config}
             attemptImages={trainImages}
             faceNames={faces}
+            selectedFaces={selectedFaces}
+            onClickFace={onClickFace}
             onRefresh={refreshFaces}
           />
         ) : (
@@ -244,22 +325,28 @@ type TrainingGridProps = {
   config: FrigateConfig;
   attemptImages: string[];
   faceNames: string[];
+  selectedFaces: string[];
+  onClickFace: (image: string) => void;
   onRefresh: () => void;
 };
 function TrainingGrid({
   config,
   attemptImages,
   faceNames,
+  selectedFaces,
+  onClickFace,
   onRefresh,
 }: TrainingGridProps) {
   return (
-    <div className="scrollbar-container flex flex-wrap gap-2 overflow-y-scroll">
+    <div className="scrollbar-container flex flex-wrap gap-2 overflow-y-scroll p-1">
       {attemptImages.map((image: string) => (
         <FaceAttempt
           key={image}
           image={image}
           faceNames={faceNames}
           threshold={config.face_recognition.threshold}
+          selected={selectedFaces.includes(image)}
+          onClick={() => onClickFace(image)}
           onRefresh={onRefresh}
         />
       ))}
@@ -271,12 +358,16 @@ type FaceAttemptProps = {
   image: string;
   faceNames: string[];
   threshold: number;
+  selected: boolean;
+  onClick: () => void;
   onRefresh: () => void;
 };
 function FaceAttempt({
   image,
   faceNames,
   threshold,
+  selected,
+  onClick,
   onRefresh,
 }: FaceAttemptProps) {
   const data = useMemo(() => {
@@ -336,30 +427,16 @@ function FaceAttempt({
       });
   }, [image, onRefresh]);
 
-  const onDelete = useCallback(() => {
-    axios
-      .post(`/faces/train/delete`, { ids: [image] })
-      .then((resp) => {
-        if (resp.status == 200) {
-          toast.success(`Successfully deleted face.`, {
-            position: "top-center",
-          });
-          onRefresh();
-        }
-      })
-      .catch((error) => {
-        const errorMessage =
-          error.response?.data?.message ||
-          error.response?.data?.detail ||
-          "Unknown error";
-        toast.error(`Failed to delete: ${errorMessage}`, {
-          position: "top-center",
-        });
-      });
-  }, [image, onRefresh]);
-
   return (
-    <div className="relative flex flex-col rounded-lg">
+    <div
+      className={cn(
+        "relative flex cursor-pointer flex-col rounded-lg outline outline-[3px]",
+        selected
+          ? "shadow-selected outline-selected"
+          : "outline-transparent duration-500",
+      )}
+      onClick={onClick}
+    >
       <div className="w-full overflow-hidden rounded-t-lg border border-t-0 *:text-card-foreground">
         <img className="size-40" src={`${baseUrl}clips/faces/train/${image}`} />
       </div>
@@ -409,15 +486,6 @@ function FaceAttempt({
               </TooltipTrigger>
               <TooltipContent>Reprocess Face</TooltipContent>
             </Tooltip>
-            <Tooltip>
-              <TooltipTrigger>
-                <LuTrash2
-                  className="size-5 cursor-pointer text-primary-variant hover:text-primary"
-                  onClick={onDelete}
-                />
-              </TooltipTrigger>
-              <TooltipContent>Delete Face Attempt</TooltipContent>
-            </Tooltip>
           </div>
         </div>
       </div>

web/src/views/events/EventView.tsx

@@ -770,7 +770,12 @@ function DetectionReview({
                       />
                     </div>
                     <div
-                      className={`review-item-ring pointer-events-none absolute inset-0 z-10 size-full rounded-lg outline outline-[3px] -outline-offset-[2.8px] ${selected ? `outline-severity_${value.severity} shadow-severity_${value.severity}` : "outline-transparent duration-500"}`}
+                      className={cn(
+                        "review-item-ring pointer-events-none absolute inset-0 z-10 size-full rounded-lg outline outline-[3px] -outline-offset-[2.8px]",
+                        selected
+                          ? `outline-severity_${value.severity} shadow-severity_${value.severity}`
+                          : "outline-transparent duration-500",
+                      )}
                     />
                   </div>
                 );