Merge branch 'blakeblackshear:dev' into dev

docs/docs/configuration/face_recognition.md

@@ -125,3 +125,7 @@ This can happen for a few different reasons, but this is usually an indicator th
 ### I see scores above the threshold in the train tab, but a sub label wasn't assigned?
 
 The Frigate considers the recognition scores across all recognition attempts for each person object. The scores are continually weighted based on the area of the face, and a sub label will only be assigned to person if a person is confidently recognized consistently. This avoids cases where a single high confidence recognition would throw off the results.
+
+### Can I use other face recognition software like DoubleTake at the same time as the built in face recognition?
+
+No, using another face recognition service will interfere with Frigate's built in face recognition. When using double-take the sub_label feature must be disabled if the built in face recognition is also desired.

frigate/data_processing/common/license_plate/mixin.py

@@ -175,6 +175,17 @@ class LicensePlateProcessingMixin:
             logger.debug("No boxes found by OCR detector model")
             return [], [], []
 
+        if len(boxes) > 0:
+            plate_left = np.min([np.min(box[:, 0]) for box in boxes])
+            plate_right = np.max([np.max(box[:, 0]) for box in boxes])
+            plate_width = plate_right - plate_left
+        else:
+            plate_width = 0
+
+        boxes = self._merge_nearby_boxes(
+            boxes, plate_width=plate_width, gap_fraction=0.1
+        )
+
         boxes = self._sort_boxes(list(boxes))
         plate_images = [self._crop_license_plate(image, x) for x in boxes]
 
@@ -297,6 +308,90 @@ class LicensePlateProcessingMixin:
         cv2.multiply(image, std, image)
         return image.transpose((2, 0, 1))[np.newaxis, ...]
 
+    def _merge_nearby_boxes(
+        self, boxes: List[np.ndarray], plate_width: float, gap_fraction: float = 0.1
+    ) -> List[np.ndarray]:
+        """
+        Merge bounding boxes that are likely part of the same license plate based on proximity,
+        with a dynamic max_gap based on the provided width of the entire license plate.
+
+        Args:
+            boxes (List[np.ndarray]): List of bounding boxes with shape (n, 4, 2), where n is the number of boxes,
+                                    each box has 4 corners, and each corner has (x, y) coordinates.
+            plate_width (float): The width of the entire license plate in pixels, used to calculate max_gap.
+            gap_fraction (float): Fraction of the plate width to use as the maximum gap.
+                                Default is 0.1 (10% of the plate width).
+
+        Returns:
+            List[np.ndarray]: List of merged bounding boxes.
+        """
+        if len(boxes) == 0:
+            return []
+
+        max_gap = plate_width * gap_fraction
+
+        # Sort boxes by top left x
+        sorted_boxes = sorted(boxes, key=lambda x: x[0][0])
+
+        merged_boxes = []
+        current_box = sorted_boxes[0]
+
+        for i in range(1, len(sorted_boxes)):
+            next_box = sorted_boxes[i]
+
+            # Calculate the horizontal gap between the current box and the next box
+            current_right = np.max(
+                current_box[:, 0]
+            )  # Rightmost x-coordinate of current box
+            next_left = np.min(next_box[:, 0])  # Leftmost x-coordinate of next box
+            horizontal_gap = next_left - current_right
+
+            # Check if the boxes are vertically aligned (similar y-coordinates)
+            current_top = np.min(current_box[:, 1])
+            current_bottom = np.max(current_box[:, 1])
+            next_top = np.min(next_box[:, 1])
+            next_bottom = np.max(next_box[:, 1])
+
+            # Consider boxes part of the same plate if they are close horizontally or overlap
+            if horizontal_gap <= max_gap and max(current_top, next_top) <= min(
+                current_bottom, next_bottom
+            ):
+                merged_points = np.vstack((current_box, next_box))
+                new_box = np.array(
+                    [
+                        [
+                            np.min(merged_points[:, 0]),
+                            np.min(merged_points[:, 1]),
+                        ],
+                        [
+                            np.max(merged_points[:, 0]),
+                            np.min(merged_points[:, 1]),
+                        ],
+                        [
+                            np.max(merged_points[:, 0]),
+                            np.max(merged_points[:, 1]),
+                        ],
+                        [
+                            np.min(merged_points[:, 0]),
+                            np.max(merged_points[:, 1]),
+                        ],
+                    ]
+                )
+                current_box = new_box
+            else:
+                # If the boxes are not close enough, add the current box to the result
+                merged_boxes.append(current_box)
+                current_box = next_box
+
+            logger.debug(
+                f"Provided plate_width: {plate_width}, max_gap: {max_gap}, horizontal_gap: {horizontal_gap}"
+            )
+
+        # Add the last box
+        merged_boxes.append(current_box)
+
+        return np.array(merged_boxes, dtype=np.int32)
+
     def _boxes_from_bitmap(
         self, output: np.ndarray, mask: np.ndarray, dest_width: int, dest_height: int
     ) -> Tuple[np.ndarray, List[float]]:
@@ -1064,14 +1159,6 @@ class LicensePlateProcessingMixin:
                 )
                 return
 
-            # don't overwrite sub label for objects that have a sub label
-            # that is not a license plate
-            if obj_data.get("sub_label") and id not in self.detected_license_plates:
-                logger.debug(
-                    f"{camera}: Not processing license plate due to existing sub label: {obj_data.get('sub_label')}."
-                )
-                return
-
             license_plate: Optional[dict[str, any]] = None
 
             if "license_plate" not in self.config.cameras[camera].objects.track:
@@ -1314,6 +1401,7 @@ class LicensePlateProcessingMixin:
                 EventMetadataTypeEnum.sub_label, (id, sub_label, avg_confidence)
             )
 
+        # always publish to recognized_license_plate field
         self.sub_label_publisher.publish(
             EventMetadataTypeEnum.recognized_license_plate,
             (id, top_plate, avg_confidence),

frigate/data_processing/real_time/face.py

@@ -393,6 +393,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
             if score <= self.face_config.unknown_score:
                 sub_label = "unknown"
 
+            if "-" in sub_label:
+                sub_label = sub_label.replace("-", "_")
+
             if self.config.face_recognition.save_attempts:
                 # write face to library
                 folder = os.path.join(FACE_DIR, "train")
@@ -460,6 +463,10 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
         if self.config.face_recognition.save_attempts:
             # write face to library
             folder = os.path.join(FACE_DIR, "train")
+
+            if "-" in sub_label:
+                sub_label = sub_label.replace("-", "_")
+
             file = os.path.join(
                 folder, f"{event_id}-{timestamp}-{sub_label}-{score}.webp"
             )