From e94b24ec399f73f9ebeb32ef6bbba3201349acda Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 4 Apr 2025 07:03:08 -0600 Subject: [PATCH 1/2] Face fixes (#17530) * Catch - in name * Add note about docs --- docs/docs/configuration/face_recognition.md | 4 ++++ frigate/data_processing/real_time/face.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md index 8dc4ac7b2e..328fc5fc3a 100644 --- a/docs/docs/configuration/face_recognition.md +++ b/docs/docs/configuration/face_recognition.md @@ -125,3 +125,7 @@ This can happen for a few different reasons, but this is usually an indicator th ### I see scores above the threshold in the train tab, but a sub label wasn't assigned? The Frigate considers the recognition scores across all recognition attempts for each person object. The scores are continually weighted based on the area of the face, and a sub label will only be assigned to person if a person is confidently recognized consistently. This avoids cases where a single high confidence recognition would throw off the results. + +### Can I use other face recognition software like DoubleTake at the same time as the built in face recognition? + +No, using another face recognition service will interfere with Frigate's built in face recognition. When using double-take the sub_label feature must be disabled if the built in face recognition is also desired. diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py index dd18aeffc5..fb55bb7d21 100644 --- a/frigate/data_processing/real_time/face.py +++ b/frigate/data_processing/real_time/face.py @@ -393,6 +393,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): if score <= self.face_config.unknown_score: sub_label = "unknown" + if "-" in sub_label: + sub_label = sub_label.replace("-", "_") + if self.config.face_recognition.save_attempts: # write face to library folder = os.path.join(FACE_DIR, "train") @@ -460,6 +463,10 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): if self.config.face_recognition.save_attempts: # write face to library folder = os.path.join(FACE_DIR, "train") + + if "-" in sub_label: + sub_label = sub_label.replace("-", "_") + file = os.path.join( folder, f"{event_id}-{timestamp}-{sub_label}-{score}.webp" ) From 7917ec611adf97787545bd6e1e68ff7c25398bc2 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 4 Apr 2025 12:42:49 -0500 Subject: [PATCH 2/2] LPR tweaks (#17536) * Merge nearby horizontal boxes * only publish to recognized plate field if object already has a sub label * don't overwrite sub labels in any situation * always publish sub label if it's a known plate --- .../common/license_plate/mixin.py | 104 ++++++++++++++++-- 1 file changed, 96 insertions(+), 8 deletions(-) diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py index 7f731704f1..778c059214 100644 --- a/frigate/data_processing/common/license_plate/mixin.py +++ b/frigate/data_processing/common/license_plate/mixin.py @@ -175,6 +175,17 @@ class LicensePlateProcessingMixin: logger.debug("No boxes found by OCR detector model") return [], [], [] + if len(boxes) > 0: + plate_left = np.min([np.min(box[:, 0]) for box in boxes]) + plate_right = np.max([np.max(box[:, 0]) for box in boxes]) + plate_width = plate_right - plate_left + else: + plate_width = 0 + + boxes = self._merge_nearby_boxes( + boxes, plate_width=plate_width, gap_fraction=0.1 + ) + boxes = self._sort_boxes(list(boxes)) plate_images = [self._crop_license_plate(image, x) for x in boxes] @@ -297,6 +308,90 @@ class LicensePlateProcessingMixin: cv2.multiply(image, std, image) return image.transpose((2, 0, 1))[np.newaxis, ...] + def _merge_nearby_boxes( + self, boxes: List[np.ndarray], plate_width: float, gap_fraction: float = 0.1 + ) -> List[np.ndarray]: + """ + Merge bounding boxes that are likely part of the same license plate based on proximity, + with a dynamic max_gap based on the provided width of the entire license plate. + + Args: + boxes (List[np.ndarray]): List of bounding boxes with shape (n, 4, 2), where n is the number of boxes, + each box has 4 corners, and each corner has (x, y) coordinates. + plate_width (float): The width of the entire license plate in pixels, used to calculate max_gap. + gap_fraction (float): Fraction of the plate width to use as the maximum gap. + Default is 0.1 (10% of the plate width). + + Returns: + List[np.ndarray]: List of merged bounding boxes. + """ + if len(boxes) == 0: + return [] + + max_gap = plate_width * gap_fraction + + # Sort boxes by top left x + sorted_boxes = sorted(boxes, key=lambda x: x[0][0]) + + merged_boxes = [] + current_box = sorted_boxes[0] + + for i in range(1, len(sorted_boxes)): + next_box = sorted_boxes[i] + + # Calculate the horizontal gap between the current box and the next box + current_right = np.max( + current_box[:, 0] + ) # Rightmost x-coordinate of current box + next_left = np.min(next_box[:, 0]) # Leftmost x-coordinate of next box + horizontal_gap = next_left - current_right + + # Check if the boxes are vertically aligned (similar y-coordinates) + current_top = np.min(current_box[:, 1]) + current_bottom = np.max(current_box[:, 1]) + next_top = np.min(next_box[:, 1]) + next_bottom = np.max(next_box[:, 1]) + + # Consider boxes part of the same plate if they are close horizontally or overlap + if horizontal_gap <= max_gap and max(current_top, next_top) <= min( + current_bottom, next_bottom + ): + merged_points = np.vstack((current_box, next_box)) + new_box = np.array( + [ + [ + np.min(merged_points[:, 0]), + np.min(merged_points[:, 1]), + ], + [ + np.max(merged_points[:, 0]), + np.min(merged_points[:, 1]), + ], + [ + np.max(merged_points[:, 0]), + np.max(merged_points[:, 1]), + ], + [ + np.min(merged_points[:, 0]), + np.max(merged_points[:, 1]), + ], + ] + ) + current_box = new_box + else: + # If the boxes are not close enough, add the current box to the result + merged_boxes.append(current_box) + current_box = next_box + + logger.debug( + f"Provided plate_width: {plate_width}, max_gap: {max_gap}, horizontal_gap: {horizontal_gap}" + ) + + # Add the last box + merged_boxes.append(current_box) + + return np.array(merged_boxes, dtype=np.int32) + def _boxes_from_bitmap( self, output: np.ndarray, mask: np.ndarray, dest_width: int, dest_height: int ) -> Tuple[np.ndarray, List[float]]: @@ -1064,14 +1159,6 @@ class LicensePlateProcessingMixin: ) return - # don't overwrite sub label for objects that have a sub label - # that is not a license plate - if obj_data.get("sub_label") and id not in self.detected_license_plates: - logger.debug( - f"{camera}: Not processing license plate due to existing sub label: {obj_data.get('sub_label')}." - ) - return - license_plate: Optional[dict[str, any]] = None if "license_plate" not in self.config.cameras[camera].objects.track: @@ -1314,6 +1401,7 @@ class LicensePlateProcessingMixin: EventMetadataTypeEnum.sub_label, (id, sub_label, avg_confidence) ) + # always publish to recognized_license_plate field self.sub_label_publisher.publish( EventMetadataTypeEnum.recognized_license_plate, (id, top_plate, avg_confidence),