Fix bug in intersection logic

frigate/test/test_video.py

@@ -10,6 +10,7 @@ from frigate.video import (
     get_cluster_candidates,
     get_cluster_region,
 )
+from frigate.util import intersection
 
 
 def draw_box(frame, box, color=(255, 0, 0), thickness=2):
@@ -63,7 +64,7 @@ def save_cluster_boundary_image(name, boxes, bounding_boxes):
     )
 
 
-class TestConfig(unittest.TestCase):
+class TestRegion(unittest.TestCase):
     def setUp(self):
         self.frame_shape = (1000, 2000)
         self.min_region_size = 160
@@ -176,3 +177,79 @@ class TestConfig(unittest.TestCase):
 
         # save_clusters_image("dont_combine", boxes, cluster_candidates, regions)
         assert len(regions) == 2
+
+
+class TestObjectBoundingBoxes(unittest.TestCase):
+    def setUp(self) -> None:
+        pass
+
+    def test_box_intersection(self):
+        box_a = [2012, 191, 2031, 205]
+        box_b = [887, 92, 985, 151]
+        box_c = [899, 128, 1080, 175]
+
+        assert intersection(box_a, box_b) == None
+        assert intersection(box_b, box_c) == [899, 128, 985, 151]
+
+    def test_consolidate_objects(self):
+        selected_objects = [
+            (
+                "car",  # valid detection, should not be consolidated
+                0.82421875,
+                (2044, 240, 2134, 297),
+                5130,
+                1.5789473684210527,
+                (2008, 188, 2168, 348),
+            ),
+            (
+                "car",  # valid detection, should not be consolidated
+                0.76953125,
+                (124, 239, 967, 719),
+                404640,
+                1.75625,
+                (0, 0, 2968, 2968),
+            ),
+            (
+                "car",  # valid detection, should not be consolidated
+                0.71484375,
+                (890, 93, 976, 150),
+                4902,
+                1.5087719298245614,
+                (854, 40, 1014, 200),
+            ),
+            (
+                "car",  # valid detection, should not be consolidated
+                0.6640625,
+                (1959, 188, 2005, 222),
+                1564,
+                1.3529411764705883,
+                (1768, 0, 2560, 792),
+            ),
+            (
+                "car",  # invalid detection, should be consolidated
+                0.65234375,
+                (2033, 193, 2048, 203),
+                150,
+                1.5,
+                (2008, 188, 2168, 348),
+            ),
+        ]
+
+        [
+            (
+                "car",
+                0.82421875,
+                (2044, 240, 2134, 297),
+                5130,
+                1.5789473684210527,
+                (2008, 188, 2168, 348),
+            ),
+            (
+                "car",
+                0.76953125,
+                (124, 239, 967, 719),
+                404640,
+                1.75625,
+                (0, 0, 2968, 2968),
+            ),
+        ]

frigate/util.py

@@ -572,7 +572,16 @@ def yuv_region_2_bgr(frame, region):
         raise
 
 
-def intersection(box_a, box_b):
+def intersection(box_a, box_b) -> Optional[list[int]]:
+    """Return intersection box or None if boxes do not intersect."""
+    if (
+        box_a[2] < box_b[0]
+        or box_a[0] > box_b[2]
+        or box_a[1] > box_b[3]
+        or box_a[3] < box_b[1]
+    ):
+        return None
+
     return (
         max(box_a[0], box_b[0]),
         max(box_a[1], box_b[1]),
@@ -589,6 +598,9 @@ def intersection_over_union(box_a, box_b):
     # determine the (x, y)-coordinates of the intersection rectangle
     intersect = intersection(box_a, box_b)
 
+    if intersect is None:
+        return 0.0
+
     # compute the area of intersection rectangle
     inter_area = max(0, intersect[2] - intersect[0] + 1) * max(
         0, intersect[3] - intersect[1] + 1

frigate/video.py

@@ -669,6 +669,40 @@ def get_cluster_region(frame_shape, min_region, cluster, boxes):
     )
 
 
+def get_consolidated_object_detections(detected_object_groups):
+    """Drop detections that overlap too much"""
+    consolidated_detections = []
+    for group in detected_object_groups.values():
+        # if the group only has 1 item, skip
+        if len(group) == 1:
+            consolidated_detections.append(group[0])
+            continue
+
+        # sort smallest to largest by area
+        sorted_by_area = sorted(group, key=lambda g: g[3])
+
+        for current_detection_idx in range(0, len(sorted_by_area)):
+            current_detection = sorted_by_area[current_detection_idx][2]
+            overlap = 0
+            for to_check_idx in range(
+                min(current_detection_idx + 1, len(sorted_by_area)),
+                len(sorted_by_area),
+            ):
+                to_check = sorted_by_area[to_check_idx][2]
+                intersect_box = intersection(current_detection, to_check)
+                # if 90% of smaller detection is inside of another detection, consolidate
+                if (
+                    intersect_box is not None
+                    and area(intersect_box) / area(current_detection) > 0.9
+                ):
+                    overlap = 1
+                    break
+            if overlap == 0:
+                consolidated_detections.append(sorted_by_area[current_detection_idx])
+
+    return consolidated_detections
+
+
 def process_frames(
     camera_name: str,
     frame_queue: mp.Queue,
@@ -849,9 +883,6 @@ def process_frames(
             # set the detections list to only include top objects
             detections = selected_objects
 
-            ## drop detections that overlap too much
-            consolidated_detections = []
-
             # if detection was run on this frame, consolidate
             if len(regions) > 0:
                 # group by name
@@ -859,35 +890,8 @@ def process_frames(
                 for detection in detections:
                     detected_object_groups[detection[0]].append(detection)
 
-                # loop over detections grouped by label
+                consolidated_detections = get_consolidated_object_detections(
-                for group in detected_object_groups.values():
+                    detected_object_groups
-                    # if the group only has 1 item, skip
-                    if len(group) == 1:
-                        consolidated_detections.append(group[0])
-                        continue
-
-                    # sort smallest to largest by area
-                    sorted_by_area = sorted(group, key=lambda g: g[3])
-
-                    for current_detection_idx in range(0, len(sorted_by_area)):
-                        current_detection = sorted_by_area[current_detection_idx][2]
-                        overlap = 0
-                        for to_check_idx in range(
-                            min(current_detection_idx + 1, len(sorted_by_area)),
-                            len(sorted_by_area),
-                        ):
-                            to_check = sorted_by_area[to_check_idx][2]
-                            # if 90% of smaller detection is inside of another detection, consolidate
-                            if (
-                                area(intersection(current_detection, to_check))
-                                / area(current_detection)
-                                > 0.9
-                            ):
-                                overlap = 1
-                                break
-                        if overlap == 0:
-                            consolidated_detections.append(
-                                sorted_by_area[current_detection_idx]
                 )
                 # now that we have refined our detections, we need to track objects
                 object_tracker.match_and_update(frame_time, consolidated_detections)