diff --git a/frigate/test/test_video.py b/frigate/test/test_video.py index c27f4c801..0accac9b5 100644 --- a/frigate/test/test_video.py +++ b/frigate/test/test_video.py @@ -10,6 +10,7 @@ from frigate.video import ( get_cluster_candidates, get_cluster_region, ) +from frigate.util import intersection def draw_box(frame, box, color=(255, 0, 0), thickness=2): @@ -63,7 +64,7 @@ def save_cluster_boundary_image(name, boxes, bounding_boxes): ) -class TestConfig(unittest.TestCase): +class TestRegion(unittest.TestCase): def setUp(self): self.frame_shape = (1000, 2000) self.min_region_size = 160 @@ -176,3 +177,79 @@ class TestConfig(unittest.TestCase): # save_clusters_image("dont_combine", boxes, cluster_candidates, regions) assert len(regions) == 2 + + +class TestObjectBoundingBoxes(unittest.TestCase): + def setUp(self) -> None: + pass + + def test_box_intersection(self): + box_a = [2012, 191, 2031, 205] + box_b = [887, 92, 985, 151] + box_c = [899, 128, 1080, 175] + + assert intersection(box_a, box_b) == None + assert intersection(box_b, box_c) == [899, 128, 985, 151] + + def test_consolidate_objects(self): + selected_objects = [ + ( + "car", # valid detection, should not be consolidated + 0.82421875, + (2044, 240, 2134, 297), + 5130, + 1.5789473684210527, + (2008, 188, 2168, 348), + ), + ( + "car", # valid detection, should not be consolidated + 0.76953125, + (124, 239, 967, 719), + 404640, + 1.75625, + (0, 0, 2968, 2968), + ), + ( + "car", # valid detection, should not be consolidated + 0.71484375, + (890, 93, 976, 150), + 4902, + 1.5087719298245614, + (854, 40, 1014, 200), + ), + ( + "car", # valid detection, should not be consolidated + 0.6640625, + (1959, 188, 2005, 222), + 1564, + 1.3529411764705883, + (1768, 0, 2560, 792), + ), + ( + "car", # invalid detection, should be consolidated + 0.65234375, + (2033, 193, 2048, 203), + 150, + 1.5, + (2008, 188, 2168, 348), + ), + ] + + [ + ( + "car", + 0.82421875, + (2044, 240, 2134, 297), + 5130, + 1.5789473684210527, + (2008, 188, 2168, 348), + ), + ( + "car", + 0.76953125, + (124, 239, 967, 719), + 404640, + 1.75625, + (0, 0, 2968, 2968), + ), + ] diff --git a/frigate/util.py b/frigate/util.py index 897aa8d21..396a0f874 100755 --- a/frigate/util.py +++ b/frigate/util.py @@ -572,7 +572,16 @@ def yuv_region_2_bgr(frame, region): raise -def intersection(box_a, box_b): +def intersection(box_a, box_b) -> Optional[list[int]]: + """Return intersection box or None if boxes do not intersect.""" + if ( + box_a[2] < box_b[0] + or box_a[0] > box_b[2] + or box_a[1] > box_b[3] + or box_a[3] < box_b[1] + ): + return None + return ( max(box_a[0], box_b[0]), max(box_a[1], box_b[1]), @@ -589,6 +598,9 @@ def intersection_over_union(box_a, box_b): # determine the (x, y)-coordinates of the intersection rectangle intersect = intersection(box_a, box_b) + if intersect is None: + return 0.0 + # compute the area of intersection rectangle inter_area = max(0, intersect[2] - intersect[0] + 1) * max( 0, intersect[3] - intersect[1] + 1 diff --git a/frigate/video.py b/frigate/video.py index 874490eae..291347ba3 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -669,6 +669,40 @@ def get_cluster_region(frame_shape, min_region, cluster, boxes): ) +def get_consolidated_object_detections(detected_object_groups): + """Drop detections that overlap too much""" + consolidated_detections = [] + for group in detected_object_groups.values(): + # if the group only has 1 item, skip + if len(group) == 1: + consolidated_detections.append(group[0]) + continue + + # sort smallest to largest by area + sorted_by_area = sorted(group, key=lambda g: g[3]) + + for current_detection_idx in range(0, len(sorted_by_area)): + current_detection = sorted_by_area[current_detection_idx][2] + overlap = 0 + for to_check_idx in range( + min(current_detection_idx + 1, len(sorted_by_area)), + len(sorted_by_area), + ): + to_check = sorted_by_area[to_check_idx][2] + intersect_box = intersection(current_detection, to_check) + # if 90% of smaller detection is inside of another detection, consolidate + if ( + intersect_box is not None + and area(intersect_box) / area(current_detection) > 0.9 + ): + overlap = 1 + break + if overlap == 0: + consolidated_detections.append(sorted_by_area[current_detection_idx]) + + return consolidated_detections + + def process_frames( camera_name: str, frame_queue: mp.Queue, @@ -849,9 +883,6 @@ def process_frames( # set the detections list to only include top objects detections = selected_objects - ## drop detections that overlap too much - consolidated_detections = [] - # if detection was run on this frame, consolidate if len(regions) > 0: # group by name @@ -859,36 +890,9 @@ def process_frames( for detection in detections: detected_object_groups[detection[0]].append(detection) - # loop over detections grouped by label - for group in detected_object_groups.values(): - # if the group only has 1 item, skip - if len(group) == 1: - consolidated_detections.append(group[0]) - continue - - # sort smallest to largest by area - sorted_by_area = sorted(group, key=lambda g: g[3]) - - for current_detection_idx in range(0, len(sorted_by_area)): - current_detection = sorted_by_area[current_detection_idx][2] - overlap = 0 - for to_check_idx in range( - min(current_detection_idx + 1, len(sorted_by_area)), - len(sorted_by_area), - ): - to_check = sorted_by_area[to_check_idx][2] - # if 90% of smaller detection is inside of another detection, consolidate - if ( - area(intersection(current_detection, to_check)) - / area(current_detection) - > 0.9 - ): - overlap = 1 - break - if overlap == 0: - consolidated_detections.append( - sorted_by_area[current_detection_idx] - ) + consolidated_detections = get_consolidated_object_detections( + detected_object_groups + ) # now that we have refined our detections, we need to track objects object_tracker.match_and_update(frame_time, consolidated_detections) # else, just update the frame times for the stationary objects