Smarter Regions

frigate/app.py

@@ -49,6 +49,7 @@ from frigate.stats import StatsEmitter, stats_init
 from frigate.storage import StorageMaintainer
 from frigate.timeline import TimelineProcessor
 from frigate.types import CameraMetricsTypes, FeatureMetricsTypes, PTZMetricsTypes
+from frigate.util.object import get_camera_regions_grid
 from frigate.version import VERSION
 from frigate.video import capture_camera, track_camera
 from frigate.watchdog import FrigateWatchdog
@@ -69,6 +70,7 @@ class FrigateApp:
         self.feature_metrics: dict[str, FeatureMetricsTypes] = {}
         self.ptz_metrics: dict[str, PTZMetricsTypes] = {}
         self.processes: dict[str, int] = {}
+        self.region_grids: dict[str, list[list[dict[str, any]]]] = {}
 
     def set_environment_vars(self) -> None:
         for key, value in self.config.environment_vars.items():
@@ -452,6 +454,10 @@ class FrigateApp:
         output_processor.start()
         logger.info(f"Output process started: {output_processor.pid}")
 
+    def init_historical_regions(self) -> None:
+        for camera in self.config.cameras.values():
+            self.region_grids[camera.name] = get_camera_regions_grid(camera)
+
     def start_camera_processors(self) -> None:
         for name, config in self.config.cameras.items():
             if not self.config.cameras[name].enabled:
@@ -471,6 +477,7 @@ class FrigateApp:
                     self.detected_frames_queue,
                     self.camera_metrics[name],
                     self.ptz_metrics[name],
+                    self.region_grids[name],
                 ),
             )
             camera_process.daemon = True
@@ -611,6 +618,7 @@ class FrigateApp:
         self.start_detectors()
         self.start_video_output_processor()
         self.start_ptz_autotracker()
+        self.init_historical_regions()
         self.start_detected_frames_processor()
         self.start_camera_processors()
         self.start_camera_capture_processes()

frigate/util/object.py

@@ -0,0 +1,145 @@
+"""Utils for reading and writing object detection data."""
+
+import logging
+
+import numpy
+
+from frigate.config import CameraConfig
+from frigate.models import Timeline
+from frigate.util.image import calculate_region
+
+logger = logging.getLogger(__name__)
+
+
+def get_camera_regions_grid(
+    camera: CameraConfig, grid_size: int = 8
+) -> list[list[dict[str, any]]]:
+    """Build a grid of expected region sizes for a camera."""
+    # create a grid
+    grid = []
+    for x in range(grid_size):
+        row = []
+        for y in range(grid_size):
+            row.append({"sizes": []})
+        grid.append(row)
+
+    timeline = (
+        Timeline.select(
+            *[
+                Timeline.camera,
+                Timeline.source,
+                Timeline.data,
+            ]
+        )
+        .where(Timeline.camera == camera.name)
+        .limit(10000)
+        .dicts()
+    )
+
+    if not timeline:
+        return grid
+
+    logger.debug(f"There are {len(timeline)} entries for {camera.name}")
+    width = camera.detect.width
+    height = camera.detect.height
+
+    logger.debug(f"The size of grid is {len(grid)} x {len(grid[grid_size - 1])}")
+    grid_coef = 1.0 / grid_size
+
+    for t in timeline:
+        if t.get("source") != "tracked_object":
+            continue
+
+        box = t["data"]["box"]
+
+        # calculate centroid position
+        x = box[0] + (box[2] / 2)
+        y = box[1] + (box[3] / 2)
+
+        x_pos = int(x * grid_size)
+        y_pos = int(y * grid_size)
+
+        calculated_region = calculate_region(
+            (height, width),
+            box[0] * width,
+            box[1] * height,
+            (box[0] + box[2]) * width,
+            (box[1] + box[3]) * height,
+            320,
+            1.35,
+        )
+        # save width of region to grid as relative
+        grid[x_pos][y_pos]["sizes"].append((calculated_region[2] - calculated_region[0]) / width)
+
+    for x in range(grid_size):
+        for y in range(grid_size):
+            cell = grid[x][y]
+            logger.debug(
+                f"stats for cell {x * grid_coef * width},{y * grid_coef * height} -> {(x + 1) * grid_coef * width},{(y + 1) * grid_coef * height} :: {len(cell['sizes'])} objects"
+            )
+
+            if len(cell["sizes"]) == 0:
+                continue
+
+            std_dev = numpy.std(cell["sizes"])
+            mean = numpy.mean(cell["sizes"])
+            logger.debug(f"std dev: {std_dev} mean: {mean}")
+            cell["std_dev"] = std_dev
+            cell["mean"] = mean
+
+    return grid
+
+
+def get_cluster_region_from_grid(frame_shape, min_region, cluster, boxes, region_grid):
+    min_x = frame_shape[1]
+    min_y = frame_shape[0]
+    max_x = 0
+    max_y = 0
+    for b in cluster:
+        min_x = min(boxes[b][0], min_x)
+        min_y = min(boxes[b][1], min_y)
+        max_x = max(boxes[b][2], max_x)
+        max_y = max(boxes[b][3], max_y)
+    return get_region_from_grid(
+        frame_shape, [min_x, min_y, max_x, max_y], min_region, region_grid
+    )
+
+
+def get_region_from_grid(
+    frame_shape: tuple[int],
+    box: list[int],
+    min_region: int,
+    region_grid: list[list[dict[str, any]]],
+) -> list[int]:
+    """Get a region for a box based on the region grid."""
+    centroid = (box[0] - (box[2] - box[0]), box[1] - (box[3] - box[1]))
+    grid_x = int(centroid[0] / frame_shape[1] * len(region_grid))
+    grid_y = int(centroid[1] / frame_shape[0] * len(region_grid))
+
+    cell = region_grid[grid_x][grid_y]
+
+    # if there is no known data, get standard region for motion box
+    if not cell or not cell["sizes"]:
+        return calculate_region(frame_shape, box[0], box[1], box[2], box[3], min_region)
+
+    calc_size = (box[2] - box[0]) / frame_shape[1]
+
+    # if region is within expected size, don't resize
+    if (cell["mean"] - cell["std_dev"]) < calc_size < (cell["mean"] + cell["std_dev"]):
+        return calculate_region(frame_shape, box[0], box[1], box[2], box[3], min_region)
+    # TODO not sure how to handle case where cluster is larger than expected region
+    elif calc_size > (cell["mean"] + cell["std_dev"]):
+        return calculate_region(frame_shape, box[0], box[1], box[2], box[3], min_region)
+
+    size = cell["mean"] * frame_shape[1]
+
+    # get region based on grid size
+    new = calculate_region(
+        frame_shape,
+        max(0, centroid[0] - size / 2),
+        max(0, centroid[1] - size / 2),
+        min(frame_shape[1], centroid[0] + size / 2),
+        min(frame_shape[0], centroid[1] + size / 2),
+        min_region,
+    )
+    return new

frigate/video.py

@@ -37,6 +37,7 @@ from frigate.util.image import (
     yuv_region_2_rgb,
     yuv_region_2_yuv,
 )
+from frigate.util.object import get_cluster_region_from_grid
 from frigate.util.services import listen
 
 logger = logging.getLogger(__name__)
@@ -457,6 +458,7 @@ def track_camera(
     detected_objects_queue,
     process_info,
     ptz_metrics,
+    region_grid,
 ):
     stop_event = mp.Event()
 
@@ -515,6 +517,7 @@ def track_camera(
         motion_enabled,
         stop_event,
         ptz_metrics,
+        region_grid,
     )
 
     logger.info(f"{name}: exiting subprocess")
@@ -559,6 +562,14 @@ def intersects_any(box_a, boxes):
     return False
 
 
+def inside_any(box_a, boxes):
+    for box in boxes:
+        # check if box_a is inside of box
+        if box_inside(box, box_a):
+            return True
+    return False
+
+
 def detect(
     detect_config: DetectConfig,
     object_detector,
@@ -623,7 +634,9 @@ def get_cluster_boundary(box, min_region):
     ]
 
 
-def get_cluster_candidates(frame_shape, min_region, boxes):
+def get_cluster_candidates(
+    frame_shape, min_region, boxes, region_grid: list[list[dict[str, any]]]
+):
     # and create a cluster of other boxes using it's max region size
     # only include boxes where the region is an appropriate(except the region could possibly be smaller?)
     # size in the cluster. in order to be in the cluster, the furthest corner needs to be within x,y offset
@@ -740,6 +753,7 @@ def process_frames(
     motion_enabled: mp.Value,
     stop_event,
     ptz_metrics: PTZMetricsTypes,
+    region_grid: list[list[dict[str, any]]],
     exit_on_empty: bool = False,
 ):
     fps = process_info["process_fps"]
@@ -815,22 +829,37 @@ def process_frames(
                 if obj["id"] not in stationary_object_ids
             ]
 
-            combined_boxes = tracked_object_boxes
+            # get consolidated regions for tracked objects
-            # only add in the motion boxes when not calibrating
-            if not motion_detector.is_calibrating():
-                combined_boxes += motion_boxes
-
-            cluster_candidates = get_cluster_candidates(
-                frame_shape, region_min_size, combined_boxes
-            )
-
             regions = [
                 get_cluster_region(
-                    frame_shape, region_min_size, candidate, combined_boxes
+                    frame_shape, region_min_size, candidate, tracked_object_boxes
+                )
+                for candidate in get_cluster_candidates(
+                    frame_shape, region_min_size, tracked_object_boxes, region_grid
                 )
-                for candidate in cluster_candidates
             ]
 
+            # only add in the motion boxes when not calibrating
+            if not motion_detector.is_calibrating():
+                # find motion boxes that are not inside tracked object regions
+                standalone_motion_boxes = [b for b in motion_boxes if not inside_any(b, regions)]
+
+                if standalone_motion_boxes:
+                    motion_clusters = get_cluster_candidates(
+                        frame_shape, region_min_size, standalone_motion_boxes, region_grid
+                    )
+                    motion_regions = [
+                        get_cluster_region_from_grid(
+                            frame_shape,
+                            region_min_size,
+                            candidate,
+                            standalone_motion_boxes,
+                            region_grid,
+                        )
+                        for candidate in motion_clusters
+                    ]
+                    regions += motion_regions
+
             # if starting up, get the next startup scan region
             if startup_scan_counter < 9:
                 ymin = int(frame_shape[0] / 3 * startup_scan_counter / 3)