Estimated object speed for zones (#16452)

* utility functions * backend config * backend object speed tracking * draw speed on debug view * basic frontend zone editor * remove line sorting * fix types * highlight line on canvas when entering value in zone edit pane * rename vars and add validation * ensure speed estimation is disabled when user adds more than 4 points * pixel velocity in debug * unit_system in config * ability to define unit system in config * save max speed to db * frontend * docs * clarify docs * utility functions * backend config * backend object speed tracking * draw speed on debug view * basic frontend zone editor * remove line sorting * fix types * highlight line on canvas when entering value in zone edit pane * rename vars and add validation * ensure speed estimation is disabled when user adds more than 4 points * pixel velocity in debug * unit_system in config * ability to define unit system in config * save max speed to db * frontend * docs * clarify docs * fix duplicates from merge * include max_estimated_speed in api responses * add units to zone edit pane * catch undefined * add average speed * clarify docs * only track average speed when object is active * rename vars * ensure points and distances are ordered clockwise * only store the last 10 speeds like score history * remove max estimated speed * update docs * update docs * fix point ordering * improve readability * docs inertia recommendation * fix point ordering * check object frame time * add velocity angle to frontend * docs clarity * add frontend speed filter * fix mqtt docs * fix mqtt docs * don't try to remove distances if they weren't already defined * don't display estimates on debug view/snapshots if object is not in a speed tracking zone * docs * implement speed_threshold for zone presence * docs for threshold * better ground plane image * improve image zone size * add inertia to speed threshold example
2026-10-03 05:16:50 +03:00 · 2025-02-10 13:23:42 -07:00
parent dd7820e4ee
commit 72209986b6
25 changed files with 1030 additions and 79 deletions
@@ -25,6 +25,8 @@ class EventsQueryParams(BaseModel):
    favorites: Optional[int] = None
    min_score: Optional[float] = None
    max_score: Optional[float] = None
+    min_speed: Optional[float] = None
+    max_speed: Optional[float] = None
    is_submitted: Optional[int] = None
    min_length: Optional[float] = None
    max_length: Optional[float] = None
@@ -51,6 +53,8 @@ class EventsSearchQueryParams(BaseModel):
    timezone: Optional[str] = "utc"
    min_score: Optional[float] = None
    max_score: Optional[float] = None
+    min_speed: Optional[float] = None
+    max_speed: Optional[float] = None
    sort: Optional[str] = None


@@ -92,6 +92,8 @@ def events(params: EventsQueryParams = Depends()):
    favorites = params.favorites
    min_score = params.min_score
    max_score = params.max_score
+    min_speed = params.min_speed
+    max_speed = params.max_speed
    is_submitted = params.is_submitted
    min_length = params.min_length
    max_length = params.max_length
@@ -226,6 +228,12 @@ def events(params: EventsQueryParams = Depends()):
    if min_score is not None:
        clauses.append((Event.data["score"] >= min_score))

+    if max_speed is not None:
+        clauses.append((Event.data["average_estimated_speed"] <= max_speed))
+
+    if min_speed is not None:
+        clauses.append((Event.data["average_estimated_speed"] >= min_speed))
+
    if min_length is not None:
        clauses.append(((Event.end_time - Event.start_time) >= min_length))

@@ -249,6 +257,10 @@ def events(params: EventsQueryParams = Depends()):
            order_by = Event.data["score"].asc()
        elif sort == "score_desc":
            order_by = Event.data["score"].desc()
+        elif sort == "speed_asc":
+            order_by = Event.data["average_estimated_speed"].asc()
+        elif sort == "speed_desc":
+            order_by = Event.data["average_estimated_speed"].desc()
        elif sort == "date_asc":
            order_by = Event.start_time.asc()
        elif sort == "date_desc":
@@ -316,7 +328,15 @@ def events_explore(limit: int = 10):
                    k: v
                    for k, v in event.data.items()
                    if k
-                    in ["type", "score", "top_score", "description", "sub_label_score"]
+                    in [
+                        "type",
+                        "score",
+                        "top_score",
+                        "description",
+                        "sub_label_score",
+                        "average_estimated_speed",
+                        "velocity_angle",
+                    ]
                },
                "event_count": label_counts[event.label],
            }
@@ -367,6 +387,8 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
    before = params.before
    min_score = params.min_score
    max_score = params.max_score
+    min_speed = params.min_speed
+    max_speed = params.max_speed
    time_range = params.time_range
    has_clip = params.has_clip
    has_snapshot = params.has_snapshot
@@ -466,6 +488,16 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
        if max_score is not None:
            event_filters.append((Event.data["score"] <= max_score))

+    if min_speed is not None and max_speed is not None:
+        event_filters.append(
+            (Event.data["average_estimated_speed"].between(min_speed, max_speed))
+        )
+    else:
+        if min_speed is not None:
+            event_filters.append((Event.data["average_estimated_speed"] >= min_speed))
+        if max_speed is not None:
+            event_filters.append((Event.data["average_estimated_speed"] <= max_speed))
+
    if time_range != DEFAULT_TIME_RANGE:
        tz_name = params.timezone
        hour_modifier, minute_modifier, _ = get_tz_modifiers(tz_name)
@@ -581,7 +613,16 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
        processed_event["data"] = {
            k: v
            for k, v in event["data"].items()
-            if k in ["type", "score", "top_score", "description"]
+            if k
+            in [
+                "type",
+                "score",
+                "top_score",
+                "description",
+                "sub_label_score",
+                "average_estimated_speed",
+                "velocity_angle",
+            ]
        }

        if event["id"] in search_results:
@@ -596,6 +637,10 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
        processed_events.sort(key=lambda x: x["score"])
    elif min_score is not None and max_score is not None and sort == "score_desc":
        processed_events.sort(key=lambda x: x["score"], reverse=True)
+    elif min_speed is not None and max_speed is not None and sort == "speed_asc":
+        processed_events.sort(key=lambda x: x["average_estimated_speed"])
+    elif min_speed is not None and max_speed is not None and sort == "speed_desc":
+        processed_events.sort(key=lambda x: x["average_estimated_speed"], reverse=True)
    elif sort == "date_asc":
        processed_events.sort(key=lambda x: x["start_time"])
    else:
@@ -1,13 +1,16 @@
 # this uses the base model because the color is an extra attribute
+import logging
 from typing import Optional, Union

 import numpy as np
-from pydantic import BaseModel, Field, PrivateAttr, field_validator
+from pydantic import BaseModel, Field, PrivateAttr, field_validator, model_validator

 from .objects import FilterConfig

 __all__ = ["ZoneConfig"]

+logger = logging.getLogger(__name__)
+

 class ZoneConfig(BaseModel):
    filters: dict[str, FilterConfig] = Field(
@@ -16,6 +19,10 @@ class ZoneConfig(BaseModel):
    coordinates: Union[str, list[str]] = Field(
        title="Coordinates polygon for the defined zone."
    )
+    distances: Optional[Union[str, list[str]]] = Field(
+        default_factory=list,
+        title="Real-world distances for the sides of quadrilateral for the defined zone.",
+    )
    inertia: int = Field(
        default=3,
        title="Number of consecutive frames required for object to be considered present in the zone.",
@@ -26,6 +33,11 @@ class ZoneConfig(BaseModel):
        ge=0,
        title="Number of seconds that an object must loiter to be considered in the zone.",
    )
+    speed_threshold: Optional[float] = Field(
+        default=None,
+        ge=0.1,
+        title="Minimum speed value for an object to be considered in the zone.",
+    )
    objects: Union[str, list[str]] = Field(
        default_factory=list,
        title="List of objects that can trigger the zone.",
@@ -49,6 +61,34 @@ class ZoneConfig(BaseModel):

        return v

+    @field_validator("distances", mode="before")
+    @classmethod
+    def validate_distances(cls, v):
+        if v is None:
+            return None
+
+        if isinstance(v, str):
+            distances = list(map(str, map(float, v.split(","))))
+        elif isinstance(v, list):
+            distances = [str(float(val)) for val in v]
+        else:
+            raise ValueError("Invalid type for distances")
+
+        if len(distances) != 4:
+            raise ValueError("distances must have exactly 4 values")
+
+        return distances
+
+    @model_validator(mode="after")
+    def check_loitering_time_constraints(self):
+        if self.loitering_time > 0 and (
+            self.speed_threshold is not None or len(self.distances) > 0
+        ):
+            logger.warning(
+                "loitering_time should not be set on a zone if speed_threshold or distances is set."
+            )
+        return self
+
    def __init__(self, **config):
        super().__init__(**config)

@@ -5,7 +5,7 @@ from pydantic import Field

 from .base import FrigateBaseModel

-__all__ = ["TimeFormatEnum", "DateTimeStyleEnum", "UIConfig"]
+__all__ = ["TimeFormatEnum", "DateTimeStyleEnum", "UnitSystemEnum", "UIConfig"]


 class TimeFormatEnum(str, Enum):
@@ -21,6 +21,11 @@ class DateTimeStyleEnum(str, Enum):
    short = "short"


+class UnitSystemEnum(str, Enum):
+    imperial = "imperial"
+    metric = "metric"
+
+
 class UIConfig(FrigateBaseModel):
    timezone: Optional[str] = Field(default=None, title="Override UI timezone.")
    time_format: TimeFormatEnum = Field(
@@ -35,3 +40,6 @@ class UIConfig(FrigateBaseModel):
    strftime_fmt: Optional[str] = Field(
        default=None, title="Override date and time format using strftime syntax."
    )
+    unit_system: UnitSystemEnum = Field(
+        default=UnitSystemEnum.metric, title="The unit system to use for measurements."
+    )
@@ -25,6 +25,9 @@ def should_update_db(prev_event: Event, current_event: Event) -> bool:
            or prev_event["entered_zones"] != current_event["entered_zones"]
            or prev_event["thumbnail"] != current_event["thumbnail"]
            or prev_event["end_time"] != current_event["end_time"]
+            or prev_event["average_estimated_speed"]
+            != current_event["average_estimated_speed"]
+            or prev_event["velocity_angle"] != current_event["velocity_angle"]
        ):
            return True
    return False
@@ -210,6 +213,8 @@ class EventProcessor(threading.Thread):
                    "score": score,
                    "top_score": event_data["top_score"],
                    "attributes": attributes,
+                    "average_estimated_speed": event_data["average_estimated_speed"],
+                    "velocity_angle": event_data["velocity_angle"],
                    "type": "object",
                    "max_severity": event_data.get("max_severity"),
                },
@@ -160,7 +160,12 @@ class CameraState:
                    box[2],
                    box[3],
                    text,
-                    f"{obj['score']:.0%} {int(obj['area'])}",
+                    f"{obj['score']:.0%} {int(obj['area'])}"
+                    + (
+                        f" {float(obj['current_estimated_speed']):.1f}"
+                        if obj["current_estimated_speed"] != 0
+                        else ""
+                    ),
                    thickness=thickness,
                    color=color,
                )
@@ -254,6 +259,7 @@ class CameraState:
            new_obj = tracked_objects[id] = TrackedObject(
                self.config.model,
                self.camera_config,
+                self.config.ui,
                self.frame_cache,
                current_detections[id],
            )
@@ -12,6 +12,7 @@ import numpy as np
 from frigate.config import (
    CameraConfig,
    ModelConfig,
+    UIConfig,
 )
 from frigate.review.types import SeverityEnum
 from frigate.util.image import (
@@ -22,6 +23,7 @@ from frigate.util.image import (
    is_better_thumbnail,
 )
 from frigate.util.object import box_inside
+from frigate.util.velocity import calculate_real_world_speed

 logger = logging.getLogger(__name__)

@@ -31,6 +33,7 @@ class TrackedObject:
        self,
        model_config: ModelConfig,
        camera_config: CameraConfig,
+        ui_config: UIConfig,
        frame_cache,
        obj_data: dict[str, any],
    ):
@@ -42,6 +45,7 @@ class TrackedObject:
        self.colormap = model_config.colormap
        self.logos = model_config.all_attribute_logos
        self.camera_config = camera_config
+        self.ui_config = ui_config
        self.frame_cache = frame_cache
        self.zone_presence: dict[str, int] = {}
        self.zone_loitering: dict[str, int] = {}
@@ -58,6 +62,10 @@ class TrackedObject:
        self.frame = None
        self.active = True
        self.pending_loitering = False
+        self.speed_history = []
+        self.current_estimated_speed = 0
+        self.average_estimated_speed = 0
+        self.velocity_angle = 0
        self.previous = self.to_dict()

    @property
@@ -129,6 +137,8 @@ class TrackedObject:
                    "region": obj_data["region"],
                    "score": obj_data["score"],
                    "attributes": obj_data["attributes"],
+                    "current_estimated_speed": self.current_estimated_speed,
+                    "velocity_angle": self.velocity_angle,
                }
                thumb_update = True

@@ -136,6 +146,7 @@ class TrackedObject:
        current_zones = []
        bottom_center = (obj_data["centroid"][0], obj_data["box"][3])
        in_loitering_zone = False
+        in_speed_zone = False

        # check each zone
        for name, zone in self.camera_config.zones.items():
@@ -144,12 +155,66 @@ class TrackedObject:
                continue
            contour = zone.contour
            zone_score = self.zone_presence.get(name, 0) + 1
+
            # check if the object is in the zone
            if cv2.pointPolygonTest(contour, bottom_center, False) >= 0:
                # if the object passed the filters once, dont apply again
                if name in self.current_zones or not zone_filtered(self, zone.filters):
-                    # an object is only considered present in a zone if it has a zone inertia of 3+
+                    # Calculate speed first if this is a speed zone
+                    if (
+                        zone.distances
+                        and obj_data["frame_time"] == current_frame_time
+                        and self.active
+                    ):
+                        speed_magnitude, self.velocity_angle = (
+                            calculate_real_world_speed(
+                                zone.contour,
+                                zone.distances,
+                                self.obj_data["estimate_velocity"],
+                                bottom_center,
+                                self.camera_config.detect.fps,
+                            )
+                        )
+
+                        if self.ui_config.unit_system == "metric":
+                            self.current_estimated_speed = (
+                                speed_magnitude * 3.6
+                            )  # m/s to km/h
+                        else:
+                            self.current_estimated_speed = (
+                                speed_magnitude * 0.681818
+                            )  # ft/s to mph
+
+                        self.speed_history.append(self.current_estimated_speed)
+                        if len(self.speed_history) > 10:
+                            self.speed_history = self.speed_history[-10:]
+
+                        self.average_estimated_speed = sum(self.speed_history) / len(
+                            self.speed_history
+                        )
+
+                        # we've exceeded the speed threshold on the zone
+                        # or we don't have a speed threshold set
+                        if (
+                            zone.speed_threshold is None
+                            or self.average_estimated_speed > zone.speed_threshold
+                        ):
+                            in_speed_zone = True
+
+                        logger.debug(
+                            f"Camera: {self.camera_config.name}, tracked object ID: {self.obj_data['id']}, "
+                            f"zone: {name}, pixel velocity: {str(tuple(np.round(self.obj_data['estimate_velocity']).flatten().astype(int)))}, "
+                            f"speed magnitude: {speed_magnitude}, velocity angle: {self.velocity_angle}, "
+                            f"estimated speed: {self.current_estimated_speed:.1f}, "
+                            f"average speed: {self.average_estimated_speed:.1f}, "
+                            f"length: {len(self.speed_history)}"
+                        )
+
+                    # Check zone entry conditions - for speed zones, require both inertia and speed
                    if zone_score >= zone.inertia:
+                        if zone.distances and not in_speed_zone:
+                            continue  # Skip zone entry for speed zones until speed threshold met
+
                        # if the zone has loitering time, update loitering status
                        if zone.loitering_time > 0:
                            in_loitering_zone = True
@@ -174,6 +239,10 @@ class TrackedObject:
                if 0 < zone_score < zone.inertia:
                    self.zone_presence[name] = zone_score - 1

+            # Reset speed if not in speed zone
+            if zone.distances and name not in current_zones:
+                self.current_estimated_speed = 0
+
        # update loitering status
        self.pending_loitering = in_loitering_zone

@@ -255,6 +324,9 @@ class TrackedObject:
            "current_attributes": self.obj_data["attributes"],
            "pending_loitering": self.pending_loitering,
            "max_severity": self.max_severity,
+            "current_estimated_speed": self.current_estimated_speed,
+            "average_estimated_speed": self.average_estimated_speed,
+            "velocity_angle": self.velocity_angle,
        }

        if include_thumbnail:
@@ -339,7 +411,12 @@ class TrackedObject:
                box[2],
                box[3],
                self.obj_data["label"],
-                f"{int(self.thumbnail_data['score'] * 100)}% {int(self.thumbnail_data['area'])}",
+                f"{int(self.thumbnail_data['score'] * 100)}% {int(self.thumbnail_data['area'])}"
+                + (
+                    f" {self.thumbnail_data['current_estimated_speed']:.1f}"
+                    if self.thumbnail_data["current_estimated_speed"] != 0
+                    else ""
+                ),
                thickness=thickness,
                color=color,
            )
@@ -0,0 +1,127 @@
+import math
+
+import numpy as np
+
+
+def order_points_clockwise(points):
+    """
+    Ensure points are sorted in clockwise order starting from the top left
+
+    :param points: Array of zone corner points in pixel coordinates
+    :return: Ordered list of points
+    """
+    top_left = min(
+        points, key=lambda p: (p[1], p[0])
+    )  # Find the top-left point (min y, then x)
+
+    # Remove the top-left point from the list of points
+    remaining_points = [p for p in points if not np.array_equal(p, top_left)]
+
+    # Sort the remaining points based on the angle relative to the top-left point
+    def angle_from_top_left(point):
+        x, y = point[0] - top_left[0], point[1] - top_left[1]
+        return math.atan2(y, x)
+
+    sorted_points = sorted(remaining_points, key=angle_from_top_left)
+
+    return [top_left] + sorted_points
+
+
+def create_ground_plane(zone_points, distances):
+    """
+    Create a ground plane that accounts for perspective distortion using real-world dimensions for each side of the zone.
+
+    :param zone_points: Array of zone corner points in pixel coordinates
+                        [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+    :param distances: Real-world dimensions ordered by A, B, C, D
+    :return: Function that calculates real-world distance per pixel at any coordinate
+    """
+    A, B, C, D = zone_points
+
+    # Calculate pixel lengths of each side
+    AB_px = np.linalg.norm(np.array(B) - np.array(A))
+    BC_px = np.linalg.norm(np.array(C) - np.array(B))
+    CD_px = np.linalg.norm(np.array(D) - np.array(C))
+    DA_px = np.linalg.norm(np.array(A) - np.array(D))
+
+    AB, BC, CD, DA = map(float, distances)
+
+    AB_scale = AB / AB_px
+    BC_scale = BC / BC_px
+    CD_scale = CD / CD_px
+    DA_scale = DA / DA_px
+
+    def distance_per_pixel(x, y):
+        """
+        Calculate the real-world distance per pixel at a given (x, y) coordinate.
+
+        :param x: X-coordinate in the image
+        :param y: Y-coordinate in the image
+        :return: Real-world distance per pixel at the given (x, y) coordinate
+        """
+        # Normalize x and y within the zone
+        x_norm = (x - A[0]) / (B[0] - A[0])
+        y_norm = (y - A[1]) / (D[1] - A[1])
+
+        # Interpolate scales horizontally and vertically
+        vertical_scale = AB_scale + (CD_scale - AB_scale) * y_norm
+        horizontal_scale = DA_scale + (BC_scale - DA_scale) * x_norm
+
+        # Combine horizontal and vertical scales
+        return (vertical_scale + horizontal_scale) / 2
+
+    return distance_per_pixel
+
+
+def calculate_real_world_speed(
+    zone_contour,
+    distances,
+    velocity_pixels,
+    position,
+    camera_fps,
+):
+    """
+    Calculate the real-world speed of a tracked object, accounting for perspective,
+    directly from the zone string.
+
+    :param zone_contour: Array of absolute zone points
+    :param distances: List of distances of each side, ordered by A, B, C, D
+    :param velocity_pixels: List of tuples representing velocity in pixels/frame
+    :param position: Current position of the object (x, y) in pixels
+    :param camera_fps: Frames per second of the camera
+    :return: speed and velocity angle direction
+    """
+    # order the zone_contour points clockwise starting at top left
+    ordered_zone_contour = order_points_clockwise(zone_contour)
+
+    # find the indices that would sort the original zone_contour to match ordered_zone_contour
+    sort_indices = [
+        np.where((zone_contour == point).all(axis=1))[0][0]
+        for point in ordered_zone_contour
+    ]
+
+    # Reorder distances to match the new order of zone_contour
+    distances = np.array(distances)
+    ordered_distances = distances[sort_indices]
+
+    ground_plane = create_ground_plane(ordered_zone_contour, ordered_distances)
+
+    if not isinstance(velocity_pixels, np.ndarray):
+        velocity_pixels = np.array(velocity_pixels)
+
+    avg_velocity_pixels = velocity_pixels.mean(axis=0)
+
+    # get the real-world distance per pixel at the object's current position and calculate real speed
+    scale = ground_plane(position[0], position[1])
+    speed_real = avg_velocity_pixels * scale * camera_fps
+
+    # euclidean speed in real-world units/second
+    speed_magnitude = np.linalg.norm(speed_real)
+
+    # movement direction
+    dx, dy = avg_velocity_pixels
+    angle = math.degrees(math.atan2(dy, dx))
+    if angle < 0:
+        angle += 360
+
+    return speed_magnitude, angle