From 8a12ef78dbcf438f14795a45e69b6e9161dee142 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 19 Oct 2023 13:41:18 -0500 Subject: [PATCH] use calibration to determine zoom levels --- frigate/app.py | 6 ++ frigate/config.py | 4 +- frigate/const.py | 2 +- frigate/ptz/autotrack.py | 144 ++++++++++++++++++++++++++++++--------- frigate/ptz/onvif.py | 12 ++++ frigate/types.py | 2 + 6 files changed, 134 insertions(+), 36 deletions(-) diff --git a/frigate/app.py b/frigate/app.py index a2e300526..2549b4ebb 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -187,6 +187,12 @@ class FrigateApp: "ptz_zoom_level": mp.Value("d", 0.0), # type: ignore[typeddict-item] # issue https://github.com/python/typeshed/issues/8799 # from mypy 0.981 onwards + "ptz_max_zoom": mp.Value("d", 0.0), # type: ignore[typeddict-item] + # issue https://github.com/python/typeshed/issues/8799 + # from mypy 0.981 onwards + "ptz_min_zoom": mp.Value("d", 0.0), # type: ignore[typeddict-item] + # issue https://github.com/python/typeshed/issues/8799 + # from mypy 0.981 onwards } self.ptz_metrics[camera_name]["ptz_stopped"].set() self.feature_metrics[camera_name] = { diff --git a/frigate/config.py b/frigate/config.py index 90740150c..3ccc76a35 100644 --- a/frigate/config.py +++ b/frigate/config.py @@ -188,8 +188,8 @@ class PtzAutotrackConfig(FrigateBaseModel): else: raise ValueError("Invalid type for movement_weights") - if len(weights) != 3: - raise ValueError("movement_weights must have exactly 3 floats") + if len(weights) != 5: + raise ValueError("movement_weights must have exactly 5 floats") return weights diff --git a/frigate/const.py b/frigate/const.py index 87da6af89..af74e74ee 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -58,6 +58,6 @@ AUTOTRACKING_MAX_AREA_RATIO = 0.5 AUTOTRACKING_MOTION_MIN_DISTANCE = 20 AUTOTRACKING_MOTION_MAX_POINTS = 500 AUTOTRACKING_MAX_MOVE_METRICS = 500 -AUTOTRACKING_ZOOM_OUT_HYSTERESIS = 2.0 +AUTOTRACKING_ZOOM_OUT_HYSTERESIS = 1.5 AUTOTRACKING_ZOOM_IN_HYSTERESIS = 0.8 AUTOTRACKING_ZOOM_EDGE_THRESHOLD = 0.05 diff --git a/frigate/ptz/autotrack.py b/frigate/ptz/autotrack.py index f27b5baa2..191db1a3f 100644 --- a/frigate/ptz/autotrack.py +++ b/frigate/ptz/autotrack.py @@ -267,12 +267,25 @@ class PtzAutoTracker: self.move_threads[camera].start() if camera_config.onvif.autotracking.movement_weights: - self.intercept[ - camera - ] = camera_config.onvif.autotracking.movement_weights[0] - self.move_coefficients[ - camera - ] = camera_config.onvif.autotracking.movement_weights[1:] + if len(camera_config.onvif.autotracking.movement_weights) == 5: + self.ptz_metrics[camera][ + "ptz_min_zoom" + ].value = camera_config.onvif.autotracking.movement_weights[0] + self.ptz_metrics[camera][ + "ptz_max_zoom" + ].value = camera_config.onvif.autotracking.movement_weights[1] + self.intercept[ + camera + ] = camera_config.onvif.autotracking.movement_weights[2] + self.move_coefficients[ + camera + ] = camera_config.onvif.autotracking.movement_weights[3:] + else: + camera_config.onvif.autotracking.enabled = False + self.ptz_metrics[camera]["ptz_autotracker_enabled"].value = False + logger.warning( + f"Autotracker recalibration is required for {camera}. Disabling autotracking." + ) if camera_config.onvif.autotracking.calibrate_on_startup: self._calibrate_camera(camera) @@ -299,11 +312,83 @@ class PtzAutoTracker: # TODO: take zooming into account too num_steps = 30 step_sizes = np.linspace(0, 1, num_steps) + zoom_in_values = [] + zoom_out_values = [] self.calibrating[camera] = True logger.info(f"Camera calibration for {camera} in progress") + # zoom levels test + if ( + self.config.cameras[camera].onvif.autotracking.zooming + != ZoomingModeEnum.disabled + ): + logger.info(f"Calibration for {camera} in progress: 0% complete") + + for i in range(2): + # absolute move to 0 - fully zoomed out + self.onvif._zoom_absolute( + camera, + self.onvif.cams[camera]["absolute_zoom_range"]["XRange"]["Min"], + 1, + ) + + while not self.ptz_metrics[camera]["ptz_stopped"].is_set(): + self.onvif.get_camera_status(camera) + + zoom_out_values.append(self.ptz_metrics[camera]["ptz_zoom_level"].value) + + self.onvif._zoom_absolute( + camera, + self.onvif.cams[camera]["absolute_zoom_range"]["XRange"]["Max"], + 1, + ) + + while not self.ptz_metrics[camera]["ptz_stopped"].is_set(): + self.onvif.get_camera_status(camera) + + zoom_in_values.append(self.ptz_metrics[camera]["ptz_zoom_level"].value) + + # relative move to -0.01 + self.onvif._move_relative( + camera, + 0, + 0, + -1e-2, + 1, + ) + + while not self.ptz_metrics[camera]["ptz_stopped"].is_set(): + self.onvif.get_camera_status(camera) + + zoom_out_values.append(self.ptz_metrics[camera]["ptz_zoom_level"].value) + + # relative move to 0.01 + self.onvif._move_relative( + camera, + 0, + 0, + 1e-2, + 1, + ) + + while not self.ptz_metrics[camera]["ptz_stopped"].is_set(): + self.onvif.get_camera_status(camera) + + zoom_in_values.append(self.ptz_metrics[camera]["ptz_zoom_level"].value) + + self.ptz_metrics[camera]["ptz_max_zoom"].value = max(zoom_in_values) + self.ptz_metrics[camera]["ptz_min_zoom"].value = min(zoom_out_values) + + logger.debug( + f'{camera}: Calibration values: max zoom: {self.ptz_metrics[camera]["ptz_max_zoom"].value}, min zoom: {self.ptz_metrics[camera]["ptz_min_zoom"].value}' + ) + + else: + self.ptz_metrics[camera]["ptz_max_zoom"].value = 1 + self.ptz_metrics[camera]["ptz_min_zoom"].value = 0 + self.onvif._move_to_preset( camera, self.config.cameras[camera].onvif.autotracking.return_preset.lower(), @@ -385,12 +470,16 @@ class PtzAutoTracker: if calibration: self.intercept[camera] = y[0] - # write the intercept and coefficients back to the config file as a comma separated string - movement_weights = np.concatenate( - ([self.intercept[camera]], self.move_coefficients[camera]) - ) + # write the min zoom, max zoom, intercept, and coefficients + # back to the config file as a comma separated string self.config.cameras[camera].onvif.autotracking.movement_weights = ", ".join( - map(str, movement_weights) + str(v) + for v in [ + self.ptz_metrics[camera]["ptz_min_zoom"].value, + self.ptz_metrics[camera]["ptz_max_zoom"].value, + self.intercept[camera], + *self.move_coefficients[camera], + ] ) logger.debug( @@ -439,10 +528,6 @@ class PtzAutoTracker: weighted_area / (camera_width * camera_height) ) ** self.zoom_factor[camera] - # self.tracked_object_metrics[camera]["target_box"] = statistics.median( - # [obj["area"] for obj in self.tracked_object_history[camera]] - # ) / (camera_width * camera_height) - if "original_target_box" not in self.tracked_object_metrics[camera]: self.tracked_object_metrics[camera][ "original_target_box" @@ -473,20 +558,9 @@ class PtzAutoTracker: continue else: - if zoom != 0: - self.tracked_object_metrics[camera][ - "last_zoom_time" - ] = frame_time - if pan != 0 or tilt != 0: - self.tracked_object_metrics[camera][ - "last_move_time" - ] = frame_time - if ( self.config.cameras[camera].onvif.autotracking.zooming == ZoomingModeEnum.relative - # this enables us to absolutely zoom if we lost an object - and self.tracked_object[camera] is not None ): self.onvif._move_relative(camera, pan, tilt, zoom, 1) @@ -762,8 +836,14 @@ class PtzAutoTracker: * AUTOTRACKING_ZOOM_IN_HYSTERESIS ) - at_max_zoom = self.ptz_metrics[camera]["ptz_zoom_level"].value == 1 - at_min_zoom = self.ptz_metrics[camera]["ptz_zoom_level"].value == 0 + at_max_zoom = ( + self.ptz_metrics[camera]["ptz_zoom_level"].value + == self.ptz_metrics[camera]["ptz_max_zoom"].value + ) + at_min_zoom = ( + self.ptz_metrics[camera]["ptz_zoom_level"].value + == self.ptz_metrics[camera]["ptz_min_zoom"].value + ) # debug zooming if debug_zooming: @@ -844,7 +924,6 @@ class PtzAutoTracker: predicted_movement_time = self._predict_movement_time(camera, pan, tilt) average_velocity, distance = self._get_valid_velocity(camera, obj) - # don't move ptz for estimates that are way too high either if distance != -1: # this box could exceed the frame boundaries if velocity is high @@ -878,7 +957,7 @@ class PtzAutoTracker: def _autotrack_move_zoom_only(self, camera, obj): camera_config = self.config.cameras[camera] - if camera_config.onvif.autotracking.zooming == ZoomingModeEnum.absolute: + if camera_config.onvif.autotracking.zooming != ZoomingModeEnum.disabled: zoom = self._get_zoom_amount(camera, obj, obj.obj_data["box"]) if zoom != 0: @@ -917,7 +996,7 @@ class PtzAutoTracker: # this is our initial zoom in on a new object if "target_box" not in self.tracked_object_metrics[camera]: zoom = target_box ** self.zoom_factor[camera] - if target_box > self.tracked_object_metrics[camera]["max_target_box"]: + if zoom > self.tracked_object_metrics[camera]["max_target_box"]: zoom = -(1 - zoom) logger.debug( f"{camera}: target box: {target_box}, max: {self.tracked_object_metrics[camera]['max_target_box']}, calc zoom: {zoom}" @@ -953,8 +1032,7 @@ class PtzAutoTracker: ) logger.debug(f"{camera}: Zoom calculation: {zoom}") if not result: - # zoom out - zoom = -(1 - abs(zoom)) if zoom > 0 else -(zoom + 1) + zoom = -(1 - abs(zoom)) if zoom > 0 else -zoom logger.debug(f"{camera}: Zooming: {result} Zoom amount: {zoom}") diff --git a/frigate/ptz/onvif.py b/frigate/ptz/onvif.py index de85972ef..212de383f 100644 --- a/frigate/ptz/onvif.py +++ b/frigate/ptz/onvif.py @@ -77,6 +77,7 @@ class OnvifController: request = ptz.create_type("GetConfigurations") configs = ptz.GetConfigurations(request)[0] + logger.debug(f"Onvif configs for {camera_name}: {configs}") request = ptz.create_type("GetConfigurationOptions") request.ConfigurationToken = profile.PTZConfiguration.token @@ -194,6 +195,17 @@ class OnvifController: if ptz_config.Spaces and ptz_config.Spaces.RelativeZoomTranslationSpace: supported_features.append("zoom-r") + try: + # get camera's zoom limits from onvif config + self.cams[camera_name][ + "relative_zoom_range" + ] = ptz_config.Spaces.RelativeZoomTranslationSpace[0] + except Exception: + if self.config.cameras[camera_name].onvif.autotracking.zooming: + self.config.cameras[camera_name].onvif.autotracking.zooming = False + logger.warning( + f"Disabling autotracking zooming for {camera_name}: Relative zoom not supported" + ) if ptz_config.Spaces and ptz_config.Spaces.AbsoluteZoomPositionSpace: supported_features.append("zoom-a") diff --git a/frigate/types.py b/frigate/types.py index d97c0e53b..ab8f07e9c 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -35,6 +35,8 @@ class PTZMetricsTypes(TypedDict): ptz_stop_time: Synchronized ptz_frame_time: Synchronized ptz_zoom_level: Synchronized + ptz_max_zoom: Synchronized + ptz_min_zoom: Synchronized class FeatureMetricsTypes(TypedDict):