diff --git a/frigate/ptz/autotrack.py b/frigate/ptz/autotrack.py
index 5b4e45c28..673c47bf9 100644
--- a/frigate/ptz/autotrack.py
+++ b/frigate/ptz/autotrack.py
@@ -786,12 +786,11 @@ class PtzAutoTracker:
         camera_width = camera_config.frame_shape[1]
         camera_height = camera_config.frame_shape[0]
 
-        if camera_config.onvif.autotracking.zooming == ZoomingModeEnum.absolute:
-            zoom = self._get_zoom_amount(camera, obj, obj.obj_data["box"])
-
         if camera_config.onvif.autotracking.zooming == ZoomingModeEnum.relative:
             target_box = obj.obj_data["area"] / (camera_width * camera_height)
-            zoom = target_box ** self.zoom_factor[camera]
+            self.previous_target_box[camera] = target_box ** self.zoom_factor[camera]
+
+        zoom = self._get_zoom_amount(camera, obj, obj.obj_data["box"])
 
         if zoom != 0:
             self._enqueue_move(camera, obj.obj_data["frame_time"], 0, 0, zoom)