diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py
index 2e9e5034f..ed6b38aaf 100644
--- a/frigate/track/norfair_tracker.py
+++ b/frigate/track/norfair_tracker.py
@@ -106,6 +106,9 @@ class NorfairTracker(ObjectTracker):
             "ymax": self.detect_config.height,
         }
 
+        # start object with a hit count of `fps` to avoid quick detection -> loss
+        next((o for o in self.tracker.tracked_objects if o.global_id == track_id)).hit_counter = self.camera_config.detect.fps
+
     def deregister(self, id, track_id):
         del self.tracked_objects[id]
         del self.disappeared[id]