diff --git a/frigate/object_detection/base.py b/frigate/object_detection/base.py
index 0b0e6c0125..aadbe500f4 100644
--- a/frigate/object_detection/base.py
+++ b/frigate/object_detection/base.py
@@ -24,7 +24,7 @@ from frigate.detectors.detector_config import (
     InputDTypeEnum,
     ModelConfig,
 )
-from frigate.util.builtin import EventsPerSecond, load_labels
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels
 from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory
 from frigate.util.process import FrigateProcess
 
@@ -129,7 +129,7 @@ class DetectorRunner(FrigateProcess):
         super().__init__(stop_event, PROCESS_PRIORITY_HIGH, name=name, daemon=True)
         self.detection_queue = detection_queue
         self.cameras = cameras
-        self.avg_speed = avg_speed
+        self.avg_speed = InferenceSpeed(avg_speed)
         self.start_time = start_time
         self.config = config
         self.detector_config = detector_config
@@ -182,7 +182,7 @@ class DetectorRunner(FrigateProcess):
             detector_publisher.publish(connection_id)
             self.start_time.value = 0.0
 
-            self.avg_speed.value = (self.avg_speed.value * 9 + duration) / 10
+            self.avg_speed.update(duration)
 
         detector_publisher.stop()
         logger.info("Exited detection process...")
@@ -203,7 +203,7 @@ class AsyncDetectorRunner(FrigateProcess):
         super().__init__(stop_event, PROCESS_PRIORITY_HIGH, name=name, daemon=True)
         self.detection_queue = detection_queue
         self.cameras = cameras
-        self.avg_speed = avg_speed
+        self.avg_speed = InferenceSpeed(avg_speed)
         self.start_time = start_time
         self.config = config
         self.detector_config = detector_config
@@ -272,7 +272,7 @@ class AsyncDetectorRunner(FrigateProcess):
             self._publisher.publish(connection_id)
 
             # update timers
-            self.avg_speed.value = (self.avg_speed.value * 9 + duration) / 10
+            self.avg_speed.update(duration)
             self.start_time.value = 0.0
 
     def run(self) -> None:
diff --git a/frigate/test/test_builtin.py b/frigate/test/test_builtin.py
index 1391c0eec7..2de63d823f 100644
--- a/frigate/test/test_builtin.py
+++ b/frigate/test/test_builtin.py
@@ -1,9 +1,10 @@
 """Tests for frigate.util.builtin helpers."""
 
 import unittest
+from multiprocessing import Value
 from unittest.mock import patch
 
-from frigate.util.builtin import EventsPerSecond
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
 
 
 class TestEventsPerSecond(unittest.TestCase):
@@ -62,5 +63,17 @@ class TestEventsPerSecond(unittest.TestCase):
             self.assertAlmostEqual(eps.eps(), 20.0)
 
 
+class TestInferenceSpeed(unittest.TestCase):
+    def test_first_inference_seeds_the_average(self) -> None:
+        metric = Value("d", 0.0)
+        speed = InferenceSpeed(metric)
+
+        speed.update(0.02)
+        self.assertEqual(metric.value, 0.02)
+
+        speed.update(0.12)
+        self.assertAlmostEqual(metric.value, 0.03)
+
+
 if __name__ == "__main__":
     unittest.main()