Reduce idle CPU from thread pools (#24652)

* reduce idle cpu from onnxruntime and opencv thread pools

ONNX Runtime's intra-op threads kept spinning for about 1 s after every run, so a CPU enrichment called once a second or more kept most cores busy. On a 16 thread machine, arcface at one call per second used 9.4 s of CPU per call with ~360 ms latency. Enrichment sessions on the CPU execution provider now set `session.intra_op.allow_spinning` to 0, which drops that to 1.9 s with no latency change. Detector sessions and GPU providers are unchanged, since disabling spinning doubled yolov9-t latency on CPU (12 to 23 ms).

The output and embeddings processes now run OpenCV single threaded, like the camera tracking process. A 4 camera birdseye redraw at 1080p drops from 13.7 to 4.0 ms of CPU, and the full frame conversion in the enrichment processors from 23 to 2.7 ms, each adding about 1 ms of wall time.

* test that only cpu enrichment sessions disable ort spinning

* patch out the neural engine in the spinning test

frigate/test/test_detection_runners.py

@@ -35,15 +35,34 @@ class TestGetOrtSessionOptions(unittest.TestCase):
             ort.GraphOptimizationLevel.ORT_ENABLE_BASIC,
         )
 
-    def test_other_models_use_defaults(self):
-        for model_type in [
-            None,
-            EnrichmentModelTypeEnum.paddleocr.value,
-            EnrichmentModelTypeEnum.arcface.value,
-            ModelTypeEnum.rfdetr.value,
-        ]:
+    def test_cpu_enrichment_models_disable_spinning(self):
+        for model_type in EnrichmentModelTypeEnum:
             with self.subTest(model_type=model_type):
-                self.assertIsNone(get_ort_session_options(model_type))
+                options = get_ort_session_options(model_type.value, cpu=True)
+
+                self.assertEqual(
+                    options.get_session_config_entry("session.intra_op.allow_spinning"),
+                    "0",
+                )
+
+    def test_gpu_enrichment_models_keep_spinning(self):
+        options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value)
+
+        with self.assertRaises(RuntimeError):
+            options.get_session_config_entry("session.intra_op.allow_spinning")
+
+    def test_arcface_keeps_default_optimization(self):
+        options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value)
+
+        self.assertEqual(
+            options.graph_optimization_level,
+            ort.GraphOptimizationLevel.ORT_ENABLE_ALL,
+        )
+
+    def test_detector_models_use_defaults(self):
+        for model_type in [None, ModelTypeEnum.rfdetr.value]:
+            with self.subTest(model_type=model_type):
+                self.assertIsNone(get_ort_session_options(model_type, cpu=True))
 
 
 class TestCudaGraphRunner(unittest.TestCase):

frigate/test/test_runner_devices.py

@@ -98,6 +98,47 @@ class TestLoadRegistry(unittest.TestCase):
         self.assertIsInstance(runner, ONNXModelRunner)
         self.assertEqual(loaded_devices["/models/arcface.onnx"], ("arcface", "CPU"))
 
+    def test_only_cpu_enrichment_sessions_disable_spinning(self):
+        for provider, expected in [
+            ("CPUExecutionProvider", "0"),
+            ("CUDAExecutionProvider", None),
+        ]:
+            with (
+                self.subTest(provider=provider),
+                patch.object(
+                    detection_runners, "is_rknn_compatible", return_value=False
+                ),
+                patch.object(
+                    detection_runners,
+                    "get_ort_providers",
+                    return_value=([provider], [{"device_id": 0}]),
+                ),
+                patch.object(
+                    detection_runners,
+                    "is_openvino_gpu_npu_available",
+                    return_value=False,
+                ),
+                patch.object(
+                    detection_runners, "get_lighter_ane_devices", return_value=[]
+                ),
+                patch.object(detection_runners.ort, "InferenceSession") as session,
+            ):
+                session.return_value.get_providers.return_value = [provider]
+
+                # paddleocr skips the CUDA graph runner, which takes no options
+                get_optimized_runner("/models/paddleocr.onnx", "GPU", "paddleocr")
+
+                options = session.call_args.kwargs["sess_options"]
+
+                try:
+                    spinning = options.get_session_config_entry(
+                        "session.intra_op.allow_spinning"
+                    )
+                except RuntimeError:
+                    spinning = None
+
+                self.assertEqual(spinning, expected)
+
 
 class TestLoadedDeviceSnapshot(unittest.TestCase):
     def setUp(self):