Reduce idle CPU from thread pools (#24652)

* reduce idle cpu from onnxruntime and opencv thread pools

ONNX Runtime's intra-op threads kept spinning for about 1 s after every run, so a CPU enrichment called once a second or more kept most cores busy. On a 16 thread machine, arcface at one call per second used 9.4 s of CPU per call with ~360 ms latency. Enrichment sessions on the CPU execution provider now set `session.intra_op.allow_spinning` to 0, which drops that to 1.9 s with no latency change. Detector sessions and GPU providers are unchanged, since disabling spinning doubled yolov9-t latency on CPU (12 to 23 ms).

The output and embeddings processes now run OpenCV single threaded, like the camera tracking process. A 4 camera birdseye redraw at 1080p drops from 13.7 to 4.0 ms of CPU, and the full frame conversion in the enrichment processors from 23 to 2.7 ms, each adding about 1 ms of wall time.

* test that only cpu enrichment sessions disable ort spinning

* patch out the neural engine in the spinning test

frigate/test/test_runner_devices.py

@@ -98,6 +98,47 @@ class TestLoadRegistry(unittest.TestCase):
         self.assertIsInstance(runner, ONNXModelRunner)
         self.assertEqual(loaded_devices["/models/arcface.onnx"], ("arcface", "CPU"))
 
+    def test_only_cpu_enrichment_sessions_disable_spinning(self):
+        for provider, expected in [
+            ("CPUExecutionProvider", "0"),
+            ("CUDAExecutionProvider", None),
+        ]:
+            with (
+                self.subTest(provider=provider),
+                patch.object(
+                    detection_runners, "is_rknn_compatible", return_value=False
+                ),
+                patch.object(
+                    detection_runners,
+                    "get_ort_providers",
+                    return_value=([provider], [{"device_id": 0}]),
+                ),
+                patch.object(
+                    detection_runners,
+                    "is_openvino_gpu_npu_available",
+                    return_value=False,
+                ),
+                patch.object(
+                    detection_runners, "get_lighter_ane_devices", return_value=[]
+                ),
+                patch.object(detection_runners.ort, "InferenceSession") as session,
+            ):
+                session.return_value.get_providers.return_value = [provider]
+
+                # paddleocr skips the CUDA graph runner, which takes no options
+                get_optimized_runner("/models/paddleocr.onnx", "GPU", "paddleocr")
+
+                options = session.call_args.kwargs["sess_options"]
+
+                try:
+                    spinning = options.get_session_config_entry(
+                        "session.intra_op.allow_spinning"
+                    )
+                except RuntimeError:
+                    spinning = None
+
+                self.assertEqual(spinning, expected)
+
 
 class TestLoadedDeviceSnapshot(unittest.TestCase):
     def setUp(self):