Reduce idle CPU from thread pools (#24652)

* reduce idle cpu from onnxruntime and opencv thread pools

ONNX Runtime's intra-op threads kept spinning for about 1 s after every run, so a CPU enrichment called once a second or more kept most cores busy. On a 16 thread machine, arcface at one call per second used 9.4 s of CPU per call with ~360 ms latency. Enrichment sessions on the CPU execution provider now set `session.intra_op.allow_spinning` to 0, which drops that to 1.9 s with no latency change. Detector sessions and GPU providers are unchanged, since disabling spinning doubled yolov9-t latency on CPU (12 to 23 ms).

The output and embeddings processes now run OpenCV single threaded, like the camera tracking process. A 4 camera birdseye redraw at 1080p drops from 13.7 to 4.0 ms of CPU, and the full frame conversion in the enrichment processors from 23 to 2.7 ms, each adding about 1 ms of wall time.

* test that only cpu enrichment sessions disable ort spinning

* patch out the neural engine in the spinning test

frigate/detectors/detection_runners.py

@@ -89,30 +89,46 @@ def is_arm64_platform() -> bool:
     return machine in ("aarch64", "arm64", "armv8", "armv7l")
 
 
-def get_ort_session_options(model_type: str | None = None) -> ort.SessionOptions | None:
+def get_ort_session_options(
+    model_type: str | None = None, cpu: bool = False
+) -> ort.SessionOptions | None:
     """Get ONNX Runtime session options with appropriate settings.
 
     Args:
         model_type: Model being loaded, used to pin its graph optimization level.
+        cpu: Whether the session runs on the CPU execution provider.
 
     Returns:
-        SessionOptions with a pinned optimization level, or None for default settings.
+        SessionOptions for enrichment models, or None for default settings.
     """
     # Import here to avoid circular imports
-    from frigate.embeddings.types import EnrichmentModelTypeEnum
+    from frigate.embeddings.types import (
+        ENRICHMENT_FOR_MODEL_TYPE,
+        EnrichmentModelTypeEnum,
+    )
+
+    if model_type not in ENRICHMENT_FOR_MODEL_TYPE:
+        return None
+
+    sess_options = ort.SessionOptions()
+
+    if cpu:
+        # idle intra-op threads otherwise spin for ~1s after every run, keeping
+        # most cores busy while an enrichment runs even once a second
+        sess_options.add_session_config_entry("session.intra_op.allow_spinning", "0")
 
     if model_type == EnrichmentModelTypeEnum.jina_v2.value:
         # below EXTENDED the CUDA EP returns an identical vector for every image,
         # and ORT_ENABLE_ALL fails to build on CPU with a SimplifiedLayerNormFusion error
-        level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
+        sess_options.graph_optimization_level = (
+            ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
+        )
     elif model_type == EnrichmentModelTypeEnum.jina_v1.value:
         # aggressive optimizations create or expect nodes that don't exist
-        level = ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
+        sess_options.graph_optimization_level = (
-    else:
+            ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
-        return None
+        )
 
-    sess_options = ort.SessionOptions()
-    sess_options.graph_optimization_level = level
     return sess_options
 
 
@@ -789,7 +805,9 @@ def get_optimized_runner(
         ONNXModelRunner(
             ort.InferenceSession(
                 model_path,
-                sess_options=get_ort_session_options(model_type),
+                sess_options=get_ort_session_options(
+                    model_type, cpu=providers[0] == "CPUExecutionProvider"
+                ),
                 providers=providers,
                 provider_options=options,
             ),

frigate/embeddings/__init__.py

@@ -10,6 +10,7 @@ from json.decoder import JSONDecodeError
 from multiprocessing.synchronize import Event as MpEvent
 from typing import Any
 
+import cv2
 import regex
 from pathvalidate import ValidationError, sanitize_filename
 
@@ -51,6 +52,11 @@ class EmbeddingProcess(FrigateProcess):
         from .maintainer import EmbeddingMaintainer
 
         self.pre_run_setup(self.config.logger)
+
+        # frame conversions and crops here are one-off calls, and OpenCV's
+        # per-core thread pool burns ~8x the CPU of running them inline
+        cv2.setNumThreads(1)
+
         maintainer = EmbeddingMaintainer(
             self.config,
             self.metrics,

frigate/output/output.py

@@ -8,6 +8,7 @@ import threading
 from multiprocessing.synchronize import Event as MpEvent
 from wsgiref.simple_server import make_server
 
+import cv2
 from ws4py.server.wsgirefserver import (
     WebSocketWSGIHandler,
     WebSocketWSGIRequestHandler,
@@ -122,6 +123,10 @@ class OutputProcess(FrigateProcess):
     def run(self) -> None:
         self.pre_run_setup(self.config.logger)
 
+        # birdseye resizes are small, and OpenCV's per-core thread pool burns
+        # ~3x the CPU of running them inline
+        cv2.setNumThreads(1)
+
         frame_manager = SharedMemoryFrameManager()
 
         # start a websocket server on 8082

frigate/test/test_detection_runners.py

@@ -35,15 +35,34 @@ class TestGetOrtSessionOptions(unittest.TestCase):
             ort.GraphOptimizationLevel.ORT_ENABLE_BASIC,
         )
 
-    def test_other_models_use_defaults(self):
+    def test_cpu_enrichment_models_disable_spinning(self):
-        for model_type in [
+        for model_type in EnrichmentModelTypeEnum:
-            None,
-            EnrichmentModelTypeEnum.paddleocr.value,
-            EnrichmentModelTypeEnum.arcface.value,
-            ModelTypeEnum.rfdetr.value,
-        ]:
             with self.subTest(model_type=model_type):
-                self.assertIsNone(get_ort_session_options(model_type))
+                options = get_ort_session_options(model_type.value, cpu=True)
+
+                self.assertEqual(
+                    options.get_session_config_entry("session.intra_op.allow_spinning"),
+                    "0",
+                )
+
+    def test_gpu_enrichment_models_keep_spinning(self):
+        options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value)
+
+        with self.assertRaises(RuntimeError):
+            options.get_session_config_entry("session.intra_op.allow_spinning")
+
+    def test_arcface_keeps_default_optimization(self):
+        options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value)
+
+        self.assertEqual(
+            options.graph_optimization_level,
+            ort.GraphOptimizationLevel.ORT_ENABLE_ALL,
+        )
+
+    def test_detector_models_use_defaults(self):
+        for model_type in [None, ModelTypeEnum.rfdetr.value]:
+            with self.subTest(model_type=model_type):
+                self.assertIsNone(get_ort_session_options(model_type, cpu=True))
 
 
 class TestCudaGraphRunner(unittest.TestCase):

frigate/test/test_runner_devices.py

@@ -98,6 +98,47 @@ class TestLoadRegistry(unittest.TestCase):
         self.assertIsInstance(runner, ONNXModelRunner)
         self.assertEqual(loaded_devices["/models/arcface.onnx"], ("arcface", "CPU"))
 
+    def test_only_cpu_enrichment_sessions_disable_spinning(self):
+        for provider, expected in [
+            ("CPUExecutionProvider", "0"),
+            ("CUDAExecutionProvider", None),
+        ]:
+            with (
+                self.subTest(provider=provider),
+                patch.object(
+                    detection_runners, "is_rknn_compatible", return_value=False
+                ),
+                patch.object(
+                    detection_runners,
+                    "get_ort_providers",
+                    return_value=([provider], [{"device_id": 0}]),
+                ),
+                patch.object(
+                    detection_runners,
+                    "is_openvino_gpu_npu_available",
+                    return_value=False,
+                ),
+                patch.object(
+                    detection_runners, "get_lighter_ane_devices", return_value=[]
+                ),
+                patch.object(detection_runners.ort, "InferenceSession") as session,
+            ):
+                session.return_value.get_providers.return_value = [provider]
+
+                # paddleocr skips the CUDA graph runner, which takes no options
+                get_optimized_runner("/models/paddleocr.onnx", "GPU", "paddleocr")
+
+                options = session.call_args.kwargs["sess_options"]
+
+                try:
+                    spinning = options.get_session_config_entry(
+                        "session.intra_op.allow_spinning"
+                    )
+                except RuntimeError:
+                    spinning = None
+
+                self.assertEqual(spinning, expected)
+
 
 class TestLoadedDeviceSnapshot(unittest.TestCase):
     def setUp(self):