Reduce idle CPU from thread pools (#24652)

* reduce idle cpu from onnxruntime and opencv thread pools

ONNX Runtime's intra-op threads kept spinning for about 1 s after every run, so a CPU enrichment called once a second or more kept most cores busy. On a 16 thread machine, arcface at one call per second used 9.4 s of CPU per call with ~360 ms latency. Enrichment sessions on the CPU execution provider now set `session.intra_op.allow_spinning` to 0, which drops that to 1.9 s with no latency change. Detector sessions and GPU providers are unchanged, since disabling spinning doubled yolov9-t latency on CPU (12 to 23 ms).

The output and embeddings processes now run OpenCV single threaded, like the camera tracking process. A 4 camera birdseye redraw at 1080p drops from 13.7 to 4.0 ms of CPU, and the full frame conversion in the enrichment processors from 23 to 2.7 ms, each adding about 1 ms of wall time.

* test that only cpu enrichment sessions disable ort spinning

* patch out the neural engine in the spinning test

frigate/detectors/detection_runners.py

@@ -89,30 +89,46 @@ def is_arm64_platform() -> bool:
     return machine in ("aarch64", "arm64", "armv8", "armv7l")
 
 
-def get_ort_session_options(model_type: str | None = None) -> ort.SessionOptions | None:
+def get_ort_session_options(
+    model_type: str | None = None, cpu: bool = False
+) -> ort.SessionOptions | None:
     """Get ONNX Runtime session options with appropriate settings.
 
     Args:
         model_type: Model being loaded, used to pin its graph optimization level.
+        cpu: Whether the session runs on the CPU execution provider.
 
     Returns:
-        SessionOptions with a pinned optimization level, or None for default settings.
+        SessionOptions for enrichment models, or None for default settings.
     """
     # Import here to avoid circular imports
-    from frigate.embeddings.types import EnrichmentModelTypeEnum
+    from frigate.embeddings.types import (
+        ENRICHMENT_FOR_MODEL_TYPE,
+        EnrichmentModelTypeEnum,
+    )
+
+    if model_type not in ENRICHMENT_FOR_MODEL_TYPE:
+        return None
+
+    sess_options = ort.SessionOptions()
+
+    if cpu:
+        # idle intra-op threads otherwise spin for ~1s after every run, keeping
+        # most cores busy while an enrichment runs even once a second
+        sess_options.add_session_config_entry("session.intra_op.allow_spinning", "0")
 
     if model_type == EnrichmentModelTypeEnum.jina_v2.value:
         # below EXTENDED the CUDA EP returns an identical vector for every image,
         # and ORT_ENABLE_ALL fails to build on CPU with a SimplifiedLayerNormFusion error
-        level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
+        sess_options.graph_optimization_level = (
+            ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
+        )
     elif model_type == EnrichmentModelTypeEnum.jina_v1.value:
         # aggressive optimizations create or expect nodes that don't exist
-        level = ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
-    else:
-        return None
+        sess_options.graph_optimization_level = (
+            ort.GraphOptimizationLevel.ORT_ENABLE_BASIC
+        )
 
-    sess_options = ort.SessionOptions()
-    sess_options.graph_optimization_level = level
     return sess_options
 
 
@@ -789,7 +805,9 @@ def get_optimized_runner(
         ONNXModelRunner(
             ort.InferenceSession(
                 model_path,
-                sess_options=get_ort_session_options(model_type),
+                sess_options=get_ort_session_options(
+                    model_type, cpu=providers[0] == "CPUExecutionProvider"
+                ),
                 providers=providers,
                 provider_options=options,
             ),