diff --git a/frigate/detectors/detection_runners.py b/frigate/detectors/detection_runners.py index 87846c698c..7277269731 100644 --- a/frigate/detectors/detection_runners.py +++ b/frigate/detectors/detection_runners.py @@ -89,30 +89,46 @@ def is_arm64_platform() -> bool: return machine in ("aarch64", "arm64", "armv8", "armv7l") -def get_ort_session_options(model_type: str | None = None) -> ort.SessionOptions | None: +def get_ort_session_options( + model_type: str | None = None, cpu: bool = False +) -> ort.SessionOptions | None: """Get ONNX Runtime session options with appropriate settings. Args: model_type: Model being loaded, used to pin its graph optimization level. + cpu: Whether the session runs on the CPU execution provider. Returns: - SessionOptions with a pinned optimization level, or None for default settings. + SessionOptions for enrichment models, or None for default settings. """ # Import here to avoid circular imports - from frigate.embeddings.types import EnrichmentModelTypeEnum + from frigate.embeddings.types import ( + ENRICHMENT_FOR_MODEL_TYPE, + EnrichmentModelTypeEnum, + ) + + if model_type not in ENRICHMENT_FOR_MODEL_TYPE: + return None + + sess_options = ort.SessionOptions() + + if cpu: + # idle intra-op threads otherwise spin for ~1s after every run, keeping + # most cores busy while an enrichment runs even once a second + sess_options.add_session_config_entry("session.intra_op.allow_spinning", "0") if model_type == EnrichmentModelTypeEnum.jina_v2.value: # below EXTENDED the CUDA EP returns an identical vector for every image, # and ORT_ENABLE_ALL fails to build on CPU with a SimplifiedLayerNormFusion error - level = ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + sess_options.graph_optimization_level = ( + ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + ) elif model_type == EnrichmentModelTypeEnum.jina_v1.value: # aggressive optimizations create or expect nodes that don't exist - level = ort.GraphOptimizationLevel.ORT_ENABLE_BASIC - else: - return None + sess_options.graph_optimization_level = ( + ort.GraphOptimizationLevel.ORT_ENABLE_BASIC + ) - sess_options = ort.SessionOptions() - sess_options.graph_optimization_level = level return sess_options @@ -789,7 +805,9 @@ def get_optimized_runner( ONNXModelRunner( ort.InferenceSession( model_path, - sess_options=get_ort_session_options(model_type), + sess_options=get_ort_session_options( + model_type, cpu=providers[0] == "CPUExecutionProvider" + ), providers=providers, provider_options=options, ), diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index a5f9f708d2..4ee8b997d8 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -10,6 +10,7 @@ from json.decoder import JSONDecodeError from multiprocessing.synchronize import Event as MpEvent from typing import Any +import cv2 import regex from pathvalidate import ValidationError, sanitize_filename @@ -51,6 +52,11 @@ class EmbeddingProcess(FrigateProcess): from .maintainer import EmbeddingMaintainer self.pre_run_setup(self.config.logger) + + # frame conversions and crops here are one-off calls, and OpenCV's + # per-core thread pool burns ~8x the CPU of running them inline + cv2.setNumThreads(1) + maintainer = EmbeddingMaintainer( self.config, self.metrics, diff --git a/frigate/output/output.py b/frigate/output/output.py index 3a76478da4..7a48f3e833 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -8,6 +8,7 @@ import threading from multiprocessing.synchronize import Event as MpEvent from wsgiref.simple_server import make_server +import cv2 from ws4py.server.wsgirefserver import ( WebSocketWSGIHandler, WebSocketWSGIRequestHandler, @@ -122,6 +123,10 @@ class OutputProcess(FrigateProcess): def run(self) -> None: self.pre_run_setup(self.config.logger) + # birdseye resizes are small, and OpenCV's per-core thread pool burns + # ~3x the CPU of running them inline + cv2.setNumThreads(1) + frame_manager = SharedMemoryFrameManager() # start a websocket server on 8082 diff --git a/frigate/test/test_detection_runners.py b/frigate/test/test_detection_runners.py index 38ea457290..21e4a18fb4 100644 --- a/frigate/test/test_detection_runners.py +++ b/frigate/test/test_detection_runners.py @@ -35,15 +35,34 @@ class TestGetOrtSessionOptions(unittest.TestCase): ort.GraphOptimizationLevel.ORT_ENABLE_BASIC, ) - def test_other_models_use_defaults(self): - for model_type in [ - None, - EnrichmentModelTypeEnum.paddleocr.value, - EnrichmentModelTypeEnum.arcface.value, - ModelTypeEnum.rfdetr.value, - ]: + def test_cpu_enrichment_models_disable_spinning(self): + for model_type in EnrichmentModelTypeEnum: with self.subTest(model_type=model_type): - self.assertIsNone(get_ort_session_options(model_type)) + options = get_ort_session_options(model_type.value, cpu=True) + + self.assertEqual( + options.get_session_config_entry("session.intra_op.allow_spinning"), + "0", + ) + + def test_gpu_enrichment_models_keep_spinning(self): + options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value) + + with self.assertRaises(RuntimeError): + options.get_session_config_entry("session.intra_op.allow_spinning") + + def test_arcface_keeps_default_optimization(self): + options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value) + + self.assertEqual( + options.graph_optimization_level, + ort.GraphOptimizationLevel.ORT_ENABLE_ALL, + ) + + def test_detector_models_use_defaults(self): + for model_type in [None, ModelTypeEnum.rfdetr.value]: + with self.subTest(model_type=model_type): + self.assertIsNone(get_ort_session_options(model_type, cpu=True)) class TestCudaGraphRunner(unittest.TestCase): diff --git a/frigate/test/test_runner_devices.py b/frigate/test/test_runner_devices.py index 8fecb611c7..ddba15e689 100644 --- a/frigate/test/test_runner_devices.py +++ b/frigate/test/test_runner_devices.py @@ -98,6 +98,47 @@ class TestLoadRegistry(unittest.TestCase): self.assertIsInstance(runner, ONNXModelRunner) self.assertEqual(loaded_devices["/models/arcface.onnx"], ("arcface", "CPU")) + def test_only_cpu_enrichment_sessions_disable_spinning(self): + for provider, expected in [ + ("CPUExecutionProvider", "0"), + ("CUDAExecutionProvider", None), + ]: + with ( + self.subTest(provider=provider), + patch.object( + detection_runners, "is_rknn_compatible", return_value=False + ), + patch.object( + detection_runners, + "get_ort_providers", + return_value=([provider], [{"device_id": 0}]), + ), + patch.object( + detection_runners, + "is_openvino_gpu_npu_available", + return_value=False, + ), + patch.object( + detection_runners, "get_lighter_ane_devices", return_value=[] + ), + patch.object(detection_runners.ort, "InferenceSession") as session, + ): + session.return_value.get_providers.return_value = [provider] + + # paddleocr skips the CUDA graph runner, which takes no options + get_optimized_runner("/models/paddleocr.onnx", "GPU", "paddleocr") + + options = session.call_args.kwargs["sess_options"] + + try: + spinning = options.get_session_config_entry( + "session.intra_op.allow_spinning" + ) + except RuntimeError: + spinning = None + + self.assertEqual(spinning, expected) + class TestLoadedDeviceSnapshot(unittest.TestCase): def setUp(self):