mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-09 07:03:02 +03:00
Josh HawkinsandGitHub
b79e043976
CI / AMD64 Build (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
* reduce idle cpu from onnxruntime and opencv thread pools ONNX Runtime's intra-op threads kept spinning for about 1 s after every run, so a CPU enrichment called once a second or more kept most cores busy. On a 16 thread machine, arcface at one call per second used 9.4 s of CPU per call with ~360 ms latency. Enrichment sessions on the CPU execution provider now set `session.intra_op.allow_spinning` to 0, which drops that to 1.9 s with no latency change. Detector sessions and GPU providers are unchanged, since disabling spinning doubled yolov9-t latency on CPU (12 to 23 ms). The output and embeddings processes now run OpenCV single threaded, like the camera tracking process. A 4 camera birdseye redraw at 1080p drops from 13.7 to 4.0 ms of CPU, and the full frame conversion in the enrichment processors from 23 to 2.7 ms, each adding about 1 ms of wall time. * test that only cpu enrichment sessions disable ort spinning * patch out the neural engine in the spinning test
108 lines
3.9 KiB
Python
108 lines
3.9 KiB
Python
"""Tests for ONNX Runtime session option selection."""
|
|
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import numpy as np
|
|
import onnxruntime as ort
|
|
|
|
from frigate.detectors.detection_runners import (
|
|
CudaGraphRunner,
|
|
get_ort_session_options,
|
|
)
|
|
from frigate.detectors.detector_config import ModelTypeEnum
|
|
from frigate.embeddings.types import EnrichmentModelTypeEnum
|
|
|
|
|
|
class TestGetOrtSessionOptions(unittest.TestCase):
|
|
def test_jina_v2_uses_extended(self):
|
|
"""jina-clip-v2 returns an identical vector for every image on the CUDA
|
|
execution provider at anything below EXTENDED."""
|
|
options = get_ort_session_options(EnrichmentModelTypeEnum.jina_v2.value)
|
|
|
|
self.assertIsNotNone(options)
|
|
self.assertEqual(
|
|
options.graph_optimization_level,
|
|
ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED,
|
|
)
|
|
|
|
def test_jina_v1_uses_basic(self):
|
|
options = get_ort_session_options(EnrichmentModelTypeEnum.jina_v1.value)
|
|
|
|
self.assertIsNotNone(options)
|
|
self.assertEqual(
|
|
options.graph_optimization_level,
|
|
ort.GraphOptimizationLevel.ORT_ENABLE_BASIC,
|
|
)
|
|
|
|
def test_cpu_enrichment_models_disable_spinning(self):
|
|
for model_type in EnrichmentModelTypeEnum:
|
|
with self.subTest(model_type=model_type):
|
|
options = get_ort_session_options(model_type.value, cpu=True)
|
|
|
|
self.assertEqual(
|
|
options.get_session_config_entry("session.intra_op.allow_spinning"),
|
|
"0",
|
|
)
|
|
|
|
def test_gpu_enrichment_models_keep_spinning(self):
|
|
options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value)
|
|
|
|
with self.assertRaises(RuntimeError):
|
|
options.get_session_config_entry("session.intra_op.allow_spinning")
|
|
|
|
def test_arcface_keeps_default_optimization(self):
|
|
options = get_ort_session_options(EnrichmentModelTypeEnum.arcface.value)
|
|
|
|
self.assertEqual(
|
|
options.graph_optimization_level,
|
|
ort.GraphOptimizationLevel.ORT_ENABLE_ALL,
|
|
)
|
|
|
|
def test_detector_models_use_defaults(self):
|
|
for model_type in [None, ModelTypeEnum.rfdetr.value]:
|
|
with self.subTest(model_type=model_type):
|
|
self.assertIsNone(get_ort_session_options(model_type, cpu=True))
|
|
|
|
|
|
class TestCudaGraphRunner(unittest.TestCase):
|
|
"""CUDA graph capture fails if the arena has to allocate during capture, so
|
|
the session is warmed up with capture disabled before the first real run."""
|
|
|
|
def setUp(self):
|
|
self.session = MagicMock()
|
|
self.session.get_outputs.return_value = [MagicMock(name="output")]
|
|
self.io_binding = self.session.io_binding.return_value
|
|
self.input = {"images": np.zeros((1, 3, 320, 320), np.float32)}
|
|
|
|
def _annotations(self) -> list[str | None]:
|
|
"""Graph annotation id passed with each run, None when unset."""
|
|
annotations = []
|
|
|
|
for call in self.session.run_with_iobinding.call_args_list:
|
|
try:
|
|
annotations.append(call.args[1].get_run_config_entry("gpu_graph_id"))
|
|
except RuntimeError:
|
|
annotations.append(None)
|
|
|
|
return annotations
|
|
|
|
def test_first_run_warms_up_with_capture_disabled(self):
|
|
with patch.object(ort.OrtValue, "ortvalue_from_numpy"):
|
|
CudaGraphRunner(self.session, 0).run(self.input)
|
|
|
|
self.assertEqual(
|
|
self._annotations(),
|
|
["-1"] * CudaGraphRunner.GRAPH_FREE_WARMUP_RUNS + [None],
|
|
)
|
|
|
|
def test_later_runs_allow_capture(self):
|
|
with patch.object(ort.OrtValue, "ortvalue_from_numpy"):
|
|
runner = CudaGraphRunner(self.session, 0)
|
|
runner.run(self.input)
|
|
self.session.run_with_iobinding.reset_mock()
|
|
runner.run(self.input)
|
|
|
|
self.assertEqual(self._annotations(), [None])
|
|
runner._input_ortvalue.update_inplace.assert_called_once()
|