mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-19 03:53:03 +03:00
Fix ONNX Runtime CPU memory arena configuration (#25003)
* Fix CPU memory arena option on ONNX Runtime sessions * Limit CPU arena correction to CPU enrichment sessions * Retain default arena in GPU runner integration regression * Preserve native detector session-options contract
This commit is contained in:
@@ -153,6 +153,8 @@ def get_ort_session_options(
|
|||||||
sess_options = ort.SessionOptions()
|
sess_options = ort.SessionOptions()
|
||||||
|
|
||||||
if cpu:
|
if cpu:
|
||||||
|
# ORT reads the CPU arena flag from session options, not provider options.
|
||||||
|
sess_options.enable_cpu_mem_arena = False
|
||||||
# idle intra-op threads otherwise spin for ~1s after every run, keeping
|
# idle intra-op threads otherwise spin for ~1s after every run, keeping
|
||||||
# most cores busy while an enrichment runs even once a second
|
# most cores busy while an enrichment runs even once a second
|
||||||
sess_options.add_session_config_entry("session.intra_op.allow_spinning", "0")
|
sess_options.add_session_config_entry("session.intra_op.allow_spinning", "0")
|
||||||
|
|||||||
@@ -59,10 +59,18 @@ class TestGetOrtSessionOptions(unittest.TestCase):
|
|||||||
ort.GraphOptimizationLevel.ORT_ENABLE_ALL,
|
ort.GraphOptimizationLevel.ORT_ENABLE_ALL,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_only_cpu_enrichment_models_disable_the_cpu_arena(self):
|
||||||
|
for model_type in EnrichmentModelTypeEnum:
|
||||||
|
for cpu in [False, True]:
|
||||||
|
with self.subTest(model_type=model_type, cpu=cpu):
|
||||||
|
options = get_ort_session_options(model_type.value, cpu=cpu)
|
||||||
|
self.assertEqual(options.enable_cpu_mem_arena, not cpu)
|
||||||
|
|
||||||
def test_detector_models_use_defaults(self):
|
def test_detector_models_use_defaults(self):
|
||||||
for model_type in [None, ModelTypeEnum.rfdetr.value]:
|
for model_type in [None, ModelTypeEnum.rfdetr.value]:
|
||||||
with self.subTest(model_type=model_type):
|
for cpu in [False, True]:
|
||||||
self.assertIsNone(get_ort_session_options(model_type, cpu=True))
|
with self.subTest(model_type=model_type, cpu=cpu):
|
||||||
|
self.assertIsNone(get_ort_session_options(model_type, cpu=cpu))
|
||||||
|
|
||||||
|
|
||||||
class TestCudaGraphRunner(unittest.TestCase):
|
class TestCudaGraphRunner(unittest.TestCase):
|
||||||
|
|||||||
@@ -140,6 +140,9 @@ class TestLoadRegistry(unittest.TestCase):
|
|||||||
spinning = None
|
spinning = None
|
||||||
|
|
||||||
self.assertEqual(spinning, expected)
|
self.assertEqual(spinning, expected)
|
||||||
|
self.assertEqual(
|
||||||
|
options.enable_cpu_mem_arena, provider != "CPUExecutionProvider"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestLoadedDeviceSnapshot(unittest.TestCase):
|
class TestLoadedDeviceSnapshot(unittest.TestCase):
|
||||||
@@ -560,4 +563,4 @@ class TestMIGraphX(unittest.TestCase):
|
|||||||
providers, options = model_util.get_ort_providers()
|
providers, options = model_util.get_ort_providers()
|
||||||
|
|
||||||
self.assertEqual(providers, ["CPUExecutionProvider"])
|
self.assertEqual(providers, ["CPUExecutionProvider"])
|
||||||
self.assertEqual(options, [{"enable_cpu_mem_arena": False}])
|
self.assertEqual(options, [{}])
|
||||||
|
|||||||
+2
-13
@@ -320,14 +320,7 @@ def get_ort_providers(
|
|||||||
device: str | None = "AUTO",
|
device: str | None = "AUTO",
|
||||||
) -> tuple[list[str], list[dict[str, Any]]]:
|
) -> tuple[list[str], list[dict[str, Any]]]:
|
||||||
if force_cpu:
|
if force_cpu:
|
||||||
return (
|
return ["CPUExecutionProvider"], [{}]
|
||||||
["CPUExecutionProvider"],
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"enable_cpu_mem_arena": False,
|
|
||||||
}
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
providers = []
|
providers = []
|
||||||
options = []
|
options = []
|
||||||
@@ -389,11 +382,7 @@ def get_ort_providers(
|
|||||||
continue
|
continue
|
||||||
elif provider == "CPUExecutionProvider":
|
elif provider == "CPUExecutionProvider":
|
||||||
providers.append(provider)
|
providers.append(provider)
|
||||||
options.append(
|
options.append({})
|
||||||
{
|
|
||||||
"enable_cpu_mem_arena": False,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
elif provider == "AzureExecutionProvider":
|
elif provider == "AzureExecutionProvider":
|
||||||
# Skip Azure provider - not typically available on local hardware
|
# Skip Azure provider - not typically available on local hardware
|
||||||
# and prevents fallback to OpenVINO when it's the first provider
|
# and prevents fallback to OpenVINO when it's the first provider
|
||||||
|
|||||||
Reference in New Issue
Block a user