mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-17 19:13:02 +03:00
Apple Silicon Macs via lighter: ONNX detector on the Neural Engine and media engine decode (#24453)
* Run ONNX models on a Mac's Neural Engine through lighter's plugin provider lighter's lighter.sh/ane device places an ONNX Runtime plugin execution provider in the container. When it is present, the ONNX session setup registers it once and opens sessions on its Neural Engine device, the same place CUDA, ROCm and OpenVINO are chosen, so the onnx detector (and any model that is not pinned to the CPU) runs there with no configuration. The hardware probe reports it as an onnx unit. * docs: hardware decode on an Apple Silicon Mac under lighter A community section on the video decoding page: lighter's lighter.sh/video device, hwaccel_args -c:v h264_v4l2m2m, and why the Raspberry Pi presets decode a single-stream camera in software. The detector docs link to it. * docs: set the lighter decoder per camera when codecs are mixed * docs: the ONNX detector on a Mac's Neural Engine under lighter * Format the Neural Engine provider setup * Fall back to the default providers when the Neural Engine cannot load a model * Apple Silicon ffmpeg presets for lighter's media engine, recommended when it is present
This commit is contained in:
@@ -25,7 +25,7 @@ class HardwareProbeTestCase(unittest.TestCase):
|
||||
self.root = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.root.cleanup)
|
||||
|
||||
for name in ("SYS_ROOT", "DEV_ROOT", "PROC_ROOT", "ETC_ROOT"):
|
||||
for name in ("SYS_ROOT", "DEV_ROOT", "PROC_ROOT", "ETC_ROOT", "LIB_ROOT"):
|
||||
sub = os.path.join(self.root.name, name.split("_")[0].lower())
|
||||
os.makedirs(sub, exist_ok=True)
|
||||
patcher = patch.object(hardware, name, sub)
|
||||
@@ -167,6 +167,29 @@ class TestNvidia(HardwareProbeTestCase):
|
||||
|
||||
|
||||
class TestAccelerators(HardwareProbeTestCase):
|
||||
def test_the_neural_engine_is_found_by_lighters_provider_library(self):
|
||||
write(os.path.join(self.lib_root, "lighter", "liblighter_ane_ep.so"))
|
||||
|
||||
with patch.dict(os.environ, clear=False) as env:
|
||||
env.pop("LIGHTER_ANE_EP", None)
|
||||
ane = self.probe()["onnx:lighter"]
|
||||
|
||||
self.assertEqual(ane.detector, "onnx")
|
||||
self.assertEqual(ane.units[0].device, "onnx")
|
||||
self.assertEqual(ane.units[0].label, "Neural Engine")
|
||||
|
||||
def test_the_neural_engine_is_found_where_lighter_ane_ep_points(self):
|
||||
library = os.path.join(self.root.name, "elsewhere", "liblighter_ane_ep.so")
|
||||
write(library)
|
||||
|
||||
with patch.dict(os.environ, {"LIGHTER_ANE_EP": library}):
|
||||
self.assertIn("onnx:lighter", self.probe())
|
||||
|
||||
def test_no_neural_engine_is_reported_without_the_library(self):
|
||||
with patch.dict(os.environ, clear=False) as env:
|
||||
env.pop("LIGHTER_ANE_EP", None)
|
||||
self.assertNotIn("onnx:lighter", self.probe())
|
||||
|
||||
def test_hailo_is_found_by_its_device_node(self):
|
||||
write(os.path.join(self.dev_root, "hailo0"))
|
||||
|
||||
|
||||
@@ -51,6 +51,16 @@ class TestFfmpegPresets(unittest.TestCase):
|
||||
" ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"])
|
||||
)
|
||||
|
||||
def test_ffmpeg_hwaccel_apple_preset_decodes_every_stream(self):
|
||||
self.default_ffmpeg["cameras"]["back"]["ffmpeg"]["hwaccel_args"] = (
|
||||
"preset-apple-silicon-h265"
|
||||
)
|
||||
frigate_config = FrigateConfig(**self.default_ffmpeg)
|
||||
cmd = " ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"])
|
||||
assert "-c:v hevc_v4l2m2m" in cmd
|
||||
assert "-c:v:1" not in cmd
|
||||
assert "scale=1920:1080" in cmd
|
||||
|
||||
def test_ffmpeg_hwaccel_not_preset(self):
|
||||
self.default_ffmpeg["cameras"]["back"]["ffmpeg"]["hwaccel_args"] = (
|
||||
"-other-hwaccel args"
|
||||
|
||||
@@ -34,6 +34,12 @@ class HwaccelRecommendationTestCase(unittest.TestCase):
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
self.sys_root = os.path.join(self.root.name, "sys")
|
||||
os.makedirs(self.sys_root)
|
||||
patcher = patch.object(hwaccel, "SYS_ROOT", self.sys_root)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
drm = patch.object(hwaccel, "enumerate_drm_devices", return_value={})
|
||||
self.drm = drm.start()
|
||||
self.addCleanup(drm.stop)
|
||||
@@ -66,6 +72,12 @@ class HwaccelRecommendationTestCase(unittest.TestCase):
|
||||
with open(os.path.join(self.proc_root, "cpuinfo"), "w") as f:
|
||||
f.write(f"processor\t: 0\nmodel name\t: {model_name}\n")
|
||||
|
||||
def write_video_device(self, vendor: str) -> None:
|
||||
device = os.path.join(self.sys_root, "class", "video4linux", "video0", "device")
|
||||
os.makedirs(device)
|
||||
with open(os.path.join(device, "vendor"), "w") as f:
|
||||
f.write(f"{vendor}\n")
|
||||
|
||||
def write_device_tree(self) -> None:
|
||||
os.makedirs(os.path.join(self.proc_root, "device-tree"), exist_ok=True)
|
||||
with open(os.path.join(self.proc_root, "device-tree", "compatible"), "w") as f:
|
||||
@@ -191,6 +203,22 @@ class TestAvailableFamilies(HwaccelRecommendationTestCase):
|
||||
self.assertIn(recommended, [family.key for family in families])
|
||||
|
||||
|
||||
class TestLighter(HwaccelRecommendationTestCase):
|
||||
def test_lighters_media_engine_is_recommended(self):
|
||||
self.write_video_device(hwaccel.LIGHTER_VIRTIO_VENDOR)
|
||||
|
||||
self.assertEqual(self.recommend(codecs={"h264"}), "apple-silicon")
|
||||
self.assertEqual(
|
||||
self.presets()["apple-silicon"],
|
||||
{"h264": "preset-apple-silicon-h264", "h265": "preset-apple-silicon-h265"},
|
||||
)
|
||||
|
||||
def test_another_virtio_media_device_is_not_lighters(self):
|
||||
self.write_video_device("0x554d4551")
|
||||
|
||||
self.assertEqual(self.available(), [])
|
||||
|
||||
|
||||
class TestCodecCoverage(HwaccelRecommendationTestCase):
|
||||
def test_a_family_carries_a_preset_per_codec(self):
|
||||
self.assertEqual(
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Tests for the device each model runner reports after loading."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -38,6 +40,7 @@ class TestRunnerDeviceName(unittest.TestCase):
|
||||
self._onnx(["OpenVINOExecutionProvider"]).device_name, "OpenVINO"
|
||||
)
|
||||
self.assertEqual(self._onnx(["CPUExecutionProvider"]).device_name, "CPU")
|
||||
self.assertEqual(self._onnx(["LighterANE"]).device_name, "Neural Engine")
|
||||
self.assertEqual(self._onnx(["ROCMExecutionProvider"]).device_name, "ROCM")
|
||||
self.assertEqual(self._onnx([]).device_name, "CPU")
|
||||
|
||||
@@ -129,3 +132,147 @@ class TestLoadedDeviceSnapshot(unittest.TestCase):
|
||||
finally:
|
||||
stop.set()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
class TestLighterANE(unittest.TestCase):
|
||||
"""lighter's plugin provider, picked up by the ONNX session setup."""
|
||||
|
||||
def setUp(self):
|
||||
loaded_devices.clear()
|
||||
self.root = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.root.cleanup)
|
||||
self.library = os.path.join(self.root.name, "liblighter_ane_ep.so")
|
||||
env = patch.dict(os.environ, {"LIGHTER_ANE_EP": self.library})
|
||||
env.start()
|
||||
self.addCleanup(env.stop)
|
||||
|
||||
def _device(self) -> MagicMock:
|
||||
device = MagicMock()
|
||||
device.ep_name = "LighterANE"
|
||||
return device
|
||||
|
||||
def test_no_devices_without_the_library(self):
|
||||
with patch.object(detection_runners.ort, "get_ep_devices") as get_ep_devices:
|
||||
self.assertEqual(detection_runners.get_lighter_ane_devices(), [])
|
||||
get_ep_devices.assert_not_called()
|
||||
|
||||
def test_the_provider_is_registered_once(self):
|
||||
open(self.library, "w").close()
|
||||
device = self._device()
|
||||
other = MagicMock()
|
||||
other.ep_name = "CPUExecutionProvider"
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
detection_runners.ort,
|
||||
"get_ep_devices",
|
||||
side_effect=[[other], [other, device], [other, device]],
|
||||
),
|
||||
patch.object(
|
||||
detection_runners.ort, "register_execution_provider_library"
|
||||
) as register,
|
||||
):
|
||||
self.assertEqual(detection_runners.get_lighter_ane_devices(), [device])
|
||||
self.assertEqual(detection_runners.get_lighter_ane_devices(), [device])
|
||||
|
||||
register.assert_called_once_with("LighterANE", self.library)
|
||||
|
||||
def test_a_provider_that_will_not_load_is_skipped(self):
|
||||
open(self.library, "w").close()
|
||||
|
||||
with (
|
||||
patch.object(detection_runners.ort, "get_ep_devices", return_value=[]),
|
||||
patch.object(
|
||||
detection_runners.ort,
|
||||
"register_execution_provider_library",
|
||||
side_effect=RuntimeError("not a provider"),
|
||||
),
|
||||
):
|
||||
self.assertEqual(detection_runners.get_lighter_ane_devices(), [])
|
||||
|
||||
def test_the_session_runs_on_the_neural_engine(self):
|
||||
device = self._device()
|
||||
session = MagicMock()
|
||||
session.get_providers.return_value = ["LighterANE", "CPUExecutionProvider"]
|
||||
options = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
|
||||
patch.object(
|
||||
detection_runners, "get_lighter_ane_devices", return_value=[device]
|
||||
),
|
||||
patch.object(
|
||||
detection_runners, "get_ort_session_options", return_value=options
|
||||
),
|
||||
patch.object(
|
||||
detection_runners.ort, "InferenceSession", return_value=session
|
||||
) as inference_session,
|
||||
):
|
||||
runner = get_optimized_runner("/models/yolo.onnx", "AUTO", "yolo-generic")
|
||||
|
||||
options.add_provider_for_devices.assert_called_once_with([device], {})
|
||||
inference_session.assert_called_once_with(
|
||||
"/models/yolo.onnx", sess_options=options
|
||||
)
|
||||
self.assertIsInstance(runner, ONNXModelRunner)
|
||||
self.assertEqual(
|
||||
loaded_devices["/models/yolo.onnx"], ("yolo-generic", "Neural Engine")
|
||||
)
|
||||
|
||||
def test_a_model_the_neural_engine_cannot_load_uses_the_default_providers(self):
|
||||
session = MagicMock()
|
||||
session.get_providers.return_value = ["CPUExecutionProvider"]
|
||||
|
||||
with (
|
||||
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
|
||||
patch.object(
|
||||
detection_runners, "get_lighter_ane_devices", return_value=[MagicMock()]
|
||||
),
|
||||
patch.object(
|
||||
detection_runners,
|
||||
"get_ort_providers",
|
||||
return_value=(["CPUExecutionProvider"], [{}]),
|
||||
),
|
||||
patch.object(
|
||||
detection_runners, "is_openvino_gpu_npu_available", return_value=False
|
||||
),
|
||||
patch.object(
|
||||
detection_runners.ort,
|
||||
"InferenceSession",
|
||||
side_effect=[RuntimeError("unsupported"), session],
|
||||
) as inference_session,
|
||||
patch.object(
|
||||
detection_runners, "get_ort_session_options", return_value=MagicMock()
|
||||
),
|
||||
self.assertLogs(detection_runners.logger, level="WARNING"),
|
||||
):
|
||||
runner = get_optimized_runner("/models/jina.onnx", "AUTO", "jina-v2")
|
||||
|
||||
self.assertEqual(inference_session.call_count, 2)
|
||||
self.assertIsInstance(runner, ONNXModelRunner)
|
||||
self.assertEqual(loaded_devices["/models/jina.onnx"], ("jina-v2", "CPU"))
|
||||
|
||||
def test_a_cpu_model_stays_on_the_cpu(self):
|
||||
session = MagicMock()
|
||||
session.get_providers.return_value = ["CPUExecutionProvider"]
|
||||
|
||||
with (
|
||||
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
|
||||
patch.object(
|
||||
detection_runners, "get_lighter_ane_devices", return_value=[MagicMock()]
|
||||
) as ane,
|
||||
patch.object(
|
||||
detection_runners,
|
||||
"get_ort_providers",
|
||||
return_value=(["CPUExecutionProvider"], [{}]),
|
||||
),
|
||||
patch.object(
|
||||
detection_runners.ort, "InferenceSession", return_value=session
|
||||
),
|
||||
patch.object(
|
||||
detection_runners, "get_ort_session_options", return_value=None
|
||||
),
|
||||
):
|
||||
get_optimized_runner("/models/arcface.onnx", "CPU", "arcface")
|
||||
|
||||
ane.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user