diff --git a/docs/docs/configuration/ffmpeg_presets.md b/docs/docs/configuration/ffmpeg_presets.md index 50236d0a97..2cd1698d95 100644 --- a/docs/docs/configuration/ffmpeg_presets.md +++ b/docs/docs/configuration/ffmpeg_presets.md @@ -17,17 +17,19 @@ Hardware acceleration arguments tell FFmpeg to decode your camera's video stream See [the hardware acceleration docs](/configuration/hardware_acceleration_video.md) for details on setting up hardware acceleration for your GPU / iGPU, then select the preset that matches your hardware. -| Preset (YAML config) | UI Label | Usage | Notes | -| --------------------- | ----------------------- | --------------------------------- | --------------------------------------------------------------- | -| preset-rpi-64-h264 | Raspberry Pi (H.264) | 64-bit Raspberry Pi, H.264 stream | | -| preset-rpi-64-h265 | Raspberry Pi (H.265) | 64-bit Raspberry Pi, H.265 stream | | -| preset-vaapi | VAAPI (Intel/AMD GPU) | Intel or AMD GPU via VAAPI | Check the hwaccel docs to ensure the correct driver is selected | -| preset-intel-qsv-h264 | Intel QuickSync (H.264) | Intel QuickSync, H.264 stream | If you have issues, use the VAAPI preset instead | -| preset-intel-qsv-h265 | Intel QuickSync (H.265) | Intel QuickSync, H.265 stream | If you have issues, use the VAAPI preset instead | -| preset-nvidia | NVIDIA GPU | NVIDIA GPU | | -| preset-jetson-h264 | NVIDIA Jetson (H.264) | NVIDIA Jetson, H.264 stream | | -| preset-jetson-h265 | NVIDIA Jetson (H.265) | NVIDIA Jetson, H.265 stream | | -| preset-rkmpp | Rockchip RKMPP | Rockchip MPP | Use an image with the `-rk` suffix and run in privileged mode | +| Preset (YAML config) | UI Label | Usage | Notes | +| ------------------------- | ----------------------- | --------------------------------------------- | --------------------------------------------------------------- | +| preset-rpi-64-h264 | Raspberry Pi (H.264) | 64-bit Raspberry Pi, H.264 stream | | +| preset-rpi-64-h265 | Raspberry Pi (H.265) | 64-bit Raspberry Pi, H.265 stream | | +| preset-apple-silicon-h264 | Apple Silicon (H.264) | Apple Silicon Mac under lighter, H.264 stream | Needs the `lighter.sh/video` device | +| preset-apple-silicon-h265 | Apple Silicon (H.265) | Apple Silicon Mac under lighter, H.265 stream | Needs the `lighter.sh/video` device | +| preset-vaapi | VAAPI (Intel/AMD GPU) | Intel or AMD GPU via VAAPI | Check the hwaccel docs to ensure the correct driver is selected | +| preset-intel-qsv-h264 | Intel QuickSync (H.264) | Intel QuickSync, H.264 stream | If you have issues, use the VAAPI preset instead | +| preset-intel-qsv-h265 | Intel QuickSync (H.265) | Intel QuickSync, H.265 stream | If you have issues, use the VAAPI preset instead | +| preset-nvidia | NVIDIA GPU | NVIDIA GPU | | +| preset-jetson-h264 | NVIDIA Jetson (H.264) | NVIDIA Jetson, H.264 stream | | +| preset-jetson-h265 | NVIDIA Jetson (H.265) | NVIDIA Jetson, H.265 stream | | +| preset-rkmpp | Rockchip RKMPP | Rockchip MPP | Use an image with the `-rk` suffix and run in privileged mode | diff --git a/docs/docs/configuration/hardware_acceleration_video.md b/docs/docs/configuration/hardware_acceleration_video.md index 80ab341479..49f8b46537 100644 --- a/docs/docs/configuration/hardware_acceleration_video.md +++ b/docs/docs/configuration/hardware_acceleration_video.md @@ -43,6 +43,10 @@ Frigate supports presets for optimal hardware accelerated video decoding: - [RKNN](#rockchip-platform): Frigate can utilize the media engine in RockChip SOCs to accelerate video decoding. +**Apple Silicon Mac** + +- [lighter](#apple-silicon-mac-lighter): Frigate can utilize the media engine in Apple Silicon Macs to accelerate video decoding, when running under the lighter container runtime. + **Other Hardware** Depending on your system, these presets may not be compatible, and you may need to use manual hwaccel args to take advantage of your hardware. More information on hardware accelerated decoding for ffmpeg can be found here: https://trac.ffmpeg.org/wiki/HWAccelIntro @@ -533,3 +537,35 @@ output_args: Make sure that your SoC supports hardware acceleration for your input stream and your input stream is h264 encoding. For example, if your camera streams with h264 encoding, your SoC must be able to de- and encode with it. If you are unsure whether your SoC meets the requirements, take a look at the datasheet. ::: + +## Apple Silicon Mac (lighter) + +[lighter](https://github.com/fieldwork-ai/lighter) is an open-source container runtime for macOS. It gives a container the Mac's media engine as a standard V4L2 decoder, backed by VideoToolbox, so Frigate decodes H.264 and H.265 streams in hardware with the ffmpeg it already ships. It works on M1 and newer Macs with lighter 0.9.2 or newer. + +Give the container the video device. With Docker Compose: + +```yaml {4-5} +services: + frigate: + ... + devices: + - lighter.sh/video=all +``` + +Or with `docker run`, add `--device lighter.sh/video=all`. + +Then set the preset for the codec your cameras stream. The decoder is specific to the codec, so if your cameras mix H.264 and H.265, set the preset for the most common codec globally and override it on the other cameras: + +```yaml +ffmpeg: + hwaccel_args: preset-apple-silicon-h264 + +cameras: + garage: # an H.265 camera + ffmpeg: + hwaccel_args: preset-apple-silicon-h265 +``` + +The presets decode on the media engine and encode the Birdseye restream and timelapses there too. Scaling to the detect resolution runs on the CPU, as ffmpeg's V4L2 decoders cannot scale. + +lighter can also run object detection on the Mac's Neural Engine; see [Apple Neural Engine (lighter)](object_detectors.md#apple-neural-engine-lighter). diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 27f03ca1f4..e2a6b9e47b 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -34,6 +34,7 @@ Frigate supports multiple different detectors that work on different types of ha **Apple Silicon** - [Apple Silicon](#apple-silicon-detector): Apple Silicon can run on M1 and newer Apple Silicon devices. +- [ONNX](#apple-neural-engine-lighter): the ONNX detector runs on the Neural Engine of M1 and newer Macs when Frigate runs under the lighter container runtime. **Intel** @@ -484,7 +485,7 @@ See [ONNX supported models](#onnx) for supported models, there are some caveats: ## ONNX -ONNX is an open format for building machine learning models, Frigate supports running ONNX models on CPU, OpenVINO, ROCm, and TensorRT. On startup Frigate will automatically try to use a GPU if one is available. +ONNX is an open format for building machine learning models, Frigate supports running ONNX models on CPU, OpenVINO, ROCm, TensorRT, and a Mac's Neural Engine. On startup Frigate will automatically try to use a GPU if one is available. :::info @@ -500,6 +501,9 @@ If the correct build is used for your GPU then the GPU will be detected and used - Nvidia GPUs will automatically be detected and used with the ONNX detector in the `-tensorrt` Frigate image. - Jetson devices will automatically be detected and used with the ONNX detector in the `-tensorrt-jp6` Frigate image. +- **Apple Silicon Mac** + - The Neural Engine will automatically be detected and used with the ONNX detector when Frigate runs under lighter with its Neural Engine device. See [Apple Neural Engine (lighter)](#apple-neural-engine-lighter). + ::: :::tip @@ -515,6 +519,22 @@ models: ::: +### Apple Neural Engine (lighter) {#apple-neural-engine-lighter} + +[lighter](https://github.com/fieldwork-ai/lighter) is an open-source container runtime for macOS. A container started with its `lighter.sh/ane` device gets an ONNX Runtime execution provider that runs models on the Mac's Neural Engine, and the ONNX detector uses it automatically, with the same models and configuration as on any other hardware. It works on M1 and newer Macs with lighter 0.9.2 or newer. + +Give the Frigate container the Neural Engine device. With Docker Compose: + +```yaml +services: + frigate: + image: ghcr.io/blakeblackshear/frigate:stable-standard-arm64 + devices: + - lighter.sh/ane=all +``` + +Or with `docker run`, add `--device lighter.sh/ane=all`. Frigate then reports the Neural Engine under **Settings > System > Detection models**, and the ONNX detector's model loads on it. lighter can also decode camera streams on the Mac's media engine; see [Video Decoding](hardware_acceleration_video.md#apple-silicon-mac-lighter). + ### Configuration {#configuration-onnx} diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 2bc95938d4..a4b106c0a3 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -78,6 +78,10 @@ Frigate supports multiple different detectors that work on different types of ha **Apple Silicon** +- [ONNX via lighter](#apple-silicon): The ONNX detector runs on the Neural Engine of M1 and newer Macs when Frigate runs in the lighter container runtime + - [Supports the same model architectures as the ONNX detector](../../configuration/object_detectors#apple-neural-engine-lighter) + - Runs inside the Frigate container, with no separate detector process to set up + - The recommended way to run Frigate on a Mac - [Apple Silicon](#apple-silicon): Apple Silicon is usable on all M1 and newer Apple Silicon devices to provide efficient and fast object detection - [Supports primarily ssdlite and mobilenet model architectures](../../configuration/object_detectors#apple-silicon-detector) - Runs well with any size models including large @@ -211,7 +215,13 @@ Inference is done with the `onnx` detector type. Speeds will vary greatly depend ### Apple Silicon -With the [Apple Silicon](../configuration/object_detectors.md#apple-silicon-detector) detector Frigate can take advantage of the NPU in M1 and newer Apple Silicon. +Frigate on a Mac is best run in the [lighter](https://github.com/fieldwork-ai/lighter) container runtime, where the [ONNX detector](../configuration/object_detectors.md#apple-neural-engine-lighter) runs on the Neural Engine of M1 and newer Macs from inside the Frigate container. There is no separate detector process to install or keep running, and the same container can decode video on the Mac's media engine. + +| Name | YOLOv9 Inference Time | YOLO-NAS Inference Time | RF-DETR Inference Time | +| ---- | -------------------------------------- | ----------------------- | ---------------------- | +| M1 | t-320: 3.3 ms s-320: 7 ms s-640: 13 ms | 320: 6.6 ms | Nano-320: 38 ms | + +Alternatively, with the [Apple Silicon](../configuration/object_detectors.md#apple-silicon-detector) detector Frigate can take advantage of the NPU in M1 and newer Apple Silicon. :::warning diff --git a/frigate/api/app.py b/frigate/api/app.py index 4078ebf082..62676bbdef 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -429,6 +429,8 @@ def ffmpeg_presets(): hwaccel_presets = [ "preset-rpi-64-h264", "preset-rpi-64-h265", + "preset-apple-silicon-h264", + "preset-apple-silicon-h265", "preset-jetson-h264", "preset-jetson-h265", "preset-rkmpp", diff --git a/frigate/detectors/detection_runners.py b/frigate/detectors/detection_runners.py index 5b5a4cba98..87846c698c 100644 --- a/frigate/detectors/detection_runners.py +++ b/frigate/detectors/detection_runners.py @@ -46,8 +46,42 @@ _PROVIDER_LABELS = { "MIGraphXExecutionProvider": "MIGraphX", "OpenVINOExecutionProvider": "OpenVINO", "CPUExecutionProvider": "CPU", + "LighterANE": "Neural Engine", } +# lighter (https://github.com/fieldwork-ai/lighter) places an ONNX Runtime plugin +# execution provider in a container started with --device lighter.sh/ane=all, +# which runs models on a Mac's Neural Engine; LIGHTER_ANE_EP names where it is +LIGHTER_ANE_EP_NAME = "LighterANE" +LIGHTER_ANE_LIBRARY = "/usr/lib/lighter/liblighter_ane_ep.so" + + +def get_lighter_ane_devices() -> list[Any]: + """Get the Neural Engine devices lighter's provider offers, registering it once. + + Returns: + The provider's ONNX Runtime devices, or an empty list without lighter's device + """ + library = os.environ.get("LIGHTER_ANE_EP", LIGHTER_ANE_LIBRARY) + + if not os.path.exists(library): + return [] + + devices = [d for d in ort.get_ep_devices() if d.ep_name == LIGHTER_ANE_EP_NAME] + + if not devices: + try: + ort.register_execution_provider_library(LIGHTER_ANE_EP_NAME, library) + except Exception as e: + logger.warning( + f"Failed to load the Neural Engine provider from {library}: {e}" + ) + return [] + + devices = [d for d in ort.get_ep_devices() if d.ep_name == LIGHTER_ANE_EP_NAME] + + return devices + def is_arm64_platform() -> bool: """Check if we're running on an ARM platform.""" @@ -689,6 +723,23 @@ def get_optimized_runner( if rknn_path: return _record_runner(model_path, model_type, RKNNModelRunner(rknn_path)) + if device != "CPU" and (ane_devices := get_lighter_ane_devices()): + sess_options = get_ort_session_options(model_type) or ort.SessionOptions() + sess_options.add_provider_for_devices(ane_devices, {}) + + try: + session = ort.InferenceSession(model_path, sess_options=sess_options) + except Exception as e: + logger.warning( + f"Failed to load {model_path} on the Neural Engine, using the default providers: {e}" + ) + else: + return _record_runner( + model_path, + model_type, + ONNXModelRunner(session, model_type=model_type), + ) + providers, options = get_ort_providers(device == "CPU", device, **kwargs) if providers[0] == "CPUExecutionProvider": diff --git a/frigate/detectors/hardware.py b/frigate/detectors/hardware.py index 1eb1111a76..10becfb219 100644 --- a/frigate/detectors/hardware.py +++ b/frigate/detectors/hardware.py @@ -25,6 +25,7 @@ SYS_ROOT = "/sys" DEV_ROOT = "/dev" PROC_ROOT = "/proc" ETC_ROOT = "/etc" +LIB_ROOT = "/usr/lib" # a Coral reports as Global Unichip until its firmware is loaded, then as Google CORAL_USB_IDS = {("1a6e", "089a"), ("18d1", "9302")} @@ -317,6 +318,19 @@ def detect_synaptics() -> DetectionHardware | None: return _hardware("synaptics", "synaptics", "Synaptics NPU", units) +def detect_lighter_ane() -> DetectionHardware | None: + """Find a Mac's Neural Engine by the provider library lighter's device places.""" + library = os.environ.get( + "LIGHTER_ANE_EP", f"{LIB_ROOT}/lighter/liblighter_ane_ep.so" + ) + if not os.path.exists(library): + return None + + # runs through onnx, whose session picks lighter's provider when it is present + units = [HardwareUnit(device="onnx", label="Neural Engine")] + return _hardware("onnx:lighter", "onnx", "Apple Neural Engine", units) + + def detect_cpu() -> DetectionHardware: """The CPU, which is always available.""" units = [HardwareUnit(device="cpu", label="CPU")] @@ -338,6 +352,7 @@ PROBES = ( detect_rockchip, detect_axengine, detect_synaptics, + detect_lighter_ane, detect_cpu, ) diff --git a/frigate/ffmpeg_presets.py b/frigate/ffmpeg_presets.py index 445cfd4df3..152ed28508 100644 --- a/frigate/ffmpeg_presets.py +++ b/frigate/ffmpeg_presets.py @@ -84,6 +84,8 @@ _user_agent_args = [ PRESETS_HW_ACCEL_DECODE = { "preset-rpi-64-h264": "-c:v:1 h264_v4l2m2m", "preset-rpi-64-h265": "-c:v:1 hevc_v4l2m2m", + "preset-apple-silicon-h264": "-c:v h264_v4l2m2m", + "preset-apple-silicon-h265": "-c:v hevc_v4l2m2m", FFMPEG_HWACCEL_VAAPI: "-hwaccel_flags allow_profile_mismatch -hwaccel vaapi -hwaccel_device {3} -hwaccel_output_format vaapi", "preset-intel-qsv-h264": f"-hwaccel qsv -qsv_device {{3}} -hwaccel_output_format qsv -c:v h264_qsv{' -bsf:v dump_extra' if LIBAVFORMAT_VERSION_MAJOR >= 61 else ''}", # https://trac.ffmpeg.org/ticket/9766#comment:17 "preset-intel-qsv-h265": f"-load_plugin hevc_hw -hwaccel qsv -qsv_device {{3}} -hwaccel_output_format qsv{' -bsf:v dump_extra' if LIBAVFORMAT_VERSION_MAJOR >= 61 else ''}", # https://trac.ffmpeg.org/ticket/9766#comment:17 @@ -120,6 +122,9 @@ PRESETS_HW_ACCEL_DECODE["preset-rk-h265"] = PRESETS_HW_ACCEL_DECODE[ PRESETS_HW_ACCEL_SCALE = { "preset-rpi-64-h264": "-r {0} -vf fps={0},scale={1}:{2}", "preset-rpi-64-h265": "-r {0} -vf fps={0},scale={1}:{2}", + # ffmpeg's v4l2m2m decoders cannot scale, so frames are scaled on the CPU + "preset-apple-silicon-h264": "-r {0} -vf fps={0},scale={1}:{2}", + "preset-apple-silicon-h265": "-r {0} -vf fps={0},scale={1}:{2}", FFMPEG_HWACCEL_VAAPI: "-r {0} -vf fps={0},scale_vaapi=w={1}:h={2},hwdownload,format=nv12", "preset-intel-qsv-h264": "-r {0} -vf vpp_qsv=w={1}:h={2}:format=nv12,hwdownload,format=nv12,fps={0},format=yuv420p", "preset-intel-qsv-h265": "-r {0} -vf vpp_qsv=w={1}:h={2}:format=nv12,hwdownload,format=nv12,fps={0},format=yuv420p", @@ -150,6 +155,8 @@ PRESETS_HW_ACCEL_SCALE["preset-rk-h265"] = PRESETS_HW_ACCEL_SCALE[FFMPEG_HWACCEL PRESETS_HW_ACCEL_ENCODE_BIRDSEYE = { "preset-rpi-64-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m {2}", "preset-rpi-64-h265": "{0} -hide_banner {1} -c:v hevc_v4l2m2m {2}", + "preset-apple-silicon-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m {2}", + "preset-apple-silicon-h265": "{0} -hide_banner {1} -c:v h264_v4l2m2m {2}", # -vaapi_device is required in addition to -hwaccel_device: this is the only # birdseye preset that uses hwupload, and ffmpeg 8 initializes filters before # the decoder creates a device, so hwupload cannot see an -hwaccel_device one. @@ -184,6 +191,8 @@ PRESETS_HW_ACCEL_ENCODE_BIRDSEYE["preset-rk-h264"] = PRESETS_HW_ACCEL_ENCODE_BIR PRESETS_HW_ACCEL_ENCODE_TIMELAPSE = { "preset-rpi-64-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m -pix_fmt yuv420p {2}", "preset-rpi-64-h265": "{0} -hide_banner {1} -c:v hevc_v4l2m2m -pix_fmt yuv420p {2}", + "preset-apple-silicon-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m -pix_fmt yuv420p {2}", + "preset-apple-silicon-h265": "{0} -hide_banner {1} -c:v hevc_v4l2m2m -pix_fmt yuv420p {2}", FFMPEG_HWACCEL_VAAPI: "{0} -hide_banner -hwaccel vaapi -hwaccel_output_format vaapi -hwaccel_device {3} {1} -c:v h264_vaapi {2}", "preset-intel-qsv-h264": "{0} -hide_banner {1} -c:v h264_qsv -profile:v high -level:v 4.1 -async_depth:v 1 {2}", "preset-intel-qsv-h265": "{0} -hide_banner {1} -c:v hevc_qsv -profile:v main -level:v 4.1 -async_depth:v 1 {2}", diff --git a/frigate/test/test_detector_hardware.py b/frigate/test/test_detector_hardware.py index 888ff1ff69..c438cb8f4f 100644 --- a/frigate/test/test_detector_hardware.py +++ b/frigate/test/test_detector_hardware.py @@ -25,7 +25,7 @@ class HardwareProbeTestCase(unittest.TestCase): self.root = tempfile.TemporaryDirectory() self.addCleanup(self.root.cleanup) - for name in ("SYS_ROOT", "DEV_ROOT", "PROC_ROOT", "ETC_ROOT"): + for name in ("SYS_ROOT", "DEV_ROOT", "PROC_ROOT", "ETC_ROOT", "LIB_ROOT"): sub = os.path.join(self.root.name, name.split("_")[0].lower()) os.makedirs(sub, exist_ok=True) patcher = patch.object(hardware, name, sub) @@ -167,6 +167,29 @@ class TestNvidia(HardwareProbeTestCase): class TestAccelerators(HardwareProbeTestCase): + def test_the_neural_engine_is_found_by_lighters_provider_library(self): + write(os.path.join(self.lib_root, "lighter", "liblighter_ane_ep.so")) + + with patch.dict(os.environ, clear=False) as env: + env.pop("LIGHTER_ANE_EP", None) + ane = self.probe()["onnx:lighter"] + + self.assertEqual(ane.detector, "onnx") + self.assertEqual(ane.units[0].device, "onnx") + self.assertEqual(ane.units[0].label, "Neural Engine") + + def test_the_neural_engine_is_found_where_lighter_ane_ep_points(self): + library = os.path.join(self.root.name, "elsewhere", "liblighter_ane_ep.so") + write(library) + + with patch.dict(os.environ, {"LIGHTER_ANE_EP": library}): + self.assertIn("onnx:lighter", self.probe()) + + def test_no_neural_engine_is_reported_without_the_library(self): + with patch.dict(os.environ, clear=False) as env: + env.pop("LIGHTER_ANE_EP", None) + self.assertNotIn("onnx:lighter", self.probe()) + def test_hailo_is_found_by_its_device_node(self): write(os.path.join(self.dev_root, "hailo0")) diff --git a/frigate/test/test_ffmpeg_presets.py b/frigate/test/test_ffmpeg_presets.py index 86fdd5f3a6..3f5a3543b6 100644 --- a/frigate/test/test_ffmpeg_presets.py +++ b/frigate/test/test_ffmpeg_presets.py @@ -51,6 +51,16 @@ class TestFfmpegPresets(unittest.TestCase): " ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"]) ) + def test_ffmpeg_hwaccel_apple_preset_decodes_every_stream(self): + self.default_ffmpeg["cameras"]["back"]["ffmpeg"]["hwaccel_args"] = ( + "preset-apple-silicon-h265" + ) + frigate_config = FrigateConfig(**self.default_ffmpeg) + cmd = " ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"]) + assert "-c:v hevc_v4l2m2m" in cmd + assert "-c:v:1" not in cmd + assert "scale=1920:1080" in cmd + def test_ffmpeg_hwaccel_not_preset(self): self.default_ffmpeg["cameras"]["back"]["ffmpeg"]["hwaccel_args"] = ( "-other-hwaccel args" diff --git a/frigate/test/test_hwaccel_recommendation.py b/frigate/test/test_hwaccel_recommendation.py index e334296ad3..2a89971bf8 100644 --- a/frigate/test/test_hwaccel_recommendation.py +++ b/frigate/test/test_hwaccel_recommendation.py @@ -34,6 +34,12 @@ class HwaccelRecommendationTestCase(unittest.TestCase): patcher.start() self.addCleanup(patcher.stop) + self.sys_root = os.path.join(self.root.name, "sys") + os.makedirs(self.sys_root) + patcher = patch.object(hwaccel, "SYS_ROOT", self.sys_root) + patcher.start() + self.addCleanup(patcher.stop) + drm = patch.object(hwaccel, "enumerate_drm_devices", return_value={}) self.drm = drm.start() self.addCleanup(drm.stop) @@ -66,6 +72,12 @@ class HwaccelRecommendationTestCase(unittest.TestCase): with open(os.path.join(self.proc_root, "cpuinfo"), "w") as f: f.write(f"processor\t: 0\nmodel name\t: {model_name}\n") + def write_video_device(self, vendor: str) -> None: + device = os.path.join(self.sys_root, "class", "video4linux", "video0", "device") + os.makedirs(device) + with open(os.path.join(device, "vendor"), "w") as f: + f.write(f"{vendor}\n") + def write_device_tree(self) -> None: os.makedirs(os.path.join(self.proc_root, "device-tree"), exist_ok=True) with open(os.path.join(self.proc_root, "device-tree", "compatible"), "w") as f: @@ -191,6 +203,22 @@ class TestAvailableFamilies(HwaccelRecommendationTestCase): self.assertIn(recommended, [family.key for family in families]) +class TestLighter(HwaccelRecommendationTestCase): + def test_lighters_media_engine_is_recommended(self): + self.write_video_device(hwaccel.LIGHTER_VIRTIO_VENDOR) + + self.assertEqual(self.recommend(codecs={"h264"}), "apple-silicon") + self.assertEqual( + self.presets()["apple-silicon"], + {"h264": "preset-apple-silicon-h264", "h265": "preset-apple-silicon-h265"}, + ) + + def test_another_virtio_media_device_is_not_lighters(self): + self.write_video_device("0x554d4551") + + self.assertEqual(self.available(), []) + + class TestCodecCoverage(HwaccelRecommendationTestCase): def test_a_family_carries_a_preset_per_codec(self): self.assertEqual( diff --git a/frigate/test/test_runner_devices.py b/frigate/test/test_runner_devices.py index 8df7be5ef5..8fecb611c7 100644 --- a/frigate/test/test_runner_devices.py +++ b/frigate/test/test_runner_devices.py @@ -1,5 +1,7 @@ """Tests for the device each model runner reports after loading.""" +import os +import tempfile import threading import unittest from unittest.mock import MagicMock, patch @@ -38,6 +40,7 @@ class TestRunnerDeviceName(unittest.TestCase): self._onnx(["OpenVINOExecutionProvider"]).device_name, "OpenVINO" ) self.assertEqual(self._onnx(["CPUExecutionProvider"]).device_name, "CPU") + self.assertEqual(self._onnx(["LighterANE"]).device_name, "Neural Engine") self.assertEqual(self._onnx(["ROCMExecutionProvider"]).device_name, "ROCM") self.assertEqual(self._onnx([]).device_name, "CPU") @@ -129,3 +132,147 @@ class TestLoadedDeviceSnapshot(unittest.TestCase): finally: stop.set() thread.join(timeout=5) + + +class TestLighterANE(unittest.TestCase): + """lighter's plugin provider, picked up by the ONNX session setup.""" + + def setUp(self): + loaded_devices.clear() + self.root = tempfile.TemporaryDirectory() + self.addCleanup(self.root.cleanup) + self.library = os.path.join(self.root.name, "liblighter_ane_ep.so") + env = patch.dict(os.environ, {"LIGHTER_ANE_EP": self.library}) + env.start() + self.addCleanup(env.stop) + + def _device(self) -> MagicMock: + device = MagicMock() + device.ep_name = "LighterANE" + return device + + def test_no_devices_without_the_library(self): + with patch.object(detection_runners.ort, "get_ep_devices") as get_ep_devices: + self.assertEqual(detection_runners.get_lighter_ane_devices(), []) + get_ep_devices.assert_not_called() + + def test_the_provider_is_registered_once(self): + open(self.library, "w").close() + device = self._device() + other = MagicMock() + other.ep_name = "CPUExecutionProvider" + + with ( + patch.object( + detection_runners.ort, + "get_ep_devices", + side_effect=[[other], [other, device], [other, device]], + ), + patch.object( + detection_runners.ort, "register_execution_provider_library" + ) as register, + ): + self.assertEqual(detection_runners.get_lighter_ane_devices(), [device]) + self.assertEqual(detection_runners.get_lighter_ane_devices(), [device]) + + register.assert_called_once_with("LighterANE", self.library) + + def test_a_provider_that_will_not_load_is_skipped(self): + open(self.library, "w").close() + + with ( + patch.object(detection_runners.ort, "get_ep_devices", return_value=[]), + patch.object( + detection_runners.ort, + "register_execution_provider_library", + side_effect=RuntimeError("not a provider"), + ), + ): + self.assertEqual(detection_runners.get_lighter_ane_devices(), []) + + def test_the_session_runs_on_the_neural_engine(self): + device = self._device() + session = MagicMock() + session.get_providers.return_value = ["LighterANE", "CPUExecutionProvider"] + options = MagicMock() + + with ( + patch.object(detection_runners, "is_rknn_compatible", return_value=False), + patch.object( + detection_runners, "get_lighter_ane_devices", return_value=[device] + ), + patch.object( + detection_runners, "get_ort_session_options", return_value=options + ), + patch.object( + detection_runners.ort, "InferenceSession", return_value=session + ) as inference_session, + ): + runner = get_optimized_runner("/models/yolo.onnx", "AUTO", "yolo-generic") + + options.add_provider_for_devices.assert_called_once_with([device], {}) + inference_session.assert_called_once_with( + "/models/yolo.onnx", sess_options=options + ) + self.assertIsInstance(runner, ONNXModelRunner) + self.assertEqual( + loaded_devices["/models/yolo.onnx"], ("yolo-generic", "Neural Engine") + ) + + def test_a_model_the_neural_engine_cannot_load_uses_the_default_providers(self): + session = MagicMock() + session.get_providers.return_value = ["CPUExecutionProvider"] + + with ( + patch.object(detection_runners, "is_rknn_compatible", return_value=False), + patch.object( + detection_runners, "get_lighter_ane_devices", return_value=[MagicMock()] + ), + patch.object( + detection_runners, + "get_ort_providers", + return_value=(["CPUExecutionProvider"], [{}]), + ), + patch.object( + detection_runners, "is_openvino_gpu_npu_available", return_value=False + ), + patch.object( + detection_runners.ort, + "InferenceSession", + side_effect=[RuntimeError("unsupported"), session], + ) as inference_session, + patch.object( + detection_runners, "get_ort_session_options", return_value=MagicMock() + ), + self.assertLogs(detection_runners.logger, level="WARNING"), + ): + runner = get_optimized_runner("/models/jina.onnx", "AUTO", "jina-v2") + + self.assertEqual(inference_session.call_count, 2) + self.assertIsInstance(runner, ONNXModelRunner) + self.assertEqual(loaded_devices["/models/jina.onnx"], ("jina-v2", "CPU")) + + def test_a_cpu_model_stays_on_the_cpu(self): + session = MagicMock() + session.get_providers.return_value = ["CPUExecutionProvider"] + + with ( + patch.object(detection_runners, "is_rknn_compatible", return_value=False), + patch.object( + detection_runners, "get_lighter_ane_devices", return_value=[MagicMock()] + ) as ane, + patch.object( + detection_runners, + "get_ort_providers", + return_value=(["CPUExecutionProvider"], [{}]), + ), + patch.object( + detection_runners.ort, "InferenceSession", return_value=session + ), + patch.object( + detection_runners, "get_ort_session_options", return_value=None + ), + ): + get_optimized_runner("/models/arcface.onnx", "CPU", "arcface") + + ane.assert_not_called() diff --git a/frigate/util/hwaccel.py b/frigate/util/hwaccel.py index 88ce16ce73..806bcb81d0 100644 --- a/frigate/util/hwaccel.py +++ b/frigate/util/hwaccel.py @@ -9,6 +9,7 @@ and resolve it per camera against that camera's detect stream. """ import logging +import os import re from pydantic import BaseModel, Field @@ -23,14 +24,20 @@ from frigate.util.services import enumerate_drm_devices logger = logging.getLogger(__name__) -# root the /proc reads use, so tests can point them at a fixture tree +# roots the /proc and /sys reads use, so tests can point them at a fixture tree PROC_ROOT = "/proc" +SYS_ROOT = "/sys" ANY_CODEC = "any" # a Raspberry Pi has no detection hardware of its own, so it gets a key here RASPBERRY_PI = "raspberrypi" +# lighter (https://github.com/fieldwork-ai/lighter) decodes on a Mac's media +# engine through virtio-media devices, which carry its virtio vendor id "LGHT" +LIGHTER_MEDIA = "lighter" +LIGHTER_VIRTIO_VENDOR = "0x4c474854" + # ffprobe names h265 streams hevc CODEC_ALIASES = {"hevc": "h265"} @@ -52,6 +59,7 @@ DECODE_HARDWARE = ( "rknn", "openvino:GPU", "onnx:amd", + LIGHTER_MEDIA, RASPBERRY_PI, ) @@ -98,6 +106,10 @@ FAMILY_RPI = HwaccelFamily( key="rpi", presets={"h264": "preset-rpi-64-h264", "h265": "preset-rpi-64-h265"}, ) +FAMILY_APPLE = HwaccelFamily( + key="apple-silicon", + presets={"h264": "preset-apple-silicon-h264", "h265": "preset-apple-silicon-h265"}, +) def _read(path: str) -> str | None: @@ -139,6 +151,20 @@ def _is_raspberry_pi() -> bool: return "raspberrypi" in compatible +def _has_lighter_media() -> bool: + video = f"{SYS_ROOT}/class/video4linux" + + try: + devices = os.listdir(video) + except OSError: + return False + + return any( + _read(f"{video}/{device}/device/vendor") == LIGHTER_VIRTIO_VENDOR + for device in devices + ) + + def _intel_families(generation: int | None) -> list[HwaccelFamily]: """vaapi drives every Intel GPU, qsv only those from gen8 on.""" if generation is not None and generation < INTEL_QSV_SUPPORTED_GEN: @@ -164,6 +190,9 @@ def _families(key: str, generation: int | None) -> list[HwaccelFamily]: if key == "onnx:amd": return [FAMILY_VAAPI] + if key == LIGHTER_MEDIA: + return [FAMILY_APPLE] + if key == RASPBERRY_PI: return [FAMILY_RPI] @@ -193,6 +222,9 @@ def _decode_hardware(detector_key: str | None) -> list[str]: """ present = {found.key for found in hardware_prober.probe()} + if _has_lighter_media(): + present.add(LIGHTER_MEDIA) + if _is_raspberry_pi(): present.add(RASPBERRY_PI) diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json index 1df503441f..c5fa7afd99 100644 --- a/web/public/locales/en/views/settings.json +++ b/web/public/locales/en/views/settings.json @@ -1567,6 +1567,8 @@ "presetLabels": { "preset-rpi-64-h264": "Raspberry Pi (H.264)", "preset-rpi-64-h265": "Raspberry Pi (H.265)", + "preset-apple-silicon-h264": "Apple Silicon (H.264)", + "preset-apple-silicon-h265": "Apple Silicon (H.265)", "preset-vaapi": "VAAPI (Intel/AMD GPU)", "preset-intel-qsv-h264": "Intel QuickSync (H.264)", "preset-intel-qsv-h265": "Intel QuickSync (H.265)", diff --git a/web/public/locales/en/views/setup.json b/web/public/locales/en/views/setup.json index 00992397cf..c96ce09626 100644 --- a/web/public/locales/en/views/setup.json +++ b/web/public/locales/en/views/setup.json @@ -48,6 +48,7 @@ "rkmpp": "RKMPP (Rockchip)", "jetson": "NVIDIA Jetson", "rpi": "V4L2 (Raspberry Pi)", + "apple-silicon": "Media engine (Apple Silicon)", "none": "None (software decoding)" } }, diff --git a/web/src/utils/health.ts b/web/src/utils/health.ts index 25213f0522..e652c94245 100644 --- a/web/src/utils/health.ts +++ b/web/src/utils/health.ts @@ -256,7 +256,13 @@ export function detectionRows({ // ------------------------------------------------------------------ hwaccel export type HwaccelFamilyKey = - "nvidia" | "vaapi" | "intel-qsv" | "rkmpp" | "jetson" | "rpi"; + | "nvidia" + | "vaapi" + | "intel-qsv" + | "rkmpp" + | "jetson" + | "rpi" + | "apple-silicon"; export type HwaccelClass = | { kind: "none" } @@ -270,6 +276,7 @@ const PRESET_FAMILIES: [string, HwaccelFamilyKey][] = [ ["preset-rk", "rkmpp"], ["preset-jetson", "jetson"], ["preset-rpi", "rpi"], + ["preset-apple-silicon", "apple-silicon"], ]; export function hwaccelFamily(value: string | string[]): HwaccelClass { @@ -294,6 +301,8 @@ const FAMILY_VENDORS: Record = { vaapi: ["intel", "amd"], rkmpp: ["rockchip"], rpi: ["rpi"], + // lighter reports no decoder usage + "apple-silicon": [], }; function decoderUsage(