diff --git a/docker/rocm/Dockerfile b/docker/rocm/Dockerfile index 653ed1e4ee..da44cf25a0 100644 --- a/docker/rocm/Dockerfile +++ b/docker/rocm/Dockerfile @@ -2,32 +2,9 @@ # https://askubuntu.com/questions/972516/debian-frontend-environment-variable ARG DEBIAN_FRONTEND=noninteractive -ARG ROCM=1 ARG HSA_OVERRIDE_GFX_VERSION ARG HSA_OVERRIDE -####################################################################### -FROM wget AS rocm - -ARG ROCM - -RUN apt update -qq && \ - apt install -y wget gpg && \ - wget -O rocm.deb https://repo.radeon.com/amdgpu-install/7.2.3/ubuntu/jammy/amdgpu-install_7.2.3.70203-1_all.deb && \ - apt install -y ./rocm.deb && \ - apt update && \ - apt install -qq -y rocm - -RUN mkdir -p /opt/rocm-dist/opt/rocm-$ROCM/lib -RUN cd /opt/rocm-$ROCM/lib && \ - cp -dpr libMIOpen*.so* libamd*.so* libhip*.so* libhsa*.so* libmigraphx*.so* librocm*.so* librocblas*.so* libroctracer*.so* librocsolver*.so* librocfft*.so* librocprofiler*.so* libroctx*.so* librocroller.so* /opt/rocm-dist/opt/rocm-$ROCM/lib/ && \ - mkdir -p /opt/rocm-dist/opt/rocm-$ROCM/lib/migraphx/lib && \ - cp -dpr migraphx/lib/* /opt/rocm-dist/opt/rocm-$ROCM/lib/migraphx/lib -RUN cd /opt/rocm-dist/opt/ && ln -s rocm-$ROCM rocm - -RUN mkdir -p /opt/rocm-dist/etc/ld.so.conf.d/ -RUN echo /opt/rocm/lib|tee /opt/rocm-dist/etc/ld.so.conf.d/rocm.conf - ####################################################################### FROM deps AS deps-prelim @@ -42,51 +19,35 @@ RUN echo "deb http://deb.debian.org/debian trixie main" > /etc/apt/sources.list. rm -f /etc/apt/sources.list.d/trixie.list && \ rm -rf /var/lib/apt/lists/* -WORKDIR /opt/frigate -COPY --from=rootfs / / - RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \ && python3 get-pip.py "pip" --break-system-packages RUN python3 -m pip config set global.break-system-packages true +# ROCm installs from AMD's wheels ahead of Frigate's code so code changes +# reuse this layer, and is pruned in the same layer so the removed files never +# reach the image. /opt/rocm links to the runtime for rocminfo and HIP headers. COPY docker/rocm/requirements-wheels-rocm.txt /requirements.txt -RUN pip3 uninstall -y onnxruntime \ - && pip3 install -r /requirements.txt +RUN --mount=type=bind,source=docker/rocm/prune_rocm.sh,target=/deps/prune_rocm.sh \ + pip3 uninstall -y onnxruntime \ + && pip3 install --no-cache-dir -r /requirements.txt \ + && ROCM_SDK_CORE="$(python3 -c 'import _rocm_sdk_core, os; print(os.path.dirname(_rocm_sdk_core.__file__))')" \ + && /deps/prune_rocm.sh "$(dirname "${ROCM_SDK_CORE}")" \ + && ln -s "${ROCM_SDK_CORE}" /opt/rocm -####################################################################### -FROM scratch AS rocm-dist - -ARG ROCM - -# Copy HIP headers required for MIOpen JIT (BuildHip) / HIPRTC at runtime -COPY --from=rocm /opt/rocm-${ROCM}/include/ /opt/rocm-${ROCM}/include/ -COPY --from=rocm /opt/rocm-$ROCM/bin/rocminfo /opt/rocm-$ROCM/bin/migraphx-driver /opt/rocm-$ROCM/bin/ -# Copy MIOpen database files for gfx10xx, gfx11xx, and gfx12xx only (RDNA2/RDNA3/RDNA4) -COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx10* /opt/rocm-$ROCM/share/miopen/db/ -COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx11* /opt/rocm-$ROCM/share/miopen/db/ -COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx12* /opt/rocm-$ROCM/share/miopen/db/ -# Copy rocBLAS library files for gfx10xx, gfx11xx, and gfx12xx only -COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*gfx10* /opt/rocm-$ROCM/lib/rocblas/library/ -COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*gfx11* /opt/rocm-$ROCM/lib/rocblas/library/ -COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*gfx12* /opt/rocm-$ROCM/lib/rocblas/library/ -COPY --from=rocm /opt/rocm-dist/ / +WORKDIR /opt/frigate +COPY --from=rootfs / / ####################################################################### FROM deps-prelim AS rocm-prelim-hsa-override0 ENV MIGRAPHX_DISABLE_MIOPEN_FUSION=1 ENV MIGRAPHX_DISABLE_SCHEDULE_PASS=1 ENV MIGRAPHX_DISABLE_REDUCE_FUSION=1 -ENV MIGRAPHX_ENABLE_HIPRTC_WORKAROUNDS=1 ENV MIOPEN_CUSTOM_CACHE_DIR=/config/model_cache/migraphx ENV MIOPEN_USER_DB_PATH=/config/model_cache/migraphx ENV AMD_COMGR_CACHE=1 ENV AMD_COMGR_CACHE_DIR=/config/model_cache/migraphx -COPY --from=rocm-dist / / - -RUN ldconfig - ####################################################################### FROM rocm-prelim-hsa-override0 as rocm-prelim-hsa-override1 diff --git a/docker/rocm/prune_rocm.sh b/docker/rocm/prune_rocm.sh new file mode 100755 index 0000000000..af08c41bc9 --- /dev/null +++ b/docker/rocm/prune_rocm.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Remove the parts of AMD's ROCm wheels that MIGraphX inference never loads: +# compilers, debuggers, profilers, management tools, and the math libraries +# (sparse, solver, FFT, random, RCCL, image processing) outside MIGraphX's +# dependencies. Runs in the same layer as the install so the files never +# reach the image. + +set -euo pipefail + +site_packages="$1" +core="${site_packages}/_rocm_sdk_core" +libraries="${site_packages}/_rocm_sdk_libraries" + +# rocminfo stays for checking that the GPU is visible in the container +find "${core}/bin" -mindepth 1 -maxdepth 1 \ + ! -name rocminfo ! -name rocm_agent_enumerator -exec rm -rf {} + + +# hiprtc compiles MIGraphX's kernels through comgr, which needs libLLVM, +# libclang-cpp, the clang resource headers, and the amdgcn device libraries +rm -rf "${core}/lib/llvm/bin" "${core}/lib/llvm/include" +find "${core}/lib/llvm/lib" -mindepth 1 -maxdepth 1 \ + ! -name 'libLLVM.so*' ! -name 'libclang-cpp.so*' ! -name clang \ + -exec rm -rf {} + +rm -rf "${core}"/lib/llvm/lib/clang/*/lib + +rm -rf \ + "${core}/lib/host-math" \ + "${core}"/lib/python3* \ + "${core}/lib/rocprofiler-sdk" \ + "${core}"/lib/librocprofiler-sdk*.so* \ + "${core}"/lib/librocprof-trace-decoder.so* \ + "${core}"/lib/librocm-dbgapi.so* \ + "${core}"/lib/librocm-debug-agent.so* \ + "${core}"/lib/libamd_smi*.so* \ + "${core}"/lib/libgoamdsmi_shim64.so* \ + "${core}"/lib/librocdecode.so* \ + "${core}"/lib/librocjpeg.so* \ + "${core}"/lib/librocdxg.so* \ + "${core}"/lib/librocjitsu_*.so* \ + "${core}"/lib/libhsa_hotswap_rocjitsu.so* \ + "${core}"/lib/*.a + +rm -rf \ + "${libraries}"/lib/librpp.so* \ + "${libraries}"/lib/librocsparse.so* \ + "${libraries}"/lib/libhipsparse.so* \ + "${libraries}"/lib/libhipsparselt.so* \ + "${libraries}/lib/rocsparse" \ + "${libraries}"/lib/librocsolver.so* \ + "${libraries}"/lib/libhipsolver.so* \ + "${libraries}"/lib/libhipblas.so* \ + "${libraries}"/lib/librocrand.so* \ + "${libraries}"/lib/libhiprand.so* \ + "${libraries}"/lib/librocfft.so* \ + "${libraries}"/lib/libhipfft.so* \ + "${libraries}"/lib/libhipfftw.so* \ + "${libraries}"/lib/librccl.so* \ + "${libraries}"/.kpack/sparse_lib_*.kpack \ + "${libraries}"/.kpack/solver_lib_*.kpack \ + "${libraries}"/.kpack/rand_lib_*.kpack \ + "${libraries}"/.kpack/fft_lib_*.kpack \ + "${libraries}"/.kpack/rccl_lib_*.kpack diff --git a/docker/rocm/requirements-wheels-rocm.txt b/docker/rocm/requirements-wheels-rocm.txt index f60b550c3f..be444b97ac 100644 --- a/docker/rocm/requirements-wheels-rocm.txt +++ b/docker/rocm/requirements-wheels-rocm.txt @@ -1 +1,19 @@ -onnxruntime-migraphx @ https://github.com/NickM-27/frigate-onnxruntime-rocm/releases/download/v7.2.3-1/onnxruntime_migraphx-1.24.4-cp311-cp311-linux_x86_64.whl \ No newline at end of file +# ROCm 10.1.0, MIGraphX, and its ONNX Runtime plugin provider from AMD's wheels. +onnxruntime == 1.29.* +onnxruntime-ep-migraphx @ https://stable.repo.amd.com/rocm/onnxruntime/whl-next/onnxruntime-ep-migraphx/onnxruntime_ep_migraphx-1.0.0%2Brocm10.1.0-cp311-cp311-manylinux_2_28_x86_64.whl +migraphx-libs @ https://stable.repo.amd.com/rocm/migraphx/whl-next/migraphx-libs/migraphx_libs-2.18.0%2Brocm10.1.0-py3-none-manylinux_2_28_x86_64.whl +rocm-sdk-core @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-core/rocm_sdk_core-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-libraries @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-libraries/rocm_sdk_libraries-10.1.0-py3-none-linux_x86_64.whl +# Kernels for each supported GPU target. MIGraphX has no gfx1031-gfx1036 kernels, +# so those GPUs run as gfx1030 with HSA_OVERRIDE_GFX_VERSION=10.3.0 +rocm-sdk-device-gfx1030 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1030/rocm_sdk_device_gfx1030-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1100 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1100/rocm_sdk_device_gfx1100-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1101 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1101/rocm_sdk_device_gfx1101-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1102 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1102/rocm_sdk_device_gfx1102-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1103 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1103/rocm_sdk_device_gfx1103-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1150 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1150/rocm_sdk_device_gfx1150-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1151 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1151/rocm_sdk_device_gfx1151-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1152 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1152/rocm_sdk_device_gfx1152-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1153 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1153/rocm_sdk_device_gfx1153-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1200 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1200/rocm_sdk_device_gfx1200-10.1.0-py3-none-linux_x86_64.whl +rocm-sdk-device-gfx1201 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1201/rocm_sdk_device_gfx1201-10.1.0-py3-none-linux_x86_64.whl diff --git a/docker/rocm/rocm.hcl b/docker/rocm/rocm.hcl index 224118818e..c383d745f2 100644 --- a/docker/rocm/rocm.hcl +++ b/docker/rocm/rocm.hcl @@ -1,6 +1,3 @@ -variable "ROCM" { - default = "7.2.3" -} variable "HSA_OVERRIDE_GFX_VERSION" { default = "" } @@ -8,12 +5,6 @@ variable "HSA_OVERRIDE" { default = "1" } -target wget { - dockerfile = "docker/main/Dockerfile" - platforms = ["linux/amd64"] - target = "wget" -} - target deps { dockerfile = "docker/main/Dockerfile" platforms = ["linux/amd64"] @@ -30,12 +21,10 @@ target rocm { dockerfile = "docker/rocm/Dockerfile" contexts = { deps = "target:deps", - wget = "target:wget", rootfs = "target:rootfs" } platforms = ["linux/amd64"] args = { - ROCM = ROCM, HSA_OVERRIDE_GFX_VERSION = HSA_OVERRIDE_GFX_VERSION, HSA_OVERRIDE = HSA_OVERRIDE } diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 9cad47ed23..1ba6fd0003 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -394,6 +394,22 @@ Note that the labelmap uses a subset of the complete COCO label set that has onl Support for AMD GPUs is provided using the [ONNX detector](#onnx). In order to utilize the AMD GPU for object detection use a frigate docker image with `-rocm` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-rocm`. +### Supported GPUs {#supported-gpus-rocm} + +The `-rocm` image includes ROCm support for these GPU chipsets, which work without any chipset override: + +| Chipset | Example GPUs | +| -------------------------------------------- | ------------------------------------------------------------------------ | +| gfx1030 (RDNA2) | Radeon RX 6800 / 6800 XT / 6900 XT / 6950 XT, Radeon PRO W6800 | +| gfx1100, gfx1101, gfx1102 (RDNA3) | Radeon RX 7900 / 7800 / 7700 / 7600 series, Radeon PRO W7000 series | +| gfx1103 (RDNA3 integrated) | Radeon 780M / 760M in Ryzen 7040, 8040, 8000G, and 200 series processors | +| gfx1150, gfx1151, gfx1152, gfx1153 (RDNA3.5) | Integrated GPUs in Ryzen AI 300 / 400 series and Ryzen AI Max processors | +| gfx1200, gfx1201 (RDNA4) | Radeon RX 9060 / 9070 series, Radeon AI PRO R9700 | + +Other RDNA2 GPUs (gfx1031 through gfx1036), such as the Radeon RX 6700 / 6600 / 6500 series and the Radeon 680M / 660M integrated GPUs, run on the gfx1030 support by [overriding the chipset](#docker-settings-for-overriding-the-gpu-chipset) to `10.3.0`. + +Older GPUs, including RDNA1 (Radeon RX 5000 series) and Vega based dedicated and integrated GPUs, are not supported. + ### Docker settings for GPU access ROCm needs access to the `/dev/kfd` and `/dev/dri` devices. When docker or frigate is not run under root then also `video` (and possibly `render` and `ssl/_ssl`) groups should be added. @@ -420,19 +436,10 @@ For reference on recommended settings see [running ROCm/pytorch in Docker](https ### Docker settings for overriding the GPU chipset -Your GPU might work just fine without any special configuration but in many cases they need manual settings. AMD/ROCm software stack comes with a limited set of GPU drivers and for newer or missing models you will have to override the chipset version to an older/generic version to get things working. - -Also AMD/ROCm does not "officially" support integrated GPUs. It still does work with most of them just fine but requires special settings. One has to configure the `HSA_OVERRIDE_GFX_VERSION` environment variable. See the [ROCm bug report](https://github.com/ROCm/ROCm/issues/1743) for context and examples. - -For the rocm frigate build there is some automatic detection: - -- gfx1031 -> 10.3.0 -- gfx1103 -> 11.0.0 - -If you have something else you might need to override the `HSA_OVERRIDE_GFX_VERSION` at Docker launch. Suppose the version you want is `10.0.0`, then you should configure it from command line as: +A GPU whose chipset isn't in the [supported GPUs](#supported-gpus-rocm) list can run on the support for a closely related chipset by setting the `HSA_OVERRIDE_GFX_VERSION` environment variable. RDNA2 GPUs other than gfx1030 (gfx1031 through gfx1036) need this, using `10.3.0`: ```bash -$ docker run -e HSA_OVERRIDE_GFX_VERSION=10.0.0 \ +$ docker run -e HSA_OVERRIDE_GFX_VERSION=10.3.0 \ ... ``` @@ -443,16 +450,17 @@ services: frigate: ... environment: - HSA_OVERRIDE_GFX_VERSION: "10.0.0" + HSA_OVERRIDE_GFX_VERSION: "10.3.0" ``` -Figuring out what version you need can be complicated as you can't tell the chipset name and driver from the AMD brand name. +GPUs in the supported list don't need an override. An override that was set for the Radeon 780M / 760M (gfx1103) on an earlier version of Frigate is no longer needed and can be removed. -- first make sure that rocm environment is running properly by running `/opt/rocm/bin/rocminfo` in the frigate container -- it should list both the CPU and the GPU with their properties -- find the chipset version you have (gfxNNN) from the output of the `rocminfo` (see below) -- use a search engine to query what `HSA_OVERRIDE_GFX_VERSION` you need for the given gfx name ("gfxNNN ROCm HSA_OVERRIDE_GFX_VERSION") -- override the `HSA_OVERRIDE_GFX_VERSION` with relevant value -- if things are not working check the frigate docker logs +You can't tell the chipset from the AMD brand name, so check it in the container: + +- first make sure that the ROCm environment is running properly by running `/opt/rocm/bin/rocminfo` in the Frigate container. It should list both the CPU and the GPU with their properties +- find the chipset version you have (gfxNNN) from the output of `rocminfo` (see below) +- if the chipset isn't in the supported list, set `HSA_OVERRIDE_GFX_VERSION` to the closest supported chipset, written as major.minor.stepping (for example gfx1030 is `10.3.0`) +- if things are not working check the Frigate docker logs #### Figuring out if AMD/ROCm is working and found your GPU diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 6a506953d2..794e9703e8 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -72,7 +72,7 @@ Frigate supports multiple different detectors that work on different types of ha **AMD** -- [ROCm](#rocm---amd-gpu): ROCm can run on AMD Discrete GPUs to provide efficient object detection +- [ROCm](#rocm---amd-gpu): ROCm can run on AMD discrete and recent integrated GPUs to provide efficient object detection - [Supports limited model architectures](../../configuration/object_detectors#amdrocm-gpu-detector) - Runs best on discrete AMD GPUs - [XDNA2 (Ryzen AI)](#amd-xdna2): AMD XDNA2 NPU (sub-watt power AI/ML processor separate to the GPU) inside Strix and other "AI" branded AMD platforms @@ -240,7 +240,7 @@ Apple Silicon can not run within a container, so a ZMQ proxy is utilized to comm ### ROCm - AMD GPU -With the [ROCm](../configuration/object_detectors.md#amdrocm-gpu-detector) detector Frigate can take advantage of many discrete AMD GPUs. +With the [ROCm](../configuration/object_detectors.md#amdrocm-gpu-detector) detector Frigate can take advantage of many AMD discrete and integrated GPUs, see the [supported GPUs](../configuration/object_detectors.md#supported-gpus-rocm). | Name | YOLOv9 Inference Time | YOLO-NAS Inference Time | RF-DETR Inference Time | | -------------- | --------------------------- | ------------------------- | ---------------------- | diff --git a/frigate/detectors/detection_runners.py b/frigate/detectors/detection_runners.py index 7277269731..80c2fc57f9 100644 --- a/frigate/detectors/detection_runners.py +++ b/frigate/detectors/detection_runners.py @@ -10,6 +10,7 @@ from typing import Any import numpy as np import onnxruntime as ort +from frigate.const import MODEL_CACHE_DIR from frigate.util.model import get_ort_providers from frigate.util.rknn_converter import auto_convert_model, is_rknn_compatible @@ -55,6 +56,9 @@ _PROVIDER_LABELS = { LIGHTER_ANE_EP_NAME = "LighterANE" LIGHTER_ANE_LIBRARY = "/usr/lib/lighter/liblighter_ane_ep.so" +# AMD's MIGraphX plugin execution provider, installed in the ROCm image +MIGRAPHX_EP_NAME = "MIGraphXExecutionProvider" + def get_lighter_ane_devices() -> list[Any]: """Get the Neural Engine devices lighter's provider offers, registering it once. @@ -83,6 +87,42 @@ def get_lighter_ane_devices() -> list[Any]: return devices +def get_migraphx_devices() -> list[Any]: + """Get the AMD GPU devices the MIGraphX provider offers, registering it once. + + Returns: + The provider's ONNX Runtime devices, or an empty list outside the ROCm + image or without an AMD GPU + """ + # Import here to avoid circular imports + from frigate.detectors.hardware import detect_amd_gpu + + try: + import onnxruntime_ep_migraphx + except ImportError: + return [] + + # ONNX Runtime finds the provider's GPUs in the same /sys/class/drm entries, + # so an AMD GPU here means the provider lists it once registered + if detect_amd_gpu() is None: + return [] + + devices = [d for d in ort.get_ep_devices() if d.ep_name == MIGRAPHX_EP_NAME] + + if not devices: + try: + ort.register_execution_provider_library( + MIGRAPHX_EP_NAME, onnxruntime_ep_migraphx.get_library_path() + ) + except Exception as e: + logger.warning("Failed to load the MIGraphX provider: %s", e) + return [] + + devices = [d for d in ort.get_ep_devices() if d.ep_name == MIGRAPHX_EP_NAME] + + return devices + + def is_arm64_platform() -> bool: """Check if we're running on an ARM platform.""" machine = platform.machine().lower() @@ -756,12 +796,41 @@ def get_optimized_runner( ONNXModelRunner(session, model_type=model_type), ) + if ( + device != "CPU" + and not ONNXModelRunner.is_migraphx_complex_model(model_type) + and (migraphx_devices := get_migraphx_devices()) + ): + migraphx_cache_dir = os.path.join(MODEL_CACHE_DIR, "migraphx/ort") + os.makedirs(migraphx_cache_dir, exist_ok=True) + sess_options = get_ort_session_options(model_type) or ort.SessionOptions() + # only the first AMD GPU is used, matching detect_amd_gpu + sess_options.add_provider_for_devices( + migraphx_devices[:1], {"cache_dir": migraphx_cache_dir} + ) + + try: + session = ort.InferenceSession(model_path, sess_options=sess_options) + except Exception as e: + logger.warning( + "Failed to load %s with MIGraphX, using the default providers: %s", + model_path, + e, + ) + else: + return _record_runner( + model_path, + model_type, + ONNXModelRunner(session, model_type=model_type), + ) + providers, options = get_ort_providers(device == "CPU", device, **kwargs) if providers[0] == "CPUExecutionProvider": # In the default image, ONNXRuntime is used so we will only get CPUExecutionProvider - # In other images we will get CUDA / ROCm which are preferred over OpenVINO - # There is currently no way to prioritize OpenVINO over CUDA / ROCm in these images + # In other images we will get CUDA which is preferred over OpenVINO, while + # MIGraphX in the ROCm image is already handled above + # There is currently no way to prioritize OpenVINO over CUDA in these images if device != "CPU" and is_openvino_gpu_npu_available(): return _record_runner( model_path, @@ -790,15 +859,6 @@ def get_optimized_runner( ), ) - if ( - providers - and providers[0] == "MIGraphXExecutionProvider" - and ONNXModelRunner.is_migraphx_complex_model(model_type) - ): - # Don't use MIGraphX for models that are not supported - providers.pop(0) - options.pop(0) - return _record_runner( model_path, model_type, diff --git a/frigate/test/test_runner_devices.py b/frigate/test/test_runner_devices.py index ddba15e689..aaee9f7ad5 100644 --- a/frigate/test/test_runner_devices.py +++ b/frigate/test/test_runner_devices.py @@ -1,12 +1,13 @@ """Tests for the device each model runner reports after loading.""" import os +import sys import tempfile import threading import unittest from unittest.mock import MagicMock, patch -from frigate.detectors import detection_runners +from frigate.detectors import detection_runners, hardware from frigate.detectors.detection_runners import ( CudaGraphRunner, ONNXModelRunner, @@ -17,6 +18,7 @@ from frigate.detectors.detection_runners import ( record_loaded_device, snapshot_loaded_devices, ) +from frigate.util import model as model_util class TestRunnerDeviceName(unittest.TestCase): @@ -317,3 +319,215 @@ class TestLighterANE(unittest.TestCase): get_optimized_runner("/models/arcface.onnx", "CPU", "arcface") ane.assert_not_called() + + +class TestMIGraphX(unittest.TestCase): + """AMD's MIGraphX plugin provider, selected through its devices in the ROCm image.""" + + def setUp(self): + loaded_devices.clear() + self.root = tempfile.TemporaryDirectory() + self.addCleanup(self.root.cleanup) + + for patcher in ( + patch.object(hardware, "detect_amd_gpu", return_value=MagicMock()), + patch.object(detection_runners, "MODEL_CACHE_DIR", self.root.name), + patch.object(detection_runners, "is_rknn_compatible", return_value=False), + patch.object(detection_runners, "get_lighter_ane_devices", return_value=[]), + ): + patcher.start() + self.addCleanup(patcher.stop) + + self.plugin = MagicMock() + self.plugin.get_library_path.return_value = "/site/libmigraphx-ep.so" + + def _device(self) -> MagicMock: + device = MagicMock() + device.ep_name = "MIGraphXExecutionProvider" + return device + + def test_no_devices_outside_the_rocm_image(self): + with ( + patch.dict(sys.modules, {"onnxruntime_ep_migraphx": None}), + patch.object( + detection_runners.ort, "register_execution_provider_library" + ) as register, + ): + self.assertEqual(detection_runners.get_migraphx_devices(), []) + self.assertEqual(detection_runners.get_migraphx_devices(), []) + + register.assert_not_called() + + def test_no_devices_without_an_amd_gpu(self): + with ( + patch.dict(sys.modules, {"onnxruntime_ep_migraphx": self.plugin}), + patch.object(hardware, "detect_amd_gpu", return_value=None), + patch.object(detection_runners.ort, "get_ep_devices") as get_ep_devices, + patch.object( + detection_runners.ort, "register_execution_provider_library" + ) as register, + ): + self.assertEqual(detection_runners.get_migraphx_devices(), []) + + get_ep_devices.assert_not_called() + register.assert_not_called() + + def test_the_provider_is_registered_once(self): + device = self._device() + other = MagicMock() + other.ep_name = "CPUExecutionProvider" + + with ( + patch.dict(sys.modules, {"onnxruntime_ep_migraphx": self.plugin}), + patch.object( + detection_runners.ort, + "get_ep_devices", + side_effect=[[other], [other, device], [other, device]], + ), + patch.object( + detection_runners.ort, "register_execution_provider_library" + ) as register, + ): + self.assertEqual(detection_runners.get_migraphx_devices(), [device]) + self.assertEqual(detection_runners.get_migraphx_devices(), [device]) + + register.assert_called_once_with( + "MIGraphXExecutionProvider", "/site/libmigraphx-ep.so" + ) + + def test_a_provider_that_will_not_load_is_skipped(self): + with ( + patch.dict(sys.modules, {"onnxruntime_ep_migraphx": self.plugin}), + patch.object(detection_runners.ort, "get_ep_devices", return_value=[]), + patch.object( + detection_runners.ort, + "register_execution_provider_library", + side_effect=RuntimeError("libamdhip64.so.7: cannot open shared object"), + ), + self.assertLogs(detection_runners.logger, level="WARNING"), + ): + self.assertEqual(detection_runners.get_migraphx_devices(), []) + + def test_the_session_runs_on_the_first_gpu(self): + device = self._device() + session = MagicMock() + session.get_providers.return_value = [ + "MIGraphXExecutionProvider", + "CPUExecutionProvider", + ] + options = MagicMock() + + with ( + patch.object( + detection_runners, + "get_migraphx_devices", + return_value=[device, self._device()], + ), + patch.object( + detection_runners, "get_ort_session_options", return_value=options + ), + patch.object( + detection_runners.ort, "InferenceSession", return_value=session + ) as inference_session, + ): + runner = get_optimized_runner("/models/yolo.onnx", "AUTO", "yolo-generic") + + cache_dir = os.path.join(self.root.name, "migraphx/ort") + options.add_provider_for_devices.assert_called_once_with( + [device], {"cache_dir": cache_dir} + ) + inference_session.assert_called_once_with( + "/models/yolo.onnx", sess_options=options + ) + self.assertTrue(os.path.isdir(cache_dir)) + self.assertIsInstance(runner, ONNXModelRunner) + self.assertEqual( + loaded_devices["/models/yolo.onnx"], ("yolo-generic", "MIGraphX") + ) + + def test_a_model_migraphx_cannot_load_uses_the_default_providers(self): + session = MagicMock() + session.get_providers.return_value = ["CPUExecutionProvider"] + + with ( + patch.object( + detection_runners, + "get_migraphx_devices", + return_value=[self._device()], + ), + patch.object( + detection_runners, + "get_ort_providers", + return_value=(["CPUExecutionProvider"], [{}]), + ), + patch.object( + detection_runners, "is_openvino_gpu_npu_available", return_value=False + ), + patch.object( + detection_runners.ort, + "InferenceSession", + side_effect=[RuntimeError("compile failed"), session], + ) as inference_session, + patch.object( + detection_runners, "get_ort_session_options", return_value=MagicMock() + ), + self.assertLogs(detection_runners.logger, level="WARNING"), + ): + runner = get_optimized_runner("/models/arcface.onnx", "AUTO", "arcface") + + self.assertEqual(inference_session.call_count, 2) + self.assertIsInstance(runner, ONNXModelRunner) + self.assertEqual(loaded_devices["/models/arcface.onnx"], ("arcface", "CPU")) + + def test_unsupported_models_and_cpu_models_skip_migraphx(self): + for model_path, device, model_type in [ + ("/models/paddleocr.onnx", "AUTO", "paddleocr"), + ("/models/dfine.onnx", "AUTO", "dfine"), + ("/models/yolo.onnx", "CPU", "yolo-generic"), + ]: + session = MagicMock() + session.get_providers.return_value = ["CPUExecutionProvider"] + + with ( + self.subTest(model_type=model_type, device=device), + patch.object( + detection_runners, + "get_migraphx_devices", + return_value=[self._device()], + ) as migraphx, + patch.object( + detection_runners, + "get_ort_providers", + return_value=(["CPUExecutionProvider"], [{}]), + ), + patch.object( + detection_runners, + "is_openvino_gpu_npu_available", + return_value=False, + ), + patch.object( + detection_runners.ort, "InferenceSession", return_value=session + ), + patch.object( + detection_runners, "get_ort_session_options", return_value=None + ), + ): + get_optimized_runner(model_path, device, model_type) + + migraphx.assert_not_called() + self.assertEqual(loaded_devices[model_path], (model_type, "CPU")) + + def test_migraphx_is_never_requested_by_name(self): + with patch.object( + model_util.ort, + "get_available_providers", + return_value=[ + "AzureExecutionProvider", + "CPUExecutionProvider", + "MIGraphXExecutionProvider", + ], + ): + providers, options = model_util.get_ort_providers() + + self.assertEqual(providers, ["CPUExecutionProvider"]) + self.assertEqual(options, [{"enable_cpu_mem_arena": False}]) diff --git a/frigate/util/model.py b/frigate/util/model.py index 80f0d29a04..8d34d422a5 100644 --- a/frigate/util/model.py +++ b/frigate/util/model.py @@ -385,15 +385,9 @@ def get_ort_providers( } ) elif provider == "MIGraphXExecutionProvider": - migraphx_cache_dir = os.path.join(MODEL_CACHE_DIR, "migraphx") - os.makedirs(migraphx_cache_dir, exist_ok=True) - - providers.append(provider) - options.append( - { - "migraphx_model_cache_dir": migraphx_cache_dir, - } - ) + # MIGraphX is a plugin provider, which ONNX Runtime can't create by + # name, so get_optimized_runner selects it through its devices + continue elif provider == "CPUExecutionProvider": providers.append(provider) options.append(