From 52ba582e54a493ddd1258905e27fddcbf06fd102 Mon Sep 17 00:00:00 2001 From: notori0us Date: Sun, 19 Apr 2026 17:03:37 -0700 Subject: [PATCH] Add Qualcomm Hexagon NPU detector (qcs6490) Adds a community-supported hardware detector for the Qualcomm Hexagon NPU on QCS6490 SoCs (e.g. Radxa Dragon Q6A) via QAIRT 2.37.1 / qai_appbuilder. Mirrors the existing community-board pattern (Rockchip / Synaptics): - frigate/detectors/plugins/qnn.py: detector plugin using yolo-generic model type, lazy SDK import, runs pre-compiled QNN context binaries from Qualcomm AI Hub. - docker/qcs6490/: Dockerfile (two-stage; rebuilds qai_appbuilder wheel inside Frigate's image to match libstdc++ ABI), bake target (qcs6490.hcl), make targets (qcs6490.mk), and a host-side user_installation.sh that installs fastrpc, the QCS6490 firmware (cDSP image + skel libs), and configures cdsprpcd. - .github/workflows/ci.yml: qcs6490_build job mirroring synaptics_build. - CODEOWNERS: /docker/qcs6490/ + qnn.py. - docs: new "Qualcomm Hexagon NPU" sections under Community Supported Detectors in object_detectors.md, plus matching entries in installation.md and hardware.md. Performance on Radxa Dragon Q6A (Hexagon v68, ~12 TOPS), YOLOv8n 640x640: ~10ms per inference under light load, ~24ms with 5 RTSP cameras live. Closes #18602. --- .github/workflows/ci.yml | 25 ++++ CODEOWNERS | 2 + docker/qcs6490/Dockerfile | 80 +++++++++++ docker/qcs6490/qcs6490.hcl | 27 ++++ docker/qcs6490/qcs6490.mk | 15 ++ docker/qcs6490/user_installation.sh | 89 ++++++++++++ docs/docs/configuration/object_detectors.md | 82 +++++++++++ docs/docs/frigate/hardware.md | 8 ++ docs/docs/frigate/installation.md | 64 +++++++++ frigate/detectors/plugins/qnn.py | 151 ++++++++++++++++++++ 10 files changed, 543 insertions(+) create mode 100644 docker/qcs6490/Dockerfile create mode 100644 docker/qcs6490/qcs6490.hcl create mode 100644 docker/qcs6490/qcs6490.mk create mode 100755 docker/qcs6490/user_installation.sh create mode 100644 frigate/detectors/plugins/qnn.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41080be5d9..c2a3ce2da6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,6 +197,31 @@ jobs: set: | synaptics.tags=${{ steps.setup.outputs.image-name }}-synaptics *.cache-from=type=gha + qcs6490_build: + runs-on: ubuntu-22.04-arm + name: Qualcomm QCS6490 Build + needs: + - arm64_build + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Set up QEMU and Buildx + id: setup + uses: ./.github/actions/setup + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push QCS6490 build + uses: docker/bake-action@v7 + with: + source: . + push: true + targets: qcs6490 + files: docker/qcs6490/qcs6490.hcl + set: | + qcs6490.tags=${{ steps.setup.outputs.image-name }}-qcs6490 + *.cache-from=type=gha # The majority of users running arm64 are rpi users, so the rpi # build should be the primary arm64 image assemble_default_build: diff --git a/CODEOWNERS b/CODEOWNERS index c37041c2c3..4b63079b74 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,3 +5,5 @@ /docker/rockchip/ @MarcA711 /docker/rocm/ @harakas /docker/hailo8l/ @spanner3003 +/docker/qcs6490/ @notori0us +/frigate/detectors/plugins/qnn.py @notori0us diff --git a/docker/qcs6490/Dockerfile b/docker/qcs6490/Dockerfile new file mode 100644 index 0000000000..562102d75f --- /dev/null +++ b/docker/qcs6490/Dockerfile @@ -0,0 +1,80 @@ +# syntax=docker/dockerfile:1.6 + +# https://askubuntu.com/questions/972516/debian-frontend-environment-variable +ARG DEBIAN_FRONTEND=noninteractive + +# Globally set pip break-system-packages option to avoid having to specify it every time +ARG PIP_BREAK_SYSTEM_PACKAGES=1 + +# QAIRT 2.37.1 runtime tarball + qai_appbuilder source pinned to a known-good +# commit that builds against this SDK version. Override at build time via +# --build-arg if a downstream maintainer hosts these elsewhere. +ARG QAIRT_RUNTIME_URL=https://github.com/notori0us/qairt-runtime/releases/download/v2.37.1.250807/qairt-runtime-2.37.1.250807-aarch64.tar.gz +ARG QAI_APPBUILDER_REF=942bc0d + +# ---------- stage: builder ---------- +# Build the qai_appbuilder Python wheel inside the Frigate image so its C++ +# ABI (libstdc++/glibc) matches Frigate's runtime stage. +FROM deps AS qcs6490-wheels +ARG DEBIAN_FRONTEND +ARG PIP_BREAK_SYSTEM_PACKAGES +ARG QAIRT_RUNTIME_URL +ARG QAI_APPBUILDER_REF + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential cmake git ca-certificates curl \ + python3-dev libyaml-0-2 \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --no-cache-dir \ + wheel==0.45.1 setuptools==80.9.0 pybind11==2.13.6 build==1.4.0 + +# Fetch QAIRT runtime (libQnnHtp*, libQnnSystem, hexagon-v68 skel, fastrpc shell). +RUN mkdir -p /opt/qairt \ + && curl -fsSL "${QAIRT_RUNTIME_URL}" | tar -C /opt/qairt -xzf - + +# Fetch qai_appbuilder source at a pinned commit. Patch out the Genie target +# (its headers don't match QAIRT 2.37.1) and ensure dist/ exists for the build. +RUN git clone --filter=blob:none https://github.com/quic/ai-engine-direct-helper.git /tmp/aedh \ + && cd /tmp/aedh \ + && git checkout "${QAI_APPBUILDER_REF}" \ + && sed -i '/add_subdirectory(genie)/d' pybind/CMakeLists.txt \ + && sed -i 's|zip_package|os.makedirs("dist", exist_ok=True)\n zip_package|' setup.py + +ENV QNN_SDK_ROOT=/opt/qairt \ + QAI_TOOLCHAINS=aarch64-oe-linux-gcc11.2 \ + QAI_HEXAGONARCH=68 +RUN cd /tmp/aedh && python3 -m build -w + +# ---------- stage: runtime ---------- +FROM deps AS qcs6490-frigate +ARG DEBIAN_FRONTEND +ARG PIP_BREAK_SYSTEM_PACKAGES + +# Runtime libs: libcdsprpc.so dlopen path + QNN SDK libs need libyaml-0.so. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libyaml-0-2 \ + && rm -rf /var/lib/apt/lists/* + +# QAIRT runtime layout. ADSP_LIBRARY_PATH below points cDSP firmware here for +# the QNN HTP backend skel (libQnnHtpV68Skel.so) and the fastrpc shell. +COPY --from=qcs6490-wheels /opt/qairt/lib /opt/qairt/lib/ +COPY --from=qcs6490-wheels /opt/qairt/hexagon-v68 /opt/qairt/hexagon-v68/ + +# Make libcdsprpc.so visible to the dynamic linker without polluting LD_LIBRARY_PATH. +RUN ln -sf /opt/qairt/lib/libcdsprpc.so /usr/lib/libcdsprpc.so && ldconfig + +# Install the qai_appbuilder wheel built in the previous stage. +RUN --mount=type=bind,from=qcs6490-wheels,source=/tmp/aedh/dist,target=/wheels \ + pip3 install --no-cache-dir /wheels/qai_appbuilder-*.whl + +WORKDIR /opt/frigate/ +COPY --from=rootfs / / + +# fastrpc separator gotcha: ADSP_LIBRARY_PATH is split by ';' (semicolon), +# not the usual ':'. The cDSP firmware also requires its skel + libc++ files +# at host paths /usr/lib/dsp/cdsp and /usr/lib/rfsa/adsp; bind-mount these +# from the host (see docs/docs/frigate/installation.md#qualcomm-platform). +ENV ADSP_LIBRARY_PATH="/opt/qairt/hexagon-v68;/usr/lib/dsp/cdsp;/usr/lib/rfsa/adsp" \ + LD_LIBRARY_PATH="/opt/qairt/lib" diff --git a/docker/qcs6490/qcs6490.hcl b/docker/qcs6490/qcs6490.hcl new file mode 100644 index 0000000000..ed3f81188f --- /dev/null +++ b/docker/qcs6490/qcs6490.hcl @@ -0,0 +1,27 @@ +target wheels { + dockerfile = "docker/main/Dockerfile" + platforms = ["linux/arm64"] + target = "wheels" +} + +target deps { + dockerfile = "docker/main/Dockerfile" + platforms = ["linux/arm64"] + target = "deps" +} + +target rootfs { + dockerfile = "docker/main/Dockerfile" + platforms = ["linux/arm64"] + target = "rootfs" +} + +target qcs6490 { + dockerfile = "docker/qcs6490/Dockerfile" + contexts = { + wheels = "target:wheels", + deps = "target:deps", + rootfs = "target:rootfs" + } + platforms = ["linux/arm64"] +} diff --git a/docker/qcs6490/qcs6490.mk b/docker/qcs6490/qcs6490.mk new file mode 100644 index 0000000000..b2ab09c98e --- /dev/null +++ b/docker/qcs6490/qcs6490.mk @@ -0,0 +1,15 @@ +BOARDS += qcs6490 + +local-qcs6490: version + docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \ + --set qcs6490.tags=frigate:latest-qcs6490 \ + --load + +build-qcs6490: version + docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \ + --set qcs6490.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qcs6490 + +push-qcs6490: build-qcs6490 + docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \ + --set qcs6490.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qcs6490 \ + --push diff --git a/docker/qcs6490/user_installation.sh b/docker/qcs6490/user_installation.sh new file mode 100755 index 0000000000..3f52f6ad39 --- /dev/null +++ b/docker/qcs6490/user_installation.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# +# Host-side setup for the Qualcomm Hexagon NPU detector on a Radxa Dragon Q6A +# (or other QCS6490 board). Installs: +# - fastrpc user-space (libcdsprpc.so, cdsprpcd, fastrpc_test) +# - Radxa firmware that ships the cDSP image + skel libs the QNN HTP +# backend dlopens at runtime +# - a transient cdsprpcd systemd service +# and disables hexagonrpcd, which holds the fastrpc devices and conflicts. +# +# Run with sudo. Logs out + back in are required for the fastrpc group to +# take effect for your user. + +set -euo pipefail + +if [ "$EUID" -ne 0 ]; then + echo "Please run as root (sudo $0)" + exit 1 +fi + +ARCH=$(dpkg --print-architecture) +if [ "$ARCH" != "arm64" ]; then + echo "This script targets arm64 (QCS6490). Detected: $ARCH" + exit 1 +fi + +apt-get update +apt-get install -y --no-install-recommends ca-certificates curl + +WORKDIR=$(mktemp -d) +trap 'rm -rf "$WORKDIR"' EXIT + +# fastrpc user-space (provides libcdsprpc.so + cdsprpcd). Not in apt. +FASTRPC_VER=1.0.4-1 +echo "==> Installing fastrpc ${FASTRPC_VER}" +for pkg in fastrpc fastrpc-tools; do + curl -fsSL -o "${WORKDIR}/${pkg}.deb" \ + "https://github.com/radxa-pkg/fastrpc/releases/download/${FASTRPC_VER}/${pkg}_${FASTRPC_VER}_arm64.deb" +done +apt-get install -y "${WORKDIR}/fastrpc.deb" "${WORKDIR}/fastrpc-tools.deb" + +# Radxa QCS6490 firmware (provides /usr/lib/dsp/cdsp/{cdsp.mbn,*_skel.so,...} +# and /usr/lib/rfsa/adsp/, both required by the cDSP at runtime). +RADXA_FW_VER=0.2.29 +echo "==> Installing radxa-firmware-qcs6490 ${RADXA_FW_VER}" +curl -fsSL -o "${WORKDIR}/radxa-firmware-qcs6490.deb" \ + "https://github.com/radxa-pkg/radxa-firmware/releases/download/${RADXA_FW_VER}/radxa-firmware-qcs6490_${RADXA_FW_VER}_all.deb" +apt-get install -y "${WORKDIR}/radxa-firmware-qcs6490.deb" + +# hexagonrpcd from the apt 'hexagonrpcd' package conflicts with cdsprpcd by +# holding /dev/fastrpc-* exclusively. We need cdsprpcd for QNN HTP. +echo "==> Disabling conflicting hexagonrpcd services" +for unit in hexagonrpcd hexagonrpcd-suspend hexagonrpcd-resume; do + systemctl disable --now "${unit}" 2>/dev/null || true +done + +echo "==> Enabling cdsprpcd" +cat >/etc/systemd/system/cdsprpcd.service <<'UNIT' +[Unit] +Description=Qualcomm cDSP FastRPC daemon +After=local-fs.target + +[Service] +Type=simple +ExecStart=/usr/bin/cdsprpcd +Restart=always + +[Install] +WantedBy=multi-user.target +UNIT +systemctl daemon-reload +systemctl enable --now cdsprpcd + +# Allow non-root containers + users to open /dev/fastrpc-*. +echo "==> Adding invoking user to fastrpc group" +TARGET_USER="${SUDO_USER:-$USER}" +if [ -n "${TARGET_USER}" ] && id "${TARGET_USER}" >/dev/null 2>&1; then + usermod -aG fastrpc "${TARGET_USER}" +fi + +echo +echo "Hexagon NPU host setup complete." +echo "Log out and back in for fastrpc group membership to take effect, then:" +echo " docker run ... ghcr.io/blakeblackshear/frigate:stable-qcs6490" +echo "Pass these to the container (devices, group, env):" +echo " --device /dev/fastrpc-cdsp --device /dev/fastrpc-cdsp-secure" +echo " --device /dev/fastrpc-adsp --device /dev/dma_heap/system" +echo " --group-add \$(getent group fastrpc | cut -d: -f3)" +echo " -v /usr/lib/dsp:/usr/lib/dsp:ro -v /usr/lib/rfsa:/usr/lib/rfsa:ro" diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 2821fb7a27..e99ce83e26 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -2045,6 +2045,88 @@ Explanation of the paramters: - **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`. - `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.2/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.2_EN.pdf). +## Qualcomm Hexagon NPU + +Hardware accelerated object detection is supported on the following Qualcomm SoCs: + +- QCS6490 (Hexagon v68, ~12 TOPS) — including the [Radxa Dragon Q6A](https://radxa.com/products/dragon/q6a/) and similar boards + +This implementation uses the [Qualcomm AI Engine Direct (QNN) SDK](https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk) (QAIRT 2.37.1) via the [`qai_appbuilder`](https://github.com/quic/ai-engine-direct-helper) Python bindings. Models are pre-compiled QNN context binaries (`.bin`) downloaded from [Qualcomm AI Hub](https://aihub.qualcomm.com/). + +:::warning + +The pre-compiled YOLOv8 weights from Qualcomm AI Hub originate from Ultralytics and are subject to the AGPL-3.0 license. They cannot be used commercially without a separate license from Ultralytics. + +::: + +### Prerequisites + +Make sure to follow the [Qualcomm specific installation instructions](/frigate/installation#qualcomm-platform). + +### Downloading a Model + +Frigate does not bundle the YOLOv8 weights. Download a QNN context binary for your SoC from Qualcomm AI Hub once and mount it into the container: + +```bash +mkdir -p ./models +# from https://aihub.qualcomm.com/compute/models/yolov8_det +# (sign in, choose target "qualcomm-qcs6490-proxy", download .bin) +mv ~/Downloads/yolov8_det.bin ./models/ +``` + +Mount `./models` into the container at `/models` and reference the file from your config. + +### Configuration + + + + +Navigate to and select **QNN** from the detector type dropdown and click **Add**. Then navigate to and configure: + +| Field | Value | +| ---------------------------------------- | --------------------------- | +| **Custom object detector model path** | `/models/yolov8_det.bin` | +| **Object Detection Model Type** | `yolo-generic` | +| **Object detection model input width** | `640` | +| **Object detection model input height** | `640` | +| **Model Input Tensor Shape** | `nhwc` | +| **Model Input D Type** | `float` | +| **Label map for custom object detector** | `/labelmap/coco-80.txt` | + + + + +```yaml +detectors: + hexagon: + type: qnn + soc_id: "6490" + +model: + path: /models/yolov8_det.bin + model_type: yolo-generic + width: 640 + height: 640 + input_tensor: nhwc + input_dtype: float + labelmap_path: /labelmap/coco-80.txt +``` + + + + +The inference time on a Radxa Dragon Q6A (QCS6490, Hexagon v68) is approximately 10–25 ms per frame at 640×640 — varying with system load and the number of cameras pumping frames into the detector. + +### Compiling Your Own Model + +To compile a different model — or to compile YOLOv8 for a Qualcomm SoC other than QCS6490 — use [Qualcomm AI Hub](https://aihub.qualcomm.com/). The workflow is: + +1. Sign in to AI Hub and find a model (for example, [YOLOv8 Detection](https://aihub.qualcomm.com/compute/models/yolov8_det)). +2. Submit a compile job for your target device (e.g. `qualcomm-qcs6490-proxy`). The job emits a QNN context binary (`.bin`) sized for that SoC's Hexagon variant. +3. Download the `.bin` and mount it into the container as above. + +The output-tensor ordering of YOLOv8 differs by SoC: QCS6490 yields `[scores, classes, boxes]` (handled by `soc_id: "6490"`); other SoCs yield `[boxes, scores, classes]` (use `soc_id: "other"`). + ## DeGirum DeGirum is a detector that can use any type of hardware listed on [their website](https://hub.degirum.com). DeGirum can be used with local hardware through a DeGirum AI Server, or through the use of `@local`. You can also connect directly to DeGirum's AI Hub to run inferences. **Please Note:** This detector _cannot_ be used for commercial purposes. diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 7df2ae0bb5..c67b3ffe04 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -295,6 +295,14 @@ The inference time of a rk3588 with all 3 cores enabled is typically 25-30 ms fo | ---------------- | ----------------------------------- | | yolov9-tiny | ~ 4 ms | +### Qualcomm Hexagon NPU + +Frigate supports hardware accelerated object detection on Qualcomm Hexagon NPUs via the [QNN detector](/configuration/object_detectors#qualcomm-hexagon-npu). Tested on the QCS6490 (Hexagon v68) on a [Radxa Dragon Q6A](https://radxa.com/products/dragon/q6a/). + +| Name | Inference Time | +| ----------------- | -------------- | +| QCS6490 / YOLOv8n | ~ 10–25 ms | + ## What does Frigate use the CPU for and what does it use a detector for? (ELI5 Version) This is taken from a [user question on reddit](https://www.reddit.com/r/homeassistant/comments/q8mgau/comment/hgqbxh5/?utm_source=share&utm_medium=web2x&context=3). Modified slightly for clarity. diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index 5d228a609a..e1a7e38ae3 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -428,6 +428,70 @@ or add these options to your `docker run` command: Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics). +### Qualcomm platform + +Hardware accelerated object detection on the Hexagon NPU is supported on the following Qualcomm SoCs: + +- QCS6490 (Hexagon v68) — including the Radxa Dragon Q6A and similar boards + +Make sure your kernel exposes the FastRPC bridges to the cDSP. On a configured board you should see: + +``` +$ ls /dev/fastrpc-* +/dev/fastrpc-adsp /dev/fastrpc-cdsp /dev/fastrpc-cdsp-secure +``` + +#### Installation + +Hexagon NPU access requires the QAIRT runtime libraries (shipped in the Frigate `-qcs6490` image), the FastRPC user-space (`libcdsprpc.so`, `cdsprpcd`), and the cDSP firmware/skel libraries that the QNN HTP backend dlopens at runtime. The latter two live on the host. We provide a convenient script to install them on Debian/Armbian-based systems. + +Follow these steps: + +1. Download [`user_installation.sh`](https://raw.githubusercontent.com/blakeblackshear/frigate/dev/docker/qcs6490/user_installation.sh). +2. Make it executable: `sudo chmod +x user_installation.sh` +3. Run the script: `sudo ./user_installation.sh` +4. Log out and back in so your user picks up the `fastrpc` group. + +The script installs the [`fastrpc`](https://github.com/radxa-pkg/fastrpc) user-space, the [`radxa-firmware-qcs6490`](https://github.com/radxa-pkg/radxa-firmware) firmware, disables the conflicting `hexagonrpcd` services, and starts a `cdsprpcd` systemd service. + +#### Setup + +Follow Frigate's default installation instructions, but use a docker image with `-qcs6490` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-qcs6490`. + +Grant the container access to the FastRPC devices and the host's cDSP firmware paths. Add the following to your `docker-compose.yml`: + +```yaml +group_add: + - "107" # fastrpc group GID. Verify with `getent group fastrpc`. +devices: + - /dev/fastrpc-cdsp + - /dev/fastrpc-cdsp-secure + - /dev/fastrpc-adsp + - /dev/dma_heap/system +volumes: + # cDSP firmware refuses to load skels from any path other than these on + # the host. Bind-mount them into the container so they appear at the + # expected locations. + - /usr/lib/dsp:/usr/lib/dsp:ro + - /usr/lib/rfsa:/usr/lib/rfsa:ro +``` + +Or, with `docker run`: + +``` +--group-add $(getent group fastrpc | cut -d: -f3) \ +--device /dev/fastrpc-cdsp \ +--device /dev/fastrpc-cdsp-secure \ +--device /dev/fastrpc-adsp \ +--device /dev/dma_heap/system \ +-v /usr/lib/dsp:/usr/lib/dsp:ro \ +-v /usr/lib/rfsa:/usr/lib/rfsa:ro +``` + +#### Configuration + +Next, configure [hardware object detection](/configuration/object_detectors#qualcomm-hexagon-npu) to complete the setup. + ### AXERA AXERA accelerators are available in an M.2 form factor, compatible with both Raspberry Pi and Orange Pi. This form factor has also been successfully tested on x86 platforms, making it a versatile choice for various computing environments. diff --git a/frigate/detectors/plugins/qnn.py b/frigate/detectors/plugins/qnn.py new file mode 100644 index 0000000000..ca8030770e --- /dev/null +++ b/frigate/detectors/plugins/qnn.py @@ -0,0 +1,151 @@ +import logging +import os +from typing import Literal + +import cv2 +import numpy as np +from pydantic import ConfigDict, Field +from typing_extensions import Annotated + +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum + +try: + from qai_appbuilder import ( + LogLevel, + PerfProfile, + ProfilingLevel, + QNNConfig, + QNNContext, + Runtime, + ) + + QNN_SUPPORT = True +except ImportError: + QNN_SUPPORT = False + +logger = logging.getLogger(__name__) + +DETECTOR_KEY = "qnn" +DEFAULT_QNN_LIB_DIR = "/opt/qairt/lib" +MAX_DETECTIONS = 20 + + +class QnnDetectorConfig(BaseDetectorConfig): + """QNN detector for Qualcomm Hexagon NPUs via QAIRT / qai_appbuilder. + + Runs pre-compiled QNN context binaries (.bin) produced by Qualcomm AI Hub + on the Hexagon NPU. Tested on QCS6490 (Hexagon v68) with YOLOv8 detection. + """ + + model_config = ConfigDict( + title="QNN", + ) + + type: Literal[DETECTOR_KEY] + qnn_lib_dir: str = Field( + default=DEFAULT_QNN_LIB_DIR, + title="Directory containing QAIRT runtime libraries (libQnnHtp.so etc.).", + ) + soc_id: str = Field( + default="6490", + title="Qualcomm SoC id. Controls output-tensor ordering of the AI Hub " + "model: '6490' yields [scores, classes, boxes]; other SoCs yield " + "[boxes, scores, classes].", + ) + conf_threshold: Annotated[float, Field(ge=0.0, le=1.0)] = 0.25 + iou_threshold: Annotated[float, Field(ge=0.0, le=1.0)] = 0.7 + + +class QnnDetector(DetectionApi): + type_key = DETECTOR_KEY + supported_models = [ModelTypeEnum.yologeneric] + + def __init__(self, detector_config: QnnDetectorConfig): + super().__init__(detector_config) + if not QNN_SUPPORT: + logger.error( + "qai_appbuilder is not installed. Use the -qcs6490 Docker image " + "variant for Qualcomm Hexagon NPU support." + ) + return + + model_path = detector_config.model.path + if not model_path or not os.path.exists(model_path): + raise FileNotFoundError(f"QNN model not found: {model_path}") + + self._input_size = detector_config.model.width + self._soc_id = detector_config.soc_id + self._conf = detector_config.conf_threshold + self._iou = detector_config.iou_threshold + + QNNConfig.Config( + detector_config.qnn_lib_dir, + Runtime.HTP, + LogLevel.WARN, + ProfilingLevel.BASIC, + ) + self._ctx = QNNContext("yolo", model_path) + PerfProfile.SetPerfProfileGlobal(PerfProfile.BURST) + logger.info( + "QNN detector loaded model=%s size=%d soc=%s", + model_path, + self._input_size, + self._soc_id, + ) + + def detect_raw(self, tensor_input: np.ndarray) -> np.ndarray: + # Frigate hands a view backed by shared-memory mmap. qai_appbuilder's + # C++ boundary segfaults on non-owning buffers — always copy. + arr = np.ascontiguousarray(tensor_input, dtype=np.float32) + if arr.ndim == 3: + arr = arr[None, ...] + if arr.size and float(arr.max()) > 1.5: + arr = arr / 255.0 + + outputs = self._ctx.Inference([arr]) + return self._decode(outputs) + + def _decode(self, outputs: list[np.ndarray]) -> np.ndarray: + if self._soc_id == "6490": + scores = np.asarray(outputs[0]).reshape(-1) + classes = np.asarray(outputs[1]).reshape(-1).astype(np.int32) + boxes = np.asarray(outputs[2]).reshape(-1, 4) + else: + boxes = np.asarray(outputs[0]).reshape(-1, 4) + scores = np.asarray(outputs[1]).reshape(-1) + classes = np.asarray(outputs[2]).reshape(-1).astype(np.int32) + + mask = scores >= self._conf + boxes, scores, classes = boxes[mask], scores[mask], classes[mask] + + out = np.zeros((MAX_DETECTIONS, 6), dtype=np.float32) + if boxes.size == 0: + return out + + cv_boxes = np.stack( + [ + boxes[:, 0], + boxes[:, 1], + boxes[:, 2] - boxes[:, 0], + boxes[:, 3] - boxes[:, 1], + ], + axis=1, + ).tolist() + idxs = cv2.dnn.NMSBoxes(cv_boxes, scores.tolist(), self._conf, self._iou) + if len(idxs) == 0: + return out + idxs = np.asarray(idxs).reshape(-1)[:MAX_DETECTIONS] + + size = float(self._input_size) + for slot, i in enumerate(idxs): + x1, y1, x2, y2 = boxes[i] + out[slot] = ( + float(classes[i]), + float(scores[i]), + float(np.clip(y1 / size, 0.0, 1.0)), + float(np.clip(x1 / size, 0.0, 1.0)), + float(np.clip(y2 / size, 0.0, 1.0)), + float(np.clip(x2 / size, 0.0, 1.0)), + ) + return out