mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-07-14 16:01:13 +03:00
Add Qualcomm Hexagon NPU detector (qcs6490)
Adds a community-supported hardware detector for the Qualcomm Hexagon NPU on QCS6490 SoCs (e.g. Radxa Dragon Q6A) via QAIRT 2.37.1 / qai_appbuilder. Mirrors the existing community-board pattern (Rockchip / Synaptics): - frigate/detectors/plugins/qnn.py: detector plugin using yolo-generic model type, lazy SDK import, runs pre-compiled QNN context binaries from Qualcomm AI Hub. - docker/qcs6490/: Dockerfile (two-stage; rebuilds qai_appbuilder wheel inside Frigate's image to match libstdc++ ABI), bake target (qcs6490.hcl), make targets (qcs6490.mk), and a host-side user_installation.sh that installs fastrpc, the QCS6490 firmware (cDSP image + skel libs), and configures cdsprpcd. - .github/workflows/ci.yml: qcs6490_build job mirroring synaptics_build. - CODEOWNERS: /docker/qcs6490/ + qnn.py. - docs: new "Qualcomm Hexagon NPU" sections under Community Supported Detectors in object_detectors.md, plus matching entries in installation.md and hardware.md. Performance on Radxa Dragon Q6A (Hexagon v68, ~12 TOPS), YOLOv8n 640x640: ~10ms per inference under light load, ~24ms with 5 RTSP cameras live. Closes #18602.
This commit is contained in:
parent
d7f42735fc
commit
52ba582e54
25
.github/workflows/ci.yml
vendored
25
.github/workflows/ci.yml
vendored
@ -197,6 +197,31 @@ jobs:
|
||||
set: |
|
||||
synaptics.tags=${{ steps.setup.outputs.image-name }}-synaptics
|
||||
*.cache-from=type=gha
|
||||
qcs6490_build:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
name: Qualcomm QCS6490 Build
|
||||
needs:
|
||||
- arm64_build
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up QEMU and Buildx
|
||||
id: setup
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Build and push QCS6490 build
|
||||
uses: docker/bake-action@v7
|
||||
with:
|
||||
source: .
|
||||
push: true
|
||||
targets: qcs6490
|
||||
files: docker/qcs6490/qcs6490.hcl
|
||||
set: |
|
||||
qcs6490.tags=${{ steps.setup.outputs.image-name }}-qcs6490
|
||||
*.cache-from=type=gha
|
||||
# The majority of users running arm64 are rpi users, so the rpi
|
||||
# build should be the primary arm64 image
|
||||
assemble_default_build:
|
||||
|
||||
@ -5,3 +5,5 @@
|
||||
/docker/rockchip/ @MarcA711
|
||||
/docker/rocm/ @harakas
|
||||
/docker/hailo8l/ @spanner3003
|
||||
/docker/qcs6490/ @notori0us
|
||||
/frigate/detectors/plugins/qnn.py @notori0us
|
||||
|
||||
80
docker/qcs6490/Dockerfile
Normal file
80
docker/qcs6490/Dockerfile
Normal file
@ -0,0 +1,80 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
|
||||
# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Globally set pip break-system-packages option to avoid having to specify it every time
|
||||
ARG PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
# QAIRT 2.37.1 runtime tarball + qai_appbuilder source pinned to a known-good
|
||||
# commit that builds against this SDK version. Override at build time via
|
||||
# --build-arg if a downstream maintainer hosts these elsewhere.
|
||||
ARG QAIRT_RUNTIME_URL=https://github.com/notori0us/qairt-runtime/releases/download/v2.37.1.250807/qairt-runtime-2.37.1.250807-aarch64.tar.gz
|
||||
ARG QAI_APPBUILDER_REF=942bc0d
|
||||
|
||||
# ---------- stage: builder ----------
|
||||
# Build the qai_appbuilder Python wheel inside the Frigate image so its C++
|
||||
# ABI (libstdc++/glibc) matches Frigate's runtime stage.
|
||||
FROM deps AS qcs6490-wheels
|
||||
ARG DEBIAN_FRONTEND
|
||||
ARG PIP_BREAK_SYSTEM_PACKAGES
|
||||
ARG QAIRT_RUNTIME_URL
|
||||
ARG QAI_APPBUILDER_REF
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
build-essential cmake git ca-certificates curl \
|
||||
python3-dev libyaml-0-2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip3 install --no-cache-dir \
|
||||
wheel==0.45.1 setuptools==80.9.0 pybind11==2.13.6 build==1.4.0
|
||||
|
||||
# Fetch QAIRT runtime (libQnnHtp*, libQnnSystem, hexagon-v68 skel, fastrpc shell).
|
||||
RUN mkdir -p /opt/qairt \
|
||||
&& curl -fsSL "${QAIRT_RUNTIME_URL}" | tar -C /opt/qairt -xzf -
|
||||
|
||||
# Fetch qai_appbuilder source at a pinned commit. Patch out the Genie target
|
||||
# (its headers don't match QAIRT 2.37.1) and ensure dist/ exists for the build.
|
||||
RUN git clone --filter=blob:none https://github.com/quic/ai-engine-direct-helper.git /tmp/aedh \
|
||||
&& cd /tmp/aedh \
|
||||
&& git checkout "${QAI_APPBUILDER_REF}" \
|
||||
&& sed -i '/add_subdirectory(genie)/d' pybind/CMakeLists.txt \
|
||||
&& sed -i 's|zip_package|os.makedirs("dist", exist_ok=True)\n zip_package|' setup.py
|
||||
|
||||
ENV QNN_SDK_ROOT=/opt/qairt \
|
||||
QAI_TOOLCHAINS=aarch64-oe-linux-gcc11.2 \
|
||||
QAI_HEXAGONARCH=68
|
||||
RUN cd /tmp/aedh && python3 -m build -w
|
||||
|
||||
# ---------- stage: runtime ----------
|
||||
FROM deps AS qcs6490-frigate
|
||||
ARG DEBIAN_FRONTEND
|
||||
ARG PIP_BREAK_SYSTEM_PACKAGES
|
||||
|
||||
# Runtime libs: libcdsprpc.so dlopen path + QNN SDK libs need libyaml-0.so.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libyaml-0-2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# QAIRT runtime layout. ADSP_LIBRARY_PATH below points cDSP firmware here for
|
||||
# the QNN HTP backend skel (libQnnHtpV68Skel.so) and the fastrpc shell.
|
||||
COPY --from=qcs6490-wheels /opt/qairt/lib /opt/qairt/lib/
|
||||
COPY --from=qcs6490-wheels /opt/qairt/hexagon-v68 /opt/qairt/hexagon-v68/
|
||||
|
||||
# Make libcdsprpc.so visible to the dynamic linker without polluting LD_LIBRARY_PATH.
|
||||
RUN ln -sf /opt/qairt/lib/libcdsprpc.so /usr/lib/libcdsprpc.so && ldconfig
|
||||
|
||||
# Install the qai_appbuilder wheel built in the previous stage.
|
||||
RUN --mount=type=bind,from=qcs6490-wheels,source=/tmp/aedh/dist,target=/wheels \
|
||||
pip3 install --no-cache-dir /wheels/qai_appbuilder-*.whl
|
||||
|
||||
WORKDIR /opt/frigate/
|
||||
COPY --from=rootfs / /
|
||||
|
||||
# fastrpc separator gotcha: ADSP_LIBRARY_PATH is split by ';' (semicolon),
|
||||
# not the usual ':'. The cDSP firmware also requires its skel + libc++ files
|
||||
# at host paths /usr/lib/dsp/cdsp and /usr/lib/rfsa/adsp; bind-mount these
|
||||
# from the host (see docs/docs/frigate/installation.md#qualcomm-platform).
|
||||
ENV ADSP_LIBRARY_PATH="/opt/qairt/hexagon-v68;/usr/lib/dsp/cdsp;/usr/lib/rfsa/adsp" \
|
||||
LD_LIBRARY_PATH="/opt/qairt/lib"
|
||||
27
docker/qcs6490/qcs6490.hcl
Normal file
27
docker/qcs6490/qcs6490.hcl
Normal file
@ -0,0 +1,27 @@
|
||||
target wheels {
|
||||
dockerfile = "docker/main/Dockerfile"
|
||||
platforms = ["linux/arm64"]
|
||||
target = "wheels"
|
||||
}
|
||||
|
||||
target deps {
|
||||
dockerfile = "docker/main/Dockerfile"
|
||||
platforms = ["linux/arm64"]
|
||||
target = "deps"
|
||||
}
|
||||
|
||||
target rootfs {
|
||||
dockerfile = "docker/main/Dockerfile"
|
||||
platforms = ["linux/arm64"]
|
||||
target = "rootfs"
|
||||
}
|
||||
|
||||
target qcs6490 {
|
||||
dockerfile = "docker/qcs6490/Dockerfile"
|
||||
contexts = {
|
||||
wheels = "target:wheels",
|
||||
deps = "target:deps",
|
||||
rootfs = "target:rootfs"
|
||||
}
|
||||
platforms = ["linux/arm64"]
|
||||
}
|
||||
15
docker/qcs6490/qcs6490.mk
Normal file
15
docker/qcs6490/qcs6490.mk
Normal file
@ -0,0 +1,15 @@
|
||||
BOARDS += qcs6490
|
||||
|
||||
local-qcs6490: version
|
||||
docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \
|
||||
--set qcs6490.tags=frigate:latest-qcs6490 \
|
||||
--load
|
||||
|
||||
build-qcs6490: version
|
||||
docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \
|
||||
--set qcs6490.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qcs6490
|
||||
|
||||
push-qcs6490: build-qcs6490
|
||||
docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \
|
||||
--set qcs6490.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qcs6490 \
|
||||
--push
|
||||
89
docker/qcs6490/user_installation.sh
Executable file
89
docker/qcs6490/user_installation.sh
Executable file
@ -0,0 +1,89 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Host-side setup for the Qualcomm Hexagon NPU detector on a Radxa Dragon Q6A
|
||||
# (or other QCS6490 board). Installs:
|
||||
# - fastrpc user-space (libcdsprpc.so, cdsprpcd, fastrpc_test)
|
||||
# - Radxa firmware that ships the cDSP image + skel libs the QNN HTP
|
||||
# backend dlopens at runtime
|
||||
# - a transient cdsprpcd systemd service
|
||||
# and disables hexagonrpcd, which holds the fastrpc devices and conflicts.
|
||||
#
|
||||
# Run with sudo. Logs out + back in are required for the fastrpc group to
|
||||
# take effect for your user.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root (sudo $0)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCH=$(dpkg --print-architecture)
|
||||
if [ "$ARCH" != "arm64" ]; then
|
||||
echo "This script targets arm64 (QCS6490). Detected: $ARCH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends ca-certificates curl
|
||||
|
||||
WORKDIR=$(mktemp -d)
|
||||
trap 'rm -rf "$WORKDIR"' EXIT
|
||||
|
||||
# fastrpc user-space (provides libcdsprpc.so + cdsprpcd). Not in apt.
|
||||
FASTRPC_VER=1.0.4-1
|
||||
echo "==> Installing fastrpc ${FASTRPC_VER}"
|
||||
for pkg in fastrpc fastrpc-tools; do
|
||||
curl -fsSL -o "${WORKDIR}/${pkg}.deb" \
|
||||
"https://github.com/radxa-pkg/fastrpc/releases/download/${FASTRPC_VER}/${pkg}_${FASTRPC_VER}_arm64.deb"
|
||||
done
|
||||
apt-get install -y "${WORKDIR}/fastrpc.deb" "${WORKDIR}/fastrpc-tools.deb"
|
||||
|
||||
# Radxa QCS6490 firmware (provides /usr/lib/dsp/cdsp/{cdsp.mbn,*_skel.so,...}
|
||||
# and /usr/lib/rfsa/adsp/, both required by the cDSP at runtime).
|
||||
RADXA_FW_VER=0.2.29
|
||||
echo "==> Installing radxa-firmware-qcs6490 ${RADXA_FW_VER}"
|
||||
curl -fsSL -o "${WORKDIR}/radxa-firmware-qcs6490.deb" \
|
||||
"https://github.com/radxa-pkg/radxa-firmware/releases/download/${RADXA_FW_VER}/radxa-firmware-qcs6490_${RADXA_FW_VER}_all.deb"
|
||||
apt-get install -y "${WORKDIR}/radxa-firmware-qcs6490.deb"
|
||||
|
||||
# hexagonrpcd from the apt 'hexagonrpcd' package conflicts with cdsprpcd by
|
||||
# holding /dev/fastrpc-* exclusively. We need cdsprpcd for QNN HTP.
|
||||
echo "==> Disabling conflicting hexagonrpcd services"
|
||||
for unit in hexagonrpcd hexagonrpcd-suspend hexagonrpcd-resume; do
|
||||
systemctl disable --now "${unit}" 2>/dev/null || true
|
||||
done
|
||||
|
||||
echo "==> Enabling cdsprpcd"
|
||||
cat >/etc/systemd/system/cdsprpcd.service <<'UNIT'
|
||||
[Unit]
|
||||
Description=Qualcomm cDSP FastRPC daemon
|
||||
After=local-fs.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/cdsprpcd
|
||||
Restart=always
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now cdsprpcd
|
||||
|
||||
# Allow non-root containers + users to open /dev/fastrpc-*.
|
||||
echo "==> Adding invoking user to fastrpc group"
|
||||
TARGET_USER="${SUDO_USER:-$USER}"
|
||||
if [ -n "${TARGET_USER}" ] && id "${TARGET_USER}" >/dev/null 2>&1; then
|
||||
usermod -aG fastrpc "${TARGET_USER}"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Hexagon NPU host setup complete."
|
||||
echo "Log out and back in for fastrpc group membership to take effect, then:"
|
||||
echo " docker run ... ghcr.io/blakeblackshear/frigate:stable-qcs6490"
|
||||
echo "Pass these to the container (devices, group, env):"
|
||||
echo " --device /dev/fastrpc-cdsp --device /dev/fastrpc-cdsp-secure"
|
||||
echo " --device /dev/fastrpc-adsp --device /dev/dma_heap/system"
|
||||
echo " --group-add \$(getent group fastrpc | cut -d: -f3)"
|
||||
echo " -v /usr/lib/dsp:/usr/lib/dsp:ro -v /usr/lib/rfsa:/usr/lib/rfsa:ro"
|
||||
@ -2045,6 +2045,88 @@ Explanation of the paramters:
|
||||
- **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`.
|
||||
- `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.2/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.2_EN.pdf).
|
||||
|
||||
## Qualcomm Hexagon NPU
|
||||
|
||||
Hardware accelerated object detection is supported on the following Qualcomm SoCs:
|
||||
|
||||
- QCS6490 (Hexagon v68, ~12 TOPS) — including the [Radxa Dragon Q6A](https://radxa.com/products/dragon/q6a/) and similar boards
|
||||
|
||||
This implementation uses the [Qualcomm AI Engine Direct (QNN) SDK](https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk) (QAIRT 2.37.1) via the [`qai_appbuilder`](https://github.com/quic/ai-engine-direct-helper) Python bindings. Models are pre-compiled QNN context binaries (`.bin`) downloaded from [Qualcomm AI Hub](https://aihub.qualcomm.com/).
|
||||
|
||||
:::warning
|
||||
|
||||
The pre-compiled YOLOv8 weights from Qualcomm AI Hub originate from Ultralytics and are subject to the AGPL-3.0 license. They cannot be used commercially without a separate license from Ultralytics.
|
||||
|
||||
:::
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Make sure to follow the [Qualcomm specific installation instructions](/frigate/installation#qualcomm-platform).
|
||||
|
||||
### Downloading a Model
|
||||
|
||||
Frigate does not bundle the YOLOv8 weights. Download a QNN context binary for your SoC from Qualcomm AI Hub once and mount it into the container:
|
||||
|
||||
```bash
|
||||
mkdir -p ./models
|
||||
# from https://aihub.qualcomm.com/compute/models/yolov8_det
|
||||
# (sign in, choose target "qualcomm-qcs6490-proxy", download .bin)
|
||||
mv ~/Downloads/yolov8_det.bin ./models/
|
||||
```
|
||||
|
||||
Mount `./models` into the container at `/models` and reference the file from your config.
|
||||
|
||||
### Configuration
|
||||
|
||||
<ConfigTabs>
|
||||
<TabItem value="ui">
|
||||
|
||||
Navigate to <NavPath path="Settings > System > Detector hardware" /> and select **QNN** from the detector type dropdown and click **Add**. Then navigate to <NavPath path="Settings > System > Detection model" /> and configure:
|
||||
|
||||
| Field | Value |
|
||||
| ---------------------------------------- | --------------------------- |
|
||||
| **Custom object detector model path** | `/models/yolov8_det.bin` |
|
||||
| **Object Detection Model Type** | `yolo-generic` |
|
||||
| **Object detection model input width** | `640` |
|
||||
| **Object detection model input height** | `640` |
|
||||
| **Model Input Tensor Shape** | `nhwc` |
|
||||
| **Model Input D Type** | `float` |
|
||||
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="yaml">
|
||||
|
||||
```yaml
|
||||
detectors:
|
||||
hexagon:
|
||||
type: qnn
|
||||
soc_id: "6490"
|
||||
|
||||
model:
|
||||
path: /models/yolov8_det.bin
|
||||
model_type: yolo-generic
|
||||
width: 640
|
||||
height: 640
|
||||
input_tensor: nhwc
|
||||
input_dtype: float
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</ConfigTabs>
|
||||
|
||||
The inference time on a Radxa Dragon Q6A (QCS6490, Hexagon v68) is approximately 10–25 ms per frame at 640×640 — varying with system load and the number of cameras pumping frames into the detector.
|
||||
|
||||
### Compiling Your Own Model
|
||||
|
||||
To compile a different model — or to compile YOLOv8 for a Qualcomm SoC other than QCS6490 — use [Qualcomm AI Hub](https://aihub.qualcomm.com/). The workflow is:
|
||||
|
||||
1. Sign in to AI Hub and find a model (for example, [YOLOv8 Detection](https://aihub.qualcomm.com/compute/models/yolov8_det)).
|
||||
2. Submit a compile job for your target device (e.g. `qualcomm-qcs6490-proxy`). The job emits a QNN context binary (`.bin`) sized for that SoC's Hexagon variant.
|
||||
3. Download the `.bin` and mount it into the container as above.
|
||||
|
||||
The output-tensor ordering of YOLOv8 differs by SoC: QCS6490 yields `[scores, classes, boxes]` (handled by `soc_id: "6490"`); other SoCs yield `[boxes, scores, classes]` (use `soc_id: "other"`).
|
||||
|
||||
## DeGirum
|
||||
|
||||
DeGirum is a detector that can use any type of hardware listed on [their website](https://hub.degirum.com). DeGirum can be used with local hardware through a DeGirum AI Server, or through the use of `@local`. You can also connect directly to DeGirum's AI Hub to run inferences. **Please Note:** This detector _cannot_ be used for commercial purposes.
|
||||
|
||||
@ -295,6 +295,14 @@ The inference time of a rk3588 with all 3 cores enabled is typically 25-30 ms fo
|
||||
| ---------------- | ----------------------------------- |
|
||||
| yolov9-tiny | ~ 4 ms |
|
||||
|
||||
### Qualcomm Hexagon NPU
|
||||
|
||||
Frigate supports hardware accelerated object detection on Qualcomm Hexagon NPUs via the [QNN detector](/configuration/object_detectors#qualcomm-hexagon-npu). Tested on the QCS6490 (Hexagon v68) on a [Radxa Dragon Q6A](https://radxa.com/products/dragon/q6a/).
|
||||
|
||||
| Name | Inference Time |
|
||||
| ----------------- | -------------- |
|
||||
| QCS6490 / YOLOv8n | ~ 10–25 ms |
|
||||
|
||||
## What does Frigate use the CPU for and what does it use a detector for? (ELI5 Version)
|
||||
|
||||
This is taken from a [user question on reddit](https://www.reddit.com/r/homeassistant/comments/q8mgau/comment/hgqbxh5/?utm_source=share&utm_medium=web2x&context=3). Modified slightly for clarity.
|
||||
|
||||
@ -428,6 +428,70 @@ or add these options to your `docker run` command:
|
||||
|
||||
Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics).
|
||||
|
||||
### Qualcomm platform
|
||||
|
||||
Hardware accelerated object detection on the Hexagon NPU is supported on the following Qualcomm SoCs:
|
||||
|
||||
- QCS6490 (Hexagon v68) — including the Radxa Dragon Q6A and similar boards
|
||||
|
||||
Make sure your kernel exposes the FastRPC bridges to the cDSP. On a configured board you should see:
|
||||
|
||||
```
|
||||
$ ls /dev/fastrpc-*
|
||||
/dev/fastrpc-adsp /dev/fastrpc-cdsp /dev/fastrpc-cdsp-secure
|
||||
```
|
||||
|
||||
#### Installation
|
||||
|
||||
Hexagon NPU access requires the QAIRT runtime libraries (shipped in the Frigate `-qcs6490` image), the FastRPC user-space (`libcdsprpc.so`, `cdsprpcd`), and the cDSP firmware/skel libraries that the QNN HTP backend dlopens at runtime. The latter two live on the host. We provide a convenient script to install them on Debian/Armbian-based systems.
|
||||
|
||||
Follow these steps:
|
||||
|
||||
1. Download [`user_installation.sh`](https://raw.githubusercontent.com/blakeblackshear/frigate/dev/docker/qcs6490/user_installation.sh).
|
||||
2. Make it executable: `sudo chmod +x user_installation.sh`
|
||||
3. Run the script: `sudo ./user_installation.sh`
|
||||
4. Log out and back in so your user picks up the `fastrpc` group.
|
||||
|
||||
The script installs the [`fastrpc`](https://github.com/radxa-pkg/fastrpc) user-space, the [`radxa-firmware-qcs6490`](https://github.com/radxa-pkg/radxa-firmware) firmware, disables the conflicting `hexagonrpcd` services, and starts a `cdsprpcd` systemd service.
|
||||
|
||||
#### Setup
|
||||
|
||||
Follow Frigate's default installation instructions, but use a docker image with `-qcs6490` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-qcs6490`.
|
||||
|
||||
Grant the container access to the FastRPC devices and the host's cDSP firmware paths. Add the following to your `docker-compose.yml`:
|
||||
|
||||
```yaml
|
||||
group_add:
|
||||
- "107" # fastrpc group GID. Verify with `getent group fastrpc`.
|
||||
devices:
|
||||
- /dev/fastrpc-cdsp
|
||||
- /dev/fastrpc-cdsp-secure
|
||||
- /dev/fastrpc-adsp
|
||||
- /dev/dma_heap/system
|
||||
volumes:
|
||||
# cDSP firmware refuses to load skels from any path other than these on
|
||||
# the host. Bind-mount them into the container so they appear at the
|
||||
# expected locations.
|
||||
- /usr/lib/dsp:/usr/lib/dsp:ro
|
||||
- /usr/lib/rfsa:/usr/lib/rfsa:ro
|
||||
```
|
||||
|
||||
Or, with `docker run`:
|
||||
|
||||
```
|
||||
--group-add $(getent group fastrpc | cut -d: -f3) \
|
||||
--device /dev/fastrpc-cdsp \
|
||||
--device /dev/fastrpc-cdsp-secure \
|
||||
--device /dev/fastrpc-adsp \
|
||||
--device /dev/dma_heap/system \
|
||||
-v /usr/lib/dsp:/usr/lib/dsp:ro \
|
||||
-v /usr/lib/rfsa:/usr/lib/rfsa:ro
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
Next, configure [hardware object detection](/configuration/object_detectors#qualcomm-hexagon-npu) to complete the setup.
|
||||
|
||||
### AXERA
|
||||
|
||||
AXERA accelerators are available in an M.2 form factor, compatible with both Raspberry Pi and Orange Pi. This form factor has also been successfully tested on x86 platforms, making it a versatile choice for various computing environments.
|
||||
|
||||
151
frigate/detectors/plugins/qnn.py
Normal file
151
frigate/detectors/plugins/qnn.py
Normal file
@ -0,0 +1,151 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Literal
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from pydantic import ConfigDict, Field
|
||||
from typing_extensions import Annotated
|
||||
|
||||
from frigate.detectors.detection_api import DetectionApi
|
||||
from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
|
||||
|
||||
try:
|
||||
from qai_appbuilder import (
|
||||
LogLevel,
|
||||
PerfProfile,
|
||||
ProfilingLevel,
|
||||
QNNConfig,
|
||||
QNNContext,
|
||||
Runtime,
|
||||
)
|
||||
|
||||
QNN_SUPPORT = True
|
||||
except ImportError:
|
||||
QNN_SUPPORT = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DETECTOR_KEY = "qnn"
|
||||
DEFAULT_QNN_LIB_DIR = "/opt/qairt/lib"
|
||||
MAX_DETECTIONS = 20
|
||||
|
||||
|
||||
class QnnDetectorConfig(BaseDetectorConfig):
|
||||
"""QNN detector for Qualcomm Hexagon NPUs via QAIRT / qai_appbuilder.
|
||||
|
||||
Runs pre-compiled QNN context binaries (.bin) produced by Qualcomm AI Hub
|
||||
on the Hexagon NPU. Tested on QCS6490 (Hexagon v68) with YOLOv8 detection.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
title="QNN",
|
||||
)
|
||||
|
||||
type: Literal[DETECTOR_KEY]
|
||||
qnn_lib_dir: str = Field(
|
||||
default=DEFAULT_QNN_LIB_DIR,
|
||||
title="Directory containing QAIRT runtime libraries (libQnnHtp.so etc.).",
|
||||
)
|
||||
soc_id: str = Field(
|
||||
default="6490",
|
||||
title="Qualcomm SoC id. Controls output-tensor ordering of the AI Hub "
|
||||
"model: '6490' yields [scores, classes, boxes]; other SoCs yield "
|
||||
"[boxes, scores, classes].",
|
||||
)
|
||||
conf_threshold: Annotated[float, Field(ge=0.0, le=1.0)] = 0.25
|
||||
iou_threshold: Annotated[float, Field(ge=0.0, le=1.0)] = 0.7
|
||||
|
||||
|
||||
class QnnDetector(DetectionApi):
|
||||
type_key = DETECTOR_KEY
|
||||
supported_models = [ModelTypeEnum.yologeneric]
|
||||
|
||||
def __init__(self, detector_config: QnnDetectorConfig):
|
||||
super().__init__(detector_config)
|
||||
if not QNN_SUPPORT:
|
||||
logger.error(
|
||||
"qai_appbuilder is not installed. Use the -qcs6490 Docker image "
|
||||
"variant for Qualcomm Hexagon NPU support."
|
||||
)
|
||||
return
|
||||
|
||||
model_path = detector_config.model.path
|
||||
if not model_path or not os.path.exists(model_path):
|
||||
raise FileNotFoundError(f"QNN model not found: {model_path}")
|
||||
|
||||
self._input_size = detector_config.model.width
|
||||
self._soc_id = detector_config.soc_id
|
||||
self._conf = detector_config.conf_threshold
|
||||
self._iou = detector_config.iou_threshold
|
||||
|
||||
QNNConfig.Config(
|
||||
detector_config.qnn_lib_dir,
|
||||
Runtime.HTP,
|
||||
LogLevel.WARN,
|
||||
ProfilingLevel.BASIC,
|
||||
)
|
||||
self._ctx = QNNContext("yolo", model_path)
|
||||
PerfProfile.SetPerfProfileGlobal(PerfProfile.BURST)
|
||||
logger.info(
|
||||
"QNN detector loaded model=%s size=%d soc=%s",
|
||||
model_path,
|
||||
self._input_size,
|
||||
self._soc_id,
|
||||
)
|
||||
|
||||
def detect_raw(self, tensor_input: np.ndarray) -> np.ndarray:
|
||||
# Frigate hands a view backed by shared-memory mmap. qai_appbuilder's
|
||||
# C++ boundary segfaults on non-owning buffers — always copy.
|
||||
arr = np.ascontiguousarray(tensor_input, dtype=np.float32)
|
||||
if arr.ndim == 3:
|
||||
arr = arr[None, ...]
|
||||
if arr.size and float(arr.max()) > 1.5:
|
||||
arr = arr / 255.0
|
||||
|
||||
outputs = self._ctx.Inference([arr])
|
||||
return self._decode(outputs)
|
||||
|
||||
def _decode(self, outputs: list[np.ndarray]) -> np.ndarray:
|
||||
if self._soc_id == "6490":
|
||||
scores = np.asarray(outputs[0]).reshape(-1)
|
||||
classes = np.asarray(outputs[1]).reshape(-1).astype(np.int32)
|
||||
boxes = np.asarray(outputs[2]).reshape(-1, 4)
|
||||
else:
|
||||
boxes = np.asarray(outputs[0]).reshape(-1, 4)
|
||||
scores = np.asarray(outputs[1]).reshape(-1)
|
||||
classes = np.asarray(outputs[2]).reshape(-1).astype(np.int32)
|
||||
|
||||
mask = scores >= self._conf
|
||||
boxes, scores, classes = boxes[mask], scores[mask], classes[mask]
|
||||
|
||||
out = np.zeros((MAX_DETECTIONS, 6), dtype=np.float32)
|
||||
if boxes.size == 0:
|
||||
return out
|
||||
|
||||
cv_boxes = np.stack(
|
||||
[
|
||||
boxes[:, 0],
|
||||
boxes[:, 1],
|
||||
boxes[:, 2] - boxes[:, 0],
|
||||
boxes[:, 3] - boxes[:, 1],
|
||||
],
|
||||
axis=1,
|
||||
).tolist()
|
||||
idxs = cv2.dnn.NMSBoxes(cv_boxes, scores.tolist(), self._conf, self._iou)
|
||||
if len(idxs) == 0:
|
||||
return out
|
||||
idxs = np.asarray(idxs).reshape(-1)[:MAX_DETECTIONS]
|
||||
|
||||
size = float(self._input_size)
|
||||
for slot, i in enumerate(idxs):
|
||||
x1, y1, x2, y2 = boxes[i]
|
||||
out[slot] = (
|
||||
float(classes[i]),
|
||||
float(scores[i]),
|
||||
float(np.clip(y1 / size, 0.0, 1.0)),
|
||||
float(np.clip(x1 / size, 0.0, 1.0)),
|
||||
float(np.clip(y2 / size, 0.0, 1.0)),
|
||||
float(np.clip(x2 / size, 0.0, 1.0)),
|
||||
)
|
||||
return out
|
||||
Loading…
Reference in New Issue
Block a user