mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-07-15 00:11:15 +03:00
Add Qualcomm Hexagon NPU detector (qcs6490)
Adds a community-supported hardware detector for the Qualcomm Hexagon NPU on QCS6490 SoCs (e.g. Radxa Dragon Q6A) via QAIRT 2.37.1 / qai_appbuilder. Mirrors the existing community-board pattern (Rockchip / Synaptics): - frigate/detectors/plugins/qnn.py: detector plugin using yolo-generic model type, lazy SDK import, runs pre-compiled QNN context binaries from Qualcomm AI Hub. - docker/qcs6490/: Dockerfile (two-stage; rebuilds qai_appbuilder wheel inside Frigate's image to match libstdc++ ABI), bake target (qcs6490.hcl), make targets (qcs6490.mk), and a host-side user_installation.sh that installs fastrpc, the QCS6490 firmware (cDSP image + skel libs), and configures cdsprpcd. - .github/workflows/ci.yml: qcs6490_build job mirroring synaptics_build. - CODEOWNERS: /docker/qcs6490/ + qnn.py. - docs: new "Qualcomm Hexagon NPU" sections under Community Supported Detectors in object_detectors.md, plus matching entries in installation.md and hardware.md. Performance on Radxa Dragon Q6A (Hexagon v68, ~12 TOPS), YOLOv8n 640x640: ~10ms per inference under light load, ~24ms with 5 RTSP cameras live. Closes #18602.
This commit is contained in:
parent
d7f42735fc
commit
52ba582e54
25
.github/workflows/ci.yml
vendored
25
.github/workflows/ci.yml
vendored
@ -197,6 +197,31 @@ jobs:
|
|||||||
set: |
|
set: |
|
||||||
synaptics.tags=${{ steps.setup.outputs.image-name }}-synaptics
|
synaptics.tags=${{ steps.setup.outputs.image-name }}-synaptics
|
||||||
*.cache-from=type=gha
|
*.cache-from=type=gha
|
||||||
|
qcs6490_build:
|
||||||
|
runs-on: ubuntu-22.04-arm
|
||||||
|
name: Qualcomm QCS6490 Build
|
||||||
|
needs:
|
||||||
|
- arm64_build
|
||||||
|
steps:
|
||||||
|
- name: Check out code
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Set up QEMU and Buildx
|
||||||
|
id: setup
|
||||||
|
uses: ./.github/actions/setup
|
||||||
|
with:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
- name: Build and push QCS6490 build
|
||||||
|
uses: docker/bake-action@v7
|
||||||
|
with:
|
||||||
|
source: .
|
||||||
|
push: true
|
||||||
|
targets: qcs6490
|
||||||
|
files: docker/qcs6490/qcs6490.hcl
|
||||||
|
set: |
|
||||||
|
qcs6490.tags=${{ steps.setup.outputs.image-name }}-qcs6490
|
||||||
|
*.cache-from=type=gha
|
||||||
# The majority of users running arm64 are rpi users, so the rpi
|
# The majority of users running arm64 are rpi users, so the rpi
|
||||||
# build should be the primary arm64 image
|
# build should be the primary arm64 image
|
||||||
assemble_default_build:
|
assemble_default_build:
|
||||||
|
|||||||
@ -5,3 +5,5 @@
|
|||||||
/docker/rockchip/ @MarcA711
|
/docker/rockchip/ @MarcA711
|
||||||
/docker/rocm/ @harakas
|
/docker/rocm/ @harakas
|
||||||
/docker/hailo8l/ @spanner3003
|
/docker/hailo8l/ @spanner3003
|
||||||
|
/docker/qcs6490/ @notori0us
|
||||||
|
/frigate/detectors/plugins/qnn.py @notori0us
|
||||||
|
|||||||
80
docker/qcs6490/Dockerfile
Normal file
80
docker/qcs6490/Dockerfile
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
# syntax=docker/dockerfile:1.6
|
||||||
|
|
||||||
|
# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
|
||||||
|
ARG DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# Globally set pip break-system-packages option to avoid having to specify it every time
|
||||||
|
ARG PIP_BREAK_SYSTEM_PACKAGES=1
|
||||||
|
|
||||||
|
# QAIRT 2.37.1 runtime tarball + qai_appbuilder source pinned to a known-good
|
||||||
|
# commit that builds against this SDK version. Override at build time via
|
||||||
|
# --build-arg if a downstream maintainer hosts these elsewhere.
|
||||||
|
ARG QAIRT_RUNTIME_URL=https://github.com/notori0us/qairt-runtime/releases/download/v2.37.1.250807/qairt-runtime-2.37.1.250807-aarch64.tar.gz
|
||||||
|
ARG QAI_APPBUILDER_REF=942bc0d
|
||||||
|
|
||||||
|
# ---------- stage: builder ----------
|
||||||
|
# Build the qai_appbuilder Python wheel inside the Frigate image so its C++
|
||||||
|
# ABI (libstdc++/glibc) matches Frigate's runtime stage.
|
||||||
|
FROM deps AS qcs6490-wheels
|
||||||
|
ARG DEBIAN_FRONTEND
|
||||||
|
ARG PIP_BREAK_SYSTEM_PACKAGES
|
||||||
|
ARG QAIRT_RUNTIME_URL
|
||||||
|
ARG QAI_APPBUILDER_REF
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
build-essential cmake git ca-certificates curl \
|
||||||
|
python3-dev libyaml-0-2 \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN pip3 install --no-cache-dir \
|
||||||
|
wheel==0.45.1 setuptools==80.9.0 pybind11==2.13.6 build==1.4.0
|
||||||
|
|
||||||
|
# Fetch QAIRT runtime (libQnnHtp*, libQnnSystem, hexagon-v68 skel, fastrpc shell).
|
||||||
|
RUN mkdir -p /opt/qairt \
|
||||||
|
&& curl -fsSL "${QAIRT_RUNTIME_URL}" | tar -C /opt/qairt -xzf -
|
||||||
|
|
||||||
|
# Fetch qai_appbuilder source at a pinned commit. Patch out the Genie target
|
||||||
|
# (its headers don't match QAIRT 2.37.1) and ensure dist/ exists for the build.
|
||||||
|
RUN git clone --filter=blob:none https://github.com/quic/ai-engine-direct-helper.git /tmp/aedh \
|
||||||
|
&& cd /tmp/aedh \
|
||||||
|
&& git checkout "${QAI_APPBUILDER_REF}" \
|
||||||
|
&& sed -i '/add_subdirectory(genie)/d' pybind/CMakeLists.txt \
|
||||||
|
&& sed -i 's|zip_package|os.makedirs("dist", exist_ok=True)\n zip_package|' setup.py
|
||||||
|
|
||||||
|
ENV QNN_SDK_ROOT=/opt/qairt \
|
||||||
|
QAI_TOOLCHAINS=aarch64-oe-linux-gcc11.2 \
|
||||||
|
QAI_HEXAGONARCH=68
|
||||||
|
RUN cd /tmp/aedh && python3 -m build -w
|
||||||
|
|
||||||
|
# ---------- stage: runtime ----------
|
||||||
|
FROM deps AS qcs6490-frigate
|
||||||
|
ARG DEBIAN_FRONTEND
|
||||||
|
ARG PIP_BREAK_SYSTEM_PACKAGES
|
||||||
|
|
||||||
|
# Runtime libs: libcdsprpc.so dlopen path + QNN SDK libs need libyaml-0.so.
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends libyaml-0-2 \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# QAIRT runtime layout. ADSP_LIBRARY_PATH below points cDSP firmware here for
|
||||||
|
# the QNN HTP backend skel (libQnnHtpV68Skel.so) and the fastrpc shell.
|
||||||
|
COPY --from=qcs6490-wheels /opt/qairt/lib /opt/qairt/lib/
|
||||||
|
COPY --from=qcs6490-wheels /opt/qairt/hexagon-v68 /opt/qairt/hexagon-v68/
|
||||||
|
|
||||||
|
# Make libcdsprpc.so visible to the dynamic linker without polluting LD_LIBRARY_PATH.
|
||||||
|
RUN ln -sf /opt/qairt/lib/libcdsprpc.so /usr/lib/libcdsprpc.so && ldconfig
|
||||||
|
|
||||||
|
# Install the qai_appbuilder wheel built in the previous stage.
|
||||||
|
RUN --mount=type=bind,from=qcs6490-wheels,source=/tmp/aedh/dist,target=/wheels \
|
||||||
|
pip3 install --no-cache-dir /wheels/qai_appbuilder-*.whl
|
||||||
|
|
||||||
|
WORKDIR /opt/frigate/
|
||||||
|
COPY --from=rootfs / /
|
||||||
|
|
||||||
|
# fastrpc separator gotcha: ADSP_LIBRARY_PATH is split by ';' (semicolon),
|
||||||
|
# not the usual ':'. The cDSP firmware also requires its skel + libc++ files
|
||||||
|
# at host paths /usr/lib/dsp/cdsp and /usr/lib/rfsa/adsp; bind-mount these
|
||||||
|
# from the host (see docs/docs/frigate/installation.md#qualcomm-platform).
|
||||||
|
ENV ADSP_LIBRARY_PATH="/opt/qairt/hexagon-v68;/usr/lib/dsp/cdsp;/usr/lib/rfsa/adsp" \
|
||||||
|
LD_LIBRARY_PATH="/opt/qairt/lib"
|
||||||
27
docker/qcs6490/qcs6490.hcl
Normal file
27
docker/qcs6490/qcs6490.hcl
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
target wheels {
|
||||||
|
dockerfile = "docker/main/Dockerfile"
|
||||||
|
platforms = ["linux/arm64"]
|
||||||
|
target = "wheels"
|
||||||
|
}
|
||||||
|
|
||||||
|
target deps {
|
||||||
|
dockerfile = "docker/main/Dockerfile"
|
||||||
|
platforms = ["linux/arm64"]
|
||||||
|
target = "deps"
|
||||||
|
}
|
||||||
|
|
||||||
|
target rootfs {
|
||||||
|
dockerfile = "docker/main/Dockerfile"
|
||||||
|
platforms = ["linux/arm64"]
|
||||||
|
target = "rootfs"
|
||||||
|
}
|
||||||
|
|
||||||
|
target qcs6490 {
|
||||||
|
dockerfile = "docker/qcs6490/Dockerfile"
|
||||||
|
contexts = {
|
||||||
|
wheels = "target:wheels",
|
||||||
|
deps = "target:deps",
|
||||||
|
rootfs = "target:rootfs"
|
||||||
|
}
|
||||||
|
platforms = ["linux/arm64"]
|
||||||
|
}
|
||||||
15
docker/qcs6490/qcs6490.mk
Normal file
15
docker/qcs6490/qcs6490.mk
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
BOARDS += qcs6490
|
||||||
|
|
||||||
|
local-qcs6490: version
|
||||||
|
docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \
|
||||||
|
--set qcs6490.tags=frigate:latest-qcs6490 \
|
||||||
|
--load
|
||||||
|
|
||||||
|
build-qcs6490: version
|
||||||
|
docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \
|
||||||
|
--set qcs6490.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qcs6490
|
||||||
|
|
||||||
|
push-qcs6490: build-qcs6490
|
||||||
|
docker buildx bake --file=docker/qcs6490/qcs6490.hcl qcs6490 \
|
||||||
|
--set qcs6490.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qcs6490 \
|
||||||
|
--push
|
||||||
89
docker/qcs6490/user_installation.sh
Executable file
89
docker/qcs6490/user_installation.sh
Executable file
@ -0,0 +1,89 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Host-side setup for the Qualcomm Hexagon NPU detector on a Radxa Dragon Q6A
|
||||||
|
# (or other QCS6490 board). Installs:
|
||||||
|
# - fastrpc user-space (libcdsprpc.so, cdsprpcd, fastrpc_test)
|
||||||
|
# - Radxa firmware that ships the cDSP image + skel libs the QNN HTP
|
||||||
|
# backend dlopens at runtime
|
||||||
|
# - a transient cdsprpcd systemd service
|
||||||
|
# and disables hexagonrpcd, which holds the fastrpc devices and conflicts.
|
||||||
|
#
|
||||||
|
# Run with sudo. Logs out + back in are required for the fastrpc group to
|
||||||
|
# take effect for your user.
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [ "$EUID" -ne 0 ]; then
|
||||||
|
echo "Please run as root (sudo $0)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ARCH=$(dpkg --print-architecture)
|
||||||
|
if [ "$ARCH" != "arm64" ]; then
|
||||||
|
echo "This script targets arm64 (QCS6490). Detected: $ARCH"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y --no-install-recommends ca-certificates curl
|
||||||
|
|
||||||
|
WORKDIR=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$WORKDIR"' EXIT
|
||||||
|
|
||||||
|
# fastrpc user-space (provides libcdsprpc.so + cdsprpcd). Not in apt.
|
||||||
|
FASTRPC_VER=1.0.4-1
|
||||||
|
echo "==> Installing fastrpc ${FASTRPC_VER}"
|
||||||
|
for pkg in fastrpc fastrpc-tools; do
|
||||||
|
curl -fsSL -o "${WORKDIR}/${pkg}.deb" \
|
||||||
|
"https://github.com/radxa-pkg/fastrpc/releases/download/${FASTRPC_VER}/${pkg}_${FASTRPC_VER}_arm64.deb"
|
||||||
|
done
|
||||||
|
apt-get install -y "${WORKDIR}/fastrpc.deb" "${WORKDIR}/fastrpc-tools.deb"
|
||||||
|
|
||||||
|
# Radxa QCS6490 firmware (provides /usr/lib/dsp/cdsp/{cdsp.mbn,*_skel.so,...}
|
||||||
|
# and /usr/lib/rfsa/adsp/, both required by the cDSP at runtime).
|
||||||
|
RADXA_FW_VER=0.2.29
|
||||||
|
echo "==> Installing radxa-firmware-qcs6490 ${RADXA_FW_VER}"
|
||||||
|
curl -fsSL -o "${WORKDIR}/radxa-firmware-qcs6490.deb" \
|
||||||
|
"https://github.com/radxa-pkg/radxa-firmware/releases/download/${RADXA_FW_VER}/radxa-firmware-qcs6490_${RADXA_FW_VER}_all.deb"
|
||||||
|
apt-get install -y "${WORKDIR}/radxa-firmware-qcs6490.deb"
|
||||||
|
|
||||||
|
# hexagonrpcd from the apt 'hexagonrpcd' package conflicts with cdsprpcd by
|
||||||
|
# holding /dev/fastrpc-* exclusively. We need cdsprpcd for QNN HTP.
|
||||||
|
echo "==> Disabling conflicting hexagonrpcd services"
|
||||||
|
for unit in hexagonrpcd hexagonrpcd-suspend hexagonrpcd-resume; do
|
||||||
|
systemctl disable --now "${unit}" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "==> Enabling cdsprpcd"
|
||||||
|
cat >/etc/systemd/system/cdsprpcd.service <<'UNIT'
|
||||||
|
[Unit]
|
||||||
|
Description=Qualcomm cDSP FastRPC daemon
|
||||||
|
After=local-fs.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/usr/bin/cdsprpcd
|
||||||
|
Restart=always
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
UNIT
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable --now cdsprpcd
|
||||||
|
|
||||||
|
# Allow non-root containers + users to open /dev/fastrpc-*.
|
||||||
|
echo "==> Adding invoking user to fastrpc group"
|
||||||
|
TARGET_USER="${SUDO_USER:-$USER}"
|
||||||
|
if [ -n "${TARGET_USER}" ] && id "${TARGET_USER}" >/dev/null 2>&1; then
|
||||||
|
usermod -aG fastrpc "${TARGET_USER}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Hexagon NPU host setup complete."
|
||||||
|
echo "Log out and back in for fastrpc group membership to take effect, then:"
|
||||||
|
echo " docker run ... ghcr.io/blakeblackshear/frigate:stable-qcs6490"
|
||||||
|
echo "Pass these to the container (devices, group, env):"
|
||||||
|
echo " --device /dev/fastrpc-cdsp --device /dev/fastrpc-cdsp-secure"
|
||||||
|
echo " --device /dev/fastrpc-adsp --device /dev/dma_heap/system"
|
||||||
|
echo " --group-add \$(getent group fastrpc | cut -d: -f3)"
|
||||||
|
echo " -v /usr/lib/dsp:/usr/lib/dsp:ro -v /usr/lib/rfsa:/usr/lib/rfsa:ro"
|
||||||
@ -2045,6 +2045,88 @@ Explanation of the paramters:
|
|||||||
- **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`.
|
- **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`.
|
||||||
- `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.2/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.2_EN.pdf).
|
- `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.2/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.2_EN.pdf).
|
||||||
|
|
||||||
|
## Qualcomm Hexagon NPU
|
||||||
|
|
||||||
|
Hardware accelerated object detection is supported on the following Qualcomm SoCs:
|
||||||
|
|
||||||
|
- QCS6490 (Hexagon v68, ~12 TOPS) — including the [Radxa Dragon Q6A](https://radxa.com/products/dragon/q6a/) and similar boards
|
||||||
|
|
||||||
|
This implementation uses the [Qualcomm AI Engine Direct (QNN) SDK](https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk) (QAIRT 2.37.1) via the [`qai_appbuilder`](https://github.com/quic/ai-engine-direct-helper) Python bindings. Models are pre-compiled QNN context binaries (`.bin`) downloaded from [Qualcomm AI Hub](https://aihub.qualcomm.com/).
|
||||||
|
|
||||||
|
:::warning
|
||||||
|
|
||||||
|
The pre-compiled YOLOv8 weights from Qualcomm AI Hub originate from Ultralytics and are subject to the AGPL-3.0 license. They cannot be used commercially without a separate license from Ultralytics.
|
||||||
|
|
||||||
|
:::
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
Make sure to follow the [Qualcomm specific installation instructions](/frigate/installation#qualcomm-platform).
|
||||||
|
|
||||||
|
### Downloading a Model
|
||||||
|
|
||||||
|
Frigate does not bundle the YOLOv8 weights. Download a QNN context binary for your SoC from Qualcomm AI Hub once and mount it into the container:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mkdir -p ./models
|
||||||
|
# from https://aihub.qualcomm.com/compute/models/yolov8_det
|
||||||
|
# (sign in, choose target "qualcomm-qcs6490-proxy", download .bin)
|
||||||
|
mv ~/Downloads/yolov8_det.bin ./models/
|
||||||
|
```
|
||||||
|
|
||||||
|
Mount `./models` into the container at `/models` and reference the file from your config.
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
<ConfigTabs>
|
||||||
|
<TabItem value="ui">
|
||||||
|
|
||||||
|
Navigate to <NavPath path="Settings > System > Detector hardware" /> and select **QNN** from the detector type dropdown and click **Add**. Then navigate to <NavPath path="Settings > System > Detection model" /> and configure:
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
| ---------------------------------------- | --------------------------- |
|
||||||
|
| **Custom object detector model path** | `/models/yolov8_det.bin` |
|
||||||
|
| **Object Detection Model Type** | `yolo-generic` |
|
||||||
|
| **Object detection model input width** | `640` |
|
||||||
|
| **Object detection model input height** | `640` |
|
||||||
|
| **Model Input Tensor Shape** | `nhwc` |
|
||||||
|
| **Model Input D Type** | `float` |
|
||||||
|
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
|
||||||
|
|
||||||
|
</TabItem>
|
||||||
|
<TabItem value="yaml">
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
detectors:
|
||||||
|
hexagon:
|
||||||
|
type: qnn
|
||||||
|
soc_id: "6490"
|
||||||
|
|
||||||
|
model:
|
||||||
|
path: /models/yolov8_det.bin
|
||||||
|
model_type: yolo-generic
|
||||||
|
width: 640
|
||||||
|
height: 640
|
||||||
|
input_tensor: nhwc
|
||||||
|
input_dtype: float
|
||||||
|
labelmap_path: /labelmap/coco-80.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
</TabItem>
|
||||||
|
</ConfigTabs>
|
||||||
|
|
||||||
|
The inference time on a Radxa Dragon Q6A (QCS6490, Hexagon v68) is approximately 10–25 ms per frame at 640×640 — varying with system load and the number of cameras pumping frames into the detector.
|
||||||
|
|
||||||
|
### Compiling Your Own Model
|
||||||
|
|
||||||
|
To compile a different model — or to compile YOLOv8 for a Qualcomm SoC other than QCS6490 — use [Qualcomm AI Hub](https://aihub.qualcomm.com/). The workflow is:
|
||||||
|
|
||||||
|
1. Sign in to AI Hub and find a model (for example, [YOLOv8 Detection](https://aihub.qualcomm.com/compute/models/yolov8_det)).
|
||||||
|
2. Submit a compile job for your target device (e.g. `qualcomm-qcs6490-proxy`). The job emits a QNN context binary (`.bin`) sized for that SoC's Hexagon variant.
|
||||||
|
3. Download the `.bin` and mount it into the container as above.
|
||||||
|
|
||||||
|
The output-tensor ordering of YOLOv8 differs by SoC: QCS6490 yields `[scores, classes, boxes]` (handled by `soc_id: "6490"`); other SoCs yield `[boxes, scores, classes]` (use `soc_id: "other"`).
|
||||||
|
|
||||||
## DeGirum
|
## DeGirum
|
||||||
|
|
||||||
DeGirum is a detector that can use any type of hardware listed on [their website](https://hub.degirum.com). DeGirum can be used with local hardware through a DeGirum AI Server, or through the use of `@local`. You can also connect directly to DeGirum's AI Hub to run inferences. **Please Note:** This detector _cannot_ be used for commercial purposes.
|
DeGirum is a detector that can use any type of hardware listed on [their website](https://hub.degirum.com). DeGirum can be used with local hardware through a DeGirum AI Server, or through the use of `@local`. You can also connect directly to DeGirum's AI Hub to run inferences. **Please Note:** This detector _cannot_ be used for commercial purposes.
|
||||||
|
|||||||
@ -295,6 +295,14 @@ The inference time of a rk3588 with all 3 cores enabled is typically 25-30 ms fo
|
|||||||
| ---------------- | ----------------------------------- |
|
| ---------------- | ----------------------------------- |
|
||||||
| yolov9-tiny | ~ 4 ms |
|
| yolov9-tiny | ~ 4 ms |
|
||||||
|
|
||||||
|
### Qualcomm Hexagon NPU
|
||||||
|
|
||||||
|
Frigate supports hardware accelerated object detection on Qualcomm Hexagon NPUs via the [QNN detector](/configuration/object_detectors#qualcomm-hexagon-npu). Tested on the QCS6490 (Hexagon v68) on a [Radxa Dragon Q6A](https://radxa.com/products/dragon/q6a/).
|
||||||
|
|
||||||
|
| Name | Inference Time |
|
||||||
|
| ----------------- | -------------- |
|
||||||
|
| QCS6490 / YOLOv8n | ~ 10–25 ms |
|
||||||
|
|
||||||
## What does Frigate use the CPU for and what does it use a detector for? (ELI5 Version)
|
## What does Frigate use the CPU for and what does it use a detector for? (ELI5 Version)
|
||||||
|
|
||||||
This is taken from a [user question on reddit](https://www.reddit.com/r/homeassistant/comments/q8mgau/comment/hgqbxh5/?utm_source=share&utm_medium=web2x&context=3). Modified slightly for clarity.
|
This is taken from a [user question on reddit](https://www.reddit.com/r/homeassistant/comments/q8mgau/comment/hgqbxh5/?utm_source=share&utm_medium=web2x&context=3). Modified slightly for clarity.
|
||||||
|
|||||||
@ -428,6 +428,70 @@ or add these options to your `docker run` command:
|
|||||||
|
|
||||||
Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics).
|
Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics).
|
||||||
|
|
||||||
|
### Qualcomm platform
|
||||||
|
|
||||||
|
Hardware accelerated object detection on the Hexagon NPU is supported on the following Qualcomm SoCs:
|
||||||
|
|
||||||
|
- QCS6490 (Hexagon v68) — including the Radxa Dragon Q6A and similar boards
|
||||||
|
|
||||||
|
Make sure your kernel exposes the FastRPC bridges to the cDSP. On a configured board you should see:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ ls /dev/fastrpc-*
|
||||||
|
/dev/fastrpc-adsp /dev/fastrpc-cdsp /dev/fastrpc-cdsp-secure
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Installation
|
||||||
|
|
||||||
|
Hexagon NPU access requires the QAIRT runtime libraries (shipped in the Frigate `-qcs6490` image), the FastRPC user-space (`libcdsprpc.so`, `cdsprpcd`), and the cDSP firmware/skel libraries that the QNN HTP backend dlopens at runtime. The latter two live on the host. We provide a convenient script to install them on Debian/Armbian-based systems.
|
||||||
|
|
||||||
|
Follow these steps:
|
||||||
|
|
||||||
|
1. Download [`user_installation.sh`](https://raw.githubusercontent.com/blakeblackshear/frigate/dev/docker/qcs6490/user_installation.sh).
|
||||||
|
2. Make it executable: `sudo chmod +x user_installation.sh`
|
||||||
|
3. Run the script: `sudo ./user_installation.sh`
|
||||||
|
4. Log out and back in so your user picks up the `fastrpc` group.
|
||||||
|
|
||||||
|
The script installs the [`fastrpc`](https://github.com/radxa-pkg/fastrpc) user-space, the [`radxa-firmware-qcs6490`](https://github.com/radxa-pkg/radxa-firmware) firmware, disables the conflicting `hexagonrpcd` services, and starts a `cdsprpcd` systemd service.
|
||||||
|
|
||||||
|
#### Setup
|
||||||
|
|
||||||
|
Follow Frigate's default installation instructions, but use a docker image with `-qcs6490` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-qcs6490`.
|
||||||
|
|
||||||
|
Grant the container access to the FastRPC devices and the host's cDSP firmware paths. Add the following to your `docker-compose.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
group_add:
|
||||||
|
- "107" # fastrpc group GID. Verify with `getent group fastrpc`.
|
||||||
|
devices:
|
||||||
|
- /dev/fastrpc-cdsp
|
||||||
|
- /dev/fastrpc-cdsp-secure
|
||||||
|
- /dev/fastrpc-adsp
|
||||||
|
- /dev/dma_heap/system
|
||||||
|
volumes:
|
||||||
|
# cDSP firmware refuses to load skels from any path other than these on
|
||||||
|
# the host. Bind-mount them into the container so they appear at the
|
||||||
|
# expected locations.
|
||||||
|
- /usr/lib/dsp:/usr/lib/dsp:ro
|
||||||
|
- /usr/lib/rfsa:/usr/lib/rfsa:ro
|
||||||
|
```
|
||||||
|
|
||||||
|
Or, with `docker run`:
|
||||||
|
|
||||||
|
```
|
||||||
|
--group-add $(getent group fastrpc | cut -d: -f3) \
|
||||||
|
--device /dev/fastrpc-cdsp \
|
||||||
|
--device /dev/fastrpc-cdsp-secure \
|
||||||
|
--device /dev/fastrpc-adsp \
|
||||||
|
--device /dev/dma_heap/system \
|
||||||
|
-v /usr/lib/dsp:/usr/lib/dsp:ro \
|
||||||
|
-v /usr/lib/rfsa:/usr/lib/rfsa:ro
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Configuration
|
||||||
|
|
||||||
|
Next, configure [hardware object detection](/configuration/object_detectors#qualcomm-hexagon-npu) to complete the setup.
|
||||||
|
|
||||||
### AXERA
|
### AXERA
|
||||||
|
|
||||||
AXERA accelerators are available in an M.2 form factor, compatible with both Raspberry Pi and Orange Pi. This form factor has also been successfully tested on x86 platforms, making it a versatile choice for various computing environments.
|
AXERA accelerators are available in an M.2 form factor, compatible with both Raspberry Pi and Orange Pi. This form factor has also been successfully tested on x86 platforms, making it a versatile choice for various computing environments.
|
||||||
|
|||||||
151
frigate/detectors/plugins/qnn.py
Normal file
151
frigate/detectors/plugins/qnn.py
Normal file
@ -0,0 +1,151 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
from pydantic import ConfigDict, Field
|
||||||
|
from typing_extensions import Annotated
|
||||||
|
|
||||||
|
from frigate.detectors.detection_api import DetectionApi
|
||||||
|
from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
|
||||||
|
|
||||||
|
try:
|
||||||
|
from qai_appbuilder import (
|
||||||
|
LogLevel,
|
||||||
|
PerfProfile,
|
||||||
|
ProfilingLevel,
|
||||||
|
QNNConfig,
|
||||||
|
QNNContext,
|
||||||
|
Runtime,
|
||||||
|
)
|
||||||
|
|
||||||
|
QNN_SUPPORT = True
|
||||||
|
except ImportError:
|
||||||
|
QNN_SUPPORT = False
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DETECTOR_KEY = "qnn"
|
||||||
|
DEFAULT_QNN_LIB_DIR = "/opt/qairt/lib"
|
||||||
|
MAX_DETECTIONS = 20
|
||||||
|
|
||||||
|
|
||||||
|
class QnnDetectorConfig(BaseDetectorConfig):
|
||||||
|
"""QNN detector for Qualcomm Hexagon NPUs via QAIRT / qai_appbuilder.
|
||||||
|
|
||||||
|
Runs pre-compiled QNN context binaries (.bin) produced by Qualcomm AI Hub
|
||||||
|
on the Hexagon NPU. Tested on QCS6490 (Hexagon v68) with YOLOv8 detection.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = ConfigDict(
|
||||||
|
title="QNN",
|
||||||
|
)
|
||||||
|
|
||||||
|
type: Literal[DETECTOR_KEY]
|
||||||
|
qnn_lib_dir: str = Field(
|
||||||
|
default=DEFAULT_QNN_LIB_DIR,
|
||||||
|
title="Directory containing QAIRT runtime libraries (libQnnHtp.so etc.).",
|
||||||
|
)
|
||||||
|
soc_id: str = Field(
|
||||||
|
default="6490",
|
||||||
|
title="Qualcomm SoC id. Controls output-tensor ordering of the AI Hub "
|
||||||
|
"model: '6490' yields [scores, classes, boxes]; other SoCs yield "
|
||||||
|
"[boxes, scores, classes].",
|
||||||
|
)
|
||||||
|
conf_threshold: Annotated[float, Field(ge=0.0, le=1.0)] = 0.25
|
||||||
|
iou_threshold: Annotated[float, Field(ge=0.0, le=1.0)] = 0.7
|
||||||
|
|
||||||
|
|
||||||
|
class QnnDetector(DetectionApi):
|
||||||
|
type_key = DETECTOR_KEY
|
||||||
|
supported_models = [ModelTypeEnum.yologeneric]
|
||||||
|
|
||||||
|
def __init__(self, detector_config: QnnDetectorConfig):
|
||||||
|
super().__init__(detector_config)
|
||||||
|
if not QNN_SUPPORT:
|
||||||
|
logger.error(
|
||||||
|
"qai_appbuilder is not installed. Use the -qcs6490 Docker image "
|
||||||
|
"variant for Qualcomm Hexagon NPU support."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
model_path = detector_config.model.path
|
||||||
|
if not model_path or not os.path.exists(model_path):
|
||||||
|
raise FileNotFoundError(f"QNN model not found: {model_path}")
|
||||||
|
|
||||||
|
self._input_size = detector_config.model.width
|
||||||
|
self._soc_id = detector_config.soc_id
|
||||||
|
self._conf = detector_config.conf_threshold
|
||||||
|
self._iou = detector_config.iou_threshold
|
||||||
|
|
||||||
|
QNNConfig.Config(
|
||||||
|
detector_config.qnn_lib_dir,
|
||||||
|
Runtime.HTP,
|
||||||
|
LogLevel.WARN,
|
||||||
|
ProfilingLevel.BASIC,
|
||||||
|
)
|
||||||
|
self._ctx = QNNContext("yolo", model_path)
|
||||||
|
PerfProfile.SetPerfProfileGlobal(PerfProfile.BURST)
|
||||||
|
logger.info(
|
||||||
|
"QNN detector loaded model=%s size=%d soc=%s",
|
||||||
|
model_path,
|
||||||
|
self._input_size,
|
||||||
|
self._soc_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def detect_raw(self, tensor_input: np.ndarray) -> np.ndarray:
|
||||||
|
# Frigate hands a view backed by shared-memory mmap. qai_appbuilder's
|
||||||
|
# C++ boundary segfaults on non-owning buffers — always copy.
|
||||||
|
arr = np.ascontiguousarray(tensor_input, dtype=np.float32)
|
||||||
|
if arr.ndim == 3:
|
||||||
|
arr = arr[None, ...]
|
||||||
|
if arr.size and float(arr.max()) > 1.5:
|
||||||
|
arr = arr / 255.0
|
||||||
|
|
||||||
|
outputs = self._ctx.Inference([arr])
|
||||||
|
return self._decode(outputs)
|
||||||
|
|
||||||
|
def _decode(self, outputs: list[np.ndarray]) -> np.ndarray:
|
||||||
|
if self._soc_id == "6490":
|
||||||
|
scores = np.asarray(outputs[0]).reshape(-1)
|
||||||
|
classes = np.asarray(outputs[1]).reshape(-1).astype(np.int32)
|
||||||
|
boxes = np.asarray(outputs[2]).reshape(-1, 4)
|
||||||
|
else:
|
||||||
|
boxes = np.asarray(outputs[0]).reshape(-1, 4)
|
||||||
|
scores = np.asarray(outputs[1]).reshape(-1)
|
||||||
|
classes = np.asarray(outputs[2]).reshape(-1).astype(np.int32)
|
||||||
|
|
||||||
|
mask = scores >= self._conf
|
||||||
|
boxes, scores, classes = boxes[mask], scores[mask], classes[mask]
|
||||||
|
|
||||||
|
out = np.zeros((MAX_DETECTIONS, 6), dtype=np.float32)
|
||||||
|
if boxes.size == 0:
|
||||||
|
return out
|
||||||
|
|
||||||
|
cv_boxes = np.stack(
|
||||||
|
[
|
||||||
|
boxes[:, 0],
|
||||||
|
boxes[:, 1],
|
||||||
|
boxes[:, 2] - boxes[:, 0],
|
||||||
|
boxes[:, 3] - boxes[:, 1],
|
||||||
|
],
|
||||||
|
axis=1,
|
||||||
|
).tolist()
|
||||||
|
idxs = cv2.dnn.NMSBoxes(cv_boxes, scores.tolist(), self._conf, self._iou)
|
||||||
|
if len(idxs) == 0:
|
||||||
|
return out
|
||||||
|
idxs = np.asarray(idxs).reshape(-1)[:MAX_DETECTIONS]
|
||||||
|
|
||||||
|
size = float(self._input_size)
|
||||||
|
for slot, i in enumerate(idxs):
|
||||||
|
x1, y1, x2, y2 = boxes[i]
|
||||||
|
out[slot] = (
|
||||||
|
float(classes[i]),
|
||||||
|
float(scores[i]),
|
||||||
|
float(np.clip(y1 / size, 0.0, 1.0)),
|
||||||
|
float(np.clip(x1 / size, 0.0, 1.0)),
|
||||||
|
float(np.clip(y2 / size, 0.0, 1.0)),
|
||||||
|
float(np.clip(x2 / size, 0.0, 1.0)),
|
||||||
|
)
|
||||||
|
return out
|
||||||
Loading…
Reference in New Issue
Block a user