Merge 8a40ea5c87 into 4ee12e6237

.github/workflows/ci.yml

@@ -197,6 +197,31 @@ jobs:
           set: |
             synaptics.tags=${{ steps.setup.outputs.image-name }}-synaptics
             *.cache-from=type=gha
+  qualcomm_build:
+    runs-on: ubuntu-22.04-arm
+    name: Qualcomm Build
+    needs:
+      - arm64_build
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Set up QEMU and Buildx
+        id: setup
+        uses: ./.github/actions/setup
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Qualcomm build
+        uses: docker/bake-action@v7
+        with:
+          source: .
+          push: true
+          targets: qualcomm
+          files: docker/qualcomm/qualcomm.hcl
+          set: |
+            qualcomm.tags=${{ steps.setup.outputs.image-name }}-qualcomm
+            *.cache-from=type=gha
   # The majority of users running arm64 are rpi users, so the rpi
   # build should be the primary arm64 image
   assemble_default_build:

CODEOWNERS

@@ -5,3 +5,4 @@
 /docker/rockchip/ @MarcA711
 /docker/rocm/ @harakas
 /docker/hailo8l/ @spanner3003
+/docker/qualcomm/ @ramalamadingdong

docker/qualcomm/Dockerfile

@@ -0,0 +1,21 @@
+# syntax=docker/dockerfile:1.6
+
+# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Globally set pip break-system-packages option to avoid having to specify it every time
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
+
+FROM wheels AS qualcomm-wheels
+ARG TARGETARCH
+
+# No extra wheels needed — ai-edge-litert is already in the base image
+# and the QNN delegate library (libQnnTFLiteDelegate.so) is provided
+# by the host via CDI bind mounts at runtime.
+
+FROM deps AS qualcomm-deps
+ARG TARGETARCH
+ARG PIP_BREAK_SYSTEM_PACKAGES
+
+WORKDIR /opt/frigate/
+COPY --from=rootfs / /

docker/qualcomm/cdi/install_cdi.py

@@ -0,0 +1,151 @@
+import argparse
+import json
+import os
+
+# The CDI descriptor is authored against the Qualcomm Linux BSP filesystem layout,
+# where the QNN/HTP runtime libraries and the Hexagon "skel" files live under
+# /usr/lib and /usr/lib/rfsa/adsp. On other officially-supported images for the
+# same boards -- notably Ubuntu for RB3 Gen 2 / Rubik Pi 3 with the QAIRT apt
+# packages (qairt-libs/qairt-tools) -- the same files are board-selected into a
+# different directory, and /usr/lib/rfsa/adsp may even contain self-referential
+# symlink loops. When a mount's authored hostPath is missing (or a broken/looping
+# symlink), look the file up by name in these fallback directories and rewrite the
+# mount to the real, resolved file so NPU offload works on both layouts.
+FALLBACK_LIB_DIRS = [
+    "/usr/lib/dsp/cdsp",  # QAIRT apt packages (board-selected from /usr/share/qcom/.../dsp/cdsp)
+]
+
+# Libraries without which HTP/NPU offload cannot work. If any of these cannot be
+# resolved we warn prominently (and abort under --strict) instead of silently
+# producing a CDI that starts fine but never offloads to the NPU.
+CRITICAL_BASENAMES = {
+    "libQnnTFLiteDelegate.so",
+    "libQnnHtp.so",
+    "libQnnHtpV68Skel.so",
+    "libQnnHtpV73Skel.so",
+    "libQnnHtpV75Skel.so",
+    "libQnnSystem.so",
+}
+
+
+def resolve_host_path(host_path):
+    """Resolve a mount's hostPath to a usable, real file/dir on this host.
+
+    Returns the original path when it already resolves (the Qualcomm Linux BSP
+    layout). If it is missing or a broken/looping symlink, search FALLBACK_LIB_DIRS
+    by basename and return the real (symlink-resolved) path, so the rewritten mount
+    is stable across reboots / apt reconfigures. Returns None if not found anywhere.
+    """
+    # os.path.exists() follows symlinks and returns False for a broken link or an
+    # ELOOP ("too many levels of symbolic links") -- exactly the treat-as-missing
+    # behavior we want before falling back.
+    if os.path.exists(host_path):
+        return host_path
+
+    basename = os.path.basename(host_path)
+    for fallback_dir in FALLBACK_LIB_DIRS:
+        candidate = os.path.join(fallback_dir, basename)
+        if candidate != host_path and os.path.exists(candidate):
+            return os.path.realpath(candidate)
+
+    return None
+
+
+parser = argparse.ArgumentParser(description="Install CDI on Dragonwing IoT boards")
+parser.add_argument("--file", type=str, required=True, help="CDI input file")
+parser.add_argument(
+    "--strict",
+    action="store_true",
+    help="Exit non-zero if any critical QNN/HTP library cannot be resolved",
+)
+args, unknown = parser.parse_known_args()
+
+if os.path.exists("/etc/cdi/cdi-hw-acc.json"):
+    print("/etc/cdi/cdi-hw-acc.json already exists. Remove it first before continuing.")
+    exit(1)
+
+if not os.path.exists(args.file):
+    print(f"{args.file} (via --file) does not exist")
+    exit(1)
+
+# Check if directory exists
+if not os.path.exists("/etc/cdi"):
+    # Check if we can create it in /etc
+    if not os.access(os.path.dirname("/etc/cdi"), os.W_OK):
+        print(
+            f"{os.path.dirname('/etc/cdi')} is not writable. Re-run this script with sudo."
+        )
+        exit(1)
+    os.mkdir("/etc/cdi")
+else:
+    # Directory exists → check if writable
+    if not os.access("/etc/cdi", os.W_OK):
+        print("/etc/cdi is not writable. Re-run this script with sudo.")
+        exit(1)
+
+with open(args.file, "r") as f:
+    cdi = json.loads(f.read())
+
+print("Resolving mount paths...")
+relocated = []
+missing = []
+for device in cdi["devices"]:
+    new_mounts = []
+
+    for mount in device["containerEdits"].get("mounts", []):
+        resolved = resolve_host_path(mount["hostPath"])
+
+        if resolved is None:
+            missing.append(mount["hostPath"])
+            print(f"    Missing  {mount['hostPath']}")
+            continue
+
+        if resolved != mount["hostPath"]:
+            relocated.append((mount["hostPath"], resolved))
+            print(f"    Resolved {mount['hostPath']} -> {resolved}")
+            # Keep containerPath as authored; only the host source moves.
+            mount["hostPath"] = resolved
+
+        new_mounts.append(mount)
+
+    device["containerEdits"]["mounts"] = new_mounts
+
+    # The QNN HTP backend (libcdsprpc) locates the Hexagon skel purely via
+    # ADSP_LIBRARY_PATH. Point it at the in-container directory where the skels are
+    # mounted so offload works regardless of the host's on-disk layout.
+    container_edits = device["containerEdits"]
+    env = container_edits.setdefault("env", [])
+    if not any(e.startswith("ADSP_LIBRARY_PATH=") for e in env):
+        env.append("ADSP_LIBRARY_PATH=/usr/lib/rfsa/adsp")
+
+print("")
+if relocated:
+    print(
+        f"Resolved {len(relocated)} mount(s) from a fallback library directory "
+        "(non-BSP image layout)."
+    )
+
+critical_missing = [m for m in missing if os.path.basename(m) in CRITICAL_BASENAMES]
+if missing:
+    print(f"WARNING: {len(missing)} expected host path(s) were missing and skipped.")
+if critical_missing:
+    print("")
+    print(
+        "WARNING: critical QNN/HTP libraries could not be found on this host -- NPU "
+        "offload will NOT work and detection would fall back to CPU:"
+    )
+    for path in critical_missing:
+        print(f"    {path}")
+    print(
+        "Confirm the QAIRT runtime is installed (e.g. the qairt-libs/qairt-tools "
+        "packages) and that the Hexagon skels exist as real files."
+    )
+    if args.strict:
+        print("Aborting due to --strict.")
+        exit(1)
+
+print("")
+print("Writing to /etc/cdi/cdi-hw-acc.json...")
+with open("/etc/cdi/cdi-hw-acc.json", "w") as f:
+    f.write(json.dumps(cdi, indent=4))
+print("Writing to /etc/cdi/cdi-hw-acc.json OK")

docker/qualcomm/qualcomm.hcl

@@ -0,0 +1,27 @@
+target wheels {
+  dockerfile = "docker/main/Dockerfile"
+  platforms = ["linux/arm64"]
+  target = "wheels"
+}
+
+target deps {
+  dockerfile = "docker/main/Dockerfile"
+  platforms = ["linux/arm64"]
+  target = "deps"
+}
+
+target rootfs {
+  dockerfile = "docker/main/Dockerfile"
+  platforms = ["linux/arm64"]
+  target = "rootfs"
+}
+
+target qualcomm {
+  dockerfile = "docker/qualcomm/Dockerfile"
+  contexts = {
+    wheels = "target:wheels",
+    deps = "target:deps",
+    rootfs = "target:rootfs"
+  }
+  platforms = ["linux/arm64"]
+}

docker/qualcomm/qualcomm.mk

@@ -0,0 +1,15 @@
+BOARDS += qualcomm
+
+local-qualcomm: version
+	docker buildx bake --file=docker/qualcomm/qualcomm.hcl qualcomm \
+		--set qualcomm.tags=frigate:latest-qualcomm \
+		--load
+
+build-qualcomm: version
+	docker buildx bake --file=docker/qualcomm/qualcomm.hcl qualcomm \
+		--set qualcomm.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qualcomm
+
+push-qualcomm: build-qualcomm
+	docker buildx bake --file=docker/qualcomm/qualcomm.hcl qualcomm \
+		--set qualcomm.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-qualcomm \
+		--push

docs/data/object_detectors_models.json

@@ -265,6 +265,19 @@
       }
     ]
   },
+  "qualcomm": {
+    "title": "Qualcomm",
+    "models": [
+      {
+        "key": "ssd_mobiledet",
+        "label": "SSD MobileDet",
+        "recommended": true,
+        "download": "The default SSD MobileDet TFLite model (`/cpu_model.tflite`) bundled with the Frigate container is used automatically. This model is INT8 quantized and compatible with the QNN HTP backend.\n\nOnly `.tflite` models are supported with this detector. A custom model must be INT8 quantized for the Hexagon NPU.",
+        "ui": "Navigate to **Settings > System > Detectors and model** and select **Qualcomm** from the detector type dropdown and click **Add**. The bundled SSD MobileDet model is used automatically.\n\nTo use a custom model, configure in the **Custom Model** tab:\n\n| Field                                    | Value                              |\n| ---------------------------------------- | ---------------------------------- |\n| **Custom object detector model path**    | `/config/your_custom_model.tflite` |\n| **Object detection model input width**   | `320`                              |\n| **Object detection model input height**  | `320`                              |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt`            |",
+        "yaml": "detectors:\n  qualcomm_npu:\n    type: qualcomm_tfl"
+      }
+    ]
+  },
   "rknn": {
     "title": "RKNN",
     "models": [

docs/docs/configuration/object_detectors.md

@@ -53,6 +53,10 @@ Frigate supports multiple different detectors that work on different types of ha
 
 - [RKNN](#rockchip-platform): RKNN models can run on Rockchip devices with included NPUs.
 
+**Qualcomm** <CommunityBadge />
+
+- [Qualcomm](#qualcomm): TFLite models can run on Qualcomm SoCs with a Hexagon NPU (e.g. IQ9100, QCS6490) via the QNN TFLite delegate.
+
 **Synaptics** <CommunityBadge />
 
 - [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs.
@@ -652,6 +656,21 @@ When configuring the Synap detector, you have to specify the model: a local **pa
 
 <ModelConfigDropdown detectorTitle="Synaptics" models={objectDetectorsModels.synaptics.models} />
 
+## Qualcomm
+
+Hardware accelerated object detection is supported on the following Qualcomm SoCs with a Hexagon NPU:
+
+- IQ9100 / IQ-9075 EVK
+- QCS6490 / RB3 Gen 2 Vision Kit / Rubik Pi 3
+
+This implementation uses the QNN TFLite delegate (`libQnnTFLiteDelegate.so`) to accelerate TFLite model inference on the Qualcomm Hexagon Tensor Processor (HTP / NPU). The delegate library and device drivers are provided by the host operating system and made available to the container via [CDI (Container Device Interface)](https://docs.docker.com/build/building/cdi/).
+
+See the [installation docs](../frigate/installation.md#qualcomm) for information on setting up CDI and configuring the hardware.
+
+### Configuration
+
+<ModelConfigDropdown detectorTitle="Qualcomm" models={objectDetectorsModels.qualcomm.models} />
+
 ## Rockchip platform
 
 Hardware accelerated object detection is supported on the following SoCs:

docs/docs/frigate/hardware.md

@@ -107,6 +107,10 @@ Frigate supports multiple different detectors that work on different types of ha
 
 - [AXEngine](#axera): axera models can run on AXERA NPUs via AXEngine, delivering highly efficient object detection.
 
+**Qualcomm** <CommunityBadge />
+
+- [Qualcomm](#qualcomm): TFLite models can run on Qualcomm SoCs with a Hexagon NPU (e.g. IQ9100, QCS6490) via the QNN TFLite delegate.
+
 :::
 
 ### Hailo-8
@@ -295,6 +299,17 @@ The inference time of a rk3588 with all 3 cores enabled is typically 25-30 ms fo
 | Name             | AXERA AX650N/AX8850N Inference Time |
 | ---------------- | ----------------------------------- |
 | yolov9-tiny      | ~ 4 ms                              |
+### Qualcomm
+
+Frigate supports hardware object detection on Qualcomm SoCs with a Hexagon NPU, including the IQ9100 and QCS6490 (RB3 Gen 2). The QNN TFLite delegate is used to accelerate inference on the HTP (Hexagon Tensor Processor).
+
+A single detector is typically sufficient for multiple camera streams. The default model is **SSD MobileDet** (INT8 quantized).
+
+| Name          | IQ9100 Inference Time | QCS6490 Inference Time |
+| ------------- | --------------------- | ---------------------- |
+| ssd_mobiledet | ~ 0.8 ms              | ~ 5.55 ms              |
+
+Detailed information is available [in the detector docs](/configuration/object_detectors#qualcomm).
 
 ## What does Frigate use the CPU for and what does it use a detector for? (ELI5 Version)
 

docs/docs/frigate/installation.md

@@ -472,6 +472,120 @@ If you are using `docker run`, add this option to your command `--device /dev/ax
 #### Configuration
 
 Finally, configure [hardware object detection](/configuration/object_detectors#axera) to complete the setup.
+### Qualcomm
+
+Hardware accelerated object detection is supported on the following Qualcomm SoCs:
+
+| Board | SoC | CDI Config File |
+| ----- | --- | --------------- |
+| IQ-9075 EVK | IQ9100 | `cdi-hw-acc-9100.json` |
+| RB3 Gen 2 Vision Kit / Rubik Pi 3 | QCS6490 | `cdi-hw-acc-6490.json` |
+
+The Qualcomm integration uses [CDI (Container Device Interface)](https://docs.docker.com/build/building/cdi/) to provide the container with access to the NPU device nodes and the QNN delegate libraries from the host.
+
+#### Prerequisites
+
+- **QNN runtime on the host**: The QNN runtime libraries (including `libQnnTFLiteDelegate.so` and `libQnnHtp.so`), the Hexagon "skel" files, and the NPU device nodes (`/dev/fastrpc-cdsp`) must be present on the host; they are bind-mounted into the container via CDI. Both officially-supported images for these boards provide them:
+  - **Qualcomm Linux BSP** — ships them under the layout the bundled CDI descriptors target (`/usr/lib`, `/usr/lib/rfsa/adsp`).
+  - **Ubuntu for RB3 Gen 2 / Rubik Pi 3** — install the `qairt-libs` and `qairt-tools` packages from the Qualcomm IoT PPA. These board-select the Hexagon skels into a different directory (`/usr/lib/dsp/cdsp`), and `/usr/lib/rfsa/adsp` may contain self-referential symlink loops; `install_cdi.py` detects this and rewrites the affected mounts to the real files automatically (see below).
+- **Docker 25.0+**: CDI device support requires Docker 25.0 or later. Check with `docker --version`.
+- **arm64 architecture**: The Qualcomm Docker image is built for `linux/arm64` only. If building locally, you must run the build on the target board itself.
+
+#### CDI Installation
+
+1. Download or copy the CDI setup files from the [Frigate repository](https://github.com/blakeblackshear/frigate/tree/dev/docker/qualcomm/cdi).
+
+2. Install the CDI configuration for your board:
+
+   ```bash
+   # IQ9100 / IQ-9075 EVK
+   sudo python3 install_cdi.py --file cdi-hw-acc-9100.json
+
+   # QCS6490 / RB3 Gen 2 Vision Kit / Rubik Pi 3
+   sudo python3 install_cdi.py --file cdi-hw-acc-6490.json
+   ```
+
+   This writes a CDI descriptor to `/etc/cdi/cdi-hw-acc.json`. For any mount whose default (BSP) path is absent or a broken symlink, the script first tries to resolve the file from the QAIRT layout (e.g. `/usr/lib/dsp/cdsp`) and rewrites the mount to the real file; paths it cannot resolve anywhere are skipped with a warning. If a **critical** QNN/HTP library is missing it prints a prominent warning — and, with `--strict`, exits non-zero — so a misconfigured host fails at install time instead of silently falling back to CPU at runtime.
+
+3. Verify the descriptor was written and that it contains the QNN delegate and Hexagon skel mounts (their absence means the QNN runtime is not installed on the host):
+
+   ```bash
+   ls -l /etc/cdi/cdi-hw-acc.json
+   grep -oE 'libQnn(TFLiteDelegate|HtpV[0-9]+Skel|System)\.so' /etc/cdi/cdi-hw-acc.json | sort -u
+   ```
+
+#### Setup
+
+Follow Frigate's default installation instructions, but use a docker image with `-qualcomm` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-qualcomm`.
+
+:::note
+
+The pre-built `stable-qualcomm` image is not yet published. Until it is available, you must build the image locally:
+
+```bash
+git clone https://github.com/blakeblackshear/frigate.git
+cd frigate
+make local-qualcomm
+```
+
+This builds `frigate:latest-qualcomm` on your device. Use `frigate:latest-qualcomm` as the image name in the examples below instead of the `ghcr.io` URL.
+
+:::
+
+Grant Docker access to your Qualcomm hardware by passing the CDI device. In your `docker-compose.yml`:
+
+```yaml
+services:
+  frigate:
+    container_name: frigate
+    restart: unless-stopped
+    image: ghcr.io/blakeblackshear/frigate:stable-qualcomm
+    devices:
+      - qualcomm.com/device=cdi-hw-acc
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - /path/to/your/config:/config
+      - /path/to/your/storage:/media/frigate
+      - type: tmpfs
+        target: /tmp/cache
+        tmpfs:
+          size: 1000000000
+    ports:
+      - "8971:8971"
+      - "8554:8554"
+      - "8555:8555/tcp"
+      - "8555:8555/udp"
+```
+
+If using `docker run`, pass the CDI device with:
+
+```bash
+docker run -d \
+  --name frigate \
+  --restart=unless-stopped \
+  --device qualcomm.com/device=cdi-hw-acc \
+  --mount type=tmpfs,target=/tmp/cache,tmpfs-size=1000000000 \
+  --shm-size=256m \
+  -v /path/to/your/storage:/media/frigate \
+  -v /path/to/your/config:/config \
+  -v /etc/localtime:/etc/localtime:ro \
+  -e FRIGATE_RTSP_PASSWORD='password' \
+  -p 8971:8971 \
+  -p 8554:8554 \
+  -p 8555:8555/tcp \
+  -p 8555:8555/udp \
+  ghcr.io/blakeblackshear/frigate:stable-qualcomm
+```
+
+:::tip
+
+Unlike other hardware integrations that pass individual `/dev/` device nodes, the Qualcomm CDI approach bundles all required device nodes, library bind mounts, and environment variables into a single `--device qualcomm.com/device=cdi-hw-acc` flag.
+
+:::
+
+#### Configuration
+
+Next, you should configure [hardware object detection](/configuration/object_detectors#qualcomm).
 
 ## Docker
 
@@ -558,6 +672,7 @@ The community supported docker image tags for the current stable version are:
 
 - `stable-tensorrt-jp6` - Frigate build optimized for Nvidia Jetson devices running Jetpack 6
 - `stable-rk` - Frigate build for SBCs with Rockchip SoC
+- `stable-qualcomm` - Frigate build for Qualcomm SoCs with Hexagon NPU (IQ9100, QCS6490)
 
 ## Home Assistant App
 

frigate/detectors/plugins/qualcomm_tfl.py

@@ -0,0 +1,48 @@
+"""Qualcomm QNN TFLite delegate detector for Qualcomm NPU acceleration."""
+
+import logging
+
+from pydantic import ConfigDict
+from typing_extensions import Literal
+
+from frigate.detectors.detection_api import DetectionApi
+from frigate.detectors.detector_config import BaseDetectorConfig
+
+from ..detector_utils import (
+    tflite_detect_raw,
+    tflite_init,
+    tflite_load_delegate_interpreter,
+)
+
+logger = logging.getLogger(__name__)
+
+# Use _tfl suffix to default to the bundled tflite model
+DETECTOR_KEY = "qualcomm_tfl"
+
+
+class QualcommDetectorConfig(BaseDetectorConfig):
+    """Qualcomm NPU detector using the QNN TFLite delegate to accelerate inference on Qualcomm SoCs with a Hexagon NPU (e.g. IQ9100, QCS6490)."""
+
+    model_config = ConfigDict(
+        title="Qualcomm",
+    )
+
+    type: Literal[DETECTOR_KEY]
+
+
+class QualcommTfl(DetectionApi):
+    type_key = DETECTOR_KEY
+
+    def __init__(self, detector_config: QualcommDetectorConfig):
+        # The QNN TFLite delegate library is provided by the host via CDI bind mounts
+        delegate_library = "libQnnTFLiteDelegate.so"
+        # Use the Hexagon Tensor Processor (HTP / NPU) backend
+        device_config = {"backend_type": "htp"}
+
+        interpreter = tflite_load_delegate_interpreter(
+            delegate_library, detector_config, device_config
+        )
+        tflite_init(self, interpreter)
+
+    def detect_raw(self, tensor_input):
+        return tflite_detect_raw(self, tensor_input)

web/public/locales/en/config/global.json

@@ -413,6 +413,10 @@
         "description": "The device to use for OpenVINO inference (e.g. 'CPU', 'GPU', 'NPU')."
       }
     },
+    "qualcomm_tfl": {
+      "label": "Qualcomm",
+      "description": "Qualcomm NPU detector using the QNN TFLite delegate to accelerate inference on Qualcomm SoCs with a Hexagon NPU (e.g. IQ9100, QCS6490)."
+    },
     "rknn": {
       "label": "RKNN",
       "description": "RKNN detector for Rockchip NPUs; runs compiled RKNN models on Rockchip hardware.",