From 7b4eaf2d10d89d5efee9be877f0fe90ddf1a6a28 Mon Sep 17 00:00:00 2001 From: shizhicheng Date: Fri, 24 Oct 2025 08:22:56 +0000 Subject: [PATCH 1/5] Initial commit for AXERA AI accelerators --- .github/workflows/ci.yml | 26 +++ docker/axcl/Dockerfile | 59 ++++++ docker/axcl/axcl.hcl | 13 ++ docker/axcl/axcl.mk | 15 ++ docker/axcl/user_installation.sh | 83 ++++++++ docs/docs/configuration/object_detectors.md | 39 ++++ docs/docs/frigate/hardware.md | 14 ++ docs/docs/frigate/installation.md | 34 ++++ frigate/detectors/plugins/axengine.py | 201 ++++++++++++++++++++ 9 files changed, 484 insertions(+) create mode 100644 docker/axcl/Dockerfile create mode 100644 docker/axcl/axcl.hcl create mode 100644 docker/axcl/axcl.mk create mode 100755 docker/axcl/user_installation.sh create mode 100644 frigate/detectors/plugins/axengine.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcf3070b5..60bcdf6b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,3 +225,29 @@ jobs: sources: | ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-amd64 ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-rpi + axera_build: + runs-on: ubuntu-22.04 + name: AXERA Build + needs: + - amd64_build + - arm64_build + steps: + - name: Check out code + uses: actions/checkout@v5 + with: + persist-credentials: false + - name: Set up QEMU and Buildx + id: setup + uses: ./.github/actions/setup + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push Axera build + uses: docker/bake-action@v6 + with: + source: . + push: true + targets: axcl + files: docker/axcl/axcl.hcl + set: | + axcl.tags=${{ steps.setup.outputs.image-name }}-axcl + *.cache-from=type=gha \ No newline at end of file diff --git a/docker/axcl/Dockerfile b/docker/axcl/Dockerfile new file mode 100644 index 000000000..86e868b61 --- /dev/null +++ b/docker/axcl/Dockerfile @@ -0,0 +1,59 @@ +# syntax=docker/dockerfile:1.6 + +# https://askubuntu.com/questions/972516/debian-frontend-environment-variable +ARG DEBIAN_FRONTEND=noninteractive + +# Globally set pip break-system-packages option to avoid having to specify it every time +ARG PIP_BREAK_SYSTEM_PACKAGES=1 + + +FROM frigate AS frigate-axcl +ARG TARGETARCH +ARG PIP_BREAK_SYSTEM_PACKAGES + +# Install axmodels +RUN mkdir -p /axmodels \ + && wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/yolov5s_320.axmodel -O /axmodels/yolov5s_320.axmodel + +# Install axpyengine +RUN wget https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3.rc1/axengine-0.1.3-py3-none-any.whl -O /axengine-0.1.3-py3-none-any.whl +RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple/ /axengine-0.1.3-py3-none-any.whl \ + && rm /axengine-0.1.3-py3-none-any.whl + +# Install axcl +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + echo "Installing x86_64 version of axcl"; \ + wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb -O /axcl.deb; \ + else \ + echo "Installing aarch64 version of axcl"; \ + wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb -O /axcl.deb; \ + fi + +RUN mkdir /unpack_axcl && \ + dpkg-deb -x /axcl.deb /unpack_axcl && \ + cp -R /unpack_axcl/usr/bin/axcl /usr/bin/ && \ + cp -R /unpack_axcl/usr/lib/axcl /usr/lib/ && \ + rm -rf /unpack_axcl /axcl.deb + + +# Install axcl ffmpeg +RUN mkdir -p /usr/lib/ffmpeg/axcl + +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffmpeg-x64 -O /usr/lib/ffmpeg/axcl/ffmpeg && \ + wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffprobe-x64 -O /usr/lib/ffmpeg/axcl/ffprobe; \ + else \ + wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffmpeg-aarch64 -O /usr/lib/ffmpeg/axcl/ffmpeg && \ + wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffprobe-aarch64 -O /usr/lib/ffmpeg/axcl/ffprobe; \ + fi + +RUN chmod +x /usr/lib/ffmpeg/axcl/ffmpeg /usr/lib/ffmpeg/axcl/ffprobe + +# Set ldconfig path +RUN echo "/usr/lib/axcl" > /etc/ld.so.conf.d/ax.conf + +# Set env +ENV PATH="$PATH:/usr/bin/axcl" +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/axcl" + +ENTRYPOINT ["sh", "-c", "ldconfig && exec /init"] \ No newline at end of file diff --git a/docker/axcl/axcl.hcl b/docker/axcl/axcl.hcl new file mode 100644 index 000000000..d7cf0d4eb --- /dev/null +++ b/docker/axcl/axcl.hcl @@ -0,0 +1,13 @@ +target frigate { + dockerfile = "docker/main/Dockerfile" + platforms = ["linux/amd64", "linux/arm64"] + target = "frigate" +} + +target axcl { + dockerfile = "docker/axcl/Dockerfile" + contexts = { + frigate = "target:frigate", + } + platforms = ["linux/amd64", "linux/arm64"] +} \ No newline at end of file diff --git a/docker/axcl/axcl.mk b/docker/axcl/axcl.mk new file mode 100644 index 000000000..e4b6d4cef --- /dev/null +++ b/docker/axcl/axcl.mk @@ -0,0 +1,15 @@ +BOARDS += axcl + +local-axcl: version + docker buildx bake --file=docker/axcl/axcl.hcl axcl \ + --set axcl.tags=frigate:latest-axcl \ + --load + +build-axcl: version + docker buildx bake --file=docker/axcl/axcl.hcl axcl \ + --set axcl.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-axcl + +push-axcl: build-axcl + docker buildx bake --file=docker/axcl/axcl.hcl axcl \ + --set axcl.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-axcl \ + --push \ No newline at end of file diff --git a/docker/axcl/user_installation.sh b/docker/axcl/user_installation.sh new file mode 100755 index 000000000..e053a5faf --- /dev/null +++ b/docker/axcl/user_installation.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Update package list and install dependencies +sudo apt-get update +sudo apt-get install -y build-essential cmake git wget pciutils kmod udev + +# Check if gcc-12 is needed +current_gcc_version=$(gcc --version | head -n1 | awk '{print $NF}') +gcc_major_version=$(echo $current_gcc_version | cut -d'.' -f1) + +if [[ $gcc_major_version -lt 12 ]]; then + echo "Current GCC version ($current_gcc_version) is lower than 12, installing gcc-12..." + sudo apt-get install -y gcc-12 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 12 + echo "GCC-12 installed and set as default" +else + echo "Current GCC version ($current_gcc_version) is sufficient, skipping GCC installation" +fi + +# Determine architecture +arch=$(uname -m) +download_url="" + +if [[ $arch == "x86_64" ]]; then + download_url="https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb" + deb_file="axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb" +elif [[ $arch == "aarch64" ]]; then + download_url="https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb" + deb_file="axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb" +else + echo "Unsupported architecture: $arch" + exit 1 +fi + +# Download AXCL driver +echo "Downloading AXCL driver for $arch..." +wget "$download_url" -O "$deb_file" + +if [ $? -ne 0 ]; then + echo "Failed to download AXCL driver" + exit 1 +fi + +# Install AXCL driver +echo "Installing AXCL driver..." +sudo dpkg -i "$deb_file" + +if [ $? -ne 0 ]; then + echo "Failed to install AXCL driver, attempting to fix dependencies..." + sudo apt-get install -f -y + sudo dpkg -i "$deb_file" + + if [ $? -ne 0 ]; then + echo "AXCL driver installation failed" + exit 1 + fi +fi + +# Update environment +echo "Updating environment..." +source /etc/profile + +# Verify installation +echo "Verifying AXCL installation..." +if command -v axcl-smi &> /dev/null; then + echo "AXCL driver detected, checking AI accelerator status..." + + axcl_output=$(axcl-smi 2>&1) + axcl_exit_code=$? + + echo "$axcl_output" + + if [ $axcl_exit_code -eq 0 ]; then + echo "AXCL driver installation completed successfully!" + else + echo "AXCL driver installed but no AI accelerator detected or communication failed." + echo "Please check if the AI accelerator is properly connected and powered on." + exit 1 + fi +else + echo "axcl-smi command not found. AXCL driver installation may have failed." + exit 1 +fi \ No newline at end of file diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index e352a6a9a..139f318d3 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -47,6 +47,11 @@ Frigate supports multiple different detectors that work on different types of ha - [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs. +**AXERA** + +- [AXEngine](#axera): axmodels can run on AXERA AI acceleration. + + **For Testing** - [CPU Detector (not recommended for actual use](#cpu-detector-not-recommended): Use a CPU to run tflite model, this is not recommended and in most cases OpenVINO can be used in CPU mode with better results. @@ -1099,6 +1104,40 @@ model: # required labelmap_path: /labelmap/coco-80.txt # required ``` +## AXERA + +Hardware accelerated object detection is supported on the following SoCs: + +- AX650N +- AX8850N + +This implementation uses the [AXera Pulsar2 Toolchain](https://huggingface.co/AXERA-TECH/Pulsar2). + +See the [installation docs](../frigate/installation.md#axera) for information on configuring the AXEngine hardware. + +### Configuration + +When configuring the AXEngine detector, you have to specify the model name. + +#### yolov5s + +A yolov5s model is provided in the container at /axmodels and is used by this detector type by default. + +Use the model configuration shown below when using the axengine detector with the default axmodel: + +```yaml +detectors: # required + axengine: # required + type: axengine # required + +model: # required + path: yolov5s_320 # required + width: 320 # required + height: 320 # required + tensor_format: bgr # required + labelmap_path: /labelmap/coco-80.txt # required +``` + ## Rockchip platform Hardware accelerated object detection is supported on the following SoCs: diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index f06f8ac7d..731de0535 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -110,6 +110,20 @@ Frigate supports multiple different detectors that work on different types of ha | ssd mobilenet | ~ 25 ms | | yolov5m | ~ 118 ms | +**Synaptics** + +- [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs to provide efficient object detection. + +::: + +### AXERA + +- **AXEngine** Default model is **yolov5s_320** + +| Name | AXERA AX650N/AX8850N Inference Time | +| ---------------- | ----------------------------------- | +| yolov5s_320 | ~ 1.676 ms | + ### Hailo-8 Frigate supports both the Hailo-8 and Hailo-8L AI Acceleration Modules on compatible hardware platforms—including the Raspberry Pi 5 with the PCIe hat from the AI kit. The Hailo detector integration in Frigate automatically identifies your hardware type and selects the appropriate default model when a custom model isn’t provided. diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index a4fd14d3c..281f87956 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -287,6 +287,40 @@ or add these options to your `docker run` command: Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics). +### AXERA + +AXERA accelerators are available in an M.2 form factor, compatible with both Raspberry Pi and Orange Pi. This form factor has also been successfully tested on x86 platforms, making it a versatile choice for various computing environments. + +#### Installation + +Using AXERA accelerators requires the installation of the AXCL driver. We provide a convenient Linux script to complete this installation. + +Follow these steps for installation: + +1. Copy or download [this script](https://github.com/ivanshi1108/assets/releases/download/v0.16.2/user_installation.sh). +2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh` +3. Run the script with `./user_installation.sh` + +#### Setup + +To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable` + +Next, grant Docker permissions to access your hardware by adding the following lines to your `docker-compose.yml` file: + +```yaml +devices: + - /dev/axcl_host + - /dev/ax_mmb_dev + - /dev/msg_userdev +``` + +If you are using `docker run`, add this option to your command `--device /dev/axcl_host --device /dev/ax_mmb_dev --device /dev/msg_userdev` + +#### Configuration + +Finally, configure [hardware object detection](/configuration/object_detectors#axera) to complete the setup. + + ## Docker Running through Docker with Docker Compose is the recommended install method. diff --git a/frigate/detectors/plugins/axengine.py b/frigate/detectors/plugins/axengine.py new file mode 100644 index 000000000..206923093 --- /dev/null +++ b/frigate/detectors/plugins/axengine.py @@ -0,0 +1,201 @@ +import logging +import os.path +import re +import urllib.request +from typing import Literal + +import cv2 +import numpy as np +from pydantic import Field + +from frigate.const import MODEL_CACHE_DIR +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum +from frigate.util.model import post_process_yolo + +import axengine as axe +from axengine import axclrt_provider_name, axengine_provider_name + +logger = logging.getLogger(__name__) + +DETECTOR_KEY = "axengine" + +CONF_THRESH = 0.65 +IOU_THRESH = 0.45 +STRIDES = [8, 16, 32] +ANCHORS = [ + [10, 13, 16, 30, 33, 23], + [30, 61, 62, 45, 59, 119], + [116, 90, 156, 198, 373, 326], +] + +class AxengineDetectorConfig(BaseDetectorConfig): + type: Literal[DETECTOR_KEY] + +class Axengine(DetectionApi): + type_key = DETECTOR_KEY + def __init__(self, config: AxengineDetectorConfig): + logger.info("__init__ axengine") + super().__init__(config) + self.height = config.model.height + self.width = config.model.width + model_path = config.model.path or "yolov5s_320" + self.session = axe.InferenceSession(f"/axmodels/{model_path}.axmodel") + + def __del__(self): + pass + + def xywh2xyxy(self, x): + # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + y = np.copy(x) + y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x + y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y + y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x + y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y + return y + + def bboxes_iou(self, boxes1, boxes2): + """calculate the Intersection Over Union value""" + boxes1 = np.array(boxes1) + boxes2 = np.array(boxes2) + + boxes1_area = (boxes1[..., 2] - boxes1[..., 0]) * ( + boxes1[..., 3] - boxes1[..., 1] + ) + boxes2_area = (boxes2[..., 2] - boxes2[..., 0]) * ( + boxes2[..., 3] - boxes2[..., 1] + ) + + left_up = np.maximum(boxes1[..., :2], boxes2[..., :2]) + right_down = np.minimum(boxes1[..., 2:], boxes2[..., 2:]) + + inter_section = np.maximum(right_down - left_up, 0.0) + inter_area = inter_section[..., 0] * inter_section[..., 1] + union_area = boxes1_area + boxes2_area - inter_area + ious = np.maximum(1.0 * inter_area / union_area, np.finfo(np.float32).eps) + + return ious + + def nms(self, proposals, iou_threshold, conf_threshold, multi_label=False): + """ + :param bboxes: (xmin, ymin, xmax, ymax, score, class) + + Note: soft-nms, https://arxiv.org/pdf/1704.04503.pdf + https://github.com/bharatsingh430/soft-nms + """ + xc = proposals[..., 4] > conf_threshold + proposals = proposals[xc] + proposals[:, 5:] *= proposals[:, 4:5] + bboxes = self.xywh2xyxy(proposals[:, :4]) + if multi_label: + mask = proposals[:, 5:] > conf_threshold + nonzero_indices = np.argwhere(mask) + if nonzero_indices.size < 0: + return + i, j = nonzero_indices.T + bboxes = np.hstack( + (bboxes[i], proposals[i, j + 5][:, None], j[:, None].astype(float)) + ) + else: + confidences = proposals[:, 5:] + conf = confidences.max(axis=1, keepdims=True) + j = confidences.argmax(axis=1)[:, None] + + new_x_parts = [bboxes, conf, j.astype(float)] + bboxes = np.hstack(new_x_parts) + + mask = conf.reshape(-1) > conf_threshold + bboxes = bboxes[mask] + + classes_in_img = list(set(bboxes[:, 5])) + bboxes = bboxes[bboxes[:, 4].argsort()[::-1][:300]] + best_bboxes = [] + + for cls in classes_in_img: + cls_mask = bboxes[:, 5] == cls + cls_bboxes = bboxes[cls_mask] + + while len(cls_bboxes) > 0: + max_ind = np.argmax(cls_bboxes[:, 4]) + best_bbox = cls_bboxes[max_ind] + best_bboxes.append(best_bbox) + cls_bboxes = np.concatenate( + [cls_bboxes[:max_ind], cls_bboxes[max_ind + 1 :]] + ) + iou = self.bboxes_iou(best_bbox[np.newaxis, :4], cls_bboxes[:, :4]) + weight = np.ones((len(iou),), dtype=np.float32) + + iou_mask = iou > iou_threshold + weight[iou_mask] = 0.0 + + cls_bboxes[:, 4] = cls_bboxes[:, 4] * weight + score_mask = cls_bboxes[:, 4] > 0.0 + cls_bboxes = cls_bboxes[score_mask] + + if len(best_bboxes) == 0: + return np.empty((0, 6)) + + best_bboxes = np.vstack(best_bboxes) + best_bboxes = best_bboxes[best_bboxes[:, 4].argsort()[::-1]] + return best_bboxes + + def sigmoid(self, x): + return np.clip(0.2 * x + 0.5, 0, 1) + + def gen_proposals(self, outputs): + new_pred = [] + anchor_grid = np.array(ANCHORS).reshape(-1, 1, 1, 3, 2) + for i, pred in enumerate(outputs): + pred = self.sigmoid(pred) + n, h, w, c = pred.shape + + pred = pred.reshape(n, h, w, 3, 85) + conv_shape = pred.shape + output_size = conv_shape[1] + conv_raw_dxdy = pred[..., 0:2] + conv_raw_dwdh = pred[..., 2:4] + xy_grid = np.meshgrid(np.arange(output_size), np.arange(output_size)) + xy_grid = np.expand_dims(np.stack(xy_grid, axis=-1), axis=2) + + xy_grid = np.tile(np.expand_dims(xy_grid, axis=0), [1, 1, 1, 3, 1]) + xy_grid = xy_grid.astype(np.float32) + pred_xy = (conv_raw_dxdy * 2.0 - 0.5 + xy_grid) * STRIDES[i] + pred_wh = (conv_raw_dwdh * 2) ** 2 * anchor_grid[i] + pred[:, :, :, :, 0:4] = np.concatenate([pred_xy, pred_wh], axis=-1) + + new_pred.append(np.reshape(pred, (-1, np.shape(pred)[-1]))) + + return np.concatenate(new_pred, axis=0) + + def post_processing(self, outputs, input_shape, threshold=0.3): + proposals = self.gen_proposals(outputs) + bboxes = self.nms(proposals, IOU_THRESH, CONF_THRESH, multi_label=True) + + """ + bboxes: [x_min, y_min, x_max, y_max, probability, cls_id] format coordinates. + """ + + results = np.zeros((20, 6), np.float32) + + for i, bbox in enumerate(bboxes): + if i >= 20: + break + coor = np.array(bbox[:4], dtype=np.int32) + score = bbox[4] + if score < threshold: + continue + class_ind = int(bbox[5]) + results[i] = [ + class_ind, + score, + max(0, bbox[1]) / input_shape[1], + max(0, bbox[0]) / input_shape[0], + min(1, bbox[3] / input_shape[1]), + min(1, bbox[2] / input_shape[0]), + ] + return results + + def detect_raw(self, tensor_input): + results = None + results = self.session.run(None, {"images": tensor_input}) + return self.post_processing(results, (self.width, self.height)) From bb45483e9e1b0925475a59565a72b84bb4ff2992 Mon Sep 17 00:00:00 2001 From: ivanshi1108 Date: Tue, 28 Oct 2025 09:54:00 +0800 Subject: [PATCH 2/5] Modify AXERA section from hardware.md Modify AXERA section and related content from hardware documentation. --- docs/docs/frigate/hardware.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 731de0535..d70018b4a 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -110,19 +110,13 @@ Frigate supports multiple different detectors that work on different types of ha | ssd mobilenet | ~ 25 ms | | yolov5m | ~ 118 ms | -**Synaptics** - -- [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs to provide efficient object detection. - -::: - ### AXERA - **AXEngine** Default model is **yolov5s_320** | Name | AXERA AX650N/AX8850N Inference Time | | ---------------- | ----------------------------------- | -| yolov5s_320 | ~ 1.676 ms | +| yolov5s_320 | ~ 1.676 ms | ### Hailo-8 From 91e17e12b72202d236fa1d0676fc57e91ee383d1 Mon Sep 17 00:00:00 2001 From: shizhicheng Date: Sun, 9 Nov 2025 13:21:17 +0000 Subject: [PATCH 3/5] Change the default detection model to YOLOv9 --- docker/axcl/Dockerfile | 2 +- docs/docs/configuration/object_detectors.md | 6 +- docs/docs/frigate/hardware.md | 4 +- frigate/detectors/plugins/axengine.py | 236 +++++++------------- 4 files changed, 90 insertions(+), 158 deletions(-) diff --git a/docker/axcl/Dockerfile b/docker/axcl/Dockerfile index 86e868b61..4a16bffaf 100644 --- a/docker/axcl/Dockerfile +++ b/docker/axcl/Dockerfile @@ -13,7 +13,7 @@ ARG PIP_BREAK_SYSTEM_PACKAGES # Install axmodels RUN mkdir -p /axmodels \ - && wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/yolov5s_320.axmodel -O /axmodels/yolov5s_320.axmodel + && wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/yolov9_tiny_u16_npu3_bgr_320x320_nhwc.axmodel -O /axmodels/yolov9_320.axmodel # Install axpyengine RUN wget https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3.rc1/axengine-0.1.3-py3-none-any.whl -O /axengine-0.1.3-py3-none-any.whl diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 139f318d3..983e3e5e7 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -1119,9 +1119,9 @@ See the [installation docs](../frigate/installation.md#axera) for information on When configuring the AXEngine detector, you have to specify the model name. -#### yolov5s +#### yolov9 -A yolov5s model is provided in the container at /axmodels and is used by this detector type by default. +A yolov9 model is provided in the container at /axmodels and is used by this detector type by default. Use the model configuration shown below when using the axengine detector with the default axmodel: @@ -1131,7 +1131,7 @@ detectors: # required type: axengine # required model: # required - path: yolov5s_320 # required + path: yolov9_320 # required width: 320 # required height: 320 # required tensor_format: bgr # required diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index d70018b4a..1b6e425d8 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -112,11 +112,11 @@ Frigate supports multiple different detectors that work on different types of ha ### AXERA -- **AXEngine** Default model is **yolov5s_320** +- **AXEngine** Default model is **yolov9** | Name | AXERA AX650N/AX8850N Inference Time | | ---------------- | ----------------------------------- | -| yolov5s_320 | ~ 1.676 ms | +| yolov9 | ~ 1.012 ms | ### Hailo-8 diff --git a/frigate/detectors/plugins/axengine.py b/frigate/detectors/plugins/axengine.py index 206923093..333c61756 100644 --- a/frigate/detectors/plugins/axengine.py +++ b/frigate/detectors/plugins/axengine.py @@ -20,14 +20,9 @@ logger = logging.getLogger(__name__) DETECTOR_KEY = "axengine" +NUM_CLASSES = 80 CONF_THRESH = 0.65 IOU_THRESH = 0.45 -STRIDES = [8, 16, 32] -ANCHORS = [ - [10, 13, 16, 30, 33, 23], - [30, 61, 62, 45, 59, 119], - [116, 90, 156, 198, 373, 326], -] class AxengineDetectorConfig(BaseDetectorConfig): type: Literal[DETECTOR_KEY] @@ -39,161 +34,98 @@ class Axengine(DetectionApi): super().__init__(config) self.height = config.model.height self.width = config.model.width - model_path = config.model.path or "yolov5s_320" + model_path = config.model.path or "yolov9_320" self.session = axe.InferenceSession(f"/axmodels/{model_path}.axmodel") def __del__(self): pass - def xywh2xyxy(self, x): - # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right - y = np.copy(x) - y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x - y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y - y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x - y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y - return y - - def bboxes_iou(self, boxes1, boxes2): - """calculate the Intersection Over Union value""" - boxes1 = np.array(boxes1) - boxes2 = np.array(boxes2) - - boxes1_area = (boxes1[..., 2] - boxes1[..., 0]) * ( - boxes1[..., 3] - boxes1[..., 1] - ) - boxes2_area = (boxes2[..., 2] - boxes2[..., 0]) * ( - boxes2[..., 3] - boxes2[..., 1] - ) - - left_up = np.maximum(boxes1[..., :2], boxes2[..., :2]) - right_down = np.minimum(boxes1[..., 2:], boxes2[..., 2:]) - - inter_section = np.maximum(right_down - left_up, 0.0) - inter_area = inter_section[..., 0] * inter_section[..., 1] - union_area = boxes1_area + boxes2_area - inter_area - ious = np.maximum(1.0 * inter_area / union_area, np.finfo(np.float32).eps) - - return ious - - def nms(self, proposals, iou_threshold, conf_threshold, multi_label=False): + def post_processing(self, raw_output, input_shape): """ - :param bboxes: (xmin, ymin, xmax, ymax, score, class) - - Note: soft-nms, https://arxiv.org/pdf/1704.04503.pdf - https://github.com/bharatsingh430/soft-nms + raw_output: [1, 1, 84, 8400] + Returns: numpy array of shape (20, 6) [class_id, score, y_min, x_min, y_max, x_max] in normalized coordinates """ - xc = proposals[..., 4] > conf_threshold - proposals = proposals[xc] - proposals[:, 5:] *= proposals[:, 4:5] - bboxes = self.xywh2xyxy(proposals[:, :4]) - if multi_label: - mask = proposals[:, 5:] > conf_threshold - nonzero_indices = np.argwhere(mask) - if nonzero_indices.size < 0: - return - i, j = nonzero_indices.T - bboxes = np.hstack( - (bboxes[i], proposals[i, j + 5][:, None], j[:, None].astype(float)) - ) - else: - confidences = proposals[:, 5:] - conf = confidences.max(axis=1, keepdims=True) - j = confidences.argmax(axis=1)[:, None] - - new_x_parts = [bboxes, conf, j.astype(float)] - bboxes = np.hstack(new_x_parts) - - mask = conf.reshape(-1) > conf_threshold - bboxes = bboxes[mask] - - classes_in_img = list(set(bboxes[:, 5])) - bboxes = bboxes[bboxes[:, 4].argsort()[::-1][:300]] - best_bboxes = [] - - for cls in classes_in_img: - cls_mask = bboxes[:, 5] == cls - cls_bboxes = bboxes[cls_mask] - - while len(cls_bboxes) > 0: - max_ind = np.argmax(cls_bboxes[:, 4]) - best_bbox = cls_bboxes[max_ind] - best_bboxes.append(best_bbox) - cls_bboxes = np.concatenate( - [cls_bboxes[:max_ind], cls_bboxes[max_ind + 1 :]] - ) - iou = self.bboxes_iou(best_bbox[np.newaxis, :4], cls_bboxes[:, :4]) - weight = np.ones((len(iou),), dtype=np.float32) - - iou_mask = iou > iou_threshold - weight[iou_mask] = 0.0 - - cls_bboxes[:, 4] = cls_bboxes[:, 4] * weight - score_mask = cls_bboxes[:, 4] > 0.0 - cls_bboxes = cls_bboxes[score_mask] - - if len(best_bboxes) == 0: - return np.empty((0, 6)) - - best_bboxes = np.vstack(best_bboxes) - best_bboxes = best_bboxes[best_bboxes[:, 4].argsort()[::-1]] - return best_bboxes - - def sigmoid(self, x): - return np.clip(0.2 * x + 0.5, 0, 1) - - def gen_proposals(self, outputs): - new_pred = [] - anchor_grid = np.array(ANCHORS).reshape(-1, 1, 1, 3, 2) - for i, pred in enumerate(outputs): - pred = self.sigmoid(pred) - n, h, w, c = pred.shape - - pred = pred.reshape(n, h, w, 3, 85) - conv_shape = pred.shape - output_size = conv_shape[1] - conv_raw_dxdy = pred[..., 0:2] - conv_raw_dwdh = pred[..., 2:4] - xy_grid = np.meshgrid(np.arange(output_size), np.arange(output_size)) - xy_grid = np.expand_dims(np.stack(xy_grid, axis=-1), axis=2) - - xy_grid = np.tile(np.expand_dims(xy_grid, axis=0), [1, 1, 1, 3, 1]) - xy_grid = xy_grid.astype(np.float32) - pred_xy = (conv_raw_dxdy * 2.0 - 0.5 + xy_grid) * STRIDES[i] - pred_wh = (conv_raw_dwdh * 2) ** 2 * anchor_grid[i] - pred[:, :, :, :, 0:4] = np.concatenate([pred_xy, pred_wh], axis=-1) - - new_pred.append(np.reshape(pred, (-1, np.shape(pred)[-1]))) - - return np.concatenate(new_pred, axis=0) - - def post_processing(self, outputs, input_shape, threshold=0.3): - proposals = self.gen_proposals(outputs) - bboxes = self.nms(proposals, IOU_THRESH, CONF_THRESH, multi_label=True) - - """ - bboxes: [x_min, y_min, x_max, y_max, probability, cls_id] format coordinates. - """ - results = np.zeros((20, 6), np.float32) - for i, bbox in enumerate(bboxes): - if i >= 20: - break - coor = np.array(bbox[:4], dtype=np.int32) - score = bbox[4] - if score < threshold: - continue - class_ind = int(bbox[5]) - results[i] = [ - class_ind, - score, - max(0, bbox[1]) / input_shape[1], - max(0, bbox[0]) / input_shape[0], - min(1, bbox[3] / input_shape[1]), - min(1, bbox[2] / input_shape[0]), - ] - return results + try: + if not isinstance(raw_output, np.ndarray): + raw_output = np.array(raw_output) + + if len(raw_output.shape) == 4 and raw_output.shape[0] == 1 and raw_output.shape[1] == 1: + raw_output = raw_output.squeeze(1) + + pred = raw_output[0].transpose(1, 0) + + bxy = pred[:, :2] + bwh = pred[:, 2:4] + cls = pred[:, 4:4 + NUM_CLASSES] + + cx = bxy[:, 0] + cy = bxy[:, 1] + w = bwh[:, 0] + h = bwh[:, 1] + + x_min = cx - w / 2 + y_min = cy - h / 2 + x_max = cx + w / 2 + y_max = cy + h / 2 + + scores = np.max(cls, axis=1) + class_ids = np.argmax(cls, axis=1) + + mask = scores >= CONF_THRESH + boxes = np.stack([x_min, y_min, x_max, y_max], axis=1)[mask] + scores = scores[mask] + class_ids = class_ids[mask] + + if len(boxes) == 0: + return results + + boxes_nms = np.stack([x_min[mask], y_min[mask], + x_max[mask] - x_min[mask], + y_max[mask] - y_min[mask]], axis=1) + + indices = cv2.dnn.NMSBoxes( + boxes_nms.tolist(), + scores.tolist(), + score_threshold=CONF_THRESH, + nms_threshold=IOU_THRESH + ) + + if len(indices) == 0: + return results + + indices = indices.flatten() + + sorted_indices = sorted(indices, key=lambda idx: scores[idx], reverse=True) + indices = sorted_indices + + valid_detections = 0 + for i, idx in enumerate(indices): + if i >= 20: + break + + x_min_val, y_min_val, x_max_val, y_max_val = boxes[idx] + score = scores[idx] + class_id = class_ids[idx] + + if score < CONF_THRESH: + continue + + results[valid_detections] = [ + float(class_id), # class_id + float(score), # score + max(0, y_min_val) / input_shape[0], # y_min + max(0, x_min_val) / input_shape[1], # x_min + min(1, y_max_val / input_shape[0]), # y_max + min(1, x_max_val / input_shape[1]) # x_max + ] + valid_detections += 1 + + return results + + except Exception as e: + return results def detect_raw(self, tensor_input): results = None From 1dee548dbce18fd641b144eb4d4952a3fb40e6e1 Mon Sep 17 00:00:00 2001 From: shizhicheng Date: Tue, 11 Nov 2025 04:40:27 +0000 Subject: [PATCH 4/5] Modifications to the YOLOv9 object detection model: The model is now dynamically downloaded to the cache directory. Post-processing is now done using Frigate's built-in `post_process_yolo`. Configuration in the relevant documentation has been updated. --- docker/axcl/Dockerfile | 4 - docs/docs/configuration/object_detectors.md | 3 +- docs/docs/frigate/hardware.md | 2 +- frigate/detectors/plugins/axengine.py | 132 +++++++------------- 4 files changed, 49 insertions(+), 92 deletions(-) diff --git a/docker/axcl/Dockerfile b/docker/axcl/Dockerfile index 4a16bffaf..83271bce8 100644 --- a/docker/axcl/Dockerfile +++ b/docker/axcl/Dockerfile @@ -11,10 +11,6 @@ FROM frigate AS frigate-axcl ARG TARGETARCH ARG PIP_BREAK_SYSTEM_PACKAGES -# Install axmodels -RUN mkdir -p /axmodels \ - && wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/yolov9_tiny_u16_npu3_bgr_320x320_nhwc.axmodel -O /axmodels/yolov9_320.axmodel - # Install axpyengine RUN wget https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3.rc1/axengine-0.1.3-py3-none-any.whl -O /axengine-0.1.3-py3-none-any.whl RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple/ /axengine-0.1.3-py3-none-any.whl \ diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 983e3e5e7..88b015c34 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -1131,7 +1131,8 @@ detectors: # required type: axengine # required model: # required - path: yolov9_320 # required + path: frigate-yolov9-tiny # required + model_type: yolo-generic # required width: 320 # required height: 320 # required tensor_format: bgr # required diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 1b6e425d8..cf7ebcdb8 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -116,7 +116,7 @@ Frigate supports multiple different detectors that work on different types of ha | Name | AXERA AX650N/AX8850N Inference Time | | ---------------- | ----------------------------------- | -| yolov9 | ~ 1.012 ms | +| yolov9-tiny | ~ 1.012 ms | ### Hailo-8 diff --git a/frigate/detectors/plugins/axengine.py b/frigate/detectors/plugins/axengine.py index 333c61756..3bbfead09 100644 --- a/frigate/detectors/plugins/axengine.py +++ b/frigate/detectors/plugins/axengine.py @@ -20,9 +20,12 @@ logger = logging.getLogger(__name__) DETECTOR_KEY = "axengine" -NUM_CLASSES = 80 -CONF_THRESH = 0.65 -IOU_THRESH = 0.45 +supported_models = { + ModelTypeEnum.yologeneric: "frigate-yolov9-tiny", +} + +model_cache_dir = os.path.join(MODEL_CACHE_DIR, "axengine_cache/") + class AxengineDetectorConfig(BaseDetectorConfig): type: Literal[DETECTOR_KEY] @@ -34,100 +37,57 @@ class Axengine(DetectionApi): super().__init__(config) self.height = config.model.height self.width = config.model.width - model_path = config.model.path or "yolov9_320" - self.session = axe.InferenceSession(f"/axmodels/{model_path}.axmodel") + model_path = config.model.path or "frigate-yolov9-tiny" + + model_props = self.parse_model_input(model_path) + + self.session = axe.InferenceSession(model_props["path"]) def __del__(self): pass - def post_processing(self, raw_output, input_shape): - """ - raw_output: [1, 1, 84, 8400] - Returns: numpy array of shape (20, 6) [class_id, score, y_min, x_min, y_max, x_max] in normalized coordinates - """ - results = np.zeros((20, 6), np.float32) + def parse_model_input(self, model_path): + model_props = {} + model_props["preset"] = True - try: - if not isinstance(raw_output, np.ndarray): - raw_output = np.array(raw_output) + model_matched = False + for model_type, pattern in supported_models.items(): + if re.match(pattern, model_path): + model_matched = True + model_props["model_type"] = model_type - if len(raw_output.shape) == 4 and raw_output.shape[0] == 1 and raw_output.shape[1] == 1: - raw_output = raw_output.squeeze(1) + if model_matched: + model_props["filename"] = model_path + f".axmodel" + model_props["path"] = model_cache_dir + model_props["filename"] - pred = raw_output[0].transpose(1, 0) - - bxy = pred[:, :2] - bwh = pred[:, 2:4] - cls = pred[:, 4:4 + NUM_CLASSES] - - cx = bxy[:, 0] - cy = bxy[:, 1] - w = bwh[:, 0] - h = bwh[:, 1] - - x_min = cx - w / 2 - y_min = cy - h / 2 - x_max = cx + w / 2 - y_max = cy + h / 2 - - scores = np.max(cls, axis=1) - class_ids = np.argmax(cls, axis=1) - - mask = scores >= CONF_THRESH - boxes = np.stack([x_min, y_min, x_max, y_max], axis=1)[mask] - scores = scores[mask] - class_ids = class_ids[mask] - - if len(boxes) == 0: - return results - - boxes_nms = np.stack([x_min[mask], y_min[mask], - x_max[mask] - x_min[mask], - y_max[mask] - y_min[mask]], axis=1) - - indices = cv2.dnn.NMSBoxes( - boxes_nms.tolist(), - scores.tolist(), - score_threshold=CONF_THRESH, - nms_threshold=IOU_THRESH + if not os.path.isfile(model_props["path"]): + self.download_model(model_props["filename"]) + else: + supported_models_str = ", ".join( + model[1:-1] for model in supported_models ) + raise Exception( + f"Model {model_path} is unsupported. Provide your own model or choose one of the following: {supported_models_str}" + ) + return model_props - if len(indices) == 0: - return results + def download_model(self, filename): + if not os.path.isdir(model_cache_dir): + os.mkdir(model_cache_dir) - indices = indices.flatten() - - sorted_indices = sorted(indices, key=lambda idx: scores[idx], reverse=True) - indices = sorted_indices - - valid_detections = 0 - for i, idx in enumerate(indices): - if i >= 20: - break - - x_min_val, y_min_val, x_max_val, y_max_val = boxes[idx] - score = scores[idx] - class_id = class_ids[idx] - - if score < CONF_THRESH: - continue - - results[valid_detections] = [ - float(class_id), # class_id - float(score), # score - max(0, y_min_val) / input_shape[0], # y_min - max(0, x_min_val) / input_shape[1], # x_min - min(1, y_max_val / input_shape[0]), # y_max - min(1, x_max_val / input_shape[1]) # x_max - ] - valid_detections += 1 - - return results - - except Exception as e: - return results + GITHUB_ENDPOINT = os.environ.get("GITHUB_ENDPOINT", "https://github.com") + urllib.request.urlretrieve( + f"{GITHUB_ENDPOINT}/ivanshi1108/assets/releases/download/v0.16.2/{filename}", + model_cache_dir + filename, + ) def detect_raw(self, tensor_input): results = None results = self.session.run(None, {"images": tensor_input}) - return self.post_processing(results, (self.width, self.height)) + if self.detector_config.model.model_type == ModelTypeEnum.yologeneric: + return post_process_yolo(results, self.width, self.height) + else: + raise ValueError( + f'Model type "{self.detector_config.model.model_type}" is currently not supported.' + ) + From e27a94ae0b0055b763d904c6434c669f76476e13 Mon Sep 17 00:00:00 2001 From: shizhicheng Date: Tue, 11 Nov 2025 05:54:19 +0000 Subject: [PATCH 5/5] Fix logical errors caused by code formatting --- frigate/detectors/plugins/axengine.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frigate/detectors/plugins/axengine.py b/frigate/detectors/plugins/axengine.py index 3bbfead09..9cde9841b 100644 --- a/frigate/detectors/plugins/axengine.py +++ b/frigate/detectors/plugins/axengine.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) DETECTOR_KEY = "axengine" supported_models = { - ModelTypeEnum.yologeneric: "frigate-yolov9-tiny", + ModelTypeEnum.yologeneric: "frigate-yolov9-.*$", } model_cache_dir = os.path.join(MODEL_CACHE_DIR, "axengine_cache/") @@ -38,9 +38,7 @@ class Axengine(DetectionApi): self.height = config.model.height self.width = config.model.width model_path = config.model.path or "frigate-yolov9-tiny" - model_props = self.parse_model_input(model_path) - self.session = axe.InferenceSession(model_props["path"]) def __del__(self): @@ -51,6 +49,7 @@ class Axengine(DetectionApi): model_props["preset"] = True model_matched = False + for model_type, pattern in supported_models.items(): if re.match(pattern, model_path): model_matched = True @@ -60,8 +59,8 @@ class Axengine(DetectionApi): model_props["filename"] = model_path + f".axmodel" model_props["path"] = model_cache_dir + model_props["filename"] - if not os.path.isfile(model_props["path"]): - self.download_model(model_props["filename"]) + if not os.path.isfile(model_props["path"]): + self.download_model(model_props["filename"]) else: supported_models_str = ", ".join( model[1:-1] for model in supported_models