From 5a87e0180af69c06ebbb1917af4b0262b5116ea3 Mon Sep 17 00:00:00 2001 From: knoffelcut Date: Mon, 27 Apr 2026 19:55:29 +0200 Subject: [PATCH 1/6] Nanodet Plus support --- frigate/detectors/detector_config.py | 1 + frigate/detectors/plugins/onnx.py | 7 ++ frigate/util/model.py | 107 +++++++++++++++++++++++++++ 3 files changed, 115 insertions(+) diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py index 5071e3a741..7e06346217 100644 --- a/frigate/detectors/detector_config.py +++ b/frigate/detectors/detector_config.py @@ -42,6 +42,7 @@ class ModelTypeEnum(str, Enum): yolox = "yolox" yolonas = "yolonas" yologeneric = "yolo-generic" + nanodet_plus = "nanodet_plus" class ModelConfig(BaseModel): diff --git a/frigate/detectors/plugins/onnx.py b/frigate/detectors/plugins/onnx.py index b9aa00fbdb..dc296e957c 100644 --- a/frigate/detectors/plugins/onnx.py +++ b/frigate/detectors/plugins/onnx.py @@ -14,6 +14,7 @@ from frigate.detectors.detector_config import ( ) from frigate.util.model import ( post_process_dfine, + post_process_nanodet_plus, post_process_rfdetr, post_process_yolo, post_process_yolox, @@ -137,6 +138,12 @@ class ONNXDetector(DetectionApi): self.grids, self.expanded_strides, ) + elif self.onnx_model_type == ModelTypeEnum.nanodet_plus: + return post_process_nanodet_plus( + tensor_output[0], + self.width, + self.height, + ) else: raise Exception( f"{self.onnx_model_type} is currently not supported for onnx. See the docs for more info on supported models." diff --git a/frigate/util/model.py b/frigate/util/model.py index 338303e2d7..64e1a28d62 100644 --- a/frigate/util/model.py +++ b/frigate/util/model.py @@ -1,5 +1,6 @@ """Model Utils""" +import functools import logging import os from typing import Any @@ -7,6 +8,7 @@ from typing import Any import cv2 import numpy as np import onnxruntime as ort +import scipy.special from frigate.const import MODEL_CACHE_DIR @@ -16,6 +18,51 @@ logger = logging.getLogger(__name__) ### Post Processing +@functools.lru_cache +def nanodet_center_priors( + input_height: int, input_width: int, strides: tuple, dtype: type +): + def get_single_level_center_priors(featmap_size, stride, dtype): + """Generate centers of a single stage feature map. + Args: + batch_size (int): Number of images in one batch. + featmap_size (tuple[int]): height and width of the feature map + stride (int): down sample stride of the feature map + dtype (obj:`torch.dtype`): data type of the tensors + device (obj:`torch.device`): device of the tensors + Return: + priors (Tensor): center priors of a single level feature map. + """ + h, w = featmap_size + x_range = (np.arange(w, dtype=dtype)) * stride + y_range = (np.arange(h, dtype=dtype)) * stride + y, x = np.meshgrid(y_range, x_range, indexing="ij") + y = y.flatten() + x = x.flatten() + strides = np.full(x.shape[0], stride) + priors = np.stack([x, y, strides, strides], axis=-1) + return priors + + featmap_sizes = [ + ( + int(np.ceil(input_height / stride)), + int(np.ceil(input_width) / stride), + ) + for stride in strides + ] + mlvl_center_priors = [ + get_single_level_center_priors( + featmap_sizes[i], + stride, + dtype, + ) + for i, stride in enumerate(strides) + ] + center_priors = np.concatenate(mlvl_center_priors, axis=0) + + return center_priors + + def post_process_dfine( tensor_output: np.ndarray, width: int, height: int ) -> np.ndarray: @@ -280,6 +327,66 @@ def post_process_yolox( return detections +def post_process_nanodet_plus( + predictions: np.ndarray, + width: int, + height: int, +): + def distance2bbox(points, distance, max_shape=None): + """Decode distance prediction to bounding box. + + Args: + points (Tensor): Shape (n, 2), [x, y]. + distance (Tensor): Distance from the given point to 4 + boundaries (left, top, right, bottom). + max_shape (tuple): Shape of the image. + + Returns: + Tensor: Decoded bboxes. + """ + x1 = points[..., 0] - distance[..., 0] + y1 = points[..., 1] - distance[..., 1] + x2 = points[..., 0] + distance[..., 2] + y2 = points[..., 1] + distance[..., 3] + if max_shape is not None: + x1 = np.clip(x1, 0, max_shape[1]) + y1 = np.clip(y1, 0, max_shape[0]) + x2 = np.clip(x2, 0, max_shape[1]) + y2 = np.clip(y2, 0, max_shape[0]) + return np.stack([x1, y1, x2, y2], -1) + + predictions = predictions[0] + + # TODO From parameters + reg_max = 7 + strides = (8, 16, 32, 64) + + num_classes = predictions.shape[-1] - 4 * (reg_max + 1) + cls_scores, bbox_preds = predictions[:, :num_classes], predictions[:, num_classes:] + + center_priors = nanodet_center_priors(height, width, strides, predictions[0].dtype) + + x = bbox_preds.reshape(bbox_preds.shape[0], 4, reg_max + 1) + x = scipy.special.softmax(x, axis=-1) + x = np.dot(x, np.linspace(0, reg_max, reg_max + 1)) + + dis_preds = x * center_priors[..., 2, None] + bboxes = distance2bbox(center_priors[..., :2], dis_preds, max_shape=(height, width)) + + class_ids = np.argmax(cls_scores, axis=1) + scores = np.max(cls_scores, axis=1) + + detections = np.zeros((20, 6), dtype=np.float32) + for i, j in enumerate(np.argsort(scores)[::-1][:20]): + detections[i, 0] = class_ids[j] + detections[i, 1] = scores[j] + detections[i, 2] = bboxes[j, 1] / height + detections[i, 3] = bboxes[j, 0] / width + detections[i, 4] = bboxes[j, 3] / height + detections[i, 5] = bboxes[j, 2] / width + return detections + + ### ONNX Utilities From a2ecbc9d6b4f0471e3ed6b99198b1b7ac0e6488d Mon Sep 17 00:00:00 2001 From: knoffelcut Date: Mon, 25 May 2026 19:55:25 +0200 Subject: [PATCH 2/6] Replace functool lru cache with dict cache --- frigate/util/model.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/frigate/util/model.py b/frigate/util/model.py index 64e1a28d62..218b3a4dd5 100644 --- a/frigate/util/model.py +++ b/frigate/util/model.py @@ -1,6 +1,5 @@ """Model Utils""" -import functools import logging import os from typing import Any @@ -18,10 +17,13 @@ logger = logging.getLogger(__name__) ### Post Processing -@functools.lru_cache -def nanodet_center_priors( +def calculate_nanodet_center_priors( input_height: int, input_width: int, strides: tuple, dtype: type ): + """ + Adapted from https://github.com/RangiLyu/nanodet/blob/be9b4a9001d7f9b6fc89c2df31ae8d428e35b4f0/nanodet/model/head/nanodet_plus_head.py + """ + def get_single_level_center_priors(featmap_size, stride, dtype): """Generate centers of a single stage feature map. Args: @@ -63,6 +65,9 @@ def nanodet_center_priors( return center_priors +nanodet_center_priors: dict[(int, int, tuple, type), np.ndarray] = {} + + def post_process_dfine( tensor_output: np.ndarray, width: int, height: int ) -> np.ndarray: @@ -332,6 +337,10 @@ def post_process_nanodet_plus( width: int, height: int, ): + """ + Adapted from https://github.com/RangiLyu/nanodet/blob/be9b4a9001d7f9b6fc89c2df31ae8d428e35b4f0/nanodet/model/head/nanodet_plus_head.py + """ + def distance2bbox(points, distance, max_shape=None): """Decode distance prediction to bounding box. @@ -357,14 +366,24 @@ def post_process_nanodet_plus( predictions = predictions[0] - # TODO From parameters + # Below two parameters are consistent with all nanodet **plus** models reg_max = 7 strides = (8, 16, 32, 64) num_classes = predictions.shape[-1] - 4 * (reg_max + 1) cls_scores, bbox_preds = predictions[:, :num_classes], predictions[:, num_classes:] - center_priors = nanodet_center_priors(height, width, strides, predictions[0].dtype) + try: + center_priors = nanodet_center_priors[ + (height, width, strides, predictions[0].dtype) + ] + except KeyError: + center_priors = calculate_nanodet_center_priors( + height, width, strides, predictions[0].dtype + ) + nanodet_center_priors[(height, width, strides, predictions[0].dtype)] = ( + center_priors + ) x = bbox_preds.reshape(bbox_preds.shape[0], 4, reg_max + 1) x = scipy.special.softmax(x, axis=-1) From 3f272e097e8a2e3bcbde9183849284b57335078f Mon Sep 17 00:00:00 2001 From: knoffelcut Date: Sat, 30 May 2026 12:40:54 +0200 Subject: [PATCH 3/6] Instructions to download NanoDet-Plus models --- docs/docs/configuration/object_detectors.md | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index e4a5082322..1d58973e2b 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -2460,3 +2460,29 @@ ARG IMG_SIZE COPY --from=build /yolov9/yolov9-${MODEL_SIZE}.onnx /yolov9-${MODEL_SIZE}-${IMG_SIZE}.onnx EOF ``` + +### Downloading NanoDet-Plus models +Compatible with the `labelmap/coco-80.txt` labelmap + +```sh +docker build . --build-arg URL_WEIGHTS=https://drive.google.com/file/d/1Dq0cTIdJDUhQxJe45z6rWncbZmOyh1Tv/view?usp=sharing --build-arg IMG_HEIGHT=320 --build-arg IMG_WIDTH=320 --build-arg CFG_PATH=config/nanodet-plus-m_320.yml --output . -f- <<'EOF' +FROM python:3.9 AS build +RUN apt-get update && apt-get install --no-install-recommends -y cmake libgl1 && rm -rf /var/lib/apt/lists/* +COPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/ +WORKDIR /nanodet_plus +ADD https://github.com/RangiLyu/nanodet.git . +RUN uv pip install --system onnx==1.18.0 onnxruntime onnx-simplifier==0.4.* onnxscript +RUN uv pip install --system "numpy<2" +RUN uv pip install --system -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --system -e . +ARG URL_WEIGHTS +RUN uv pip install --system gdown +RUN gdown --fuzzy ${URL_WEIGHTS} -O nanodet_plus.pth +ARG IMG_HEIGHT +ARG IMG_WIDTH +ARG CFG_PATH +RUN python tools/export_onnx.py --cfg_path=${CFG_PATH} --model_path=nanodet_plus.pth --input_shape=${IMG_HEIGHT},${IMG_WIDTH} --out_path=nanodet_plus.onnx +FROM scratch +COPY --from=build /nanodet_plus/nanodet_plus.onnx nanodet_plus.onnx +EOF +``` From 6b80061cfd2bf6346bbf9cb9c76194c730d694cb Mon Sep 17 00:00:00 2001 From: knoffelcut Date: Sat, 30 May 2026 18:41:08 +0200 Subject: [PATCH 4/6] NanoDet-Plus OpenVINO compatibility --- frigate/detectors/plugins/openvino.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frigate/detectors/plugins/openvino.py b/frigate/detectors/plugins/openvino.py index 1e9fb1ab10..2be8e527cc 100644 --- a/frigate/detectors/plugins/openvino.py +++ b/frigate/detectors/plugins/openvino.py @@ -10,6 +10,7 @@ from frigate.detectors.detection_runners import OpenVINOModelRunner from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum from frigate.util.model import ( post_process_dfine, + post_process_nanodet_plus, post_process_rfdetr, post_process_yolo, ) @@ -43,6 +44,7 @@ class OvDetector(DetectionApi): ModelTypeEnum.yolonas, ModelTypeEnum.yologeneric, ModelTypeEnum.yolox, + ModelTypeEnum.nanodet_plus, ] def __init__(self, detector_config: OvDetectorConfig): @@ -238,3 +240,9 @@ class OvDetector(DetectionApi): object_detected[6], object_detected[5], object_detected[:4] ) return detections + elif self.ov_model_type == ModelTypeEnum.nanodet_plus: + return post_process_nanodet_plus( + outputs[0], + self.width, + self.height, + ) From 32d1c94b8675a892a5279073f22aa686c18cfd48 Mon Sep 17 00:00:00 2001 From: knoffelcut Date: Wed, 3 Jun 2026 21:02:59 +0200 Subject: [PATCH 5/6] NanoDet-Plus documentation --- docs/docs/configuration/object_detectors.md | 88 ++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 1d58973e2b..2b74aaa9a5 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -494,7 +494,8 @@ detectors: | [YOLO-NAS](#yolo-nas) | ✅ | ✅ | | | [MobileNet v2](#ssdlite-mobilenet-v2) | ✅ | ✅ | Fast and lightweight model, less accurate than larger models | | [YOLOX](#yolox) | ✅ | ? | | -| [D-FINE / DEIMv2](#d-fine--deimv2) | ❌ | ❌ | | +| [D-FINE](#d-fine) | ❌ | ❌ | | +| [NanoDet-Plus](#nanodet-plus) | ? | ? | | #### SSDLite MobileNet v2 @@ -791,6 +792,44 @@ Note that the labelmap uses a subset of the complete COCO label set that has onl +#### NanoDet-Plus +[NanoDet-Plus](https://github.com/RangiLyu/nanodet) is a lightweight object detection model that achieves +good accuracy on CPUs given its small footprint. + +Script to export an ONNX model for use in Frigate is provided in [the models section](#downloading-nanodet-plus-models). + +:::warning + +NanoDet-Plus has not been tested in GPU nor NPU modes. + +::: + +
+ NanoDet-Plus Setup & Config + +After placing the exported onnx model in your config/model_cache folder, you can use the following configuration: + +```yaml +detectors: + ov: + type: openvino + device: CPU + +model: + model_type: nanodet_plus + width: 320 + height: 320 + input_tensor: nchw + input_dtype: float + input_pixel_format: bgr + path: /config/model_cache/nanodet_plus.onnx + labelmap_path: /labelmap/coco-80.txt +``` + +Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. + +
+ ## Apple Silicon detector The NPU in Apple Silicon can't be accessed from within a container, so the [Apple Silicon detector client](https://github.com/frigate-nvr/apple-silicon-detector) must first be setup. It is recommended to use the Frigate docker image with `-standard-arm64` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-standard-arm64`. @@ -1029,6 +1068,7 @@ detectors: | [YOLO-NAS](#yolo-nas-1) | ⚠️ | ⚠️ | Not supported by CUDA Graphs | | [YOLOX](#yolox-1) | ✅ | ✅ | Supports CUDA Graphs for optimal Nvidia performance | | [D-FINE / DEIMv2](#d-fine--deimv2-1) | ⚠️ | ❌ | Not supported by CUDA Graphs | +| [NanoDet-Plus](#nanodet-plus-1) | ✅ | ? | Supports CUDA Graphs for optimal Nvidia performance | There is no default model provided, the following formats are supported: @@ -1311,6 +1351,42 @@ model: Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. +#### NanoDet-Plus +[NanoDet-Plus](https://github.com/RangiLyu/nanodet) is a lightweight object detection model that achieves +good accuracy on CPUs given its small footprint. + +Script to export an ONNX model for use in Frigate is provided in [the models section](#downloading-nanodet-plus-models). +:::warning + +NanoDet-Plus has not been tested on AMD GPUs. + +::: + +
+ NanoDet-Plus Setup & Config + +After placing the exported onnx model in your config/model_cache folder, you can use the following configuration: + +```yaml +detectors: + onnx: + type: onnx + +model: + model_type: nanodet_plus + width: 320 + height: 320 + input_tensor: nchw + input_dtype: float + input_pixel_format: bgr + path: /config/model_cache/nanodet_plus.onnx + labelmap_path: /labelmap/coco-80.txt +``` + +Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. + +
+ ## CPU Detector (not recommended) The CPU detector type runs a TensorFlow Lite model utilizing the CPU without hardware acceleration. It is recommended to use a hardware accelerated detector type instead for better performance. To configure a CPU based detector, set the `"type"` attribute to `"cpu"`. @@ -2462,6 +2538,16 @@ EOF ``` ### Downloading NanoDet-Plus models +NanoDet-Plus can be downloaded using the command below. Copy and paste the complete command to your terminal to export +the model as `nanodet_plus.onnx` in the current working directory. The command builds the NanoDet-Plus environment, +downloads the specified model and converts it to ONNX. + +The below command is configured to use the smallest model provided by the authors, NanoDet-Plus-m-320. Other models +can be specified by changing the `URL_WEIGHTS` link to the appropriate pretrained weights URL from +[NanoDet-Plus Model Zoo](https://github.com/RangiLyu/nanodet#model-zoo). Remember to change the `IMG_HEIGHT`, +`IMG_WIDTH` and `CFG_PATH` ([configuration files](https://github.com/RangiLyu/nanodet/tree/main/config)) parameters +accordingly. + Compatible with the `labelmap/coco-80.txt` labelmap ```sh From 87f55be8057906c321cd1db82108dac7fb188d40 Mon Sep 17 00:00:00 2001 From: knoffelcut Date: Sun, 7 Jun 2026 20:20:53 +0200 Subject: [PATCH 6/6] Fix removed DEIMv2 reference --- docs/docs/configuration/object_detectors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 2b74aaa9a5..edfd4c9dc0 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -494,7 +494,7 @@ detectors: | [YOLO-NAS](#yolo-nas) | ✅ | ✅ | | | [MobileNet v2](#ssdlite-mobilenet-v2) | ✅ | ✅ | Fast and lightweight model, less accurate than larger models | | [YOLOX](#yolox) | ✅ | ? | | -| [D-FINE](#d-fine) | ❌ | ❌ | | +| [D-FINE / DEIMv2](#d-fine--deimv2) | ❌ | ❌ | | | [NanoDet-Plus](#nanodet-plus) | ? | ? | | #### SSDLite MobileNet v2