diff --git a/docs/data/object_detectors_models.json b/docs/data/object_detectors_models.json new file mode 100644 index 0000000000..31806a69ad --- /dev/null +++ b/docs/data/object_detectors_models.json @@ -0,0 +1,349 @@ +{ + "edgeTPU": { + "title": "EdgeTPU", + "models": [ + { + "key": "mobiledet", + "label": "Mobiledet", + "recommended": true, + "download": "A TensorFlow Lite model is provided in the container at `/edgetpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with `model.path`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **EdgeTPU** from the detector type dropdown and click **Add**, then set device to `usb`.", + "yaml": "detectors:\n coral:\n type: edgetpu\n device: usb" + }, + { + "key": "yolov9", + "label": "YOLOv9", + "recommended": false, + "download": "[Download the model](https://github.com/dbro/frigate-detector-edgetpu-yolo9/releases/download/v1.0/yolov9-s-relu6-best_320_int8_edgetpu.tflite), bind mount the file into the container, and provide the path with `model.path`. Note that the linked model requires a 17-label [labelmap file](https://raw.githubusercontent.com/dbro/frigate-detector-edgetpu-yolo9/refs/heads/main/labels-coco17.txt) that includes only 17 COCO classes.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **EdgeTPU** from the detector type dropdown and click **Add**, then set device to `usb`. Then on the same page, in the **Custom Model** tab, configure the model settings:\n\n| Field | Value |\n| ---------------------------------------- | ----------------------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (should match the imgsize of the model) |\n| **Object detection model input height** | `320` (should match the imgsize of the model) |\n| **Custom object detector model path** | `/config/model_cache/yolov9-s-relu6-best_320_int8_edgetpu.tflite` |\n| **Label map for custom object detector** | `/config/labels-coco17.txt` |", + "yaml": "detectors:\n coral:\n type: edgetpu\n device: usb\n\nmodel:\n model_type: yolo-generic\n width: 320 # <--- should match the imgsize of the model, typically 320\n height: 320 # <--- should match the imgsize of the model, typically 320\n path: /config/model_cache/yolov9-s-relu6-best_320_int8_edgetpu.tflite\n labelmap_path: /config/labels-coco17.txt" + } + ] + }, + "hailo8l": { + "title": "Hailo-8/Hailo-8L", + "models": [ + { + "key": "yolo", + "label": "YOLO", + "recommended": true, + "download": "If no custom model path or URL is provided, the Hailo detector automatically downloads the default model (YOLOv6n) from the Hailo Model Zoo on first startup based on the detected hardware. Once cached under `/config/model_cache/hailo`, the model works fully offline.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **Hailo-8/Hailo-8L** from the detector type dropdown and click **Add**, then set device to `PCIe`. Then on the same page, in the **Custom Model** tab, configure the model settings:\n\n| Field | Value |\n| ---------------------------------------- | ----------------------- |\n| **Object detection model input width** | `320` |\n| **Object detection model input height** | `320` |\n| **Model Input Tensor Shape** | `nhwc` |\n| **Model Input Pixel Color Format** | `rgb` |\n| **Model Input D Type** | `int` |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |\n\nThe detector automatically selects the default model based on your hardware. Optionally, specify a local model path or URL to override.", + "yaml": "detectors:\n hailo:\n type: hailo8l\n device: PCIe\n\nmodel:\n width: 320\n height: 320\n input_tensor: nhwc\n input_pixel_format: rgb\n input_dtype: int\n model_type: yolo-generic\n labelmap_path: /labelmap/coco-80.txt\n\n # The detector automatically selects the default model based on your hardware:\n # - For Hailo-8 hardware: YOLOv6n (default: yolov6n.hef)\n # - For Hailo-8L hardware: YOLOv6n (default: yolov6n.hef)\n #\n # Optionally, you can specify a local model path to override the default.\n # If a local path is provided and the file exists, it will be used instead of downloading.\n # Example:\n # path: /config/model_cache/hailo/yolov6n.hef\n #\n # You can also override using a custom URL:\n # path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8/yolov6n.hef\n # just make sure to give it the write configuration based on the model" + }, + { + "key": "ssd", + "label": "SSD MobileNet v1", + "recommended": false, + "download": "For SSD-based models, provide either a model path or URL to your compiled SSD model. The integration will first check the local path before downloading if necessary. The model file is cached under `/config/model_cache/hailo`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **Hailo-8/Hailo-8L** from the detector type dropdown and click **Add**, then set device to `PCIe`. Then on the same page, in the **Custom Model** tab, configure the model settings:\n\n| Field | Value |\n| --------------------------------------- | ------ |\n| **Object detection model input width** | `300` |\n| **Object detection model input height** | `300` |\n| **Model Input Tensor Shape** | `nhwc` |\n| **Model Input Pixel Color Format** | `rgb` |\n| **Object Detection Model Type** | `ssd` |\n\nSpecify the local model path or URL for SSD MobileNet v1.", + "yaml": "detectors:\n hailo:\n type: hailo8l\n device: PCIe\n\nmodel:\n width: 300\n height: 300\n input_tensor: nhwc\n input_pixel_format: rgb\n model_type: ssd\n # Specify the local model path (if available) or URL for SSD MobileNet v1.\n # Example with a local path:\n # path: /config/model_cache/h8l_cache/ssd_mobilenet_v1.hef\n #\n # Or override using a custom URL:\n # path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8l/ssd_mobilenet_v1.hef" + } + ] + }, + "openvino": { + "title": "OpenVINO", + "models": [ + { + "key": "ssd", + "label": "SSDLite MobileNet v2", + "recommended": true, + "download": "An OpenVINO model is provided in the container at `/openvino-model/ssdlite_mobilenet_v2.xml` and is used by this detector type by default. The model comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU` (or `NPU`). Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ------------------------------------------ |\n| **Object detection model input width** | `300` |\n| **Object detection model input height** | `300` |\n| **Model Input Tensor Shape** | `nhwc` |\n| **Model Input Pixel Color Format** | `bgr` |\n| **Custom object detector model path** | `/openvino-model/ssdlite_mobilenet_v2.xml` |\n| **Label map for custom object detector** | `/openvino-model/coco_91cl_bkgr.txt` |", + "yaml": "detectors:\n ov:\n type: openvino\n device: GPU # Or NPU\n\nmodel:\n width: 300\n height: 300\n input_tensor: nhwc\n input_pixel_format: bgr\n path: /openvino-model/ssdlite_mobilenet_v2.xml\n labelmap_path: /openvino-model/coco_91cl_bkgr.txt" + }, + { + "key": "yolov9", + "label": "YOLOv9", + "recommended": false, + "download": "YOLOv9 model can be exported as ONNX using the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=t` and `IMG_SIZE=320` in the first line to the [model size](https://github.com/WongKinYiu/yolov9#performance) you would like to convert (available model sizes are `t`, `s`, `m`, `c`, and `e`, common image sizes are `320` and `640`).\n\n```sh\ndocker build . --build-arg MODEL_SIZE=t --build-arg IMG_SIZE=320 --output . -f- <<'EOF'\nFROM python:3.11 AS build\nRUN apt-get update && apt-get install --no-install-recommends -y cmake libgl1 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/\nWORKDIR /yolov9\nADD https://github.com/WongKinYiu/yolov9.git .\nRUN uv pip install --system -r requirements.txt\nRUN uv pip install --system onnx==1.18.0 onnxruntime onnx-simplifier==0.4.* onnxscript\nARG MODEL_SIZE\nARG IMG_SIZE\nADD https://github.com/WongKinYiu/yolov9/releases/download/v0.1/yolov9-${MODEL_SIZE}-converted.pt yolov9-${MODEL_SIZE}.pt\nRUN sed -i \"s/ckpt = torch.load(attempt_download(w), map_location='cpu')/ckpt = torch.load(attempt_download(w), map_location='cpu', weights_only=False)/g\" models/experimental.py\nRUN python3 export.py --weights ./yolov9-${MODEL_SIZE}.pt --imgsz ${IMG_SIZE} --simplify --include onnx\nFROM scratch\nARG MODEL_SIZE\nARG IMG_SIZE\nCOPY --from=build /yolov9/yolov9-${MODEL_SIZE}.onnx /yolov9-${MODEL_SIZE}-${IMG_SIZE}.onnx\nEOF\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU` (or `NPU`). Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (should match the imgsize set during model export) |\n| **Object detection model input height** | `320` (should match the imgsize set during model export) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/yolo.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n ov:\n type: openvino\n device: GPU # or NPU\n\nmodel:\n model_type: yolo-generic\n width: 320 # <--- should match the imgsize set during model export\n height: 320 # <--- should match the imgsize set during model export\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/yolo.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolo-legacy", + "label": "YOLO (v3, v4, v7)", + "recommended": false, + "download": "To export as ONNX:\n\n```sh\ngit clone https://github.com/NateMeyer/tensorrt_demos\ncd tensorrt_demos/yolo\n./download_yolo.sh\npython3 yolo_to_onnx.py -m yolov7-320\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU` (or `NPU`). Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (should match the imgsize set during model export) |\n| **Object detection model input height** | `320` (should match the imgsize set during model export) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/yolo.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n ov:\n type: openvino\n device: GPU # or NPU\n\nmodel:\n model_type: yolo-generic\n width: 320 # <--- should match the imgsize set during model export\n height: 320 # <--- should match the imgsize set during model export\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/yolo.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolonas", + "label": "YOLO-NAS", + "recommended": false, + "download": "You can build and download a compatible model with pre-trained weights using [this notebook](https://github.com/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb) which can be run directly in [Google Colab](https://colab.research.google.com/github/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb).\n\n:::warning\n\nThe pre-trained YOLO-NAS weights from DeciAI are subject to their license and can't be used commercially. For more information, see: https://docs.deci.ai/super-gradients/latest/LICENSE.YOLONAS.html\n\n:::\n\nThe input image size in this notebook is set to 320x320. This results in lower CPU usage and faster inference times without impacting performance in most cases due to the way Frigate crops video frames to areas of interest before running detection. The notebook and config can be updated to 640x640 if desired.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ------------------------------------------------- |\n| **Object Detection Model Type** | `yolonas` |\n| **Object detection model input width** | `320` (should match whatever was set in notebook) |\n| **Object detection model input height** | `320` (should match whatever was set in notebook) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input Pixel Color Format** | `bgr` |\n| **Custom object detector model path** | `/config/yolo_nas_s.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n ov:\n type: openvino\n device: GPU\n\nmodel:\n model_type: yolonas\n width: 320 # <--- should match whatever was set in notebook\n height: 320 # <--- should match whatever was set in notebook\n input_tensor: nchw\n input_pixel_format: bgr\n path: /config/yolo_nas_s.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolox", + "label": "YOLOX", + "recommended": false, + "download": "YOLOx models can be downloaded [from the YOLOx repo](https://github.com/Megvii-BaseDetection/YOLOX/tree/main/demo/ONNXRuntime).", + "ui": "Navigate to **Settings > System > Detectors and model** and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ------------------------------------- | -------------------------------- |\n| **Object Detection Model Type** | `yolox` |\n| **Custom object detector model path** | path to your YOLOX ONNX model |", + "yaml": "detectors:\n ov:\n type: openvino\n device: GPU\n\nmodel:\n model_type: yolox\n path: /config/model_cache/yolox.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "rfdetr", + "label": "RF-DETR", + "recommended": false, + "download": "RF-DETR can be exported as ONNX by running the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=Nano` in the first line to `Nano`, `Small`, or `Medium` size.\n\n```sh\ndocker build . --build-arg MODEL_SIZE=Nano --rm --output . -f- <<'EOF'\nFROM python:3.12 AS build\nRUN apt-get update && apt-get install --no-install-recommends -y libgl1 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/\nWORKDIR /rfdetr\nRUN uv pip install --system rfdetr[onnxexport] torch==2.8.0 onnx==1.19.1 transformers==4.57.6 onnxscript\nARG MODEL_SIZE\nRUN python3 -c \"from rfdetr import RFDETR${MODEL_SIZE}; x = RFDETR${MODEL_SIZE}(resolution=320); x.export(simplify=True)\"\nFROM scratch\nARG MODEL_SIZE\nCOPY --from=build /rfdetr/output/inference_model.onnx /rfdetr-${MODEL_SIZE}.onnx\nEOF\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| --------------------------------------- | --------------------------------- |\n| **Object Detection Model Type** | `rfdetr` |\n| **Object detection model input width** | `320` |\n| **Object detection model input height** | `320` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/rfdetr.onnx` |", + "yaml": "detectors:\n ov:\n type: openvino\n device: GPU\n\nmodel:\n model_type: rfdetr\n width: 320\n height: 320\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/rfdetr.onnx" + }, + { + "key": "dfine", + "label": "D-FINE / DEIMv2", + "recommended": false, + "download": "#### D-FINE\n\nD-FINE can be exported as ONNX by running the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=s` in the first line to `s`, `m`, or `l` size.\n\n```sh\ndocker build . --build-arg MODEL_SIZE=s --output . -f- <<'EOF'\nFROM python:3.11 AS build\nRUN apt-get update && apt-get install --no-install-recommends -y libgl1 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.8.0 /uv /bin/\nWORKDIR /dfine\nRUN git clone https://github.com/Peterande/D-FINE.git .\nRUN uv pip install --system -r requirements.txt\nRUN uv pip install --system onnx onnxruntime onnxsim onnxscript\n# Create output directory and download checkpoint\nRUN mkdir -p output\nARG MODEL_SIZE\nRUN wget https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_${MODEL_SIZE}_obj2coco.pth -O output/dfine_${MODEL_SIZE}_obj2coco.pth\n# Modify line 58 of export_onnx.py to change batch size to 1\nRUN sed -i '58s/data = torch.rand(.*)/data = torch.rand(1, 3, 640, 640)/' tools/deployment/export_onnx.py\nRUN python3 tools/deployment/export_onnx.py -c configs/dfine/objects365/dfine_hgnetv2_${MODEL_SIZE}_obj2coco.yml -r output/dfine_${MODEL_SIZE}_obj2coco.pth\nFROM scratch\nARG MODEL_SIZE\nCOPY --from=build /dfine/output/dfine_${MODEL_SIZE}_obj2coco.onnx /dfine-${MODEL_SIZE}.onnx\nEOF\n```\n\n#### DEIMv2\n\n[DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) can be exported as ONNX by running the command below. Pretrained weights are available on Hugging Face for two backbone families:\n\n- **HGNetv2** (smaller/faster): `atto`, `femto`, `pico`, `n`\n- **DINOv3** (larger/more accurate): `s`, `m`, `l`, `x`\n\nSet `BACKBONE` and `MODEL_SIZE` in the first line to match your desired variant. Hugging Face model names use uppercase (e.g. `HGNetv2_N`, `DINOv3_S`), while config files use lowercase (e.g. `hgnetv2_n`, `dinov3_s`).\n\n```sh\ndocker build . --rm --build-arg BACKBONE=hgnetv2 --build-arg MODEL_SIZE=n --output . -f- <<'EOF'\nFROM python:3.11-slim AS build\nRUN apt-get update && apt-get install --no-install-recommends -y git libgl1 libglib2.0-0 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.8.0 /uv /bin/\nWORKDIR /deimv2\nRUN git clone https://github.com/Intellindust-AI-Lab/DEIMv2.git .\n# Install CPU-only PyTorch first to avoid pulling CUDA variant\nRUN uv pip install --no-cache --system torch torchvision --index-url https://download.pytorch.org/whl/cpu\nRUN uv pip install --no-cache --system -r requirements.txt\nRUN uv pip install --no-cache --system onnx safetensors huggingface_hub\nRUN mkdir -p output\nARG BACKBONE\nARG MODEL_SIZE\n# Download from Hugging Face and convert safetensors to pth\nRUN python3 -c \"\\\nfrom huggingface_hub import hf_hub_download; \\\nfrom safetensors.torch import load_file; \\\nimport torch; \\\nbackbone = '${BACKBONE}'.replace('hgnetv2','HGNetv2').replace('dinov3','DINOv3'); \\\nsize = '${MODEL_SIZE}'.upper(); \\\nst = load_file(hf_hub_download('Intellindust/DEIMv2_' + backbone + '_' + size + '_COCO', 'model.safetensors')); \\\ntorch.save({'model': st}, 'output/deimv2.pth')\"\nRUN sed -i \"s/data = torch.rand(2/data = torch.rand(1/\" tools/deployment/export_onnx.py\n# HuggingFace safetensors omits frozen constants that the model constructor initializes\nRUN sed -i \"s/cfg.model.load_state_dict(state)/cfg.model.load_state_dict(state, strict=False)/\" tools/deployment/export_onnx.py\nRUN python3 tools/deployment/export_onnx.py -c configs/deimv2/deimv2_${BACKBONE}_${MODEL_SIZE}_coco.yml -r output/deimv2.pth\nFROM scratch\nARG BACKBONE\nARG MODEL_SIZE\nCOPY --from=build /deimv2/output/deimv2.onnx /deimv2_${BACKBONE}_${MODEL_SIZE}.onnx\nEOF\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `CPU`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ---------------------------------- |\n| **Object Detection Model Type** | `dfine` |\n| **Object detection model input width** | `640` |\n| **Object detection model input height** | `640` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/dfine-s.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n ov:\n type: openvino\n device: CPU\n\nmodel:\n model_type: dfine\n width: 640\n height: 640\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/dfine-s.onnx\n labelmap_path: /labelmap/coco-80.txt" + } + ] + }, + "appleSilicon": { + "title": "Apple Silicon", + "models": [ + { + "key": "yolov9", + "label": "YOLOv9", + "recommended": true, + "download": "YOLOv9 model can be exported as ONNX using the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=t` and `IMG_SIZE=320` in the first line to the [model size](https://github.com/WongKinYiu/yolov9#performance) you would like to convert (available model sizes are `t`, `s`, `m`, `c`, and `e`, common image sizes are `320` and `640`).\n\n```sh\ndocker build . --build-arg MODEL_SIZE=t --build-arg IMG_SIZE=320 --output . -f- <<'EOF'\nFROM python:3.11 AS build\nRUN apt-get update && apt-get install --no-install-recommends -y cmake libgl1 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/\nWORKDIR /yolov9\nADD https://github.com/WongKinYiu/yolov9.git .\nRUN uv pip install --system -r requirements.txt\nRUN uv pip install --system onnx==1.18.0 onnxruntime onnx-simplifier==0.4.* onnxscript\nARG MODEL_SIZE\nARG IMG_SIZE\nADD https://github.com/WongKinYiu/yolov9/releases/download/v0.1/yolov9-${MODEL_SIZE}-converted.pt yolov9-${MODEL_SIZE}.pt\nRUN sed -i \"s/ckpt = torch.load(attempt_download(w), map_location='cpu')/ckpt = torch.load(attempt_download(w), map_location='cpu', weights_only=False)/g\" models/experimental.py\nRUN python3 export.py --weights ./yolov9-${MODEL_SIZE}.pt --imgsz ${IMG_SIZE} --simplify --include onnx\nFROM scratch\nARG MODEL_SIZE\nARG IMG_SIZE\nCOPY --from=build /yolov9/yolov9-${MODEL_SIZE}.onnx /yolov9-${MODEL_SIZE}-${IMG_SIZE}.onnx\nEOF\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ZMQ IPC** from the detector type dropdown and click **Add**, then set the endpoint to `tcp://host.docker.internal:5555`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (should match the imgsize set during model export) |\n| **Object detection model input height** | `320` (should match the imgsize set during model export) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/yolo.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n apple-silicon:\n type: zmq\n endpoint: tcp://host.docker.internal:5555\n\nmodel:\n model_type: yolo-generic\n width: 320 # <--- should match the imgsize set during model export\n height: 320 # <--- should match the imgsize set during model export\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/yolo.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolo-legacy", + "label": "YOLO (v3, v4, v7)", + "recommended": false, + "download": "To export as ONNX:\n\n```sh\ngit clone https://github.com/NateMeyer/tensorrt_demos\ncd tensorrt_demos/yolo\n./download_yolo.sh\npython3 yolo_to_onnx.py -m yolov7-320\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ZMQ IPC** from the detector type dropdown and click **Add**, then set the endpoint to `tcp://host.docker.internal:5555`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (should match the imgsize set during model export) |\n| **Object detection model input height** | `320` (should match the imgsize set during model export) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/yolo.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n apple-silicon:\n type: zmq\n endpoint: tcp://host.docker.internal:5555\n\nmodel:\n model_type: yolo-generic\n width: 320 # <--- should match the imgsize set during model export\n height: 320 # <--- should match the imgsize set during model export\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/yolo.onnx\n labelmap_path: /labelmap/coco-80.txt" + } + ] + }, + "onnx": { + "title": "ONNX", + "models": [ + { + "key": "yolov9", + "label": "YOLOv9", + "recommended": true, + "download": "YOLOv9 model can be exported as ONNX using the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=t` and `IMG_SIZE=320` in the first line to the [model size](https://github.com/WongKinYiu/yolov9#performance) you would like to convert (available model sizes are `t`, `s`, `m`, `c`, and `e`, common image sizes are `320` and `640`).\n\n```sh\ndocker build . --build-arg MODEL_SIZE=t --build-arg IMG_SIZE=320 --output . -f- <<'EOF'\nFROM python:3.11 AS build\nRUN apt-get update && apt-get install --no-install-recommends -y cmake libgl1 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/\nWORKDIR /yolov9\nADD https://github.com/WongKinYiu/yolov9.git .\nRUN uv pip install --system -r requirements.txt\nRUN uv pip install --system onnx==1.18.0 onnxruntime onnx-simplifier==0.4.* onnxscript\nARG MODEL_SIZE\nARG IMG_SIZE\nADD https://github.com/WongKinYiu/yolov9/releases/download/v0.1/yolov9-${MODEL_SIZE}-converted.pt yolov9-${MODEL_SIZE}.pt\nRUN sed -i \"s/ckpt = torch.load(attempt_download(w), map_location='cpu')/ckpt = torch.load(attempt_download(w), map_location='cpu', weights_only=False)/g\" models/experimental.py\nRUN python3 export.py --weights ./yolov9-${MODEL_SIZE}.pt --imgsz ${IMG_SIZE} --simplify --include onnx\nFROM scratch\nARG MODEL_SIZE\nARG IMG_SIZE\nCOPY --from=build /yolov9/yolov9-${MODEL_SIZE}.onnx /yolov9-${MODEL_SIZE}-${IMG_SIZE}.onnx\nEOF\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (should match the imgsize set during model export) |\n| **Object detection model input height** | `320` (should match the imgsize set during model export) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/yolo.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n onnx:\n type: onnx\n\nmodel:\n model_type: yolo-generic\n width: 320 # <--- should match the imgsize set during model export\n height: 320 # <--- should match the imgsize set during model export\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/yolo.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "rfdetr", + "label": "RF-DETR", + "recommended": false, + "download": "RF-DETR can be exported as ONNX by running the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=Nano` in the first line to `Nano`, `Small`, or `Medium` size.\n\n```sh\ndocker build . --build-arg MODEL_SIZE=Nano --rm --output . -f- <<'EOF'\nFROM python:3.12 AS build\nRUN apt-get update && apt-get install --no-install-recommends -y libgl1 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/\nWORKDIR /rfdetr\nRUN uv pip install --system rfdetr[onnxexport] torch==2.8.0 onnx==1.19.1 transformers==4.57.6 onnxscript\nARG MODEL_SIZE\nRUN python3 -c \"from rfdetr import RFDETR${MODEL_SIZE}; x = RFDETR${MODEL_SIZE}(resolution=320); x.export(simplify=True)\"\nFROM scratch\nARG MODEL_SIZE\nCOPY --from=build /rfdetr/output/inference_model.onnx /rfdetr-${MODEL_SIZE}.onnx\nEOF\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| --------------------------------------- | --------------------------------- |\n| **Object Detection Model Type** | `rfdetr` |\n| **Object detection model input width** | `320` |\n| **Object detection model input height** | `320` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/rfdetr.onnx` |", + "yaml": "detectors:\n onnx:\n type: onnx\n\nmodel:\n model_type: rfdetr\n width: 320\n height: 320\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/rfdetr.onnx" + }, + { + "key": "yolonas", + "label": "YOLO-NAS", + "recommended": false, + "download": "You can build and download a compatible model with pre-trained weights using [this notebook](https://github.com/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb) which can be run directly in [Google Colab](https://colab.research.google.com/github/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb).\n\n:::warning\n\nThe pre-trained YOLO-NAS weights from DeciAI are subject to their license and can't be used commercially. For more information, see: https://docs.deci.ai/super-gradients/latest/LICENSE.YOLONAS.html\n\n:::\n\nThe input image size in this notebook is set to 320x320. This results in lower CPU usage and faster inference times without impacting performance in most cases due to the way Frigate crops video frames to areas of interest before running detection. The notebook and config can be updated to 640x640 if desired.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ------------------------------------------------- |\n| **Object Detection Model Type** | `yolonas` |\n| **Object detection model input width** | `320` (should match whatever was set in notebook) |\n| **Object detection model input height** | `320` (should match whatever was set in notebook) |\n| **Model Input Pixel Color Format** | `bgr` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Custom object detector model path** | `/config/yolo_nas_s.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n onnx:\n type: onnx\n\nmodel:\n model_type: yolonas\n width: 320 # <--- should match whatever was set in notebook\n height: 320 # <--- should match whatever was set in notebook\n input_pixel_format: bgr\n input_tensor: nchw\n path: /config/yolo_nas_s.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolox", + "label": "YOLOX", + "recommended": false, + "download": "YOLOx models can be downloaded [from the YOLOx repo](https://github.com/Megvii-BaseDetection/YOLOX/tree/main/demo/ONNXRuntime).", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------------- |\n| **Object Detection Model Type** | `yolox` |\n| **Object detection model input width** | `416` (should match the imgsize set during model export) |\n| **Object detection model input height** | `416` (should match the imgsize set during model export) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float_denorm` |\n| **Custom object detector model path** | `/config/model_cache/yolox_tiny.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n onnx:\n type: onnx\n\nmodel:\n model_type: yolox\n width: 416 # <--- should match the imgsize set during model export\n height: 416 # <--- should match the imgsize set during model export\n input_tensor: nchw\n input_dtype: float_denorm\n path: /config/model_cache/yolox_tiny.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "dfine", + "label": "D-FINE / DEIMv2", + "recommended": false, + "download": "#### Downloading D-FINE Model\n\nD-FINE can be exported as ONNX by running the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=s` in the first line to `s`, `m`, or `l` size.\n\n```sh\ndocker build . --build-arg MODEL_SIZE=s --output . -f- <<'EOF'\nFROM python:3.11 AS build\nRUN apt-get update && apt-get install --no-install-recommends -y libgl1 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.8.0 /uv /bin/\nWORKDIR /dfine\nRUN git clone https://github.com/Peterande/D-FINE.git .\nRUN uv pip install --system -r requirements.txt\nRUN uv pip install --system onnx onnxruntime onnxsim onnxscript\n# Create output directory and download checkpoint\nRUN mkdir -p output\nARG MODEL_SIZE\nRUN wget https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_${MODEL_SIZE}_obj2coco.pth -O output/dfine_${MODEL_SIZE}_obj2coco.pth\n# Modify line 58 of export_onnx.py to change batch size to 1\nRUN sed -i '58s/data = torch.rand(.*)/data = torch.rand(1, 3, 640, 640)/' tools/deployment/export_onnx.py\nRUN python3 tools/deployment/export_onnx.py -c configs/dfine/objects365/dfine_hgnetv2_${MODEL_SIZE}_obj2coco.yml -r output/dfine_${MODEL_SIZE}_obj2coco.pth\nFROM scratch\nARG MODEL_SIZE\nCOPY --from=build /dfine/output/dfine_${MODEL_SIZE}_obj2coco.onnx /dfine-${MODEL_SIZE}.onnx\nEOF\n```\n\n#### Downloading DEIMv2 Model\n\n[DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) can be exported as ONNX by running the command below. Pretrained weights are available on Hugging Face for two backbone families:\n\n- **HGNetv2** (smaller/faster): `atto`, `femto`, `pico`, `n`\n- **DINOv3** (larger/more accurate): `s`, `m`, `l`, `x`\n\nSet `BACKBONE` and `MODEL_SIZE` in the first line to match your desired variant. Hugging Face model names use uppercase (e.g. `HGNetv2_N`, `DINOv3_S`), while config files use lowercase (e.g. `hgnetv2_n`, `dinov3_s`).\n\n```sh\ndocker build . --rm --build-arg BACKBONE=hgnetv2 --build-arg MODEL_SIZE=n --output . -f- <<'EOF'\nFROM python:3.11-slim AS build\nRUN apt-get update && apt-get install --no-install-recommends -y git libgl1 libglib2.0-0 && rm -rf /var/lib/apt/lists/*\nCOPY --from=ghcr.io/astral-sh/uv:0.8.0 /uv /bin/\nWORKDIR /deimv2\nRUN git clone https://github.com/Intellindust-AI-Lab/DEIMv2.git .\n# Install CPU-only PyTorch first to avoid pulling CUDA variant\nRUN uv pip install --no-cache --system torch torchvision --index-url https://download.pytorch.org/whl/cpu\nRUN uv pip install --no-cache --system -r requirements.txt\nRUN uv pip install --no-cache --system onnx safetensors huggingface_hub\nRUN mkdir -p output\nARG BACKBONE\nARG MODEL_SIZE\n# Download from Hugging Face and convert safetensors to pth\nRUN python3 -c \"\\\nfrom huggingface_hub import hf_hub_download; \\\nfrom safetensors.torch import load_file; \\\nimport torch; \\\nbackbone = '${BACKBONE}'.replace('hgnetv2','HGNetv2').replace('dinov3','DINOv3'); \\\nsize = '${MODEL_SIZE}'.upper(); \\\nst = load_file(hf_hub_download('Intellindust/DEIMv2_' + backbone + '_' + size + '_COCO', 'model.safetensors')); \\\ntorch.save({'model': st}, 'output/deimv2.pth')\"\nRUN sed -i \"s/data = torch.rand(2/data = torch.rand(1/\" tools/deployment/export_onnx.py\n# HuggingFace safetensors omits frozen constants that the model constructor initializes\nRUN sed -i \"s/cfg.model.load_state_dict(state)/cfg.model.load_state_dict(state, strict=False)/\" tools/deployment/export_onnx.py\nRUN python3 tools/deployment/export_onnx.py -c configs/deimv2/deimv2_${BACKBONE}_${MODEL_SIZE}_coco.yml -r output/deimv2.pth\nFROM scratch\nARG BACKBONE\nARG MODEL_SIZE\nCOPY --from=build /deimv2/output/deimv2.onnx /deimv2_${BACKBONE}_${MODEL_SIZE}.onnx\nEOF\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ------------------------------------------- |\n| **Object Detection Model Type** | `dfine` |\n| **Object detection model input width** | `640` |\n| **Object detection model input height** | `640` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/dfine_m_obj2coco.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n onnx:\n type: onnx\n\nmodel:\n model_type: dfine\n width: 640\n height: 640\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/dfine_m_obj2coco.onnx\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolo-legacy", + "label": "YOLO (v3, v4, v7)", + "recommended": false, + "download": "To export as ONNX:\n\n```sh\ngit clone https://github.com/NateMeyer/tensorrt_demos\ncd tensorrt_demos/yolo\n./download_yolo.sh\npython3 yolo_to_onnx.py -m yolov7-320\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (should match the imgsize set during model export) |\n| **Object detection model input height** | `320` (should match the imgsize set during model export) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Custom object detector model path** | `/config/model_cache/yolo.onnx` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n onnx:\n type: onnx\n\nmodel:\n model_type: yolo-generic\n width: 320 # <--- should match the imgsize set during model export\n height: 320 # <--- should match the imgsize set during model export\n input_tensor: nchw\n input_dtype: float\n path: /config/model_cache/yolo.onnx\n labelmap_path: /labelmap/coco-80.txt" + } + ] + }, + "cpu": { + "title": "CPU", + "models": [ + { + "key": "ssd", + "label": "MobileNet v2", + "recommended": true, + "download": "A TensorFlow Lite model is provided in the container at `/cpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with `model.path`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **CPU** from the detector type dropdown and click **Add**. Configure the number of threads and click **Add** again to add additional CPU detectors as needed (one per camera is recommended).\n\n| Field | Value |\n| ----------------- | ----- |\n| **Detector type** | `cpu` |\n| **Num threads** | `3` |", + "yaml": "detectors:\n cpu1:\n type: cpu\n num_threads: 3" + } + ] + }, + "deepstack": { + "title": "DeepStack / CodeProject.AI", + "models": [ + { + "key": "yolo", + "label": "YOLO", + "recommended": true, + "download": "This detector runs object detection over the network against a CodeProject.AI or DeepStack server, so no model is downloaded into Frigate itself. Visit the [CodeProject.AI official website](https://www.codeproject.com/Articles/5322557/CodeProject-AI-Server-AI-the-easy-way) to download and install the AI server on your preferred device (e.g. Raspberry Pi, Nvidia Jetson, or other compatible hardware) before configuring the detector.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **DeepStack** from the detector type dropdown and click **Add**. Set the API URL to point to your CodeProject.AI server (e.g., `http://:/v1/vision/detection`).\n\n| Field | Value |\n| ------------- | ---------------------------------------------------------------------- |\n| **API URL** | `http://:/v1/vision/detection` |\n| **API Timeout** | `0.1` (seconds) |", + "yaml": "detectors:\n deepstack:\n api_url: http://:/v1/vision/detection\n type: deepstack\n api_timeout: 0.1 # seconds" + } + ] + }, + "memryx": { + "title": "MemryX", + "models": [ + { + "key": "yolonas", + "label": "YOLO-NAS", + "recommended": true, + "download": "The [YOLO-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) model included in this detector is downloaded automatically and compiled to DFP with [mx_nc](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage).\n\n**Note:** The default model for the MemryX detector is YOLO-NAS 320x320.\n\nThe input size for **YOLO-NAS** can be set to either **320x320** (default) or **640x640**.\n\n- The default size of **320x320** is optimized for lower CPU usage and faster inference times.\n\nMemryX `.dfp` models are automatically downloaded at runtime, if enabled, to the container at `/memryx_models/model_folder/`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ------------------------------------------------- |\n| **Object Detection Model Type** | `yolonas` |\n| **Object detection model input width** | `320` (can be set to `640` for higher resolution) |\n| **Object detection model input height** | `320` (can be set to `640` for higher resolution) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n memx0:\n type: memryx\n device: PCIe:0\n\nmodel:\n model_type: yolonas\n width: 320 # (Can be set to 640 for higher resolution)\n height: 320 # (Can be set to 640 for higher resolution)\n input_tensor: nchw\n input_dtype: float\n labelmap_path: /labelmap/coco-80.txt\n # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.\n # path: /config/yolonas.zip\n # The .zip file must contain:\n # \u251c\u2500\u2500 yolonas.dfp (a file ending with .dfp)\n # \u2514\u2500\u2500 yolonas_post.onnx (optional; only if the model includes a cropped post-processing network)" + }, + { + "key": "yolov9", + "label": "YOLOv9", + "recommended": false, + "download": "The YOLOv9s model included in this detector is downloaded from [the original GitHub](https://github.com/WongKinYiu/yolov9) and compiled to DFP with [mx_nc](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage).\n\nMemryX `.dfp` models are automatically downloaded at runtime, if enabled, to the container at `/memryx_models/model_folder/`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ------------------------------------------------- |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` (can be set to `640` for higher resolution) |\n| **Object detection model input height** | `320` (can be set to `640` for higher resolution) |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n memx0:\n type: memryx\n device: PCIe:0\n\nmodel:\n model_type: yolo-generic\n width: 320 # (Can be set to 640 for higher resolution)\n height: 320 # (Can be set to 640 for higher resolution)\n input_tensor: nchw\n input_dtype: float\n labelmap_path: /labelmap/coco-80.txt\n # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.\n # path: /config/yolov9.zip\n # The .zip file must contain:\n # \u251c\u2500\u2500 yolov9.dfp (a file ending with .dfp)" + }, + { + "key": "yolox", + "label": "YOLOX", + "recommended": false, + "download": "The model is sourced from the [OpenCV Model Zoo](https://github.com/opencv/opencv_zoo) and precompiled to DFP.\n\nMemryX `.dfp` models are automatically downloaded at runtime, if enabled, to the container at `/memryx_models/model_folder/`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ----------------------- |\n| **Object Detection Model Type** | `yolox` |\n| **Object detection model input width** | `640` |\n| **Object detection model input height** | `640` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float_denorm` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n memx0:\n type: memryx\n device: PCIe:0\n\nmodel:\n model_type: yolox\n width: 640\n height: 640\n input_tensor: nchw\n input_dtype: float_denorm\n labelmap_path: /labelmap/coco-80.txt\n # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.\n # path: /config/yolox.zip\n # The .zip file must contain:\n # \u251c\u2500\u2500 yolox.dfp (a file ending with .dfp)" + }, + { + "key": "ssd", + "label": "SSDLite MobileNet v2", + "recommended": false, + "download": "The model is sourced from the [OpenMMLab Model Zoo](https://mmdeploy-oss.openmmlab.com/model/mmdet-det/ssdlite-e8679f.onnx) and has been converted to DFP.\n\nMemryX `.dfp` models are automatically downloaded at runtime, if enabled, to the container at `/memryx_models/model_folder/`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ----------------------- |\n| **Object Detection Model Type** | `ssd` |\n| **Object detection model input width** | `320` |\n| **Object detection model input height** | `320` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input D Type** | `float` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n memx0:\n type: memryx\n device: PCIe:0\n\nmodel:\n model_type: ssd\n width: 320\n height: 320\n input_tensor: nchw\n input_dtype: float\n labelmap_path: /labelmap/coco-80.txt\n # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.\n # path: /config/ssdlite_mobilenet.zip\n # The .zip file must contain:\n # \u251c\u2500\u2500 ssdlite_mobilenet.dfp (a file ending with .dfp)\n # \u2514\u2500\u2500 ssdlite_mobilenet_post.onnx (optional; only if the model includes a cropped post-processing network)" + } + ] + }, + "tensorrt": { + "title": "TensorRT", + "models": [ + { + "key": "yolo-legacy", + "label": "YOLO (v3, v4, v7)", + "recommended": true, + "download": "The model used for TensorRT must be preprocessed on the same hardware platform that it will run on, so Frigate generates the `.trt` model file on-device at startup. Processed models are stored in the `/config/model_cache` folder.\n\nBy default no models are generated. Set the `YOLO_MODELS` environment variable in Docker to one or more comma-separated model names (from the available `yolov3`/`yolov4`/`yolov7` models) and each one will be generated on startup if the corresponding `{model}.trt` file is not already present in `model_cache` (delete it to force regeneration). On Jetson devices with DLAs (Xavier or Orin), append `-dla` to a model name to generate a DLA model. If your GPU does not support FP16 operations, pass `USE_FP16=False` to disable it.\n\nAn example `docker-compose.yml` fragment that converts the `yolov7-320` and `yolov7x-640` models:\n\n```yml\nfrigate:\n environment:\n - YOLO_MODELS=yolov7-320,yolov7x-640\n - USE_FP16=false\n```", + "ui": "Navigate to **Settings > System > Detectors and model** and select **TensorRT** from the detector type dropdown and click **Add**, then set the device to `0` (the default GPU index). Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ------------------------------------------------------------ |\n| **Custom object detector model path** | `/config/model_cache/tensorrt/yolov7-320.trt` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |\n| **Model Input Tensor Shape** | `nchw` |\n| **Model Input Pixel Color Format** | `rgb` |\n| **Object detection model input width** | `320` (MUST match the chosen model, e.g., yolov7-320 -> 320) |\n| **Object detection model input height** | `320` (MUST match the chosen model, e.g., yolov7-320 -> 320) |", + "yaml": "detectors:\n tensorrt:\n type: tensorrt\n device: 0 #This is the default, select the first GPU\n\nmodel:\n path: /config/model_cache/tensorrt/yolov7-320.trt\n labelmap_path: /labelmap/coco-80.txt\n input_tensor: nchw\n input_pixel_format: rgb\n width: 320 # MUST match the chosen model i.e yolov7-320 -> 320, yolov4-416 -> 416\n height: 320 # MUST match the chosen model i.e yolov7-320 -> 320 yolov4-416 -> 416" + } + ] + }, + "synaptics": { + "title": "Synaptics", + "models": [ + { + "key": "ssd", + "label": "SSD MobileNet", + "recommended": true, + "download": "A synap model is provided in the container at `/mobilenet.synap` and is used by this detector type by default. The model comes from the [Synap-release Github](https://github.com/synaptics-astra/synap-release/tree/v1.5.0/models/dolphin/object_detection/coco/model/mobilenet224_full80).", + "ui": "Navigate to **Settings > System > Detectors and model** and select **Synaptics** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ---------------------------- |\n| **Custom object detector model path** | `/synaptics/mobilenet.synap` |\n| **Object detection model input width** | `224` |\n| **Object detection model input height** | `224` |\n| **Model Input Tensor Shape** | `nhwc` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors: # required\n synap_npu: # required\n type: synaptics # required\n\nmodel: # required\n path: /synaptics/mobilenet.synap # required\n width: 224 # required\n height: 224 # required\n input_tensor: nhwc # default value (optional. If you change the model, it is required)\n labelmap_path: /labelmap/coco-80.txt # required" + } + ] + }, + "rknn": { + "title": "RKNN", + "models": [ + { + "key": "yolov9", + "label": "YOLOv9", + "recommended": true, + "download": "If no custom model is provided, the RKNN detector downloads a default model from GitHub on first startup. Once cached, the model works fully offline. All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space.\n\nYou can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models.", + "ui": "Navigate to **Settings > System > Detectors and model** and, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | -------------------------------------------------- |\n| **Custom object detector model path** | `frigate-fp16-yolov9-t` (or other yolov9 variants) |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` |\n| **Object detection model input height** | `320` |\n| **Model Input Tensor Shape** | `nhwc` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "model: # required\n # name of model (will be automatically downloaded) or path to your own .rknn model file\n # possible values are:\n # - frigate-fp16-yolov9-t\n # - frigate-fp16-yolov9-s\n # - frigate-fp16-yolov9-m\n # - frigate-fp16-yolov9-c\n # - frigate-fp16-yolov9-e\n # your yolo_model.rknn\n path: frigate-fp16-yolov9-t\n model_type: yolo-generic\n width: 320\n height: 320\n input_tensor: nhwc\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolonas", + "label": "YOLO-NAS", + "recommended": false, + "download": "If no custom model is provided, the RKNN detector downloads a default model from GitHub on first startup. Once cached, the model works fully offline. All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space.\n\nYou can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models.\n\n**Note:** The pre-trained YOLO-NAS weights from DeciAI are subject to their license and can't be used commercially. For more information, see: https://docs.deci.ai/super-gradients/latest/LICENSE.YOLONAS.html", + "ui": "Navigate to **Settings > System > Detectors and model** and, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ----------------------------------------------------------------------- |\n| **Custom object detector model path** | `deci-fp16-yolonas_s` (or `deci-fp16-yolonas_m`, `deci-fp16-yolonas_l`) |\n| **Object Detection Model Type** | `yolonas` |\n| **Object detection model input width** | `320` |\n| **Object detection model input height** | `320` |\n| **Model Input Pixel Color Format** | `bgr` |\n| **Model Input Tensor Shape** | `nhwc` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "model: # required\n # name of model (will be automatically downloaded) or path to your own .rknn model file\n # possible values are:\n # - deci-fp16-yolonas_s\n # - deci-fp16-yolonas_m\n # - deci-fp16-yolonas_l\n # your yolonas_model.rknn\n path: deci-fp16-yolonas_s\n model_type: yolonas\n width: 320\n height: 320\n input_pixel_format: bgr\n input_tensor: nhwc\n labelmap_path: /labelmap/coco-80.txt" + }, + { + "key": "yolox", + "label": "YOLOx", + "recommended": false, + "download": "If no custom model is provided, the RKNN detector downloads a default model from GitHub on first startup. Once cached, the model works fully offline. All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space.\n\nYou can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models.", + "ui": "Navigate to **Settings > System > Detectors and model** and, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ---------------------------------------------- |\n| **Custom object detector model path** | `rock-i8-yolox_nano` (or other yolox variants) |\n| **Object Detection Model Type** | `yolox` |\n| **Object detection model input width** | `416` |\n| **Object detection model input height** | `416` |\n| **Model Input Tensor Shape** | `nhwc` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "model: # required\n # name of model (will be automatically downloaded) or path to your own .rknn model file\n # possible values are:\n # - rock-i8-yolox_nano\n # - rock-i8-yolox_tiny\n # - rock-fp16-yolox_nano\n # - rock-fp16-yolox_tiny\n # your yolox_model.rknn\n path: rock-i8-yolox_nano\n model_type: yolox\n width: 416\n height: 416\n input_tensor: nhwc\n labelmap_path: /labelmap/coco-80.txt" + } + ] + }, + "axengine": { + "title": "AXEngine", + "models": [ + { + "key": "yolov9", + "label": "YOLOv9", + "recommended": true, + "download": "A yolov9 axmodel is provided in the container at `/axmodels` and is used by this detector type by default. The AXEngine detector downloads its default model from HuggingFace on first startup; once cached, the model works fully offline.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **AXEngine NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure:\n\n| Field | Value |\n| ---------------------------------------- | ----------------------- |\n| **Custom object detector model path** | `frigate-yolov9-tiny` |\n| **Object Detection Model Type** | `yolo-generic` |\n| **Object detection model input width** | `320` |\n| **Object detection model input height** | `320` |\n| **Model Input D Type** | `int` |\n| **Model Input Pixel Color Format** | `bgr` |\n| **Label map for custom object detector** | `/labelmap/coco-80.txt` |", + "yaml": "detectors:\n axengine:\n type: axengine\n\nmodel:\n path: frigate-yolov9-tiny\n model_type: yolo-generic\n width: 320\n height: 320\n input_dtype: int\n input_pixel_format: bgr\n labelmap_path: /labelmap/coco-80.txt" + } + ] + }, + "degirumAiServer": { + "title": "DeGirum AI Server", + "models": [ + { + "key": "ai-server-inference", + "label": "AI Server Inference", + "recommended": true, + "download": "Launch a DeGirum AI server as a Docker container, then point the detector at it. Add this to your `docker-compose.yml`:\n\n```yaml\ndegirum_detector:\n container_name: degirum\n image: degirum/aiserver:latest\n privileged: true\n ports:\n - \"8778:8778\"\n```\n\nSet `location` to the server's service name, container name, or `host:port`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **DeGirum** from the detector type dropdown and click **Add**.\n\n| Field | Value |\n| --- | --- |\n| **Location** | `degirum` |\n| **Zoo** | `degirum/public` |\n| **Token** | your AI Hub token (optional for the public zoo) |\n", + "yaml": "degirum_detector:\n type: degirum\n location: degirum\n zoo: degirum/public\n token: dg_example_token\n" + } + ] + }, + "degirumLocal": { + "title": "DeGirum Local", + "models": [ + { + "key": "local-inference", + "label": "Local Inference", + "recommended": true, + "download": "Run hardware directly inside the Frigate container with `@local`, removing the AI server hop. The matching device runtime (e.g. the Hailo runtime) must be installed in the container; confirm it with `degirum sys-info`.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **DeGirum** from the detector type dropdown and click **Add**.\n\n| Field | Value |\n| --- | --- |\n| **Location** | `@local` |\n| **Zoo** | `degirum/public` |\n| **Token** | your AI Hub token (optional for the public zoo) |\n", + "yaml": "degirum_detector:\n type: degirum\n location: @local\n zoo: degirum/public\n token: dg_example_token\n" + } + ] + }, + "degirumCloud": { + "title": "DeGirum AI Hub Cloud", + "models": [ + { + "key": "ai-hub-cloud-inference", + "label": "AI Hub Cloud Inference", + "recommended": true, + "download": "Run inferences on DeGirum's [AI Hub](https://hub.degirum.com) cloud with `@cloud`. Sign up, create an access token, and set it as `token`. Network latency may require lowering your detection fps.", + "ui": "Navigate to **Settings > System > Detectors and model** and select **DeGirum** from the detector type dropdown and click **Add**.\n\n| Field | Value |\n| --- | --- |\n| **Location** | `@cloud` |\n| **Zoo** | `degirum/public` |\n| **Token** | your AI Hub token (optional for the public zoo) |\n", + "yaml": "degirum_detector:\n type: degirum\n location: @cloud\n zoo: degirum/public\n token: dg_example_token\n" + } + ] + } +} diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 4bd99ed8f1..51a9b213a0 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -4,9 +4,11 @@ title: Object Detectors --- import CommunityBadge from '@site/src/components/CommunityBadge'; -import ConfigTabs from "@site/src/components/ConfigTabs"; -import TabItem from "@theme/TabItem"; -import NavPath from "@site/src/components/NavPath"; +import ConfigTabs from '@site/src/components/ConfigTabs'; +import TabItem from '@theme/TabItem'; +import NavPath from '@site/src/components/NavPath'; +import ModelConfigDropdown from '@site/src/components/ModelConfigDropdown'; +import objectDetectorsModels from '@site/data/object_detectors_models.json'; ### Supported hardware @@ -236,68 +238,9 @@ detectors: -### EdgeTPU Supported Models +### Configuration -| Model | Notes | -| ----------------------- | ------------------------------------------- | -| [Mobiledet](#mobiledet) | Default model | -| [YOLOv9](#yolov9) | More accurate but slower than default model | - -#### Mobiledet - -A TensorFlow Lite model is provided in the container at `/edgetpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with `model.path`. - -#### YOLOv9 - -YOLOv9 models that are compiled for TensorFlow Lite and properly quantized are supported, but not included by default. [Instructions](#yolov9-for-google-coral-support) for downloading a model with support for the Google Coral. - -:::tip - -**Frigate+ Users:** Follow the [instructions](/integrations/plus#use-models) to set a model ID in your config file. - -::: - -
- YOLOv9 Setup & Config - -After placing the downloaded files for the tflite model and labels in your config folder, use the following configuration: - - - - -Navigate to and select **EdgeTPU** from the detector type dropdown and click **Add**, then set device to `usb`. Then on the same page, in the **Custom Model** tab, configure the model settings: - -| Field | Value | -| ---------------------------------------- | ----------------------------------------------------------------- | -| **Object Detection Model Type** | `yolo-generic` | -| **Object detection model input width** | `320` (should match the imgsize of the model) | -| **Object detection model input height** | `320` (should match the imgsize of the model) | -| **Custom object detector model path** | `/config/model_cache/yolov9-s-relu6-best_320_int8_edgetpu.tflite` | -| **Label map for custom object detector** | `/config/labels-coco17.txt` | - - - - -```yaml -detectors: - coral: - type: edgetpu - device: usb - -model: - model_type: yolo-generic - width: 320 # <--- should match the imgsize of the model, typically 320 - height: 320 # <--- should match the imgsize of the model, typically 320 - path: /config/model_cache/yolov9-s-relu6-best_320_int8_edgetpu.tflite - labelmap_path: /config/labels-coco17.txt -``` - - - - -Note that due to hardware limitations of the Coral, the labelmap is a subset of the COCO labels and includes only 17 object classes. - -
+ --- @@ -318,145 +261,7 @@ If no custom model is provided, the Hailo detector downloads a default model fro When configuring the Hailo detector, you have two options to specify the model: a local **path** or a **URL**. If both are provided, the detector will first check for the model at the given local path. If the file is not found, it will download the model from the specified URL. The model file is cached under `/config/model_cache/hailo`. -#### YOLO - -Use this configuration for YOLO-based models. When no custom model path or URL is provided, the detector automatically downloads the default model based on the detected hardware: - -- **Hailo-8 hardware:** Uses **YOLOv6n** (default: `yolov6n.hef`) -- **Hailo-8L hardware:** Uses **YOLOv6n** (default: `yolov6n.hef`) - - - - -Navigate to and select **Hailo-8/Hailo-8L** from the detector type dropdown and click **Add**, then set device to `PCIe`. Then on the same page, in the **Custom Model** tab, configure the model settings: - -| Field | Value | -| ---------------------------------------- | ----------------------- | -| **Object detection model input width** | `320` | -| **Object detection model input height** | `320` | -| **Model Input Tensor Shape** | `nhwc` | -| **Model Input Pixel Color Format** | `rgb` | -| **Model Input D Type** | `int` | -| **Object Detection Model Type** | `yolo-generic` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - -The detector automatically selects the default model based on your hardware. Optionally, specify a local model path or URL to override. - - - - -```yaml -detectors: - hailo: - type: hailo8l - device: PCIe - -model: - width: 320 - height: 320 - input_tensor: nhwc - input_pixel_format: rgb - input_dtype: int - model_type: yolo-generic - labelmap_path: /labelmap/coco-80.txt - - # The detector automatically selects the default model based on your hardware: - # - For Hailo-8 hardware: YOLOv6n (default: yolov6n.hef) - # - For Hailo-8L hardware: YOLOv6n (default: yolov6n.hef) - # - # Optionally, you can specify a local model path to override the default. - # If a local path is provided and the file exists, it will be used instead of downloading. - # Example: - # path: /config/model_cache/hailo/yolov6n.hef - # - # You can also override using a custom URL: - # path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8/yolov6n.hef - # just make sure to give it the write configuration based on the model -``` - - - - -#### SSD - -For SSD-based models, provide either a model path or URL to your compiled SSD model. The integration will first check the local path before downloading if necessary. - - - - -Navigate to and select **Hailo-8/Hailo-8L** from the detector type dropdown and click **Add**, then set device to `PCIe`. Then on the same page, in the **Custom Model** tab, configure the model settings: - -| Field | Value | -| --------------------------------------- | ------ | -| **Object detection model input width** | `300` | -| **Object detection model input height** | `300` | -| **Model Input Tensor Shape** | `nhwc` | -| **Model Input Pixel Color Format** | `rgb` | -| **Object Detection Model Type** | `ssd` | - -Specify the local model path or URL for SSD MobileNet v1. - - - - -```yaml -detectors: - hailo: - type: hailo8l - device: PCIe - -model: - width: 300 - height: 300 - input_tensor: nhwc - input_pixel_format: rgb - model_type: ssd - # Specify the local model path (if available) or URL for SSD MobileNet v1. - # Example with a local path: - # path: /config/model_cache/h8l_cache/ssd_mobilenet_v1.hef - # - # Or override using a custom URL: - # path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8l/ssd_mobilenet_v1.hef -``` - - - - -#### Custom Models - -The Hailo detector supports all YOLO models compiled for Hailo hardware that include post-processing. You can specify a custom URL or a local path to download or use your model directly. If both are provided, the detector checks the local path first. - - - - -Navigate to and select **Hailo-8/Hailo-8L** from the detector type dropdown and click **Add**, then set device to `PCIe`. Then on the same page, in the **Custom Model** tab, configure the model settings to match your custom model dimensions and format. - - - - -```yaml -detectors: - hailo: - type: hailo8l - device: PCIe - -model: - width: 640 - height: 640 - input_tensor: nhwc - input_pixel_format: rgb - input_dtype: int - model_type: yolo-generic - labelmap_path: /labelmap/coco-80.txt - # Optional: Specify a local model path. - # path: /config/model_cache/hailo/custom_model.hef - # - # Alternatively, or as a fallback, provide a custom URL: - # path: https://custom-model-url.com/path/to/model.hef -``` - - - + For additional ready-to-use models, please visit: https://github.com/hailo-ai/hailo_model_zoo @@ -481,14 +286,6 @@ OpenVINO is supported on 6th Gen Intel platforms (Skylake) and newer. It will al When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming GPU resources are available. An example configuration would be: - - - -Navigate to and select **OpenVINO** from the detector type dropdown and click **Add** to add multiple detectors, each targeting `GPU` or `NPU`. - - - - ```yaml detectors: ov_0: @@ -499,316 +296,13 @@ detectors: device: GPU # or NPU ``` - - - ::: -### OpenVINO Supported Models +### Configuration -| Model | GPU | NPU | Notes | -| ------------------------------------- | --- | --- | ------------------------------------------------------------ | -| [YOLOv9](#yolo-v3-v4-v7-v9) | ✅ | ✅ | Recommended for GPU & NPU | -| [RF-DETR](#rf-detr) | ✅ | ✅ | Requires XE iGPU or Arc | -| [YOLO-NAS](#yolo-nas) | ✅ | ✅ | | -| [MobileNet v2](#ssdlite-mobilenet-v2) | ✅ | ✅ | Fast and lightweight model, less accurate than larger models | -| [YOLOX](#yolox) | ✅ | ? | | -| [D-FINE / DEIMv2](#d-fine--deimv2) | ❌ | ❌ | | + -#### SSDLite MobileNet v2 - -An OpenVINO model is provided in the container at `/openvino-model/ssdlite_mobilenet_v2.xml` and is used by this detector type by default. The model comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model. - -
- MobileNet v2 Config - -Use the model configuration shown below when using the OpenVINO detector with the default OpenVINO model: - - - - -Navigate to and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU` (or `NPU`). Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ------------------------------------------ | -| **Object detection model input width** | `300` | -| **Object detection model input height** | `300` | -| **Model Input Tensor Shape** | `nhwc` | -| **Model Input Pixel Color Format** | `bgr` | -| **Custom object detector model path** | `/openvino-model/ssdlite_mobilenet_v2.xml` | -| **Label map for custom object detector** | `/openvino-model/coco_91cl_bkgr.txt` | - - - - -```yaml -detectors: - ov: - type: openvino - device: GPU # Or NPU - -model: - width: 300 - height: 300 - input_tensor: nhwc - input_pixel_format: bgr - path: /openvino-model/ssdlite_mobilenet_v2.xml - labelmap_path: /openvino-model/coco_91cl_bkgr.txt -``` - - - - -
- -#### YOLOX - -This detector also supports YOLOX. Frigate does not come with any YOLOX models preloaded, so you will need to supply your own models. - -#### YOLO-NAS - -[YOLO-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) models are supported, but not included by default. See [the models section](#downloading-yolo-nas-model) for more information on downloading the YOLO-NAS model for use in Frigate. - -
- YOLO-NAS Setup & Config - -After placing the downloaded onnx model in your config folder, use the following configuration: - - - - -Navigate to and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ------------------------------------------------- | -| **Object Detection Model Type** | `yolonas` | -| **Object detection model input width** | `320` (should match whatever was set in notebook) | -| **Object detection model input height** | `320` (should match whatever was set in notebook) | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input Pixel Color Format** | `bgr` | -| **Custom object detector model path** | `/config/yolo_nas_s.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - ov: - type: openvino - device: GPU - -model: - model_type: yolonas - width: 320 # <--- should match whatever was set in notebook - height: 320 # <--- should match whatever was set in notebook - input_tensor: nchw - input_pixel_format: bgr - path: /config/yolo_nas_s.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - - -Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. - -
- -#### YOLO (v3, v4, v7, v9) - -YOLOv3, YOLOv4, YOLOv7, and [YOLOv9](https://github.com/WongKinYiu/yolov9) models are supported, but not included by default. - -:::tip - -The YOLO detector has been designed to support YOLOv3, YOLOv4, YOLOv7, and YOLOv9 models, but may support other YOLO model architectures as well. - -::: - -
- YOLOv Setup & Config - -:::warning - -If you are using a Frigate+ model, you should not define any of the below `model` parameters in your config except for `path`. See [the Frigate+ model docs](/plus/first_model#step-3-set-your-model-id-in-the-config) for more information on setting up your model. - -::: - -After placing the downloaded onnx model in your config folder, use the following configuration: - - - - -Navigate to and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU` (or `NPU`). Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | -------------------------------------------------------- | -| **Object Detection Model Type** | `yolo-generic` | -| **Object detection model input width** | `320` (should match the imgsize set during model export) | -| **Object detection model input height** | `320` (should match the imgsize set during model export) | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Custom object detector model path** | `/config/model_cache/yolo.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - ov: - type: openvino - device: GPU # or NPU - -model: - model_type: yolo-generic - width: 320 # <--- should match the imgsize set during model export - height: 320 # <--- should match the imgsize set during model export - input_tensor: nchw - input_dtype: float - path: /config/model_cache/yolo.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - - -Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. - -
- -#### RF-DETR - -[RF-DETR](https://github.com/roboflow/rf-detr) is a DETR based model. The ONNX exported models are supported, but not included by default. See [the models section](#downloading-rf-detr-model) for more information on downloading the RF-DETR model for use in Frigate. - -:::warning - -Due to the size and complexity of the RF-DETR model, it is only recommended to be run with discrete Arc Graphics Cards. - -::: - -
- RF-DETR Setup & Config - -After placing the downloaded onnx model in your `config/model_cache` folder, use the following configuration: - - - - -Navigate to and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `GPU`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| --------------------------------------- | --------------------------------- | -| **Object Detection Model Type** | `rfdetr` | -| **Object detection model input width** | `320` | -| **Object detection model input height** | `320` | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Custom object detector model path** | `/config/model_cache/rfdetr.onnx` | - - - - -```yaml -detectors: - ov: - type: openvino - device: GPU - -model: - model_type: rfdetr - width: 320 - height: 320 - input_tensor: nchw - input_dtype: float - path: /config/model_cache/rfdetr.onnx -``` - - - - -
- -#### D-FINE / DEIMv2 - -[D-FINE](https://github.com/Peterande/D-FINE) and [DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) are DETR based models that share the same ONNX input/output format. The ONNX exported models are supported, but not included by default. See the models section for downloading [D-FINE](#downloading-d-fine-model) or [DEIMv2](#downloading-deimv2-model) for use in Frigate. - -:::warning - -Currently D-FINE / DEIMv2 models only run on OpenVINO in CPU mode, GPUs currently fail to compile the model - -::: - -
- D-FINE Setup & Config - -After placing the downloaded onnx model in your config/model_cache folder, use the following configuration: - - - - -Navigate to and select **OpenVINO** from the detector type dropdown and click **Add**, then set device to `CPU`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ---------------------------------- | -| **Object Detection Model Type** | `dfine` | -| **Object detection model input width** | `640` | -| **Object detection model input height** | `640` | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Custom object detector model path** | `/config/model_cache/dfine-s.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - ov: - type: openvino - device: CPU - -model: - model_type: dfine - width: 640 - height: 640 - input_tensor: nchw - input_dtype: float - path: /config/model_cache/dfine-s.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - - -Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. - -
- -
- DEIMv2 Setup & Config - -After placing the downloaded onnx model in your `config/model_cache` folder, you can use the following configuration: - -```yaml -detectors: - ov: - type: openvino - device: CPU - -model: - model_type: dfine - width: 640 - height: 640 - input_tensor: nchw - input_dtype: float - path: /config/model_cache/deimv2_hgnetv2_n.onnx - labelmap_path: /labelmap/coco-80.txt -``` - -Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. - -
+--- ## Apple Silicon detector @@ -823,76 +317,7 @@ The NPU in Apple Silicon can't be accessed from within a container, so the [Appl Using the detector config below will connect to the client: - - - -Navigate to and select **ZMQ IPC** from the detector type dropdown and click **Add**, then set the endpoint to `tcp://host.docker.internal:5555`. - - - - -```yaml -detectors: - apple-silicon: - type: zmq - endpoint: tcp://host.docker.internal:5555 -``` - - - - -### Apple Silicon Supported Models - -There is no default model provided, the following formats are supported: - -#### YOLO (v3, v4, v7, v9) - -YOLOv3, YOLOv4, YOLOv7, and [YOLOv9](https://github.com/WongKinYiu/yolov9) models are supported, but not included by default. - -:::tip - -The YOLO detector has been designed to support YOLOv3, YOLOv4, YOLOv7, and YOLOv9 models, but may support other YOLO model architectures as well. See [the models section](#downloading-yolo-models) for more information on downloading YOLO models for use in Frigate. - -::: - -When Frigate is started with the following config it will connect to the detector client and transfer the model automatically: - - - - -Navigate to and select **ZMQ IPC** from the detector type dropdown and click **Add**, then set the endpoint to `tcp://host.docker.internal:5555`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | -------------------------------------------------------- | -| **Object Detection Model Type** | `yolo-generic` | -| **Object detection model input width** | `320` (should match the imgsize set during model export) | -| **Object detection model input height** | `320` (should match the imgsize set during model export) | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Custom object detector model path** | `/config/model_cache/yolo.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - apple-silicon: - type: zmq - endpoint: tcp://host.docker.internal:5555 - -model: - model_type: yolo-generic - width: 320 # <--- should match the imgsize set during model export - height: 320 # <--- should match the imgsize set during model export - input_tensor: nchw - input_dtype: float - path: /config/model_cache/yolo.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - + Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. @@ -976,7 +401,7 @@ We unset the `HSA_OVERRIDE_GFX_VERSION` to prevent an existing override from mes $ docker exec -it frigate /bin/bash -c '(unset HSA_OVERRIDE_GFX_VERSION && /opt/rocm/bin/rocminfo |grep gfx)' ``` -### ROCm Supported Models +### Configuration :::tip @@ -989,11 +414,13 @@ The AMD GPU kernel is known problematic especially when converting models to mxr ::: -See [ONNX supported models](#supported-models) for supported models, there are some caveats: +See [ONNX supported models](#onnx) for supported models, there are some caveats: - D-FINE / DEIMv2 models are not supported - YOLO-NAS models are known to not run well on integrated GPUs + + ## ONNX ONNX is an open format for building machine learning models, Frigate supports running ONNX models on CPU, OpenVINO, ROCm, and TensorRT. On startup Frigate will automatically try to use a GPU if one is available. @@ -1018,14 +445,6 @@ If the correct build is used for your GPU then the GPU will be detected and used When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming GPU resources are available. An example configuration would be: - - - -Navigate to and select **ONNX** from the detector type dropdown and click **Add** to add multiple detectors. - - - - ```yaml detectors: onnx_0: @@ -1034,301 +453,13 @@ detectors: type: onnx ``` - - - ::: -### ONNX Supported Models +### Configuration -| Model | Nvidia GPU | AMD GPU | Notes | -| ------------------------------------ | ---------- | ------- | --------------------------------------------------- | -| [YOLOv9](#yolo-v3-v4-v7-v9-2) | ✅ | ✅ | Supports CUDA Graphs for optimal Nvidia performance | -| [RF-DETR](#rf-detr) | ✅ | ⚠️ | Supports CUDA Graphs for optimal Nvidia performance | -| [YOLO-NAS](#yolo-nas-1) | ⚠️ | ⚠️ | Not supported by CUDA Graphs | -| [YOLOX](#yolox-1) | ✅ | ✅ | Supports CUDA Graphs for optimal Nvidia performance | -| [D-FINE / DEIMv2](#d-fine--deimv2-1) | ⚠️ | ❌ | Not supported by CUDA Graphs | + -There is no default model provided, the following formats are supported: - -#### YOLO-NAS - -[YOLO-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) models are supported, but not included by default. See [the models section](#downloading-yolo-nas-model) for more information on downloading the YOLO-NAS model for use in Frigate. - -
- YOLO-NAS Setup & Config - -:::warning - -If you are using a Frigate+ YOLO-NAS model, you should not define any of the below `model` parameters in your config except for `path`. See [the Frigate+ model docs](/plus/first_model#step-3-set-your-model-id-in-the-config) for more information on setting up your model. - -::: - -After placing the downloaded onnx model in your config folder, use the following configuration: - - - - -Navigate to and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ------------------------------------------------- | -| **Object Detection Model Type** | `yolonas` | -| **Object detection model input width** | `320` (should match whatever was set in notebook) | -| **Object detection model input height** | `320` (should match whatever was set in notebook) | -| **Model Input Pixel Color Format** | `bgr` | -| **Model Input Tensor Shape** | `nchw` | -| **Custom object detector model path** | `/config/yolo_nas_s.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - onnx: - type: onnx - -model: - model_type: yolonas - width: 320 # <--- should match whatever was set in notebook - height: 320 # <--- should match whatever was set in notebook - input_pixel_format: bgr - input_tensor: nchw - path: /config/yolo_nas_s.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - - -
- -#### YOLO (v3, v4, v7, v9) - -YOLOv3, YOLOv4, YOLOv7, and [YOLOv9](https://github.com/WongKinYiu/yolov9) models are supported, but not included by default. - -:::tip - -The YOLO detector has been designed to support YOLOv3, YOLOv4, YOLOv7, and YOLOv9 models, but may support other YOLO model architectures as well. See [the models section](#downloading-yolo-models) for more information on downloading YOLO models for use in Frigate. - -::: - -
- YOLOv Setup & Config - -:::warning - -If you are using a Frigate+ model, you should not define any of the below `model` parameters in your config except for `path`. See [the Frigate+ model docs](/plus/first_model#step-3-set-your-model-id-in-the-config) for more information on setting up your model. - -::: - -After placing the downloaded onnx model in your config folder, use the following configuration: - - - - -Navigate to and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | -------------------------------------------------------- | -| **Object Detection Model Type** | `yolo-generic` | -| **Object detection model input width** | `320` (should match the imgsize set during model export) | -| **Object detection model input height** | `320` (should match the imgsize set during model export) | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Custom object detector model path** | `/config/model_cache/yolo.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - onnx: - type: onnx - -model: - model_type: yolo-generic - width: 320 # <--- should match the imgsize set during model export - height: 320 # <--- should match the imgsize set during model export - input_tensor: nchw - input_dtype: float - path: /config/model_cache/yolo.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - - -
- -Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. - -#### YOLOx - -[YOLOx](https://github.com/Megvii-BaseDetection/YOLOX) models are supported, but not included by default. See [the models section](#downloading-yolo-models) for more information on downloading the YOLOx model for use in Frigate. - -
- YOLOx Setup & Config - -After placing the downloaded onnx model in your config folder, use the following configuration: - - - - -Navigate to and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | -------------------------------------------------------- | -| **Object Detection Model Type** | `yolox` | -| **Object detection model input width** | `416` (should match the imgsize set during model export) | -| **Object detection model input height** | `416` (should match the imgsize set during model export) | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float_denorm` | -| **Custom object detector model path** | `/config/model_cache/yolox_tiny.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - onnx: - type: onnx - -model: - model_type: yolox - width: 416 # <--- should match the imgsize set during model export - height: 416 # <--- should match the imgsize set during model export - input_tensor: nchw - input_dtype: float_denorm - path: /config/model_cache/yolox_tiny.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - - -Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. - -
- -#### RF-DETR - -[RF-DETR](https://github.com/roboflow/rf-detr) is a DETR based model. The ONNX exported models are supported, but not included by default. See [the models section](#downloading-rf-detr-model) for more information on downloading the RF-DETR model for use in Frigate. - -
- RF-DETR Setup & Config - -After placing the downloaded onnx model in your `config/model_cache` folder, use the following configuration: - - - - -Navigate to and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| --------------------------------------- | --------------------------------- | -| **Object Detection Model Type** | `rfdetr` | -| **Object detection model input width** | `320` | -| **Object detection model input height** | `320` | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Custom object detector model path** | `/config/model_cache/rfdetr.onnx` | - - - - -```yaml -detectors: - onnx: - type: onnx - -model: - model_type: rfdetr - width: 320 - height: 320 - input_tensor: nchw - input_dtype: float - path: /config/model_cache/rfdetr.onnx -``` - - - - -
- -#### D-FINE / DEIMv2 - -[D-FINE](https://github.com/Peterande/D-FINE) and [DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) are DETR based models that share the same ONNX input/output format. The ONNX exported models are supported, but not included by default. See the models section for downloading [D-FINE](#downloading-d-fine-model) or [DEIMv2](#downloading-deimv2-model) for use in Frigate. - -
- D-FINE Setup & Config - -After placing the downloaded onnx model in your `config/model_cache` folder, use the following configuration: - - - - -Navigate to and select **ONNX** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ------------------------------------------- | -| **Object Detection Model Type** | `dfine` | -| **Object detection model input width** | `640` | -| **Object detection model input height** | `640` | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Custom object detector model path** | `/config/model_cache/dfine_m_obj2coco.onnx` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - onnx: - type: onnx - -model: - model_type: dfine - width: 640 - height: 640 - input_tensor: nchw - input_dtype: float - path: /config/model_cache/dfine_m_obj2coco.onnx - labelmap_path: /labelmap/coco-80.txt -``` - - - - -
- -
- DEIMv2 Setup & Config - -After placing the downloaded onnx model in your `config/model_cache` folder, you can use the following configuration: - -```yaml -detectors: - onnx: - type: onnx - -model: - model_type: dfine - width: 640 - height: 640 - input_tensor: nchw - input_dtype: float - path: /config/model_cache/deimv2_hgnetv2_n.onnx - labelmap_path: /labelmap/coco-80.txt -``` - -
- -Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. +--- ## CPU Detector (not recommended) @@ -1344,29 +475,9 @@ The number of threads used by the interpreter can be specified using the `"num_t A TensorFlow Lite model is provided in the container at `/cpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with `model.path`. - - +### Configuration -Navigate to and select **CPU** from the detector type dropdown and click **Add**. Configure the number of threads and click **Add** again to add additional CPU detectors as needed (one per camera is recommended). - - - - -```yaml -detectors: - cpu1: - type: cpu - num_threads: 3 - cpu2: - type: cpu - num_threads: 3 - -model: - path: "/custom_model.tflite" -``` - - - + When using CPU detectors, you can add one CPU detector per camera. Adding more detectors than the number of cameras should not improve performance. @@ -1380,24 +491,9 @@ To get started with CodeProject.AI, visit their [official website](https://www.c To integrate CodeProject.AI into Frigate, configure the detector as follows: - - +### Configuration -Navigate to and select **DeepStack** from the detector type dropdown and click **Add**. Set the API URL to point to your CodeProject.AI server (e.g., `http://:/v1/vision/detection`). - - - - -```yaml -detectors: - deepstack: - api_url: http://:/v1/vision/detection - type: deepstack - api_timeout: 0.1 # seconds -``` - - - + Replace `` and `` with the IP address and port of your CodeProject.AI server. @@ -1415,255 +511,7 @@ To configure a MemryX detector, simply set the `type` attribute to `memryx` and ### Configuration -To configure the MemryX detector, use the following example configuration: - -#### Single PCIe MemryX MX3 - - - - -Navigate to and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. - - - - -```yaml -detectors: - memx0: - type: memryx - device: PCIe:0 -``` - - - - -#### Multiple PCIe MemryX MX3 Modules - - - - -Navigate to and select **MemryX** from the detector type dropdown and click **Add** to add multiple detectors, specifying `PCIe:0`, `PCIe:1`, `PCIe:2`, etc. as the device for each. - - - - -```yaml -detectors: - memx0: - type: memryx - device: PCIe:0 - - memx1: - type: memryx - device: PCIe:1 - - memx2: - type: memryx - device: PCIe:2 -``` - - - - -### Supported Models - -MemryX `.dfp` models are automatically downloaded at runtime, if enabled, to the container at `/memryx_models/model_folder/`. - -#### YOLO-NAS - -The [YOLO-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) model included in this detector is downloaded from the [Models Section](#downloading-yolo-nas-model) and compiled to DFP with [mx_nc](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage). - -**Note:** The default model for the MemryX detector is YOLO-NAS 320x320. - -The input size for **YOLO-NAS** can be set to either **320x320** (default) or **640x640**. - -- The default size of **320x320** is optimized for lower CPU usage and faster inference times. - -##### Configuration - -Below is the recommended configuration for using the **YOLO-NAS** (small) model with the MemryX detector: - - - - -Navigate to and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ------------------------------------------------- | -| **Object Detection Model Type** | `yolonas` | -| **Object detection model input width** | `320` (can be set to `640` for higher resolution) | -| **Object detection model input height** | `320` (can be set to `640` for higher resolution) | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - memx0: - type: memryx - device: PCIe:0 - -model: - model_type: yolonas - width: 320 # (Can be set to 640 for higher resolution) - height: 320 # (Can be set to 640 for higher resolution) - input_tensor: nchw - input_dtype: float - labelmap_path: /labelmap/coco-80.txt - # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. - # path: /config/yolonas.zip - # The .zip file must contain: - # ├── yolonas.dfp (a file ending with .dfp) - # └── yolonas_post.onnx (optional; only if the model includes a cropped post-processing network) -``` - - - - -#### YOLOv9 - -The YOLOv9s model included in this detector is downloaded from [the original GitHub](https://github.com/WongKinYiu/yolov9) like in the [Models Section](#yolov9-1) and compiled to DFP with [mx_nc](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage). - -##### Configuration - -Below is the recommended configuration for using the **YOLOv9** (small) model with the MemryX detector: - - - - -Navigate to and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ------------------------------------------------- | -| **Object Detection Model Type** | `yolo-generic` | -| **Object detection model input width** | `320` (can be set to `640` for higher resolution) | -| **Object detection model input height** | `320` (can be set to `640` for higher resolution) | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - memx0: - type: memryx - device: PCIe:0 - -model: - model_type: yolo-generic - width: 320 # (Can be set to 640 for higher resolution) - height: 320 # (Can be set to 640 for higher resolution) - input_tensor: nchw - input_dtype: float - labelmap_path: /labelmap/coco-80.txt - # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. - # path: /config/yolov9.zip - # The .zip file must contain: - # ├── yolov9.dfp (a file ending with .dfp) -``` - - - - -#### YOLOX - -The model is sourced from the [OpenCV Model Zoo](https://github.com/opencv/opencv_zoo) and precompiled to DFP. - -##### Configuration - -Below is the recommended configuration for using the **YOLOX** (small) model with the MemryX detector: - - - - -Navigate to and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ----------------------- | -| **Object Detection Model Type** | `yolox` | -| **Object detection model input width** | `640` | -| **Object detection model input height** | `640` | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float_denorm` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - memx0: - type: memryx - device: PCIe:0 - -model: - model_type: yolox - width: 640 - height: 640 - input_tensor: nchw - input_dtype: float_denorm - labelmap_path: /labelmap/coco-80.txt - # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. - # path: /config/yolox.zip - # The .zip file must contain: - # ├── yolox.dfp (a file ending with .dfp) -``` - - - - -#### SSDLite MobileNet v2 - -The model is sourced from the [OpenMMLab Model Zoo](https://mmdeploy-oss.openmmlab.com/model/mmdet-det/ssdlite-e8679f.onnx) and has been converted to DFP. - -##### Configuration - -Below is the recommended configuration for using the **SSDLite MobileNet v2** model with the MemryX detector: - - - - -Navigate to and select **MemryX** from the detector type dropdown and click **Add**, then set device to `PCIe:0`. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ----------------------- | -| **Object Detection Model Type** | `ssd` | -| **Object detection model input width** | `320` | -| **Object detection model input height** | `320` | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input D Type** | `float` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - memx0: - type: memryx - device: PCIe:0 - -model: - model_type: ssd - width: 320 - height: 320 - input_tensor: nchw - input_dtype: float - labelmap_path: /labelmap/coco-80.txt - # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. - # path: /config/ssdlite_mobilenet.zip - # The .zip file must contain: - # ├── ssdlite_mobilenet.dfp (a file ending with .dfp) - # └── ssdlite_mobilenet_post.onnx (optional; only if the model includes a cropped post-processing network) -``` - - - + #### Using a Custom Model @@ -1784,40 +632,7 @@ The TensorRT detector uses `.trt` model files that are located in `/config/model Use the config below to work with generated TRT models: - - - -Navigate to and select **TensorRT** from the detector type dropdown and click **Add**, then set the device to `0` (the default GPU index). Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ------------------------------------------------------------ | -| **Custom object detector model path** | `/config/model_cache/tensorrt/yolov7-320.trt` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | -| **Model Input Tensor Shape** | `nchw` | -| **Model Input Pixel Color Format** | `rgb` | -| **Object detection model input width** | `320` (MUST match the chosen model, e.g., yolov7-320 -> 320) | -| **Object detection model input height** | `320` (MUST match the chosen model, e.g., yolov7-320 -> 320) | - - - - -```yaml -detectors: - tensorrt: - type: tensorrt - device: 0 #This is the default, select the first GPU - -model: - path: /config/model_cache/tensorrt/yolov7-320.trt - labelmap_path: /labelmap/coco-80.txt - input_tensor: nchw - input_pixel_format: rgb - width: 320 # MUST match the chosen model i.e yolov7-320 -> 320, yolov4-416 -> 416 - height: 320 # MUST match the chosen model i.e yolov7-320 -> 320 yolov4-416 -> 416 -``` - - - + ## Synaptics @@ -1835,43 +650,7 @@ See the [installation docs](../frigate/installation.md#synaptics) for informatio When configuring the Synap detector, you have to specify the model: a local **path**. -#### SSD Mobilenet - -A synap model is provided in the container at /mobilenet.synap and is used by this detector type by default. The model comes from [Synap-release Github](https://github.com/synaptics-astra/synap-release/tree/v1.5.0/models/dolphin/object_detection/coco/model/mobilenet224_full80). - -Use the model configuration shown below when using the synaptics detector with the default synap model: - - - - -Navigate to and select **Synaptics** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ---------------------------- | -| **Custom object detector model path** | `/synaptics/mobilenet.synap` | -| **Object detection model input width** | `224` | -| **Object detection model input height** | `224` | -| **Model Input Tensor Shape** | `nhwc` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: # required - synap_npu: # required - type: synaptics # required - -model: # required - path: /synaptics/mobilenet.synap # required - width: 224 # required - height: 224 # required - input_tensor: nhwc # default value (optional. If you change the model, it is required) - labelmap_path: /labelmap/coco-80.txt # required -``` - - - + ## Rockchip platform @@ -1895,14 +674,6 @@ If no custom model is provided, the RKNN detector downloads a default model from When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming NPU resources are available. An example configuration would be: - - - -Navigate to and select **RKNN** from the detector type dropdown and click **Add** to add multiple detectors, each with `num_cores` set to `0` for automatic selection. - - - - ```yaml detectors: rknn_0: @@ -1913,9 +684,6 @@ detectors: num_cores: 0 ``` - - - ::: ### Prerequisites @@ -1937,27 +705,6 @@ $ cat /sys/kernel/debug/rknpu/load This `config.yml` shows all relevant options to configure the detector and explains them. All values shown are the default values (except for two). Lines that are required at least to use the detector are labeled as required, all other lines are optional. - - - -Navigate to and select **RKNN** from the detector type dropdown and click **Add**. Set `num_cores` to `0` for automatic selection (increase for better performance on multicore NPUs, e.g., set to `3` on rk3588). - - - - -```yaml -detectors: # required - rknn: # required - type: rknn # required - # number of NPU cores to use - # 0 means choose automatically - # increase for better performance if you have a multicore NPU e.g. set to 3 on rk3588 - num_cores: 0 -``` - - - - The inference time was determined on a rk3588 with 3 NPU cores. | Model | Size in mb | Inference time in ms | @@ -1972,130 +719,7 @@ The inference time was determined on a rk3588 with 3 NPU cores. - All models are automatically downloaded and stored in the folder `config/model_cache/rknn_cache`. After upgrading Frigate, you should remove older models to free up space. - You can also provide your own `.rknn` model. You should not save your own models in the `rknn_cache` folder, store them directly in the `model_cache` folder or another subfolder. To convert a model to `.rknn` format see the `rknn-toolkit2` (requires a x86 machine). Note, that there is only post-processing for the supported models. -#### YOLO-NAS - - - - -Navigate to and, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ----------------------------------------------------------------------- | -| **Custom object detector model path** | `deci-fp16-yolonas_s` (or `deci-fp16-yolonas_m`, `deci-fp16-yolonas_l`) | -| **Object Detection Model Type** | `yolonas` | -| **Object detection model input width** | `320` | -| **Object detection model input height** | `320` | -| **Model Input Pixel Color Format** | `bgr` | -| **Model Input Tensor Shape** | `nhwc` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -model: # required - # name of model (will be automatically downloaded) or path to your own .rknn model file - # possible values are: - # - deci-fp16-yolonas_s - # - deci-fp16-yolonas_m - # - deci-fp16-yolonas_l - # your yolonas_model.rknn - path: deci-fp16-yolonas_s - model_type: yolonas - width: 320 - height: 320 - input_pixel_format: bgr - input_tensor: nhwc - labelmap_path: /labelmap/coco-80.txt -``` - - - - -:::warning - -The pre-trained YOLO-NAS weights from DeciAI are subject to their license and can't be used commercially. For more information, see: https://docs.deci.ai/super-gradients/latest/LICENSE.YOLONAS.html - -::: - -#### YOLO (v9) - - - - -Navigate to and, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | -------------------------------------------------- | -| **Custom object detector model path** | `frigate-fp16-yolov9-t` (or other yolov9 variants) | -| **Object Detection Model Type** | `yolo-generic` | -| **Object detection model input width** | `320` | -| **Object detection model input height** | `320` | -| **Model Input Tensor Shape** | `nhwc` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -model: # required - # name of model (will be automatically downloaded) or path to your own .rknn model file - # possible values are: - # - frigate-fp16-yolov9-t - # - frigate-fp16-yolov9-s - # - frigate-fp16-yolov9-m - # - frigate-fp16-yolov9-c - # - frigate-fp16-yolov9-e - # your yolo_model.rknn - path: frigate-fp16-yolov9-t - model_type: yolo-generic - width: 320 - height: 320 - input_tensor: nhwc - labelmap_path: /labelmap/coco-80.txt -``` - - - - -#### YOLOx - - - - -Navigate to and, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ---------------------------------------------- | -| **Custom object detector model path** | `rock-i8-yolox_nano` (or other yolox variants) | -| **Object Detection Model Type** | `yolox` | -| **Object detection model input width** | `416` | -| **Object detection model input height** | `416` | -| **Model Input Tensor Shape** | `nhwc` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -model: # required - # name of model (will be automatically downloaded) or path to your own .rknn model file - # possible values are: - # - rock-i8-yolox_nano - # - rock-i8-yolox_tiny - # - rock-fp16-yolox_nano - # - rock-fp16-yolox_tiny - # your yolox_model.rknn - path: rock-i8-yolox_nano - model_type: yolox - width: 416 - height: 416 - input_tensor: nhwc - labelmap_path: /labelmap/coco-80.txt -``` - - - + ### Converting your own onnx model to rknn format @@ -2154,24 +778,7 @@ All supported hardware will automatically be found on your AI server host as lon Once completed, configure the detector as follows: - - - -Navigate to and select **DeGirum** from the detector type dropdown and click **Add**. Set the location to your AI server (e.g., service name, container name, or `host:port`), the zoo to `degirum/public`, and provide your authentication token if needed. - - - - -```yaml -degirum_detector: - type: degirum - location: degirum # Set to service name (degirum_detector), container_name (degirum), or a host:port (192.168.29.4:8778) - zoo: degirum/public # DeGirum's public model zoo. Zoo name should be in format "workspace/zoo_name". degirum/public is available to everyone, so feel free to use it if you don't know where to start. If you aren't pulling a model from the AI Hub, leave this and 'token' blank. - token: dg_example_token # For authentication with the AI Hub. Get this token through the "tokens" section on the main page of the [AI Hub](https://hub.degirum.com). This can be left blank if you're pulling a model from the public zoo and running inferences on your local hardware using @local or a local DeGirum AI Server -``` - - - + Setting up a model in the `config.yml` is similar to setting up an AI server. You can set it to: @@ -2197,24 +804,7 @@ It is also possible to eliminate the need for an AI server and run the hardware 2. To double check the runtime is detected by the DeGirum detector, make sure the `degirum sys-info` command properly shows whatever runtimes you mean to install. 3. Create a DeGirum detector in your configuration. - - - -Navigate to and select **DeGirum** from the detector type dropdown and click **Add**. Set the location to `@local`, the zoo to `degirum/public`, and provide your authentication token. - - - - -```yaml -degirum_detector: - type: degirum - location: "@local" # For accessing AI Hub devices and models - zoo: degirum/public # DeGirum's public model zoo. Zoo name should be in format "workspace/zoo_name". degirum/public is available to everyone, so feel free to use it if you don't know where to start. - token: dg_example_token # For authentication with the AI Hub. Get this token through the "tokens" section on the main page of the [AI Hub](https://hub.degirum.com). This can be left blank if you're pulling a model from the public zoo and running inferences on your local hardware using @local or a local DeGirum AI Server -``` - - - + Once `degirum_detector` is setup, you can choose a model through 'model' section in the `config.yml` file. @@ -2234,24 +824,7 @@ If you do not possess whatever hardware you want to run, there's also the option 2. Get an access token. 3. Create a DeGirum detector in your configuration. - - - -Navigate to and select **DeGirum** from the detector type dropdown and click **Add**. Set the location to `@cloud`, the zoo to `degirum/public`, and provide your authentication token. - - - - -```yaml -degirum_detector: - type: degirum - location: "@cloud" # For accessing AI Hub devices and models - zoo: degirum/public # DeGirum's public model zoo. Zoo name should be in format "workspace/zoo_name". degirum/public is available to everyone, so feel free to use it if you don't know where to start. - token: dg_example_token # For authentication with the AI Hub. Get this token through the "tokens" section on the main page of the (AI Hub)[https://hub.degirum.com). -``` - - - + Once `degirum_detector` is setup, you can choose a model through 'model' section in the `config.yml` file. @@ -2284,198 +857,4 @@ The AXEngine detector downloads its default model from HuggingFace on first star When configuring the AXEngine detector, you have to specify the model name. -#### yolov9 - -A yolov9 model is provided in the container at `/axmodels` and is used by this detector type by default. - -Use the model configuration shown below when using the axengine detector with the default axmodel: - - - - -Navigate to and select **AXEngine NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure: - -| Field | Value | -| ---------------------------------------- | ----------------------- | -| **Custom object detector model path** | `frigate-yolov9-tiny` | -| **Object Detection Model Type** | `yolo-generic` | -| **Object detection model input width** | `320` | -| **Object detection model input height** | `320` | -| **Model Input D Type** | `int` | -| **Model Input Pixel Color Format** | `bgr` | -| **Label map for custom object detector** | `/labelmap/coco-80.txt` | - - - - -```yaml -detectors: - axengine: - type: axengine - -model: - path: frigate-yolov9-tiny - model_type: yolo-generic - width: 320 - height: 320 - input_dtype: int - input_pixel_format: bgr - labelmap_path: /labelmap/coco-80.txt -``` - - - - -# Models - -Some model types are not included in Frigate by default. - -## Downloading Models - -Here are some tips for getting different model types - -### Downloading D-FINE Model - -D-FINE can be exported as ONNX by running the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=s` in the first line to `s`, `m`, or `l` size. - -```sh -docker build . --build-arg MODEL_SIZE=s --output . -f- <<'EOF' -FROM python:3.11 AS build -RUN apt-get update && apt-get install --no-install-recommends -y libgl1 && rm -rf /var/lib/apt/lists/* -COPY --from=ghcr.io/astral-sh/uv:0.8.0 /uv /bin/ -WORKDIR /dfine -RUN git clone https://github.com/Peterande/D-FINE.git . -RUN uv pip install --system -r requirements.txt -RUN uv pip install --system onnx onnxruntime onnxsim onnxscript -# Create output directory and download checkpoint -RUN mkdir -p output -ARG MODEL_SIZE -RUN wget https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_${MODEL_SIZE}_obj2coco.pth -O output/dfine_${MODEL_SIZE}_obj2coco.pth -# Modify line 58 of export_onnx.py to change batch size to 1 -RUN sed -i '58s/data = torch.rand(.*)/data = torch.rand(1, 3, 640, 640)/' tools/deployment/export_onnx.py -RUN python3 tools/deployment/export_onnx.py -c configs/dfine/objects365/dfine_hgnetv2_${MODEL_SIZE}_obj2coco.yml -r output/dfine_${MODEL_SIZE}_obj2coco.pth -FROM scratch -ARG MODEL_SIZE -COPY --from=build /dfine/output/dfine_${MODEL_SIZE}_obj2coco.onnx /dfine-${MODEL_SIZE}.onnx -EOF -``` - -### Downloading DEIMv2 Model - -[DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) can be exported as ONNX by running the command below. Pretrained weights are available on Hugging Face for two backbone families: - -- **HGNetv2** (smaller/faster): `atto`, `femto`, `pico`, `n` -- **DINOv3** (larger/more accurate): `s`, `m`, `l`, `x` - -Set `BACKBONE` and `MODEL_SIZE` in the first line to match your desired variant. Hugging Face model names use uppercase (e.g. `HGNetv2_N`, `DINOv3_S`), while config files use lowercase (e.g. `hgnetv2_n`, `dinov3_s`). - -```sh -docker build . --rm --build-arg BACKBONE=hgnetv2 --build-arg MODEL_SIZE=n --output . -f- <<'EOF' -FROM python:3.11-slim AS build -RUN apt-get update && apt-get install --no-install-recommends -y git libgl1 libglib2.0-0 && rm -rf /var/lib/apt/lists/* -COPY --from=ghcr.io/astral-sh/uv:0.8.0 /uv /bin/ -WORKDIR /deimv2 -RUN git clone https://github.com/Intellindust-AI-Lab/DEIMv2.git . -# Install CPU-only PyTorch first to avoid pulling CUDA variant -RUN uv pip install --no-cache --system torch torchvision --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache --system -r requirements.txt -RUN uv pip install --no-cache --system onnx safetensors huggingface_hub -RUN mkdir -p output -ARG BACKBONE -ARG MODEL_SIZE -# Download from Hugging Face and convert safetensors to pth -RUN python3 -c "\ -from huggingface_hub import hf_hub_download; \ -from safetensors.torch import load_file; \ -import torch; \ -backbone = '${BACKBONE}'.replace('hgnetv2','HGNetv2').replace('dinov3','DINOv3'); \ -size = '${MODEL_SIZE}'.upper(); \ -st = load_file(hf_hub_download('Intellindust/DEIMv2_' + backbone + '_' + size + '_COCO', 'model.safetensors')); \ -torch.save({'model': st}, 'output/deimv2.pth')" -RUN sed -i "s/data = torch.rand(2/data = torch.rand(1/" tools/deployment/export_onnx.py -# HuggingFace safetensors omits frozen constants that the model constructor initializes -RUN sed -i "s/cfg.model.load_state_dict(state)/cfg.model.load_state_dict(state, strict=False)/" tools/deployment/export_onnx.py -RUN python3 tools/deployment/export_onnx.py -c configs/deimv2/deimv2_${BACKBONE}_${MODEL_SIZE}_coco.yml -r output/deimv2.pth -FROM scratch -ARG BACKBONE -ARG MODEL_SIZE -COPY --from=build /deimv2/output/deimv2.onnx /deimv2_${BACKBONE}_${MODEL_SIZE}.onnx -EOF -``` - -### Downloading RF-DETR Model - -RF-DETR can be exported as ONNX by running the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=Nano` in the first line to `Nano`, `Small`, or `Medium` size. - -```sh -docker build . --build-arg MODEL_SIZE=Nano --rm --output . -f- <<'EOF' -FROM python:3.12 AS build -RUN apt-get update && apt-get install --no-install-recommends -y libgl1 && rm -rf /var/lib/apt/lists/* -COPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/ -WORKDIR /rfdetr -RUN uv pip install --system rfdetr[onnxexport] torch==2.8.0 onnx==1.19.1 transformers==4.57.6 onnxscript -ARG MODEL_SIZE -RUN python3 -c "from rfdetr import RFDETR${MODEL_SIZE}; x = RFDETR${MODEL_SIZE}(resolution=320); x.export(simplify=True)" -FROM scratch -ARG MODEL_SIZE -COPY --from=build /rfdetr/output/inference_model.onnx /rfdetr-${MODEL_SIZE}.onnx -EOF -``` - -### Downloading YOLO-NAS Model - -You can build and download a compatible model with pre-trained weights using [this notebook](https://github.com/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb) which can be run directly in [Google Colab](https://colab.research.google.com/github/blakeblackshear/frigate/blob/dev/notebooks/YOLO_NAS_Pretrained_Export.ipynb). - -:::warning - -The pre-trained YOLO-NAS weights from DeciAI are subject to their license and can't be used commercially. For more information, see: https://docs.deci.ai/super-gradients/latest/LICENSE.YOLONAS.html - -::: - -The input image size in this notebook is set to 320x320. This results in lower CPU usage and faster inference times without impacting performance in most cases due to the way Frigate crops video frames to areas of interest before running detection. The notebook and config can be updated to 640x640 if desired. - -### Downloading YOLO Models - -#### YOLOx - -YOLOx models can be downloaded [from the YOLOx repo](https://github.com/Megvii-BaseDetection/YOLOX/tree/main/demo/ONNXRuntime). - -#### YOLOv3, YOLOv4, and YOLOv7 - -To export as ONNX: - -```sh -git clone https://github.com/NateMeyer/tensorrt_demos -cd tensorrt_demos/yolo -./download_yolo.sh -python3 yolo_to_onnx.py -m yolov7-320 -``` - -#### YOLOv9 for Google Coral Support - -[Download the model](https://github.com/dbro/frigate-detector-edgetpu-yolo9/releases/download/v1.0/yolov9-s-relu6-best_320_int8_edgetpu.tflite), bind mount the file into the container, and provide the path with `model.path`. Note that the linked model requires a 17-label [labelmap file](https://raw.githubusercontent.com/dbro/frigate-detector-edgetpu-yolo9/refs/heads/main/labels-coco17.txt) that includes only 17 COCO classes. - -#### YOLOv9 for other detectors - -YOLOv9 model can be exported as ONNX using the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=t` and `IMG_SIZE=320` in the first line to the [model size](https://github.com/WongKinYiu/yolov9#performance) you would like to convert (available model sizes are `t`, `s`, `m`, `c`, and `e`, common image sizes are `320` and `640`). - -```sh -docker build . --build-arg MODEL_SIZE=t --build-arg IMG_SIZE=320 --output . -f- <<'EOF' -FROM python:3.11 AS build -RUN apt-get update && apt-get install --no-install-recommends -y cmake libgl1 && rm -rf /var/lib/apt/lists/* -COPY --from=ghcr.io/astral-sh/uv:0.10.4 /uv /bin/ -WORKDIR /yolov9 -ADD https://github.com/WongKinYiu/yolov9.git . -RUN uv pip install --system -r requirements.txt -RUN uv pip install --system onnx==1.18.0 onnxruntime onnx-simplifier==0.4.* onnxscript -ARG MODEL_SIZE -ARG IMG_SIZE -ADD https://github.com/WongKinYiu/yolov9/releases/download/v0.1/yolov9-${MODEL_SIZE}-converted.pt yolov9-${MODEL_SIZE}.pt -RUN sed -i "s/ckpt = torch.load(attempt_download(w), map_location='cpu')/ckpt = torch.load(attempt_download(w), map_location='cpu', weights_only=False)/g" models/experimental.py -RUN python3 export.py --weights ./yolov9-${MODEL_SIZE}.pt --imgsz ${IMG_SIZE} --simplify --include onnx -FROM scratch -ARG MODEL_SIZE -ARG IMG_SIZE -COPY --from=build /yolov9/yolov9-${MODEL_SIZE}.onnx /yolov9-${MODEL_SIZE}-${IMG_SIZE}.onnx -EOF -``` + diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 6e98d1b7bf..abe4630653 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -68,26 +68,26 @@ Frigate supports multiple different detectors that work on different types of ha **AMD** - [ROCm](#rocm---amd-gpu): ROCm can run on AMD Discrete GPUs to provide efficient object detection - - [Supports limited model architectures](../../configuration/object_detectors#rocm-supported-models) + - [Supports limited model architectures](../../configuration/object_detectors#amdrocm-gpu-detector) - Runs best on discrete AMD GPUs **Apple Silicon** - [Apple Silicon](#apple-silicon): Apple Silicon is usable on all M1 and newer Apple Silicon devices to provide efficient and fast object detection - - [Supports primarily ssdlite and mobilenet model architectures](../../configuration/object_detectors#apple-silicon-supported-models) + - [Supports primarily ssdlite and mobilenet model architectures](../../configuration/object_detectors#apple-silicon-detector) - Runs well with any size models including large - Runs via ZMQ proxy which adds some latency, only recommended for local connection **Intel** - [OpenVino](#openvino---intel): OpenVino can run on Intel Arc GPUs, Intel integrated GPUs, and Intel NPUs to provide efficient object detection. - - [Supports majority of model architectures](../../configuration/object_detectors#openvino-supported-models) + - [Supports majority of model architectures](../../configuration/object_detectors#openvino-detector) - Runs best with tiny, small, or medium models **Nvidia** - [Nvidia GPU](#nvidia-gpus): Nvidia GPUs can provide efficient object detection. - - [Supports majority of model architectures via ONNX](../../configuration/object_detectors#onnx-supported-models) + - [Supports majority of model architectures via ONNX](../../configuration/object_detectors#onnx) - Runs well with any size models including large - [Jetson](#nvidia-jetson): Jetson devices are supported via the TensorRT or ONNX detectors when running Jetpack 6. diff --git a/docs/package.json b/docs/package.json index e57d7a1540..1649df7e0c 100644 --- a/docs/package.json +++ b/docs/package.json @@ -29,6 +29,7 @@ "docusaurus-plugin-openapi-docs": "^4.5.1", "docusaurus-theme-openapi-docs": "^4.5.1", "js-yaml": "^4.1.1", + "marked": "^16.4.2", "prism-react-renderer": "^2.4.1", "raw-loader": "^4.0.2", "react": "^18.3.1", diff --git a/docs/src/components/ModelConfigDropdown/index.jsx b/docs/src/components/ModelConfigDropdown/index.jsx new file mode 100644 index 0000000000..737f47ec23 --- /dev/null +++ b/docs/src/components/ModelConfigDropdown/index.jsx @@ -0,0 +1,171 @@ +import React, { useState } from "react"; +import CodeBlock from "@theme/CodeBlock"; +import ConfigTabs from "@site/src/components/ConfigTabs"; +import TabItem from "@theme/TabItem"; +import { marked } from "marked"; +import styles from "./styles.module.css"; + +marked.setOptions({ gfm: true }); + +/** + * @typedef {Object} Model + * @property {string} key + * @property {string} label + * @property {boolean} recommended + * @property {string} download Markdown for the "download the model" step. + * @property {string} ui Markdown for the Frigate UI configuration step. + * @property {string} yaml Raw YAML for the configuration step. + */ + +// Render a markdown string to React nodes. Fenced code blocks become Docusaurus +// CodeBlock components (so they get syntax highlighting and a copy button); +// everything else is marked-parsed to HTML. +function renderBlocks(md, keyPrefix) { + if (!md.trim()) return []; + const tokens = marked.lexer(md); + const nodes = []; + let buffer = []; + let idx = 0; + + const flush = () => { + if (buffer.length) { + buffer.links = tokens.links; + nodes.push( +
, + ); + buffer = []; + } + }; + + tokens.forEach((token) => { + if (token.type === "code") { + flush(); + const language = (token.lang || "text").split(/\s+/)[0]; + nodes.push( + + {token.text} + , + ); + } else { + buffer.push(token); + } + }); + flush(); + return nodes; +} + +// marked does not understand Docusaurus admonitions (:::warning ... :::), so +// render those blocks ourselves and render everything around them normally. +function renderMarkdown(md) { + if (!md) return null; + const admonition = /:::(\w+)[ \t]*([^\n]*)\n([\s\S]*?)\n:::/g; + const nodes = []; + let lastIndex = 0; + let match; + let k = 0; + while ((match = admonition.exec(md)) !== null) { + nodes.push(...renderBlocks(md.slice(lastIndex, match.index), `seg${k}`)); + const [, type, title, body] = match; + const heading = (title || type).trim(); + nodes.push( +
+
{heading}
+ {renderBlocks(body, `adm${k}`)} +
, + ); + lastIndex = admonition.lastIndex; + k++; + } + nodes.push(...renderBlocks(md.slice(lastIndex), `seg${k}`)); + return nodes; +} + +function Markdown({ children }) { + return
{renderMarkdown(children)}
; +} + +function RecommendedBadge() { + return Recommended; +} + +/** + * @param {{ models: Model[] }} props + */ +export default function ModelConfigDropdown({ models }) { + const [selectedModelIndex, setSelectedModelIndex] = useState(0); + const [isOpen, setIsOpen] = useState(false); + + const selectedModel = models[selectedModelIndex]; + const hasChoices = models.length > 1; + + const handleModelSelect = (index) => { + setSelectedModelIndex(index); + setIsOpen(false); + }; + + return ( +
+
+
+

Step 1 — Choose a model

+
setIsOpen(!isOpen) : undefined} + > +
+ + {selectedModel.label} + {selectedModel.recommended && } + + {hasChoices && ( + {isOpen ? "▲" : "▼"} + )} +
+
+ + {isOpen && hasChoices && ( +
+ {models.map((model, index) => ( +
handleModelSelect(index)} + > + {model.label} + {model.recommended && } +
+ ))} +
+ )} +
+ +
+

Step 2 — Download the model

+ {selectedModel.download} +
+ +
+

Step 3 — Configure the detector

+ + + {selectedModel.ui} + + + {selectedModel.yaml} + + +
+
+
+ ); +} diff --git a/docs/src/components/ModelConfigDropdown/styles.module.css b/docs/src/components/ModelConfigDropdown/styles.module.css new file mode 100644 index 0000000000..06f0333344 --- /dev/null +++ b/docs/src/components/ModelConfigDropdown/styles.module.css @@ -0,0 +1,275 @@ +/* =================================================================== + ModelConfigDropdown — styles + =================================================================== */ + +.wrapper { + margin: 1.5rem 0; +} + +/* --- Dropdown button --- */ + +.dropdown { + display: inline-block; + width: 360px; + max-width: 100%; + text-align: left; + border: 1px solid var(--ifm-color-emphasis-400); + border-radius: 8px; + background: var(--ifm-background-color); + cursor: pointer; + transition: + border-color 0.2s, + box-shadow 0.2s; +} + +[data-theme="light"] .dropdown { + border: 1px solid #d0d7de; + background: #fff; +} + +[data-theme="dark"] .dropdown { + border: 1px solid var(--ifm-color-emphasis-300); + background: #21262d; +} + +.dropdown:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 3px var(--ifm-color-primary-lightest); +} + +[data-theme="dark"] .dropdown:hover { + box-shadow: 0 0 0 3px var(--ifm-color-primary-lightest); +} + +.dropdown.open { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 3px var(--ifm-color-primary-lightest); +} + +[data-theme="dark"] .dropdown.open { + border-color: var(--ifm-color-primary); +} + +/* Single-model detectors render the label without a clickable menu. */ +.dropdown.static { + cursor: default; +} + +.dropdown.static:hover { + border-color: var(--ifm-color-emphasis-400); + box-shadow: none; +} + +[data-theme="light"] .dropdown.static:hover { + border-color: #d0d7de; +} + +[data-theme="dark"] .dropdown.static:hover { + border-color: var(--ifm-color-emphasis-300); +} + +.dropdownContent { + display: flex; + justify-content: space-between; + align-items: center; + gap: 1rem; + padding: 0.8rem 1rem; +} + +/* --- Model menu --- */ + +.menu { + margin-top: 0.25rem; + width: 360px; + max-width: 100%; + border: 1px solid var(--ifm-color-emphasis-400); + border-radius: 8px; + overflow: hidden; + background: var(--ifm-background-color); +} + +[data-theme="light"] .menu { + border: 1px solid #d0d7de; + background: #fff; +} + +[data-theme="dark"] .menu { + border: 1px solid var(--ifm-color-emphasis-300); + background: #21262d; +} + +.menuItem { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.6rem 1rem; + cursor: pointer; + font-size: 0.95rem; + color: var(--ifm-font-color-base); + transition: background 0.15s; +} + +.menuItem:not(:last-child) { + border-bottom: 1px solid var(--ifm-color-emphasis-200); +} + +.menuItem:hover { + background: var(--ifm-color-emphasis-100); +} + +.menuItemActive { + font-weight: var(--ifm-font-weight-semibold); + background: var(--ifm-color-primary-lightest); +} + +[data-theme="dark"] .menuItem:hover { + background: #2b3139; +} + +[data-theme="dark"] .menuItemActive { + background: #2b3139; +} + +.modelName { + font-weight: var(--ifm-font-weight-semibold); + color: var(--ifm-font-color-base); + font-size: 1rem; + display: flex; + align-items: center; + gap: 0.5rem; + white-space: nowrap; +} + +.recommendedBadge { + display: inline-block; + background: var(--ifm-color-success); + color: #fff; + font-size: 0.7rem; + font-weight: 600; + padding: 2px 8px; + border-radius: 12px; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.arrow { + font-size: 0.7rem; + color: var(--ifm-font-color-secondary); + transition: transform 0.2s; +} + +.dropdown.open .arrow { + transform: rotate(180deg); +} + +/* --- Panel --- */ + +.panel { + margin-top: 0.5rem; + border: 1px solid var(--ifm-color-emphasis-400); + border-radius: 8px; + overflow: hidden; + background: var(--ifm-background-color); +} + +[data-theme="light"] .panel { + border: 1px solid #d0d7de; + background: #fff; +} + +[data-theme="dark"] .panel { + border: 1px solid var(--ifm-color-emphasis-300); + background: #21262d; +} + +/* --- Steps --- */ + +.step { + padding: 1rem; +} + +.step:not(:last-child) { + border-bottom: 1px solid var(--ifm-color-emphasis-200); +} + +.stepTitle { + margin: 0 0 0.75rem 0; + font-size: 1rem; + font-weight: 600; + color: var(--ifm-font-color-base); +} + +/* Rendered markdown (download + Frigate UI instructions). */ + +.markdown { + font-size: 0.9rem; + line-height: 1.6; +} + +.markdown > :last-child { + margin-bottom: 0; +} + +.markdown a { + color: var(--ifm-color-primary); + text-decoration: underline; + text-underline-offset: 2px; +} + +.markdown table { + display: table; + width: 100%; + margin: 0.75rem 0; + font-size: 0.85rem; +} + +/* Docusaurus-style admonitions rendered from markdown. */ + +.admonition { + margin: 0.75rem 0; + padding: 0.75rem 1rem; + border-left: 4px solid var(--ifm-color-info); + border-radius: 4px; + background: var(--ifm-color-info-contrast-background); + font-size: 0.85rem; +} + +.admonition > :last-child { + margin-bottom: 0; +} + +.admonitionTitle { + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.5px; + font-size: 0.75rem; + margin-bottom: 0.4rem; + color: var(--ifm-color-info); +} + +.admonition_warning { + border-left-color: var(--ifm-color-warning); + background: var(--ifm-color-warning-contrast-background); +} + +.admonition_warning .admonitionTitle { + color: var(--ifm-color-warning-dark); +} + +.admonition_danger { + border-left-color: var(--ifm-color-danger); + background: var(--ifm-color-danger-contrast-background); +} + +.admonition_danger .admonitionTitle { + color: var(--ifm-color-danger-dark); +} + +.admonition_tip { + border-left-color: var(--ifm-color-success); + background: var(--ifm-color-success-contrast-background); +} + +.admonition_tip .admonitionTitle { + color: var(--ifm-color-success-dark); +}