administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-09-24 18:26:51 +03:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Refactor detector and model management (#23995)

Browse Source 
* Refactor detector and model management

* Fix model resolution field
This commit is contained in:
Nicolas Mowen
2026-09-12 07:30:04 -06:00
parent fd76eb6c6f
commit 8fe35ace3b
63 changed files with 2052 additions and 1152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/data/object_detectors_models.yaml
+411 -379
View File
@@ -7,10 +7,9 @@ edgeTPU:
download: A TensorFlow Lite model is provided in the container at `/edgetpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with `model.path`. download: A TensorFlow Lite model is provided in the container at `/edgetpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with `model.path`.
ui: Navigate to **Settings > System > Detectors and model** and select **EdgeTPU** from the detector type dropdown and click **Add**, then set device to `usb`. ui: Navigate to **Settings > System > Detectors and model** and select **EdgeTPU** from the detector type dropdown and click **Add**, then set device to `usb`.
yaml: |- yaml: |-
detectors: models:
coral: - devices:
type: edgetpu - edgetpu:usb
device: usb
- key: yolov9 - key: yolov9
label: YOLOv9 label: YOLOv9
recommended: false recommended: false
@@ -29,17 +28,14 @@ edgeTPU:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
coral: - devices:
type: edgetpu - edgetpu:usb
device: usb model_type: yolo-generic
width: 320 # <--- should match the imgsize of the model, typically 320
model: height: 320 # <--- should match the imgsize of the model, typically 320
model_type: yolo-generic path: /config/model_cache/yolov9-s-relu6-best_320_int8_edgetpu.tflite
width: 320 # <--- should match the imgsize of the model, typically 320 labelmap_path: /config/labels-coco17.txt
height: 320 # <--- should match the imgsize of the model, typically 320
path: /config/model_cache/yolov9-s-relu6-best_320_int8_edgetpu.tflite
labelmap_path: /config/labels-coco17.txt
hailo8l: hailo8l:
title: Hailo-8/Hailo-8L title: Hailo-8/Hailo-8L
models: models:
@@ -62,32 +58,29 @@ hailo8l:
The detector automatically selects the default model based on your hardware. Optionally, specify a local model path or URL to override. The detector automatically selects the default model based on your hardware. Optionally, specify a local model path or URL to override.
yaml: |- yaml: |-
detectors: models:
hailo: - devices:
type: hailo8l - hailo8l:PCIe
device: PCIe width: 320
height: 320
input_tensor: nhwc
input_pixel_format: rgb
input_dtype: int
model_type: yolo-generic
labelmap_path: /labelmap/coco-80.txt
model: # The detector automatically selects the default model based on your hardware:
width: 320 # - For Hailo-8 hardware: YOLOv6n (default: yolov6n.hef)
height: 320 # - For Hailo-8L hardware: YOLOv6n (default: yolov6n.hef)
input_tensor: nhwc #
input_pixel_format: rgb # Optionally, you can specify a local model path to override the default.
input_dtype: int # If a local path is provided and the file exists, it will be used instead of downloading.
model_type: yolo-generic # Example:
labelmap_path: /labelmap/coco-80.txt # path: /config/model_cache/hailo/yolov6n.hef
#
# The detector automatically selects the default model based on your hardware: # You can also override using a custom URL:
# - For Hailo-8 hardware: YOLOv6n (default: yolov6n.hef) # path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8/yolov6n.hef
# - For Hailo-8L hardware: YOLOv6n (default: yolov6n.hef) # just make sure to give it the write configuration based on the model
#
# Optionally, you can specify a local model path to override the default.
# If a local path is provided and the file exists, it will be used instead of downloading.
# Example:
# path: /config/model_cache/hailo/yolov6n.hef
#
# You can also override using a custom URL:
# path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8/yolov6n.hef
# just make sure to give it the write configuration based on the model
- key: ssd - key: ssd
label: SSD MobileNet v1 label: SSD MobileNet v1
recommended: false recommended: false
@@ -106,23 +99,20 @@ hailo8l:
Specify the local model path or URL for SSD MobileNet v1. Specify the local model path or URL for SSD MobileNet v1.
yaml: |- yaml: |-
detectors: models:
hailo: - devices:
type: hailo8l - hailo8l:PCIe
device: PCIe width: 300
height: 300
model: input_tensor: nhwc
width: 300 input_pixel_format: rgb
height: 300 model_type: ssd
input_tensor: nhwc # Specify the local model path (if available) or URL for SSD MobileNet v1.
input_pixel_format: rgb # Example with a local path:
model_type: ssd # path: /config/model_cache/h8l_cache/ssd_mobilenet_v1.hef
# Specify the local model path (if available) or URL for SSD MobileNet v1. #
# Example with a local path: # Or override using a custom URL:
# path: /config/model_cache/h8l_cache/ssd_mobilenet_v1.hef # path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8l/ssd_mobilenet_v1.hef
#
# Or override using a custom URL:
# path: https://hailo-model-zoo.s3.eu-west-2.amazonaws.com/ModelZoo/Compiled/v2.14.0/hailo8l/ssd_mobilenet_v1.hef
openvino: openvino:
title: OpenVINO title: OpenVINO
models: models:
@@ -166,19 +156,16 @@ openvino:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
ov: - devices:
type: openvino - openvino:GPU
device: GPU # or NPU model_type: yolo-generic
width: 320 # <--- should match the imgsize set during model export
model: height: 320 # <--- should match the imgsize set during model export
model_type: yolo-generic input_tensor: nchw
width: 320 # <--- should match the imgsize set during model export input_dtype: float
height: 320 # <--- should match the imgsize set during model export path: /config/model_cache/yolo.onnx # use the filename you generated above
input_tensor: nchw labelmap_path: /labelmap/coco-80.txt
input_dtype: float
path: /config/model_cache/yolo.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
- key: ssd - key: ssd
label: SSDLite MobileNet v2 label: SSDLite MobileNet v2
recommended: false recommended: false
@@ -197,18 +184,15 @@ openvino:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `ssd` (Frigate's default value) | | **Object Detection Model Type** | `ssd` (Frigate's default value) |
yaml: |- yaml: |-
detectors: models:
ov: - devices:
type: openvino - openvino:GPU
device: GPU # Or NPU width: 300
height: 300
model: input_tensor: nhwc
width: 300 input_pixel_format: bgr
height: 300 path: /openvino-model/ssdlite_mobilenet_v2.xml
input_tensor: nhwc labelmap_path: /openvino-model/coco_91cl_bkgr.txt
input_pixel_format: bgr
path: /openvino-model/ssdlite_mobilenet_v2.xml
labelmap_path: /openvino-model/coco_91cl_bkgr.txt
- key: yolo-legacy - key: yolo-legacy
label: YOLO (v3, v4, v7) label: YOLO (v3, v4, v7)
recommended: false recommended: false
@@ -235,19 +219,16 @@ openvino:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
ov: - devices:
type: openvino - openvino:GPU
device: GPU # or NPU model_type: yolo-generic
width: 320 # <--- should match the imgsize set during model export
model: height: 320 # <--- should match the imgsize set during model export
model_type: yolo-generic input_tensor: nchw
width: 320 # <--- should match the imgsize set during model export input_dtype: float
height: 320 # <--- should match the imgsize set during model export path: /config/model_cache/yolo.onnx # use the filename you generated above
input_tensor: nchw labelmap_path: /labelmap/coco-80.txt
input_dtype: float
path: /config/model_cache/yolo.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
- key: yolonas - key: yolonas
label: YOLO-NAS label: YOLO-NAS
recommended: false recommended: false
@@ -275,19 +256,16 @@ openvino:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolonas` | | **Object Detection Model Type** | `yolonas` |
yaml: |- yaml: |-
detectors: models:
ov: - devices:
type: openvino - openvino:GPU
device: GPU model_type: yolonas
width: 320 # <--- should match whatever was set in notebook
model: height: 320 # <--- should match whatever was set in notebook
model_type: yolonas input_tensor: nchw
width: 320 # <--- should match whatever was set in notebook input_pixel_format: bgr
height: 320 # <--- should match whatever was set in notebook path: /config/yolo_nas_s.onnx
input_tensor: nchw labelmap_path: /labelmap/coco-80.txt
input_pixel_format: bgr
path: /config/yolo_nas_s.onnx
labelmap_path: /labelmap/coco-80.txt
- key: yolox - key: yolox
label: YOLOX label: YOLOX
recommended: false recommended: false
@@ -303,15 +281,12 @@ openvino:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolox` | | **Object Detection Model Type** | `yolox` |
yaml: |- yaml: |-
detectors: models:
ov: - devices:
type: openvino - openvino:GPU
device: GPU model_type: yolox
path: /config/model_cache/yolox.onnx # use the filename you generated above
model: labelmap_path: /labelmap/coco-80.txt
model_type: yolox
path: /config/model_cache/yolox.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
- key: rfdetr - key: rfdetr
label: RF-DETR label: RF-DETR
recommended: false recommended: false
@@ -345,18 +320,15 @@ openvino:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `rfdetr` | | **Object Detection Model Type** | `rfdetr` |
yaml: |- yaml: |-
detectors: models:
ov: - devices:
type: openvino - openvino:GPU
device: GPU model_type: rfdetr
width: 320
model: height: 320
model_type: rfdetr input_tensor: nchw
width: 320 input_dtype: float
height: 320 path: /config/model_cache/rfdetr.onnx # use the filename you generated above
input_tensor: nchw
input_dtype: float
path: /config/model_cache/rfdetr.onnx # use the filename you generated above
- key: dfine - key: dfine
label: D-FINE / DEIMv2 label: D-FINE / DEIMv2
recommended: false recommended: false
@@ -443,19 +415,16 @@ openvino:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `dfine` | | **Object Detection Model Type** | `dfine` |
yaml: |- yaml: |-
detectors: models:
ov: - devices:
type: openvino - openvino:CPU
device: CPU model_type: dfine
width: 640
model: height: 640
model_type: dfine input_tensor: nchw
width: 640 input_dtype: float
height: 640 path: /config/model_cache/dfine-s.onnx # use the filename you generated above
input_tensor: nchw labelmap_path: /labelmap/coco-80.txt
input_dtype: float
path: /config/model_cache/dfine-s.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
appleSilicon: appleSilicon:
title: Apple Silicon title: Apple Silicon
models: models:
@@ -499,19 +468,16 @@ appleSilicon:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
apple-silicon: - devices:
type: zmq - zmq:tcp://host.docker.internal:5555
endpoint: tcp://host.docker.internal:5555 model_type: yolo-generic
width: 320 # <--- should match the imgsize set during model export
model: height: 320 # <--- should match the imgsize set during model export
model_type: yolo-generic input_tensor: nchw
width: 320 # <--- should match the imgsize set during model export input_dtype: float
height: 320 # <--- should match the imgsize set during model export path: /config/model_cache/yolo.onnx # use the filename you generated above
input_tensor: nchw labelmap_path: /labelmap/coco-80.txt
input_dtype: float
path: /config/model_cache/yolo.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
- key: yolo-legacy - key: yolo-legacy
label: YOLO (v3, v4, v7) label: YOLO (v3, v4, v7)
recommended: false recommended: false
@@ -538,19 +504,16 @@ appleSilicon:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
apple-silicon: - devices:
type: zmq - zmq:tcp://host.docker.internal:5555
endpoint: tcp://host.docker.internal:5555 model_type: yolo-generic
width: 320 # <--- should match the imgsize set during model export
model: height: 320 # <--- should match the imgsize set during model export
model_type: yolo-generic input_tensor: nchw
width: 320 # <--- should match the imgsize set during model export input_dtype: float
height: 320 # <--- should match the imgsize set during model export path: /config/model_cache/yolo.onnx # use the filename you generated above
input_tensor: nchw labelmap_path: /labelmap/coco-80.txt
input_dtype: float
path: /config/model_cache/yolo.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
onnx: onnx:
title: ONNX title: ONNX
models: models:
@@ -594,18 +557,16 @@ onnx:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
onnx: - devices:
type: onnx - onnx
model_type: yolo-generic
model: width: 320 # <--- should match the imgsize set during model export
model_type: yolo-generic height: 320 # <--- should match the imgsize set during model export
width: 320 # <--- should match the imgsize set during model export input_tensor: nchw
height: 320 # <--- should match the imgsize set during model export input_dtype: float
input_tensor: nchw path: /config/model_cache/yolo.onnx # use the filename you generated above
input_dtype: float labelmap_path: /labelmap/coco-80.txt
path: /config/model_cache/yolo.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
- key: rfdetr - key: rfdetr
label: RF-DETR label: RF-DETR
recommended: false recommended: false
@@ -639,17 +600,15 @@ onnx:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `rfdetr` | | **Object Detection Model Type** | `rfdetr` |
yaml: |- yaml: |-
detectors: models:
onnx: - devices:
type: onnx - onnx
model_type: rfdetr
model: width: 320
model_type: rfdetr height: 320
width: 320 input_tensor: nchw
height: 320 input_dtype: float
input_tensor: nchw path: /config/model_cache/rfdetr.onnx # use the filename you generated above
input_dtype: float
path: /config/model_cache/rfdetr.onnx # use the filename you generated above
- key: yolonas - key: yolonas
label: YOLO-NAS label: YOLO-NAS
recommended: false recommended: false
@@ -677,18 +636,16 @@ onnx:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolonas` | | **Object Detection Model Type** | `yolonas` |
yaml: |- yaml: |-
detectors: models:
onnx: - devices:
type: onnx - onnx
model_type: yolonas
model: width: 320 # <--- should match whatever was set in notebook
model_type: yolonas height: 320 # <--- should match whatever was set in notebook
width: 320 # <--- should match whatever was set in notebook input_pixel_format: bgr
height: 320 # <--- should match whatever was set in notebook input_tensor: nchw
input_pixel_format: bgr path: /config/yolo_nas_s.onnx
input_tensor: nchw labelmap_path: /labelmap/coco-80.txt
path: /config/yolo_nas_s.onnx
labelmap_path: /labelmap/coco-80.txt
- key: yolox - key: yolox
label: YOLOX label: YOLOX
recommended: false recommended: false
@@ -707,18 +664,16 @@ onnx:
| **Model Input D Type** | `float_denorm` | | **Model Input D Type** | `float_denorm` |
| **Object Detection Model Type** | `yolox` | | **Object Detection Model Type** | `yolox` |
yaml: |- yaml: |-
detectors: models:
onnx: - devices:
type: onnx - onnx
model_type: yolox
model: width: 416 # <--- should match the imgsize set during model export
model_type: yolox height: 416 # <--- should match the imgsize set during model export
width: 416 # <--- should match the imgsize set during model export input_tensor: nchw
height: 416 # <--- should match the imgsize set during model export input_dtype: float_denorm
input_tensor: nchw path: /config/model_cache/yolox_tiny.onnx # use the filename you generated above
input_dtype: float_denorm labelmap_path: /labelmap/coco-80.txt
path: /config/model_cache/yolox_tiny.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
- key: dfine - key: dfine
label: D-FINE / DEIMv2 label: D-FINE / DEIMv2
recommended: false recommended: false
@@ -805,18 +760,16 @@ onnx:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `dfine` | | **Object Detection Model Type** | `dfine` |
yaml: |- yaml: |-
detectors: models:
onnx: - devices:
type: onnx - onnx
model_type: dfine
model: width: 640
model_type: dfine height: 640
width: 640 input_tensor: nchw
height: 640 input_dtype: float
input_tensor: nchw path: /config/model_cache/dfine_m_obj2coco.onnx # use the filename you generated above
input_dtype: float labelmap_path: /labelmap/coco-80.txt
path: /config/model_cache/dfine_m_obj2coco.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
- key: yolo-legacy - key: yolo-legacy
label: YOLO (v3, v4, v7) label: YOLO (v3, v4, v7)
recommended: false recommended: false
@@ -843,18 +796,16 @@ onnx:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
onnx: - devices:
type: onnx - onnx
model_type: yolo-generic
model: width: 320 # <--- should match the imgsize set during model export
model_type: yolo-generic height: 320 # <--- should match the imgsize set during model export
width: 320 # <--- should match the imgsize set during model export input_tensor: nchw
height: 320 # <--- should match the imgsize set during model export input_dtype: float
input_tensor: nchw path: /config/model_cache/yolo.onnx # use the filename you generated above
input_dtype: float labelmap_path: /labelmap/coco-80.txt
path: /config/model_cache/yolo.onnx # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
cpu: cpu:
title: CPU title: CPU
models: models:
@@ -870,10 +821,9 @@ cpu:
| **Detector type** | `cpu` | | **Detector type** | `cpu` |
| **Num threads** | `3` | | **Num threads** | `3` |
yaml: |- yaml: |-
detectors: models:
cpu1: - devices:
type: cpu - cpu:3
num_threads: 3
deepstack: deepstack:
title: DeepStack / CodeProject.AI title: DeepStack / CodeProject.AI
models: models:
@@ -889,11 +839,9 @@ deepstack:
| **API URL** | `http://<your_codeproject_ai_server_ip>:<port>/v1/vision/detection` | | **API URL** | `http://<your_codeproject_ai_server_ip>:<port>/v1/vision/detection` |
| **API Timeout** | `0.1` (seconds) | | **API Timeout** | `0.1` (seconds) |
yaml: |- yaml: |-
detectors: models:
deepstack: - devices:
api_url: http://<your_codeproject_ai_server_ip>:<port>/v1/vision/detection - deepstack:http://<your_codeproject_ai_server_ip>:<port>/v1/vision/detection
type: deepstack
api_timeout: 0.1 # seconds
memryx: memryx:
title: MemryX title: MemryX
models: models:
@@ -923,23 +871,20 @@ memryx:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolonas` | | **Object Detection Model Type** | `yolonas` |
yaml: |- yaml: |-
detectors: models:
memx0: - devices:
type: memryx - memryx:PCIe:0
device: PCIe:0 model_type: yolonas
width: 320 # (Can be set to 640 for higher resolution)
model: height: 320 # (Can be set to 640 for higher resolution)
model_type: yolonas input_tensor: nchw
width: 320 # (Can be set to 640 for higher resolution) input_dtype: float
height: 320 # (Can be set to 640 for higher resolution) labelmap_path: /labelmap/coco-80.txt
input_tensor: nchw # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.
input_dtype: float # path: /config/yolonas.zip
labelmap_path: /labelmap/coco-80.txt # The .zip file must contain:
# Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. # ├── yolonas.dfp (a file ending with .dfp)
# path: /config/yolonas.zip # └── yolonas_post.onnx (optional; only if the model includes a cropped post-processing network)
# The .zip file must contain:
# ├── yolonas.dfp (a file ending with .dfp)
# └── yolonas_post.onnx (optional; only if the model includes a cropped post-processing network)
- key: yolov9 - key: yolov9
label: YOLOv9 label: YOLOv9
recommended: false recommended: false
@@ -960,22 +905,19 @@ memryx:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
detectors: models:
memx0: - devices:
type: memryx - memryx:PCIe:0
device: PCIe:0 model_type: yolo-generic
width: 320 # (Can be set to 640 for higher resolution)
model: height: 320 # (Can be set to 640 for higher resolution)
model_type: yolo-generic input_tensor: nchw
width: 320 # (Can be set to 640 for higher resolution) input_dtype: float
height: 320 # (Can be set to 640 for higher resolution) labelmap_path: /labelmap/coco-80.txt
input_tensor: nchw # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.
input_dtype: float # path: /config/yolov9.zip
labelmap_path: /labelmap/coco-80.txt # The .zip file must contain:
# Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. # ├── yolov9.dfp (a file ending with .dfp)
# path: /config/yolov9.zip
# The .zip file must contain:
# ├── yolov9.dfp (a file ending with .dfp)
- key: yolox - key: yolox
label: YOLOX label: YOLOX
recommended: false recommended: false
@@ -996,22 +938,19 @@ memryx:
| **Model Input D Type** | `float_denorm` | | **Model Input D Type** | `float_denorm` |
| **Object Detection Model Type** | `yolox` | | **Object Detection Model Type** | `yolox` |
yaml: |- yaml: |-
detectors: models:
memx0: - devices:
type: memryx - memryx:PCIe:0
device: PCIe:0 model_type: yolox
width: 640
model: height: 640
model_type: yolox input_tensor: nchw
width: 640 input_dtype: float_denorm
height: 640 labelmap_path: /labelmap/coco-80.txt
input_tensor: nchw # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.
input_dtype: float_denorm # path: /config/yolox.zip
labelmap_path: /labelmap/coco-80.txt # The .zip file must contain:
# Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. # ├── yolox.dfp (a file ending with .dfp)
# path: /config/yolox.zip
# The .zip file must contain:
# ├── yolox.dfp (a file ending with .dfp)
- key: ssd - key: ssd
label: SSDLite MobileNet v2 label: SSDLite MobileNet v2
recommended: false recommended: false
@@ -1032,23 +971,20 @@ memryx:
| **Model Input D Type** | `float` | | **Model Input D Type** | `float` |
| **Object Detection Model Type** | `ssd` | | **Object Detection Model Type** | `ssd` |
yaml: |- yaml: |-
detectors: models:
memx0: - devices:
type: memryx - memryx:PCIe:0
device: PCIe:0 model_type: ssd
width: 320
model: height: 320
model_type: ssd input_tensor: nchw
width: 320 input_dtype: float
height: 320 labelmap_path: /labelmap/coco-80.txt
input_tensor: nchw # Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model.
input_dtype: float # path: /config/ssdlite_mobilenet.zip
labelmap_path: /labelmap/coco-80.txt # The .zip file must contain:
# Optional: The model is normally fetched through the runtime, so 'path' can be omitted unless you want to use a custom or local model. # ├── ssdlite_mobilenet.dfp (a file ending with .dfp)
# path: /config/ssdlite_mobilenet.zip # └── ssdlite_mobilenet_post.onnx (optional; only if the model includes a cropped post-processing network)
# The .zip file must contain:
# ├── ssdlite_mobilenet.dfp (a file ending with .dfp)
# └── ssdlite_mobilenet_post.onnx (optional; only if the model includes a cropped post-processing network)
tensorrt: tensorrt:
title: TensorRT title: TensorRT
models: models:
@@ -1082,18 +1018,15 @@ tensorrt:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `ssd` (Frigate's default value) | | **Object Detection Model Type** | `ssd` (Frigate's default value) |
yaml: |- yaml: |-
detectors: models:
tensorrt: - devices:
type: tensorrt - tensorrt:0
device: 0 #This is the default, select the first GPU path: /config/model_cache/tensorrt/yolov7-320.trt # use the filename you generated above
labelmap_path: /labelmap/coco-80.txt
model: input_tensor: nchw
path: /config/model_cache/tensorrt/yolov7-320.trt # use the filename you generated above input_pixel_format: rgb
labelmap_path: /labelmap/coco-80.txt width: 320 # MUST match the chosen model i.e yolov7-320 -> 320, yolov4-416 -> 416
input_tensor: nchw height: 320 # MUST match the chosen model i.e yolov7-320 -> 320 yolov4-416 -> 416
input_pixel_format: rgb
width: 320 # MUST match the chosen model i.e yolov7-320 -> 320, yolov4-416 -> 416
height: 320 # MUST match the chosen model i.e yolov7-320 -> 320 yolov4-416 -> 416
synaptics: synaptics:
title: Synaptics title: Synaptics
models: models:
@@ -1115,16 +1048,15 @@ synaptics:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `ssd` (Frigate's default value) | | **Object Detection Model Type** | `ssd` (Frigate's default value) |
yaml: |- yaml: |-
detectors: # required models:
synap_npu: # required - # required
type: synaptics # required devices:
- synaptics
model: # required path: /synaptics/mobilenet.synap # required
path: /synaptics/mobilenet.synap # required width: 224 # required
width: 224 # required height: 224 # required
height: 224 # required input_tensor: nhwc # default value (optional. If you change the model, it is required)
input_tensor: nhwc # default value (optional. If you change the model, it is required) labelmap_path: /labelmap/coco-80.txt # required
labelmap_path: /labelmap/coco-80.txt # required
rknn: rknn:
title: RKNN title: RKNN
models: models:
@@ -1149,21 +1081,23 @@ rknn:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
model: # required models:
# name of model (will be automatically downloaded) or path to your own .rknn model file - devices:
# possible values are: - rknn
# - frigate-fp16-yolov9-t # name of model (will be automatically downloaded) or path to your own .rknn model file
# - frigate-fp16-yolov9-s # possible values are:
# - frigate-fp16-yolov9-m # - frigate-fp16-yolov9-t
# - frigate-fp16-yolov9-c # - frigate-fp16-yolov9-s
# - frigate-fp16-yolov9-e # - frigate-fp16-yolov9-m
# your yolo_model.rknn # - frigate-fp16-yolov9-c
path: frigate-fp16-yolov9-t # - frigate-fp16-yolov9-e
model_type: yolo-generic # your yolo_model.rknn
width: 320 path: frigate-fp16-yolov9-t
height: 320 model_type: yolo-generic
input_tensor: nhwc width: 320
labelmap_path: /labelmap/coco-80.txt height: 320
input_tensor: nhwc
labelmap_path: /labelmap/coco-80.txt
- key: yolonas - key: yolonas
label: YOLO-NAS label: YOLO-NAS
recommended: false recommended: false
@@ -1187,20 +1121,22 @@ rknn:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolonas` | | **Object Detection Model Type** | `yolonas` |
yaml: |- yaml: |-
model: # required models:
# name of model (will be automatically downloaded) or path to your own .rknn model file - devices:
# possible values are: - rknn
# - deci-fp16-yolonas_s # name of model (will be automatically downloaded) or path to your own .rknn model file
# - deci-fp16-yolonas_m # possible values are:
# - deci-fp16-yolonas_l # - deci-fp16-yolonas_s
# your yolonas_model.rknn # - deci-fp16-yolonas_m
path: deci-fp16-yolonas_s # - deci-fp16-yolonas_l
model_type: yolonas # your yolonas_model.rknn
width: 320 path: deci-fp16-yolonas_s
height: 320 model_type: yolonas
input_pixel_format: bgr width: 320
input_tensor: nhwc height: 320
labelmap_path: /labelmap/coco-80.txt input_pixel_format: bgr
input_tensor: nhwc
labelmap_path: /labelmap/coco-80.txt
- key: yolox - key: yolox
label: YOLOx label: YOLOx
recommended: false recommended: false
@@ -1222,20 +1158,22 @@ rknn:
| **Model Input D Type** | `int` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolox` | | **Object Detection Model Type** | `yolox` |
yaml: |- yaml: |-
model: # required models:
# name of model (will be automatically downloaded) or path to your own .rknn model file - devices:
# possible values are: - rknn
# - rock-i8-yolox_nano # name of model (will be automatically downloaded) or path to your own .rknn model file
# - rock-i8-yolox_tiny # possible values are:
# - rock-fp16-yolox_nano # - rock-i8-yolox_nano
# - rock-fp16-yolox_tiny # - rock-i8-yolox_tiny
# your yolox_model.rknn # - rock-fp16-yolox_nano
path: rock-i8-yolox_nano # - rock-fp16-yolox_tiny
model_type: yolox # your yolox_model.rknn
width: 416 path: rock-i8-yolox_nano
height: 416 model_type: yolox
input_tensor: nhwc width: 416
labelmap_path: /labelmap/coco-80.txt height: 416
input_tensor: nhwc
labelmap_path: /labelmap/coco-80.txt
axengine: axengine:
title: AXEngine title: AXEngine
models: models:
@@ -1257,6 +1195,7 @@ axengine:
| **Model Input D Type** | `int` | | **Model Input D Type** | `int` |
| **Object Detection Model Type** | `yolo-generic` | | **Object Detection Model Type** | `yolo-generic` |
yaml: |- yaml: |-
<<<<<<< HEAD
detectors: detectors:
axengine: axengine:
type: axengine type: axengine
@@ -1269,3 +1208,96 @@ axengine:
input_dtype: int input_dtype: int
input_pixel_format: bgr input_pixel_format: bgr
labelmap_path: /labelmap/coco-80.txt labelmap_path: /labelmap/coco-80.txt
=======
models:
- devices:
- axengine
path: frigate-yolov9-tiny
model_type: yolo-generic
width: 320
height: 320
input_dtype: int
input_pixel_format: bgr
labelmap_path: /labelmap/coco-80.txt
degirumAiServer:
title: DeGirum AI Server
models:
- key: ai-server-inference
label: AI Server Inference
recommended: true
download: |-
Launch a DeGirum AI server as a Docker container, then point the detector at it. Add this to your `docker-compose.yml`:
```yaml
degirum_detector:
container_name: degirum
image: degirum/aiserver:latest
privileged: true
ports:
- "8778:8778"
```
Set `location` to the server's service name, container name, or `host:port`.
ui: |
Navigate to **Settings > System > Detectors and model** and select **DeGirum** from the detector type dropdown and click **Add**.
| Field | Value |
| --- | --- |
| **Location** | `degirum` |
| **Zoo** | `degirum/public` |
| **Token** | your AI Hub token (optional for the public zoo) |
yaml: |
models:
- devices:
- degirum:degirum
path: ./mobilenet_v2_ssd_coco--300x300_quant_n2x_orca1_1
width: 300
height: 300
input_pixel_format: rgb
degirumLocal:
title: DeGirum Local
models:
- key: local-inference
label: Local Inference
recommended: true
download: Run hardware directly inside the Frigate container with `@local`, removing the AI server hop. The matching device runtime (e.g. the Hailo runtime) must be installed in the container; confirm it with `degirum sys-info`.
ui: |
Navigate to **Settings > System > Detectors and model** and select **DeGirum** from the detector type dropdown and click **Add**.
| Field | Value |
| --- | --- |
| **Location** | `@local` |
| **Zoo** | `degirum/public` |
| **Token** | your AI Hub token (optional for the public zoo) |
yaml: |
models:
- devices:
- degirum:@local
path: ./mobilenet_v2_ssd_coco--300x300_quant_n2x_orca1_1
width: 300
height: 300
input_pixel_format: rgb
degirumCloud:
title: DeGirum AI Hub Cloud
models:
- key: ai-hub-cloud-inference
label: AI Hub Cloud Inference
recommended: true
download: Run inferences on DeGirum's [AI Hub](https://hub.degirum.com) cloud with `@cloud`. Sign up, create an access token, and set it as `token`. Network latency may require lowering your detection fps.
ui: |
Navigate to **Settings > System > Detectors and model** and select **DeGirum** from the detector type dropdown and click **Add**.
| Field | Value |
| --- | --- |
| **Location** | `@cloud` |
| **Zoo** | `degirum/public` |
| **Token** | your AI Hub token (optional for the public zoo) |
yaml: |
models:
- devices:
- degirum:@cloud
path: ./mobilenet_v2_ssd_coco--300x300_quant_n2x_orca1_1
width: 300
height: 300
input_pixel_format: rgb
>>>>>>> 34363affa (Refactor detector and model management)
docs/docs/configuration/advanced/reference.md
+52 -47
View File
@@ -56,17 +56,6 @@ mqtt:
# 2 = exactly once # 2 = exactly once
qos: 0 qos: 0
# Optional: Detectors configuration. Defaults to a single CPU detector
detectors:
# Required: name of the detector
detector_name:
# Required: type of the detector
# Frigate provides many types, see https://docs.frigate.video/configuration/object_detectors for more details (default: shown below)
# Additional detector types can also be plugged in.
# Detectors may require additional configuration.
# Refer to the Detectors configuration page for more information.
type: cpu
# Optional: Database configuration # Optional: Database configuration
database: database:
# The path to store the SQLite DB (default: shown below) # The path to store the SQLite DB (default: shown below)
@@ -157,44 +146,56 @@ auth:
- front_door - front_door
- back_yard - back_yard
# Optional: model modifications # Optional: object detection models. Defaults to a single model on a CPU detector.
# NOTE: The default values are for the EdgeTPU detector. # NOTE: The default values are for the EdgeTPU detector.
# Other detectors will require the model config to be set. # Other detectors will require the model config to be set.
model: models:
# Required: path to the model. Frigate+ models use plus://<model_id> (default: automatic based on detector) # Optional: the camera environment this model is for (default: shown below)
path: /edgetpu_model.tflite # Cameras select a model by setting detect -> scene to a matching value, and
# Required: path to the labelmap (default: shown below) # a model with a scene of all is used by any camera that does not set one.
labelmap_path: /labelmap.txt # Valid values are all, indoor, outdoor, indoor_thermal, outdoor_thermal
# Required: Object detection model input width (default: shown below) - scene: all
width: 320 # Required: hardware this model runs on, as <detector> or <detector>:<device>
# Required: Object detection model input height (default: shown below) # See https://docs.frigate.video/configuration/object_detectors for the
height: 320 # detectors available and the devices each one accepts. All of a model's
# Required: Object detection model input colorspace # devices must use the same detector. Listing the same device more than once
# Valid values are rgb, bgr, or yuv. (default: shown below) # runs additional inference processes on it.
input_pixel_format: rgb devices:
# Required: Object detection model input tensor format - edgetpu:pci:0
# Valid values are nhwc, nchw, hwnc, or hwcn (default: shown below) # Required: path to the model. Frigate+ models use plus://<model_id> (default: automatic based on detector)
input_tensor: nhwc path: /edgetpu_model.tflite
# Optional: Data type of the model input tensor # Required: path to the labelmap (default: shown below)
# Valid values are float, float_denorm, or int (default: shown below) labelmap_path: /labelmap.txt
input_dtype: int # Required: Object detection model input width (default: shown below)
# Required: Object detection model architecture, used by detectors that support more width: 320
# than one model type (openvino, onnx, rknn, memryx, axengine, synaptics, and others) # Required: Object detection model input height (default: shown below)
# Valid values are ssd, yolox, yolonas, yolo-generic, rfdetr, dfine (default: shown below) height: 320
model_type: ssd # Required: Object detection model input colorspace
# Required: Label name modifications. These are merged into the standard labelmap. # Valid values are rgb, bgr, or yuv. (default: shown below)
labelmap: input_pixel_format: rgb
2: vehicle # Required: Object detection model input tensor format
# Optional: Map of object labels to their attribute labels (default: depends on model) # Valid values are nhwc, nchw, hwnc, or hwcn (default: shown below)
attributes_map: input_tensor: nhwc
person: # Optional: Data type of the model input tensor
- amazon # Valid values are float, float_denorm, or int (default: shown below)
- face input_dtype: int
car: # Required: Object detection model architecture, used by detectors that support more
- amazon # than one model type (openvino, onnx, rknn, memryx, axengine, synaptics, and others)
- fedex # Valid values are ssd, yolox, yolonas, yolo-generic, rfdetr, dfine (default: shown below)
- license_plate model_type: ssd
- ups # Required: Label name modifications. These are merged into the standard labelmap.
labelmap:
2: vehicle
# Optional: Map of object labels to their attribute labels (default: depends on model)
attributes_map:
person:
- amazon
- face
car:
- amazon
- fedex
- license_plate
- ups
# Optional: Audio Events Configuration # Optional: Audio Events Configuration
# NOTE: Can be overridden at the camera level # NOTE: Can be overridden at the camera level
@@ -314,6 +315,10 @@ detect:
width: 1280 width: 1280
# Optional: height of the frame for the input with the detect role (default: use native stream resolution) # Optional: height of the frame for the input with the detect role (default: use native stream resolution)
height: 720 height: 720
# Optional: the environment this camera looks at, which picks the model it runs on
# (default: the model with a scene of all)
# Valid values are all, indoor, outdoor, indoor_thermal, outdoor_thermal
scene: outdoor
# Optional: desired fps for your camera for the input with the detect role (default: shown below) # Optional: desired fps for your camera for the input with the detect role (default: shown below)
# NOTE: Recommended value of 5. Ideally, try and reduce your FPS on the camera. # NOTE: Recommended value of 5. Ideally, try and reduce your FPS on the camera.
fps: 5 fps: 5
docs/docs/configuration/advanced/system.md
+17 -15
View File
@@ -192,12 +192,14 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and open
```yaml ```yaml
# Optional: model config # Optional: model config
model: models:
path: /path/to/model - devices:
width: 320 - openvino:GPU
height: 320 path: /path/to/model
input_tensor: "nhwc" width: 320
input_pixel_format: "bgr" height: 320
input_tensor: "nhwc"
input_pixel_format: "bgr"
``` ```
</TabItem> </TabItem>
@@ -214,15 +216,15 @@ If the labelmap is customized then the labels used for alerts will need to be ad
The labelmap can be customized to your needs. A common reason to do this is to combine multiple object types that are easily confused when you don't need to be as granular such as car/truck. By default, truck is renamed to car because they are often confused. You cannot add new object types, but you can change the names of existing objects in the model. The labelmap can be customized to your needs. A common reason to do this is to combine multiple object types that are easily confused when you don't need to be as granular such as car/truck. By default, truck is renamed to car because they are often confused. You cannot add new object types, but you can change the names of existing objects in the model.
```yaml ```yaml
model: models:
labelmap: - labelmap:
2: vehicle 2: vehicle
3: vehicle 3: vehicle
5: vehicle 5: vehicle
7: vehicle 7: vehicle
15: animal 15: animal
16: animal 16: animal
17: animal 17: animal
``` ```
Note that if you rename objects in the labelmap, you will also need to update your `objects -> track` list as well. Note that if you rename objects in the labelmap, you will also need to update your `objects -> track` list as well.
docs/docs/configuration/config.md
+12 -17
View File
@@ -172,10 +172,9 @@ mqtt:
ffmpeg: ffmpeg:
hwaccel_args: preset-rpi-64-h264 hwaccel_args: preset-rpi-64-h264
detectors: models:
coral: - devices:
type: edgetpu - edgetpu:usb
device: usb
record: record:
enabled: True enabled: True
@@ -249,10 +248,9 @@ mqtt:
ffmpeg: ffmpeg:
hwaccel_args: preset-vaapi hwaccel_args: preset-vaapi
detectors: models:
coral: - devices:
type: edgetpu - edgetpu:usb
device: usb
record: record:
enabled: True enabled: True
@@ -329,15 +327,12 @@ mqtt:
ffmpeg: ffmpeg:
hwaccel_args: preset-vaapi hwaccel_args: preset-vaapi
detectors: models:
ov: - devices:
type: openvino - openvino:AUTO
device: AUTO width: 300
height: 300
model: input_tensor: nhwc
width: 300
height: 300
input_tensor: nhwc
input_pixel_format: bgr input_pixel_format: bgr
path: /openvino-model/ssdlite_mobilenet_v2.xml path: /openvino-model/ssdlite_mobilenet_v2.xml
labelmap_path: /openvino-model/coco_91cl_bkgr.txt labelmap_path: /openvino-model/coco_91cl_bkgr.txt
docs/docs/configuration/object_detectors.md
+201 -59
View File
@@ -68,12 +68,66 @@ Frigate supports multiple different detectors that work on different types of ha
:::note :::note
Multiple detectors can not be mixed for object detection (ex: OpenVINO and Coral EdgeTPU can not be used for object detection at the same time). A single model can not be spread across different detector types (ex: OpenVINO and Coral EdgeTPU can not run the same model at the same time). Configuring more than one model, each on its own detector type, is supported.
This does not affect using hardware for accelerating other tasks such as [semantic search](./semantic_search.md) This does not affect using hardware for accelerating other tasks such as [semantic search](./semantic_search.md)
::: :::
### Configuring models and hardware
Object detection is configured with a `models` list. Each entry describes one model and the hardware it runs on:
```yaml
models:
- devices:
- openvino:GPU
path: /config/model_cache/yolov9-s.onnx
model_type: yolo-generic
width: 320
height: 320
```
Each entry in `devices` is a detector type, optionally followed by a colon and a device for that detector, such as `edgetpu:pci:0`, `openvino:NPU`, or `tensorrt:0`. The per-detector sections below document the device values each one accepts. Listing several devices runs the model on all of them, and listing the **same** device more than once runs additional inference processes against it, which can improve throughput on hardware that keeps up with more than one stream:
```yaml
models:
- devices:
- openvino:GPU
- openvino:GPU
```
Coral EdgeTPU and MemryX accelerators can only be opened by one process, so those devices can not be repeated.
### Running more than one model
Cameras can be split across models by scene, which is useful when indoor and outdoor cameras benefit from differently trained models. Each model declares the `scene` it is for, and each camera picks one with `detect -> scene`:
```yaml
models:
- scene: outdoor
path: plus://your-outdoor-model
devices:
- edgetpu:pci:0
- scene: indoor
path: /config/model_cache/indoor.onnx
model_type: yolo-generic
devices:
- openvino:GPU
cameras:
driveway:
detect:
scene: outdoor
...
hallway:
detect:
scene: indoor
...
```
Available scenes are `all`, `indoor`, `outdoor`, `indoor_thermal`, and `outdoor_thermal`. A model with a scene of `all` is used by every camera that does not set one, and `all` is the default when a model does not declare a scene. Changing a camera's scene requires a restart.
### Choosing a model size ### Choosing a model size
Along with picking a detector for your hardware, you will choose a model's **input resolution** (such as `320x320` or `640x640`) and, for model families like YOLOv9, a **variant size** (`tiny`, `small`, etc.). Both affect the balance between accuracy and the inference time your hardware can sustain. Along with picking a detector for your hardware, you will choose a model's **input resolution** (such as `320x320` or `640x640`) and, for model families like YOLOv9, a **variant size** (`tiny`, `small`, etc.). Both affect the balance between accuracy and the inference time your hardware can sustain.
@@ -92,11 +146,11 @@ The best detection accuracy comes from a model trained on images that look like
# Officially Supported Detectors # Officially Supported Detectors
Frigate provides a number of builtin detector types. By default, Frigate will use a single CPU detector. Other detectors may require additional configuration as described below. When using multiple detectors they will run in dedicated processes, but pull from a common queue of detection requests from across all cameras. Frigate provides a number of builtin detector types. By default, Frigate will use a single CPU detector. Other detectors may require additional configuration as described below. Each of a model's devices runs in a dedicated process, and they pull from a common queue of detection requests from the cameras assigned to that model.
## Edge TPU Detector ## Edge TPU Detector
The Edge TPU detector type runs TensorFlow Lite models utilizing the Google Coral delegate for hardware acceleration. To configure an Edge TPU detector, set the `"type"` attribute to `"edgetpu"`. The Edge TPU detector type runs TensorFlow Lite models utilizing the Google Coral delegate for hardware acceleration. To use it, prefix a model's device with `edgetpu`.
The Edge TPU device can be specified using the `"device"` attribute according to the [Documentation for the TensorFlow Lite Python API](https://coral.ai/docs/edgetpu/multiple-edgetpu/#using-the-tensorflow-lite-python-api). If not set, the delegate will use the first device it finds. The Edge TPU device can be specified using the `"device"` attribute according to the [Documentation for the TensorFlow Lite Python API](https://coral.ai/docs/edgetpu/multiple-edgetpu/#using-the-tensorflow-lite-python-api). If not set, the delegate will use the first device it finds.
@@ -117,10 +171,9 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and selec
<TabItem value="yaml"> <TabItem value="yaml">
```yaml ```yaml
detectors: models:
coral: - devices:
type: edgetpu - edgetpu:usb
device: usb
``` ```
</TabItem> </TabItem>
@@ -137,13 +190,10 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and selec
<TabItem value="yaml"> <TabItem value="yaml">
```yaml ```yaml
detectors: models:
coral1: - devices:
type: edgetpu - edgetpu:usb:0
device: usb:0 - edgetpu:usb:1
coral2:
type: edgetpu
device: usb:1
``` ```
</TabItem> </TabItem>
@@ -162,10 +212,9 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and selec
<TabItem value="yaml"> <TabItem value="yaml">
```yaml ```yaml
detectors: models:
coral: - devices:
type: edgetpu - 'edgetpu:'
device: ""
``` ```
</TabItem> </TabItem>
@@ -182,10 +231,9 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and selec
<TabItem value="yaml"> <TabItem value="yaml">
```yaml ```yaml
detectors: models:
coral: - devices:
type: edgetpu - edgetpu:pci
device: pci
``` ```
</TabItem> </TabItem>
@@ -202,13 +250,10 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and selec
<TabItem value="yaml"> <TabItem value="yaml">
```yaml ```yaml
detectors: models:
coral1: - devices:
type: edgetpu - edgetpu:pci:0
device: pci:0 - edgetpu:pci:1
coral2:
type: edgetpu
device: pci:1
``` ```
</TabItem> </TabItem>
@@ -225,13 +270,10 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and selec
<TabItem value="yaml"> <TabItem value="yaml">
```yaml ```yaml
detectors: models:
coral_usb: - devices:
type: edgetpu - edgetpu:usb
device: usb - edgetpu:pci
coral_pci:
type: edgetpu
device: pci
``` ```
</TabItem> </TabItem>
@@ -273,7 +315,7 @@ Hailo8 supports all models in the Hailo Model Zoo that include HailoRT post-proc
## OpenVINO Detector ## OpenVINO Detector
The OpenVINO detector type runs an OpenVINO IR model on AMD and Intel CPUs, Intel GPUs and Intel NPUs. To configure an OpenVINO detector, set the `"type"` attribute to `"openvino"`. The OpenVINO detector type runs an OpenVINO IR model on AMD and Intel CPUs, Intel GPUs and Intel NPUs. To use it, prefix a model's device with `openvino`.
The OpenVINO device to be used is specified using the `"device"` attribute according to the naming conventions in the [Device Documentation](https://docs.openvino.ai/2025/openvino-workflow/running-inference/inference-devices-and-modes.html). The most common devices are `CPU`, `GPU`, or `NPU`. The OpenVINO device to be used is specified using the `"device"` attribute according to the naming conventions in the [Device Documentation](https://docs.openvino.ai/2025/openvino-workflow/running-inference/inference-devices-and-modes.html). The most common devices are `CPU`, `GPU`, or `NPU`.
@@ -286,13 +328,10 @@ OpenVINO is supported on 6th Gen Intel platforms (Skylake) and newer. It will al
When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming GPU resources are available. An example configuration would be: When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming GPU resources are available. An example configuration would be:
```yaml ```yaml
detectors: models:
ov_0: - devices:
type: openvino - openvino:GPU # or NPU
device: GPU # or NPU - openvino:GPU # or NPU
ov_1:
type: openvino
device: GPU # or NPU
``` ```
::: :::
@@ -313,6 +352,12 @@ Intel NPUs cannot be used under Home Assistant OS, which does not include the NP
## Apple Silicon detector ## Apple Silicon detector
:::warning
The network-based detectors (Deepstack, DeGirum, and the Apple Silicon client) are being reworked. Their extra options no longer have a place in the config, so only the endpoint carried in the device string is honored right now: Deepstack ignores `api_key` and `api_timeout`, DeGirum ignores `zoo` and `token`, and the Apple Silicon client ignores `request_timeout_ms` and `linger_ms`. Anything else is dropped when your config is migrated.
:::
The NPU in Apple Silicon can't be accessed from within a container, so the [Apple Silicon detector client](https://github.com/frigate-nvr/apple-silicon-detector) must first be setup. It is recommended to use the Frigate docker image with `-standard-arm64` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-standard-arm64`. The NPU in Apple Silicon can't be accessed from within a container, so the [Apple Silicon detector client](https://github.com/frigate-nvr/apple-silicon-detector) must first be setup. It is recommended to use the Frigate docker image with `-standard-arm64` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-standard-arm64`.
### Setup {#setup-apple-silicon} ### Setup {#setup-apple-silicon}
@@ -453,11 +498,10 @@ If the correct build is used for your GPU then the GPU will be detected and used
When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming GPU resources are available. An example configuration would be: When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming GPU resources are available. An example configuration would be:
```yaml ```yaml
detectors: models:
onnx_0: - devices:
type: onnx - onnx
onnx_1: - onnx
type: onnx
``` ```
::: :::
@@ -470,7 +514,7 @@ detectors:
## CPU Detector (not recommended) ## CPU Detector (not recommended)
The CPU detector type runs a TensorFlow Lite model utilizing the CPU without hardware acceleration. It is recommended to use a hardware accelerated detector type instead for better performance. To configure a CPU based detector, set the `"type"` attribute to `"cpu"`. The CPU detector type runs a TensorFlow Lite model utilizing the CPU without hardware acceleration. It is recommended to use a hardware accelerated detector type instead for better performance. To use it, set a model's device to `cpu`.
:::danger :::danger
@@ -480,7 +524,7 @@ The CPU detector is not recommended for general use. If you do not have GPU or E
The number of threads used by the interpreter can be specified using the `"num_threads"` attribute, and defaults to `3.` The number of threads used by the interpreter can be specified using the `"num_threads"` attribute, and defaults to `3.`
A TensorFlow Lite model is provided in the container at `/cpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with `model.path`. A TensorFlow Lite model is provided in the container at `/cpu_model.tflite` and is used by this detector type by default. To provide your own model, bind mount the file into the container and provide the path with the model's `path`.
### Configuration {#configuration-cpu} ### Configuration {#configuration-cpu}
@@ -490,6 +534,12 @@ When using CPU detectors, you can add one CPU detector per camera. Adding more d
## Deepstack / CodeProject.AI Server Detector ## Deepstack / CodeProject.AI Server Detector
:::warning
The network-based detectors (Deepstack, DeGirum, and the Apple Silicon client) are being reworked. Their extra options no longer have a place in the config, so only the endpoint carried in the device string is honored right now: Deepstack ignores `api_key` and `api_timeout`, DeGirum ignores `zoo` and `token`, and the Apple Silicon client ignores `request_timeout_ms` and `linger_ms`. Anything else is dropped when your config is migrated.
:::
The Deepstack / CodeProject.AI Server detector for Frigate allows you to integrate Deepstack and CodeProject.AI object detection capabilities into Frigate. CodeProject.AI and DeepStack are open-source AI platforms that can be run on various devices such as the Raspberry Pi, Nvidia Jetson, and other compatible hardware. It is important to note that the integration is performed over the network, so the inference times may not be as fast as native Frigate detectors, but it still provides an efficient and reliable solution for object detection and tracking. The Deepstack / CodeProject.AI Server detector for Frigate allows you to integrate Deepstack and CodeProject.AI object detection capabilities into Frigate. CodeProject.AI and DeepStack are open-source AI platforms that can be run on various devices such as the Raspberry Pi, Nvidia Jetson, and other compatible hardware. It is important to note that the integration is performed over the network, so the inference times may not be as fast as native Frigate detectors, but it still provides an efficient and reliable solution for object detection and tracking.
### Setup {#setup-deepstack} ### Setup {#setup-deepstack}
@@ -552,7 +602,7 @@ For detailed instructions on compiling models, refer to the [MemryX Compiler](ht
3. Depending on the model, the compiler may also generate a cropped post-processing network. If present, it will be named with the suffix `_post.onnx`. 3. Depending on the model, the compiler may also generate a cropped post-processing network. If present, it will be named with the suffix `_post.onnx`.
4. Bind-mount the `.zip` file into the container and specify its path using `model.path` in your config. 4. Bind-mount the `.zip` file into the container and specify its path using the model's `path` in your config.
5. Update `labelmap_path` to match your custom model's labels. 5. Update `labelmap_path` to match your custom model's labels.
@@ -682,13 +732,10 @@ If no custom model is provided, the RKNN detector downloads a default model from
When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming NPU resources are available. An example configuration would be: When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming NPU resources are available. An example configuration would be:
```yaml ```yaml
detectors: models:
rknn_0: - devices:
type: rknn - rknn:0
num_cores: 0 - rknn:0
rknn_1:
type: rknn
num_cores: 0
``` ```
::: :::
@@ -762,6 +809,101 @@ Explanation of the parameters:
- **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`. - **example**: Specifying `output_name = "frigate-{quant}-{input_basename}-{soc}-v{tk_version}"` could result in a model called `frigate-i8-my_model-rk3588-v2.3.0.rknn`.
- `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.2/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.2_EN.pdf). - `config`: Configuration passed to `rknn-toolkit2` for model conversion. For an explanation of all available parameters have a look at section "2.2. Model configuration" of [this manual](https://github.com/MarcA711/rknn-toolkit2/releases/download/v2.3.2/03_Rockchip_RKNPU_API_Reference_RKNN_Toolkit2_V2.3.2_EN.pdf).
<<<<<<< HEAD
=======
## DeGirum
:::warning
The network-based detectors (Deepstack, DeGirum, and the Apple Silicon client) are being reworked. Their extra options no longer have a place in the config, so only the endpoint carried in the device string is honored right now: Deepstack ignores `api_key` and `api_timeout`, DeGirum ignores `zoo` and `token`, and the Apple Silicon client ignores `request_timeout_ms` and `linger_ms`. Anything else is dropped when your config is migrated.
:::
DeGirum is a detector that can use any type of hardware listed on [their website](https://hub.degirum.com). DeGirum can be used with local hardware through a DeGirum AI Server, or through the use of `@local`. You can also connect directly to DeGirum's AI Hub to run inferences. **Please Note:** This detector _cannot_ be used for commercial purposes.
### Configuration {#configuration-degirum}
#### AI Server Inference
Before starting with the config file for this section, you must first launch an AI server. DeGirum has an AI server ready to use as a docker container. Add this to your `docker-compose.yml` to get started:
```yaml
degirum_detector:
container_name: degirum
image: degirum/aiserver:latest
privileged: true
ports:
- "8778:8778"
```
All supported hardware will automatically be found on your AI server host as long as relevant runtimes and drivers are properly installed on your machine. Refer to [DeGirum's docs site](https://docs.degirum.com/pysdk/runtimes-and-drivers) if you have any trouble.
Once completed, configure the detector as follows:
<ModelConfigDropdown detectorTitle="DeGirum" models={objectDetectorsModels.degirumAiServer.models} />
The model is set on the same `models` entry as the DeGirum device. You can set it to:
- A model listed on the [AI Hub](https://hub.degirum.com)
- If this is what you choose to do, the correct model will be downloaded onto your machine before running.
- A local directory acting as a zoo. See DeGirum's docs site [for more information](https://docs.degirum.com/pysdk/user-guide-pysdk/organizing-models#model-zoo-directory-structure).
- A path to some model.json.
```yaml
models:
- devices:
- degirum:<location>
path: ./mobilenet_v2_ssd_coco--300x300_quant_n2x_orca1_1 # directory to model .json and file
width: 300 # width is in the model name as the first number in the "int"x"int" section
height: 300 # height is in the model name as the second number in the "int"x"int" section
input_pixel_format: rgb/bgr # look at the model.json to figure out which to put here
```
#### Local Inference
It is also possible to eliminate the need for an AI server and run the hardware directly. The benefit of this approach is that you eliminate any bottlenecks that occur when transferring prediction results from the AI server docker container to the frigate one. However, the method of implementing local inference is different for every device and hardware combination, so it's usually more trouble than it's worth. A general guideline to achieve this would be:
1. Ensuring that the frigate docker container has the runtime you want to use. So for instance, running `@local` for Hailo means making sure the container you're using has the Hailo runtime installed.
2. To double check the runtime is detected by the DeGirum detector, make sure the `degirum sys-info` command properly shows whatever runtimes you mean to install.
3. Create a DeGirum detector in your configuration.
<ModelConfigDropdown detectorTitle="DeGirum" models={objectDetectorsModels.degirumLocal.models} />
Once the DeGirum device is set up, you can choose a model on the same `models` entry in the `config.yml` file.
```yaml
models:
- devices:
- degirum:<location>
path: mobilenet_v2_ssd_coco--300x300_quant_n2x_orca1_1
width: 300 # width is in the model name as the first number in the "int"x"int" section
height: 300 # height is in the model name as the second number in the "int"x"int" section
input_pixel_format: rgb/bgr # look at the model.json to figure out which to put here
```
#### AI Hub Cloud Inference
If you do not possess whatever hardware you want to run, there's also the option to run cloud inferences. Do note that your detection fps might need to be lowered as network latency does significantly slow down this method of detection. For use with Frigate, we highly recommend using a local AI server as described above. To set up cloud inferences,
1. Sign up at [DeGirum's AI Hub](https://hub.degirum.com).
2. Get an access token.
3. Create a DeGirum detector in your configuration.
<ModelConfigDropdown detectorTitle="DeGirum" models={objectDetectorsModels.degirumCloud.models} />
Once the DeGirum device is set up, you can choose a model on the same `models` entry in the `config.yml` file.
```yaml
models:
- devices:
- degirum:<location>
path: mobilenet_v2_ssd_coco--300x300_quant_n2x_orca1_1
width: 300 # width is in the model name as the first number in the "int"x"int" section
height: 300 # height is in the model name as the second number in the "int"x"int" section
input_pixel_format: rgb/bgr # look at the model.json to figure out which to put here
```
>>>>>>> 34363affa (Refactor detector and model management)
## AXERA ## AXERA
Hardware accelerated object detection is supported on the following SoCs: Hardware accelerated object detection is supported on the following SoCs:
docs/docs/guides/getting_started.md
+13 -18
View File
@@ -222,15 +222,12 @@ You need to refer to **Configure hardware acceleration** above to enable the con
```yaml {3-6,9-15,20-21} ```yaml {3-6,9-15,20-21}
mqtt: ... mqtt: ...
detectors: # <---- add detectors models: # <---- add models
ov: - devices:
type: openvino # <---- use openvino detector - openvino:GPU # <---- use the openvino detector on the GPU
device: GPU # We will use the default MobileNet_v2 model from OpenVINO.
width: 300
# We will use the default MobileNet_v2 model from OpenVINO. height: 300
model:
width: 300
height: 300
input_tensor: nhwc input_tensor: nhwc
input_pixel_format: bgr input_pixel_format: bgr
path: /openvino-model/ssdlite_mobilenet_v2.xml path: /openvino-model/ssdlite_mobilenet_v2.xml
@@ -281,10 +278,9 @@ Navigate to <NavPath path="Settings > System > Detectors and model" /> and add a
```yaml {3-6,11-12} ```yaml {3-6,11-12}
mqtt: ... mqtt: ...
detectors: # <---- add detectors models: # <---- add models
coral: - devices:
type: edgetpu - edgetpu:usb
device: usb
cameras: cameras:
name_of_your_camera: name_of_your_camera:
@@ -321,10 +317,9 @@ If you are using YAML to configure Frigate instead of the UI, your configuration
mqtt: mqtt:
enabled: False enabled: False
detectors: models:
coral: - devices:
type: edgetpu - edgetpu:usb
device: usb
cameras: cameras:
name_of_your_camera: name_of_your_camera:
@@ -357,7 +352,7 @@ In order to review activity in the Frigate UI, recordings need to be enabled.
```yaml {16-17} ```yaml {16-17}
mqtt: ... mqtt: ...
detectors: ... models: ...
cameras: cameras:
name_of_your_camera: name_of_your_camera:
docs/docs/integrations/plus.md
+10 -10
View File
@@ -62,10 +62,9 @@ Once you have [requested your first model](../plus/first_model.md) and gotten yo
You can either choose the new model from the <NavPath path="Settings > System > Detectors and model" /> pane in the Frigate UI (the **Frigate+ Model** tab), or manually set the model at the root level in your config: You can either choose the new model from the <NavPath path="Settings > System > Detectors and model" /> pane in the Frigate UI (the **Frigate+ Model** tab), or manually set the model at the root level in your config:
```yaml ```yaml
detectors: ... models:
- devices: ...
model: path: plus://<your_model_id>
path: plus://<your_model_id>
``` ```
:::note :::note
@@ -79,10 +78,11 @@ Models are downloaded into the `/config/model_cache` folder and only downloaded
If needed, you can override the labelmap for Frigate+ models. This is not recommended as renaming labels will break the Submit to Frigate+ feature if the labels are not available in Frigate+. If needed, you can override the labelmap for Frigate+ models. This is not recommended as renaming labels will break the Submit to Frigate+ feature if the labels are not available in Frigate+.
```yaml ```yaml
model: models:
path: plus://<your_model_id> - devices: ...
labelmap: path: plus://<your_model_id>
3: animal labelmap:
4: animal 3: animal
5: animal 4: animal
5: animal
``` ```
docs/docs/plus/first_model.md
+3 -4
View File
@@ -36,10 +36,9 @@ Navigate to <NavPath path="Settings > System > Detectors and model" />. In the *
<TabItem value="yaml"> <TabItem value="yaml">
```yaml ```yaml
detectors: ... models:
- devices: ...
model: path: plus://<your_model_id>
path: plus://<your_model_id>
``` ```
:::tip :::tip
frigate/api/app.py
+24 -28
View File
@@ -292,10 +292,6 @@ def config(request: Request):
config: dict[str, dict[str, Any]] = config_obj.model_dump( config: dict[str, dict[str, Any]] = config_obj.model_dump(
mode="json", warnings="none", exclude_none=True mode="json", warnings="none", exclude_none=True
) )
config["detectors"] = {
name: detector.model_dump(mode="json", warnings="none", exclude_none=True)
for name, detector in config_obj.detectors.items()
}
# remove environment_vars for non-admin users # remove environment_vars for non-admin users
if request.headers.get("remote-role") != "admin": if request.headers.get("remote-role") != "admin":
@@ -376,31 +372,28 @@ def config(request: Request):
config["go2rtc"]["streams"][stream_name] = cleaned config["go2rtc"]["streams"][stream_name] = cleaned
config["plus"] = {"enabled": request.app.frigate_config.plus_api.is_active()} config["plus"] = {"enabled": request.app.frigate_config.plus_api.is_active()}
config["model"]["colormap"] = config_obj.model.colormap
config["model"]["all_attributes"] = config_obj.model.all_attributes
config["model"]["non_logo_attributes"] = config_obj.model.non_logo_attributes
# Add model plus data if plus is enabled for index, model in enumerate(config_obj.models):
if config["plus"]["enabled"]: model_dict = config["models"][index]
model_path = config.get("model", {}).get("path") model_dict["colormap"] = model.colormap
if model_path: model_dict["all_attributes"] = model.all_attributes
model_json_path = FilePath(model_path).with_suffix(".json") model_dict["non_logo_attributes"] = model.non_logo_attributes
model_dict["labelmap"] = model.merged_labelmap
if not config["plus"]["enabled"]:
continue
# Add model plus data if plus is enabled
model_dict["plus"] = None
if model.path:
model_json_path = FilePath(model.path).with_suffix(".json")
try: try:
with open(model_json_path) as f: with open(model_json_path) as f:
model_plus_data = json.load(f) model_dict["plus"] = json.load(f)
config["model"]["plus"] = model_plus_data except (FileNotFoundError, json.JSONDecodeError):
except FileNotFoundError: pass
config["model"]["plus"] = None
except json.JSONDecodeError:
config["model"]["plus"] = None
else:
config["model"]["plus"] = None
# use merged labelamp
for detector_config in config["detectors"].values():
detector_config["model"]["labelmap"] = (
request.app.frigate_config.model.merged_labelmap
)
return JSONResponse(content=config) return JSONResponse(content=config)
@@ -1360,11 +1353,14 @@ def plusModels(request: Request, filterByCurrentModelDetector: bool = False):
modelList = models["list"] modelList = models["list"]
config: FrigateConfig = request.app.frigate_config
primary_model = config.primary_model
# current model type # current model type
modelType = request.app.frigate_config.model.model_type modelType = primary_model.model_type
# current detectorType for comparing to supportedDetectors # current detectorType for comparing to supportedDetectors
detectorType = list(request.app.frigate_config.detectors.values())[0].type detectorType = config.devices_for_model(primary_model)[0].detector
validModels = [] validModels = []
frigate/api/media.py
+1 -1
View File
@@ -941,7 +941,7 @@ async def event_snapshot(
timestamp_style=request.app.frigate_config.cameras[ timestamp_style=request.app.frigate_config.cameras[
event.camera event.camera
].timestamp_style, ].timestamp_style,
colormap=request.app.frigate_config.model.colormap, colormap=request.app.frigate_config.model_for_camera(event.camera).colormap,
) )
except DoesNotExist: except DoesNotExist:
# see if the object is currently being tracked # see if the object is currently being tracked
frigate/app.py
+39 -22
View File
@@ -49,6 +49,8 @@ from frigate.debug_replay import (
DebugReplayManager, DebugReplayManager,
cleanup_replay_cameras, cleanup_replay_cameras,
) )
from frigate.detectors.detector_config import SceneEnum
from frigate.detectors.device import build_detector_config, runner_names
from frigate.embeddings import EmbeddingProcess, EmbeddingsContext from frigate.embeddings import EmbeddingProcess, EmbeddingsContext
from frigate.events.audio import AudioProcessor from frigate.events.audio import AudioProcessor
from frigate.events.cleanup import EventCleanup from frigate.events.cleanup import EventCleanup
@@ -69,6 +71,7 @@ from frigate.models import (
User, User,
) )
from frigate.object_detection.base import ObjectDetectProcess from frigate.object_detection.base import ObjectDetectProcess
from frigate.object_detection.util import detection_frame_size
from frigate.output.output import OutputProcess from frigate.output.output import OutputProcess
from frigate.ptz.autotrack import PtzAutoTrackerThread from frigate.ptz.autotrack import PtzAutoTrackerThread
from frigate.ptz.onvif import OnvifController from frigate.ptz.onvif import OnvifController
@@ -98,7 +101,9 @@ class FrigateApp:
self.metrics_manager = manager self.metrics_manager = manager
self.audio_process: mp.Process | None = None self.audio_process: mp.Process | None = None
self.stop_event = stop_event self.stop_event = stop_event
self.detection_queue: Queue = mp.Queue() self.detection_queues: dict[SceneEnum, Queue] = {
model.scene: mp.Queue() for model in config.models
}
self.detectors: dict[str, ObjectDetectProcess] = {} self.detectors: dict[str, ObjectDetectProcess] = {}
self.detection_shms: list[mp.shared_memory.SharedMemory] = [] self.detection_shms: list[mp.shared_memory.SharedMemory] = []
self.log_queue: Queue = mp.Queue() self.log_queue: Queue = mp.Queue()
@@ -344,20 +349,19 @@ class FrigateApp:
self.dispatcher.profile_manager = self.profile_manager self.dispatcher.profile_manager = self.profile_manager
def start_detectors(self) -> None: def start_detectors(self) -> None:
model_cameras: dict[SceneEnum, list[str]] = {
model.scene: [] for model in self.config.models
}
for name in self.config.cameras.keys(): for name in self.config.cameras.keys():
model = self.config.model_for_camera(name)
model_cameras[model.scene].append(name)
try: try:
largest_frame = max(
[
det.model.height * det.model.width * 3
if det.model is not None
else 320
for det in self.config.detectors.values()
]
)
shm_in = UntrackedSharedMemory( shm_in = UntrackedSharedMemory(
name=name, name=name,
create=True, create=True,
size=largest_frame, size=detection_frame_size(model),
) )
except FileExistsError: except FileExistsError:
shm_in = UntrackedSharedMemory(name=name) shm_in = UntrackedSharedMemory(name=name)
@@ -372,15 +376,26 @@ class FrigateApp:
self.detection_shms.append(shm_in) self.detection_shms.append(shm_in)
self.detection_shms.append(shm_out) self.detection_shms.append(shm_out)
for name, detector_config in self.config.detectors.items(): # a device may be listed more than once to run additional inference
self.detectors[name] = ObjectDetectProcess( # processes on it, so names are only unique once de-duplicated
name, all_devices = [
self.detection_queue, device
list(self.config.cameras.keys()), for model in self.config.models
self.config, for device in self.config.devices_for_model(model)
detector_config, ]
self.stop_event, names = iter(runner_names(all_devices))
)
for model in self.config.models:
for device in self.config.devices_for_model(model):
name = next(names)
self.detectors[name] = ObjectDetectProcess(
name,
self.detection_queues[model.scene],
model_cameras[model.scene],
self.config,
build_detector_config(device, model),
self.stop_event,
)
def start_ptz_autotracker(self) -> None: def start_ptz_autotracker(self) -> None:
self.ptz_autotracker_thread = PtzAutoTrackerThread( self.ptz_autotracker_thread = PtzAutoTrackerThread(
@@ -411,7 +426,7 @@ class FrigateApp:
def start_camera_processor(self) -> None: def start_camera_processor(self) -> None:
self.camera_maintainer = CameraMaintainer( self.camera_maintainer = CameraMaintainer(
self.config, self.config,
self.detection_queue, self.detection_queues,
self.detected_frames_queue, self.detected_frames_queue,
self.camera_metrics, self.camera_metrics,
self.ptz_metrics, self.ptz_metrics,
@@ -675,8 +690,10 @@ class FrigateApp:
for detector in self.detectors.values(): for detector in self.detectors.values():
detector.stop() detector.stop()
empty_and_close_queue(self.detection_queue) for detection_queue in self.detection_queues.values():
logger.info("Detection queue closed") empty_and_close_queue(detection_queue)
logger.info("Detection queues closed")
self.detected_frames_processor.join() self.detected_frames_processor.join()
empty_and_close_queue(self.detected_frames_queue) empty_and_close_queue(self.detected_frames_queue)
frigate/camera/activity_manager.py
+2 -1
View File
@@ -18,6 +18,7 @@ from frigate.config.camera.updater import (
CameraConfigUpdateEnum, CameraConfigUpdateEnum,
CameraConfigUpdateSubscriber, CameraConfigUpdateSubscriber,
) )
from frigate.detectors.detector_config import NON_LOGO_ATTRIBUTES
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -178,7 +179,7 @@ class CameraActivityManager:
return return
for label in camera_config.objects.track: for label in camera_config.objects.track:
if label in self.config.model.non_logo_attributes: if label in NON_LOGO_ATTRIBUTES:
continue continue
new_count = all_objects[label] new_count = all_objects[label]
frigate/camera/maintainer.py
+12 -16
View File
@@ -15,7 +15,9 @@ from frigate.config.camera.updater import (
CameraConfigUpdateSubscriber, CameraConfigUpdateSubscriber,
) )
from frigate.const import REPLAY_CAMERA_PREFIX from frigate.const import REPLAY_CAMERA_PREFIX
from frigate.detectors.detector_config import SceneEnum
from frigate.models import Regions from frigate.models import Regions
from frigate.object_detection.util import detection_frame_size
from frigate.util.builtin import empty_and_close_queue from frigate.util.builtin import empty_and_close_queue
from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory
from frigate.util.object import get_camera_regions_grid from frigate.util.object import get_camera_regions_grid
@@ -29,7 +31,7 @@ class CameraMaintainer(threading.Thread):
def __init__( def __init__(
self, self,
config: FrigateConfig, config: FrigateConfig,
detection_queue: Queue, detection_queues: dict[SceneEnum, Queue],
detected_frames_queue: Queue, detected_frames_queue: Queue,
camera_metrics: DictProxy, camera_metrics: DictProxy,
ptz_metrics: dict[str, PTZMetrics], ptz_metrics: dict[str, PTZMetrics],
@@ -38,7 +40,7 @@ class CameraMaintainer(threading.Thread):
): ):
super().__init__(name="camera_processor") super().__init__(name="camera_processor")
self.config = config self.config = config
self.detection_queue = detection_queue self.detection_queues = detection_queues
self.detected_frames_queue = detected_frames_queue self.detected_frames_queue = detected_frames_queue
self.stop_event = stop_event self.stop_event = stop_event
self.camera_metrics = camera_metrics self.camera_metrics = camera_metrics
@@ -79,10 +81,11 @@ class CameraMaintainer(threading.Thread):
# create or update region grids for each camera # create or update region grids for each camera
for camera in self.config.cameras.values(): for camera in self.config.cameras.values():
assert camera.name is not None assert camera.name is not None
model = self.config.model_for_camera(camera.name)
self.region_grids[camera.name] = get_camera_regions_grid( self.region_grids[camera.name] = get_camera_regions_grid(
camera.name, camera.name,
camera.detect, camera.detect,
max(self.config.model.width, self.config.model.height), max(model.width, model.height),
) )
def __calculate_shm_frame_count(self) -> int: def __calculate_shm_frame_count(self) -> int:
@@ -114,6 +117,7 @@ class CameraMaintainer(threading.Thread):
return return
camera_stop_event = self.__ensure_camera_stop_event(name) camera_stop_event = self.__ensure_camera_stop_event(name)
model = self.config.model_for_camera(name)
if runtime: if runtime:
self.camera_metrics[name] = CameraMetrics(self.metrics_manager) self.camera_metrics[name] = CameraMetrics(self.metrics_manager)
@@ -123,32 +127,24 @@ class CameraMaintainer(threading.Thread):
self.region_grids[name] = get_camera_regions_grid( self.region_grids[name] = get_camera_regions_grid(
name, name,
config.detect, config.detect,
max(self.config.model.width, self.config.model.height), max(model.width, model.height),
) )
try: try:
largest_frame = max(
[
det.model.height * det.model.width * 3
if det.model is not None
else 320
for det in self.config.detectors.values()
]
)
UntrackedSharedMemory(name=f"out-{name}", create=True, size=20 * 6 * 4) UntrackedSharedMemory(name=f"out-{name}", create=True, size=20 * 6 * 4)
UntrackedSharedMemory( UntrackedSharedMemory(
name=name, name=name,
create=True, create=True,
size=largest_frame, size=detection_frame_size(model),
) )
except FileExistsError: except FileExistsError:
pass pass
camera_process = CameraTracker( camera_process = CameraTracker(
config, config,
self.config.model, model,
self.config.model.merged_labelmap, model.merged_labelmap,
self.detection_queue, self.detection_queues[model.scene],
self.detected_frames_queue, self.detected_frames_queue,
self.camera_metrics[name], self.camera_metrics[name],
self.ptz_metrics[name], self.ptz_metrics[name],
frigate/camera/state.py
+6 -11
View File
@@ -40,6 +40,7 @@ class CameraState:
self.name = name self.name = name
self.config = config self.config = config
self.camera_config = config.cameras[name] self.camera_config = config.cameras[name]
self.model = config.model_for_camera(name)
self.frame_manager = frame_manager self.frame_manager = frame_manager
self.best_objects: dict[str, TrackedObject] = {} self.best_objects: dict[str, TrackedObject] = {}
self.tracked_objects: dict[str, TrackedObject] = {} self.tracked_objects: dict[str, TrackedObject] = {}
@@ -106,9 +107,7 @@ class CameraState:
thickness = 1 thickness = 1
else: else:
thickness = 2 thickness = 2
color = self.config.model.colormap.get( color = self.model.colormap.get(obj["label"], (255, 255, 255))
obj["label"], (255, 255, 255)
)
else: else:
thickness = 1 thickness = 1
color = (255, 0, 0) color = (255, 0, 0)
@@ -130,9 +129,7 @@ class CameraState:
and obj["frame_time"] == frame_time and obj["frame_time"] == frame_time
): ):
thickness = 5 thickness = 5
color = self.config.model.colormap.get( color = self.model.colormap.get(obj["label"], (255, 255, 255))
obj["label"], (255, 255, 255)
)
# debug autotracking zooming - show the zoom factor box # debug autotracking zooming - show the zoom factor box
if ( if (
@@ -266,9 +263,7 @@ class CameraState:
if draw_options.get("paths"): if draw_options.get("paths"):
for obj in tracked_objects.values(): for obj in tracked_objects.values():
if obj["frame_time"] == frame_time and obj["path_data"]: if obj["frame_time"] == frame_time and obj["path_data"]:
color = self.config.model.colormap.get( color = self.model.colormap.get(obj["label"], (255, 255, 255))
obj["label"], (255, 255, 255)
)
path_points = [ path_points = [
( (
@@ -371,7 +366,7 @@ class CameraState:
for id in new_ids: for id in new_ids:
logger.debug(f"{self.name}: New tracked object ID: {id}") logger.debug(f"{self.name}: New tracked object ID: {id}")
new_obj = tracked_objects[id] = TrackedObject( new_obj = tracked_objects[id] = TrackedObject(
self.config.model, self.model,
self.camera_config, self.camera_config,
self.config.ui, self.config.ui,
self.frame_cache, self.frame_cache,
@@ -515,7 +510,7 @@ class CameraState:
sub_label = None sub_label = None
if obj.obj_data.get("sub_label"): if obj.obj_data.get("sub_label"):
if obj.obj_data["sub_label"][0] in self.config.model.all_attributes: if obj.obj_data["sub_label"][0] in self.model.all_attributes:
label = obj.obj_data["sub_label"][0] label = obj.obj_data["sub_label"][0]
else: else:
label = f"{object_type}-verified" label = f"{object_type}-verified"
frigate/comms/dispatcher.py
+2 -1
View File
@@ -261,10 +261,11 @@ class Dispatcher:
if camera not in self.config.cameras: if camera not in self.config.cameras:
return None return None
model = self.config.model_for_camera(camera)
grid = get_camera_regions_grid( grid = get_camera_regions_grid(
camera, camera,
self.config.cameras[camera].detect, self.config.cameras[camera].detect,
max(self.config.model.width, self.config.model.height), max(model.width, model.height),
) )
return grid return grid
frigate/config/camera/detect.py
+7
View File
@@ -1,5 +1,7 @@
from pydantic import Field, model_validator from pydantic import Field, model_validator
from frigate.detectors.detector_config import SceneEnum
from ..base import FrigateBaseModel from ..base import FrigateBaseModel
__all__ = ["DetectConfig", "StationaryConfig", "StationaryMaxFramesConfig"] __all__ = ["DetectConfig", "StationaryConfig", "StationaryMaxFramesConfig"]
@@ -60,6 +62,11 @@ class DetectConfig(FrigateBaseModel):
title="Detect width", title="Detect width",
description="Width (pixels) of frames used for the detect stream; leave empty to use the native stream resolution.", description="Width (pixels) of frames used for the detect stream; leave empty to use the native stream resolution.",
) )
scene: SceneEnum | None = Field(
default=None,
title="Detect scene",
description="The environment this camera looks at, used to pick which of the configured models runs on it. Defaults to the model with a scene of 'all'.",
)
fps: int = Field( fps: int = Field(
default=5, default=5,
title="Detect FPS", title="Detect FPS",
frigate/config/config.py
+226 -59
View File
@@ -11,7 +11,6 @@ from pydantic import (
BaseModel, BaseModel,
ConfigDict, ConfigDict,
Field, Field,
TypeAdapter,
ValidationInfo, ValidationInfo,
field_validator, field_validator,
model_validator, model_validator,
@@ -19,8 +18,9 @@ from pydantic import (
from ruamel.yaml import YAML from ruamel.yaml import YAML
from frigate.const import REGEX_JSON from frigate.const import REGEX_JSON
from frigate.detectors import DetectorConfig, ModelConfig from frigate.detectors import ModelConfig
from frigate.detectors.detector_config import BaseDetectorConfig from frigate.detectors.detector_config import SceneEnum
from frigate.detectors.device import DeviceParseError, DeviceSpec, parse_device
from frigate.plus import PlusApi from frigate.plus import PlusApi
from frigate.util.builtin import ( from frigate.util.builtin import (
deep_merge, deep_merge,
@@ -79,9 +79,14 @@ logger = logging.getLogger(__name__)
yaml = YAML() yaml = YAML()
# Pydantic field default applied when an existing config omits `detectors:`. # Pydantic field default applied when an existing config omits `models:`.
# Kept as cpu tflite for backwards compatibility with 0.17 configs. # Kept as cpu tflite for backwards compatibility with 0.17 configs.
DEFAULT_DETECTORS = {"cpu": {"type": "cpu"}} DEFAULT_MODELS = [{"devices": ["cpu"]}]
def _default_models() -> list[ModelConfig]:
return [ModelConfig.model_validate(model) for model in DEFAULT_MODELS]
# Used by the openvino branch below and rendered into the new-config YAML # Used by the openvino branch below and rendered into the new-config YAML
# template so first-time setups default to openvino on CPU. # template so first-time setups default to openvino on CPU.
@@ -93,7 +98,7 @@ DEFAULT_MODEL = {
"path": "/openvino-model/ssdlite_mobilenet_v2.xml", "path": "/openvino-model/ssdlite_mobilenet_v2.xml",
"labelmap_path": "/openvino-model/coco_91cl_bkgr.txt", "labelmap_path": "/openvino-model/coco_91cl_bkgr.txt",
} }
NEW_CONFIG_DETECTORS = {"ov": {"type": "openvino", "device": "CPU"}} NEW_CONFIG_MODELS = [{"devices": ["openvino:CPU"], **DEFAULT_MODEL}]
DEFAULT_DETECT_DIMENSIONS = {"width": 1280, "height": 720} DEFAULT_DETECT_DIMENSIONS = {"width": 1280, "height": 720}
@@ -109,7 +114,7 @@ DEFAULT_CONFIG = f"""
mqtt: mqtt:
enabled: False enabled: False
{_render_default_yaml({"detectors": NEW_CONFIG_DETECTORS, "model": DEFAULT_MODEL})} {_render_default_yaml({"models": NEW_CONFIG_MODELS})}
cameras: {{}} # No cameras defined, UI wizard should be used cameras: {{}} # No cameras defined, UI wizard should be used
version: {CURRENT_CONFIG_VERSION} version: {CURRENT_CONFIG_VERSION}
""" """
@@ -520,16 +525,11 @@ class FrigateConfig(FrigateBaseModel):
description="User interface preferences such as timezone, time/date formatting, and units.", description="User interface preferences such as timezone, time/date formatting, and units.",
) )
# Detector config # Detection model config
detectors: dict[str, BaseDetectorConfig] = Field( models: list[ModelConfig] = Field(
default=DEFAULT_DETECTORS, default_factory=_default_models,
title="Detector hardware", title="Detection models",
description="Configuration for object detectors (CPU, GPU, ONNX backends) and any detector-specific model settings.", description="Object detection models and the hardware each one runs on. Cameras pick a model by matching their detect.scene against a model's scene.",
)
model: ModelConfig = Field(
default_factory=ModelConfig,
title="Detection model",
description="Settings to configure a custom object detection model and its input shape.",
) )
# GenAI config (named provider configs: name -> GenAIConfig) # GenAI config (named provider configs: name -> GenAIConfig)
@@ -644,11 +644,202 @@ class FrigateConfig(FrigateBaseModel):
) )
_plus_api: PlusApi _plus_api: PlusApi
_model_devices: dict[SceneEnum, list[DeviceSpec]]
_camera_models: dict[str, ModelConfig]
_all_attributes: list[str]
_all_attribute_logos: list[str]
_all_attributes_map: dict[str, list[str]]
_all_labels: set[str]
@property @property
def plus_api(self) -> PlusApi: def plus_api(self) -> PlusApi:
return self._plus_api return self._plus_api
@property
def all_attributes(self) -> list[str]:
"""Every attribute label across all configured models."""
return self._all_attributes
@property
def all_attribute_logos(self) -> list[str]:
"""Every logo attribute label across all configured models."""
return self._all_attribute_logos
@property
def all_attributes_map(self) -> dict[str, list[str]]:
"""Object label to attribute labels, merged across all configured models."""
return self._all_attributes_map
@property
def all_labels(self) -> set[str]:
"""Every object label across all configured models."""
return self._all_labels
@property
def primary_model(self) -> ModelConfig:
"""The model used when no specific camera is in play."""
for model in self.models:
if model.scene == SceneEnum.all:
return model
return self.models[0]
def model_for_camera(self, camera_name: str) -> ModelConfig:
"""Get the detection model a camera runs on.
Args:
camera_name: Name of the camera
Returns:
The model matching the camera's detect scene
"""
return self._camera_models[camera_name]
def devices_for_model(self, model: ModelConfig) -> list[DeviceSpec]:
"""Get the parsed hardware devices a model runs on.
Args:
model: One of the configured models
Returns:
The parsed device specs, in config order
"""
return self._model_devices[model.scene]
def _load_model(self, model: ModelConfig, detector: str) -> ModelConfig:
"""Apply detector specific defaults to a model and load its weights and labels.
Args:
model: The configured model
detector: The detector type the model runs on
Returns:
The loaded model
"""
model_config = model.model_dump(exclude_unset=True, warnings="none")
if "path" not in model_config:
if detector == "cpu" or detector.endswith("_tfl"):
model_config["path"] = "/cpu_model.tflite"
elif detector == "edgetpu":
model_config["path"] = "/edgetpu_model.tflite"
elif detector == "openvino":
for default_key, default_value in DEFAULT_MODEL.items():
model_config.setdefault(default_key, default_value)
loaded = ModelConfig.model_validate(model_config)
loaded.check_and_load_plus_model(self.plus_api, detector)
loaded.compute_model_hash()
return loaded
def _load_models(self) -> None:
"""Validate the configured models and load each one."""
if not self.models:
raise ValueError("At least one model must be configured under models")
model_devices: dict[SceneEnum, list[DeviceSpec]] = {}
# device string -> the scene of the model that already claimed it
claimed_devices: dict[str, SceneEnum] = {}
for index, model in enumerate(self.models):
scene = model.scene.value
if model.scene in model_devices:
raise ValueError(
f"Multiple models are configured with a scene of '{scene}'. Each model must use a different scene."
)
if not model.devices:
raise ValueError(
f"Model '{scene}' must list at least one entry under devices."
)
try:
devices = [parse_device(device) for device in model.devices]
except DeviceParseError as err:
raise ValueError(
f"Model '{scene}' has an invalid device: {err}"
) from err
detectors = {device.detector for device in devices}
if len(detectors) > 1:
raise ValueError(
f"Model '{scene}' mixes the {', '.join(sorted(detectors))} detectors. All of a model's devices must use the same detector."
)
for device in devices:
if device.raw in claimed_devices and not device.shareable:
other = claimed_devices[device.raw]
where = (
f"twice by model '{scene}'"
if other == model.scene
else f"by both the '{other.value}' and '{scene}' models"
)
raise ValueError(
f"Device '{device.raw}' is used {where}, but it can only run one detection process."
)
claimed_devices[device.raw] = model.scene
self.models[index] = self._load_model(model, devices[0].detector)
model_devices[model.scene] = devices
attributes: set[str] = set()
attribute_logos: set[str] = set()
attributes_map: dict[str, set[str]] = {}
labels: set[str] = set()
for model in self.models:
attributes.update(model.all_attributes)
attribute_logos.update(model.all_attribute_logos)
labels.update(model.merged_labelmap.values())
for label, label_attributes in model.attributes_map.items():
attributes_map.setdefault(label, set()).update(label_attributes)
self._model_devices = model_devices
self._all_attributes = sorted(attributes)
self._all_attribute_logos = sorted(attribute_logos)
self._all_attributes_map = {
label: sorted(label_attributes)
for label, label_attributes in sorted(attributes_map.items())
}
self._all_labels = labels
def _resolve_camera_model(self, name: str, scene: SceneEnum | None) -> ModelConfig:
"""Resolve which model a camera runs on.
Args:
name: Name of the camera
scene: The camera's configured detect scene, if any
Returns:
The model the camera runs on
"""
by_scene = {model.scene: model for model in self.models}
if scene is not None:
model = by_scene.get(scene)
if model is None:
raise ValueError(
f"Camera '{name}' has a detect scene of '{scene.value}', but no model is configured for that scene."
)
return model
default = by_scene.get(SceneEnum.all) or (
self.models[0] if len(self.models) == 1 else None
)
if default is None:
raise ValueError(
f"Camera '{name}' must set detect -> scene, because more than one model is configured and none of them uses a scene of 'all'."
)
return default
@model_validator(mode="after") @model_validator(mode="after")
def post_validation(self, info: ValidationInfo) -> Self: def post_validation(self, info: ValidationInfo) -> Self:
# Load plus api from context, if possible. # Load plus api from context, if possible.
@@ -693,8 +884,10 @@ class FrigateConfig(FrigateBaseModel):
"'embeddings' in its roles for semantic search." "'embeddings' in its roles for semantic search."
) )
self._load_models()
# set default min_score for object attributes # set default min_score for object attributes
for attribute in self.model.all_attributes: for attribute in self.all_attributes:
existing = self.objects.filters.get(attribute) existing = self.objects.filters.get(attribute)
if existing is None: if existing is None:
self.objects.filters[attribute] = FilterConfig(min_score=0.7) self.objects.filters[attribute] = FilterConfig(min_score=0.7)
@@ -744,44 +937,7 @@ class FrigateConfig(FrigateBaseModel):
exclude_unset=True, exclude_unset=True,
) )
for key, detector in self.detectors.items(): self._camera_models = {}
adapter = TypeAdapter(DetectorConfig)
model_dict = (
detector
if isinstance(detector, dict)
else detector.model_dump(warnings="none")
)
detector_config: BaseDetectorConfig = adapter.validate_python(model_dict)
# users should not set model themselves
if detector_config.model:
logger.warning(
"The model key should be specified at the root level of the config, not under detectors. The nested model key will be ignored."
)
detector_config.model = None
model_config = self.model.model_dump(exclude_unset=True, warnings="none")
if detector_config.model_path:
model_config["path"] = detector_config.model_path
if "path" not in model_config:
if detector_config.type == "cpu" or detector_config.type.endswith(
"_tfl"
):
model_config["path"] = "/cpu_model.tflite"
elif detector_config.type == "edgetpu":
model_config["path"] = "/edgetpu_model.tflite"
elif detector_config.type == "openvino":
for default_key, default_value in DEFAULT_MODEL.items():
model_config.setdefault(default_key, default_value)
model = ModelConfig.model_validate(model_config)
model.check_and_load_plus_model(self.plus_api, detector_config.type)
model.compute_model_hash()
labelmap_objects = model.merged_labelmap.values()
detector_config.model = model
self.detectors[key] = detector_config
for name, camera in self.cameras.items(): for name, camera in self.cameras.items():
modified_global_config = global_config.copy() modified_global_config = global_config.copy()
@@ -808,6 +964,9 @@ class FrigateConfig(FrigateBaseModel):
{"name": name, **merged_config} {"name": name, **merged_config}
) )
camera_model = self._resolve_camera_model(name, camera_config.detect.scene)
self._camera_models[name] = camera_model
if camera_config.ffmpeg.hwaccel_args == "auto": if camera_config.ffmpeg.hwaccel_args == "auto":
camera_config.ffmpeg.hwaccel_args = self.ffmpeg.hwaccel_args camera_config.ffmpeg.hwaccel_args = self.ffmpeg.hwaccel_args
@@ -1028,7 +1187,7 @@ class FrigateConfig(FrigateBaseModel):
verify_profile_overrides_match_base(camera_config) verify_profile_overrides_match_base(camera_config)
verify_autotrack_zones(camera_config) verify_autotrack_zones(camera_config)
verify_motion_and_detect(camera_config) verify_motion_and_detect(camera_config)
verify_objects_track(camera_config, labelmap_objects) verify_objects_track(camera_config, camera_model.merged_labelmap.values())
verify_lpr_and_face(self, camera_config) verify_lpr_and_face(self, camera_config)
# Validate camera profiles reference top-level profile definitions # Validate camera profiles reference top-level profile definitions
@@ -1045,8 +1204,16 @@ class FrigateConfig(FrigateBaseModel):
config.name = name config.name = name
self.objects.parse_all_objects(self.cameras) self.objects.parse_all_objects(self.cameras)
self.model.create_colormap(sorted(self.objects.all_objects))
self.model.check_and_load_plus_model(self.plus_api) # every model shares one colormap so a label is drawn the same color no
# matter which model detected it, so filter attributes across all models
# rather than letting each model filter with only its own
colored_labels = sorted(
set(self.objects.all_objects) - set(self.all_attributes)
)
for model in self.models:
model.create_colormap(colored_labels)
# Check audio transcription and audio detection requirements # Check audio transcription and audio detection requirements
if self.audio_transcription.enabled: if self.audio_transcription.enabled:
frigate/data_processing/common/license_plate/mixin.py
+1 -1
View File
@@ -72,7 +72,7 @@ class LicensePlateProcessingMixin:
# Object config # Object config
self.lp_objects: list[str] = [] self.lp_objects: list[str] = []
for obj, attributes in self.config.model.attributes_map.items(): for obj, attributes in self.config.all_attributes_map.items():
if "license_plate" in attributes: if "license_plate" in attributes:
self.lp_objects.append(obj) self.lp_objects.append(obj)
frigate/data_processing/post/review_descriptions.py
+2 -2
View File
@@ -234,8 +234,8 @@ class ReviewDescriptionProcessor(PostProcessorApi):
final_data, final_data,
thumbs, thumbs,
camera_config.review.genai, camera_config.review.genai,
list(self.config.model.merged_labelmap.values()), sorted(self.config.all_labels),
self.config.model.all_attributes, self.config.all_attributes,
), ),
).start() ).start()
frigate/detectors/detector_config.py
+34 -5
View File
@@ -3,7 +3,7 @@ import json
import logging import logging
import os import os
from enum import Enum from enum import Enum
from typing import Any from typing import Any, ClassVar
import requests import requests
from pydantic import BaseModel, ConfigDict, Field from pydantic import BaseModel, ConfigDict, Field
@@ -15,6 +15,9 @@ from frigate.util.builtin import generate_color_palette, load_labels
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# attributes that are recognized rather than shown as a logo
NON_LOGO_ATTRIBUTES = ["face", "license_plate"]
class PixelFormatEnum(str, Enum): class PixelFormatEnum(str, Enum):
rgb = "rgb" rgb = "rgb"
@@ -44,7 +47,27 @@ class ModelTypeEnum(str, Enum):
yologeneric = "yolo-generic" yologeneric = "yolo-generic"
class SceneEnum(str, Enum):
"""The camera environment a detection model is intended for."""
all = "all"
indoor = "indoor"
outdoor = "outdoor"
indoor_thermal = "indoor_thermal"
outdoor_thermal = "outdoor_thermal"
class ModelConfig(BaseModel): class ModelConfig(BaseModel):
scene: SceneEnum = Field(
default=SceneEnum.all,
title="Model scene",
description="The camera environment this model is used for. Cameras select a model by setting detect.scene to a matching value, and 'all' is used by any camera that does not set one.",
)
devices: list[str] = Field(
default_factory=list,
title="Detection hardware",
description="Hardware this model runs on, as '<detector>' or '<detector>:<device>' (for example 'edgetpu:pci:0' or 'openvino:GPU'). Listing the same device more than once runs additional inference processes on it.",
)
path: str | None = Field( path: str | None = Field(
None, None,
title="Custom object detector model path", title="Custom object detector model path",
@@ -111,7 +134,7 @@ class ModelConfig(BaseModel):
@property @property
def non_logo_attributes(self) -> list[str]: def non_logo_attributes(self) -> list[str]:
return ["face", "license_plate"] return NON_LOGO_ATTRIBUTES
@property @property
def all_attributes(self) -> list[str]: def all_attributes(self) -> list[str]:
@@ -201,9 +224,7 @@ class ModelConfig(BaseModel):
unique_attributes.update(attributes) unique_attributes.update(attributes)
self._all_attributes = list(unique_attributes) self._all_attributes = list(unique_attributes)
self._all_attribute_logos = list( self._all_attribute_logos = list(unique_attributes - set(NON_LOGO_ATTRIBUTES))
unique_attributes - set(["face", "license_plate"])
)
self._merged_labelmap = { self._merged_labelmap = {
**{int(key): val for key, val in model_info["labelMap"].items()}, **{int(key): val for key, val in model_info["labelMap"].items()},
@@ -234,6 +255,14 @@ class ModelConfig(BaseModel):
class BaseDetectorConfig(BaseModel): class BaseDetectorConfig(BaseModel):
# how the trailing part of a device string ("openvino:GPU" -> "GPU") maps onto
# this detector's fields, and whether the same device may be listed more than
# once to run additional inference processes against it. Most accelerators
# multiplex fine, so this is opt-out rather than opt-in.
device_spec_field: ClassVar[str] = "device"
device_spec_type: ClassVar[type] = str
shareable: ClassVar[bool] = True
# the type field must be defined in all subclasses # the type field must be defined in all subclasses
type: str = Field( type: str = Field(
default="cpu", default="cpu",
frigate/detectors/detector_types.py
+19 -1
View File
@@ -2,7 +2,7 @@ import importlib
import logging import logging
import pkgutil import pkgutil
from enum import Enum from enum import Enum
from typing import Annotated, Union from typing import Annotated, Union, get_args
from pydantic import Field from pydantic import Field
@@ -39,3 +39,21 @@ DetectorConfig = Annotated[
Union[tuple(BaseDetectorConfig.__subclasses__())], # noqa: UP007 Union[tuple(BaseDetectorConfig.__subclasses__())], # noqa: UP007
Field(discriminator="type"), Field(discriminator="type"),
] ]
def _discriminator_value(config_class: type[BaseDetectorConfig]) -> str | None:
"""Read the Literal value of a detector config class' type field."""
field = config_class.model_fields.get("type")
if field is None:
return None
values = get_args(field.annotation)
return values[0] if values else None
config_types: dict[str, type[BaseDetectorConfig]] = {
key: config_class
for config_class in BaseDetectorConfig.__subclasses__()
if (key := _discriminator_value(config_class)) is not None
}
frigate/detectors/device.py
+113
View File
@@ -0,0 +1,113 @@
"""Parsing of detection hardware device strings."""
import logging
from dataclasses import dataclass
from pydantic import TypeAdapter, ValidationError
from frigate.detectors.detector_config import BaseDetectorConfig, ModelConfig
from frigate.detectors.detector_types import DetectorConfig, config_types
logger = logging.getLogger(__name__)
_detector_adapter: TypeAdapter[BaseDetectorConfig] = TypeAdapter(DetectorConfig)
@dataclass(frozen=True)
class DeviceSpec:
"""A parsed `<detector>` or `<detector>:<device>` string."""
raw: str
detector: str
device: str | None
@property
def shareable(self) -> bool:
"""Whether this device may be listed more than once."""
return config_types[self.detector].shareable
class DeviceParseError(ValueError):
pass
def parse_device(raw: str) -> DeviceSpec:
"""Parse a device string into its detector type and detector specific device.
Args:
raw: The configured device string, for example 'edgetpu:pci:0'
Returns:
The parsed spec
Raises:
DeviceParseError: If the detector type is unknown or the device is not
valid for that detector
"""
detector, separator, device = raw.partition(":")
if detector not in config_types:
raise DeviceParseError(
f"'{raw}' does not name a known detector. Available detectors are {', '.join(sorted(config_types))}"
)
spec = DeviceSpec(raw=raw, detector=detector, device=device if separator else None)
# surface a bad device now rather than when the detection process starts
build_detector_config(spec, None)
return spec
def build_detector_config(
spec: DeviceSpec, model: ModelConfig | None
) -> BaseDetectorConfig:
"""Build the detector config a device string describes.
Args:
spec: The parsed device spec
model: The model this detector runs, if it has been resolved yet
Returns:
The validated detector config
Raises:
DeviceParseError: If the device is not valid for this detector type
"""
config: dict[str, object] = {"type": spec.detector, "model": model}
if spec.device is not None:
config_class = config_types[spec.detector]
try:
config[config_class.device_spec_field] = config_class.device_spec_type(
spec.device
)
except ValueError as err:
raise DeviceParseError(
f"'{spec.raw}' is not a valid {spec.detector} device: {err}"
) from err
try:
return _detector_adapter.validate_python(config)
except ValidationError as err:
raise DeviceParseError(f"'{spec.raw}' is not a valid device: {err}") from err
def runner_names(devices: list[DeviceSpec]) -> list[str]:
"""Build a unique name for each device, since a shareable device may repeat.
Args:
devices: Every device spec across every configured model, in config order
Returns:
A name per device, suffixed with '#2', '#3', etc. on repeats
"""
names: list[str] = []
seen: dict[str, int] = {}
for spec in devices:
count = seen.get(spec.raw, 0) + 1
seen[spec.raw] = count
names.append(spec.raw if count == 1 else f"{spec.raw}#{count}")
return names
frigate/detectors/plugins/cpu_tfl.py
+4 -1
View File
@@ -1,5 +1,5 @@
import logging import logging
from typing import Literal from typing import ClassVar, Literal
from pydantic import ConfigDict, Field from pydantic import ConfigDict, Field
@@ -27,6 +27,9 @@ class CpuDetectorConfig(BaseDetectorConfig):
title="CPU", title="CPU",
) )
device_spec_field: ClassVar[str] = "num_threads"
device_spec_type: ClassVar[type] = int
type: Literal[DETECTOR_KEY] type: Literal[DETECTOR_KEY]
num_threads: int = Field( num_threads: int = Field(
default=3, default=3,
frigate/detectors/plugins/edgetpu_tfl.py
+4 -1
View File
@@ -1,7 +1,7 @@
import logging import logging
import math import math
import os import os
from typing import Literal from typing import ClassVar, Literal
import cv2 import cv2
import numpy as np import numpy as np
@@ -28,6 +28,9 @@ class EdgeTpuDetectorConfig(BaseDetectorConfig):
title="EdgeTPU", title="EdgeTPU",
) )
# a TPU can only be opened by one process
shareable: ClassVar[bool] = False
type: Literal[DETECTOR_KEY] type: Literal[DETECTOR_KEY]
device: str = Field( device: str = Field(
default=None, default=None,
frigate/detectors/plugins/memryx.py
+4 -1
View File
@@ -5,7 +5,7 @@ import shutil
import urllib.request import urllib.request
import zipfile import zipfile
from queue import Queue from queue import Queue
from typing import Literal from typing import ClassVar, Literal
import cv2 import cv2
import numpy as np import numpy as np
@@ -37,6 +37,9 @@ class MemryXDetectorConfig(BaseDetectorConfig):
title="MemryX", title="MemryX",
) )
# an accelerator can only be opened by one process
shareable: ClassVar[bool] = False
type: Literal[DETECTOR_KEY] type: Literal[DETECTOR_KEY]
device: str = Field( device: str = Field(
default="PCIe", default="PCIe",
frigate/detectors/plugins/openvino.py
+1 -1
View File
@@ -28,7 +28,7 @@ class OvDetectorConfig(BaseDetectorConfig):
type: Literal[DETECTOR_KEY] type: Literal[DETECTOR_KEY]
device: str = Field( device: str = Field(
default=None, default="AUTO",
title="Device Type", title="Device Type",
description="The device to use for OpenVINO inference (e.g. 'CPU', 'GPU', 'NPU').", description="The device to use for OpenVINO inference (e.g. 'CPU', 'GPU', 'NPU').",
) )
frigate/detectors/plugins/rknn.py
+4 -1
View File
@@ -2,7 +2,7 @@ import logging
import os.path import os.path
import re import re
import urllib.request import urllib.request
from typing import Literal from typing import ClassVar, Literal
import cv2 import cv2
import numpy as np import numpy as np
@@ -35,6 +35,9 @@ class RknnDetectorConfig(BaseDetectorConfig):
title="RKNN", title="RKNN",
) )
device_spec_field: ClassVar[str] = "num_cores"
device_spec_type: ClassVar[type] = int
type: Literal[DETECTOR_KEY] type: Literal[DETECTOR_KEY]
num_cores: int = Field( num_cores: int = Field(
default=0, default=0,
frigate/detectors/plugins/tensorrt.py
+3 -1
View File
@@ -14,7 +14,7 @@ try:
except ModuleNotFoundError: except ModuleNotFoundError:
TRT_SUPPORT = False TRT_SUPPORT = False
from typing import Literal from typing import ClassVar, Literal
from pydantic import ConfigDict, Field from pydantic import ConfigDict, Field
@@ -53,6 +53,8 @@ class TensorRTDetectorConfig(BaseDetectorConfig):
title="TensorRT", title="TensorRT",
) )
device_spec_type: ClassVar[type] = int
type: Literal[DETECTOR_KEY] type: Literal[DETECTOR_KEY]
device: int = Field( device: int = Field(
default=0, title="GPU Device Index", description="The GPU device index to use." default=0, title="GPU Device Index", description="The GPU device index to use."
frigate/events/maintainer.py
+5 -8
View File
@@ -159,7 +159,8 @@ class EventProcessor(threading.Thread):
if width is None or height is None: if width is None or height is None:
return return
first_detector = list(self.config.detectors.values())[0] camera_model = self.config.model_for_camera(camera)
camera_detector = self.config.devices_for_model(camera_model)[0].detector
start_time = event_data["start_time"] start_time = event_data["start_time"]
end_time = ( end_time = (
@@ -229,13 +230,9 @@ class EventProcessor(threading.Thread):
Event.thumbnail: event_data.get("thumbnail"), Event.thumbnail: event_data.get("thumbnail"),
Event.has_clip: event_data["has_clip"], Event.has_clip: event_data["has_clip"],
Event.has_snapshot: event_data["has_snapshot"], Event.has_snapshot: event_data["has_snapshot"],
Event.model_hash: first_detector.model.model_hash Event.model_hash: camera_model.model_hash,
if first_detector.model Event.model_type: camera_model.model_type,
else None, Event.detector_type: camera_detector,
Event.model_type: first_detector.model.model_type
if first_detector.model
else None,
Event.detector_type: first_detector.type,
Event.data: { Event.data: {
"box": box, "box": box,
"region": region, "region": region,
frigate/object_detection/util.py
+13 -1
View File
@@ -5,7 +5,19 @@ import threading
from numpy import ndarray from numpy import ndarray
from frigate.detectors.detector_config import InputTensorEnum from frigate.detectors.detector_config import InputTensorEnum, ModelConfig
def detection_frame_size(model: ModelConfig) -> int:
"""Get the shared memory size a camera needs to hand frames to a model.
Args:
model: The model the camera runs on
Returns:
Size in bytes of one model input frame
"""
return model.height * model.width * 3
class RequestStore: class RequestStore:
frigate/review/maintainer.py
+2 -2
View File
@@ -481,7 +481,7 @@ class ReviewSegmentMaintainer(threading.Thread):
if not object["sub_label"]: if not object["sub_label"]:
segment.detections[object["id"]] = object["label"] segment.detections[object["id"]] = object["label"]
elif object["sub_label"][0] in self.config.model.all_attributes: elif object["sub_label"][0] in self.config.all_attributes:
segment.detections[object["id"]] = object["sub_label"][0] segment.detections[object["id"]] = object["sub_label"][0]
else: else:
segment.detections[object["id"]] = f"{object['label']}-verified" segment.detections[object["id"]] = f"{object['label']}-verified"
@@ -619,7 +619,7 @@ class ReviewSegmentMaintainer(threading.Thread):
for object in activity.get_all_objects(): for object in activity.get_all_objects():
if not object["sub_label"]: if not object["sub_label"]:
detections[object["id"]] = object["label"] detections[object["id"]] = object["label"]
elif object["sub_label"][0] in self.config.model.all_attributes: elif object["sub_label"][0] in self.config.all_attributes:
detections[object["id"]] = object["sub_label"][0] detections[object["id"]] = object["sub_label"][0]
else: else:
detections[object["id"]] = f"{object['label']}-verified" detections[object["id"]] = f"{object['label']}-verified"
frigate/stats/util.py
+14 -13
View File
@@ -322,19 +322,20 @@ async def set_gpu_stats(
async def set_npu_usages(config: FrigateConfig, all_stats: dict[str, Any]) -> None: async def set_npu_usages(config: FrigateConfig, all_stats: dict[str, Any]) -> None:
stats: dict[str, dict] = {} stats: dict[str, dict] = {}
for detector in config.detectors.values(): for model in config.models:
if detector.type == "rknn": for device in config.devices_for_model(model):
# Rockchip NPU usage if device.detector == "rknn":
rk_usage = get_rockchip_npu_stats() # Rockchip NPU usage
stats["rockchip"] = rk_usage rk_usage = get_rockchip_npu_stats()
elif detector.type == "openvino" and detector.device == "NPU": stats["rockchip"] = rk_usage
# OpenVINO NPU usage elif device.detector == "openvino" and device.device == "NPU":
ov_usage = get_openvino_npu_stats() # OpenVINO NPU usage
stats["openvino"] = ov_usage ov_usage = get_openvino_npu_stats()
elif detector.type == "axengine": stats["openvino"] = ov_usage
# AXERA NPU usage elif device.detector == "axengine":
axcl_usage = get_axcl_npu_stats() # AXERA NPU usage
stats["axengine"] = axcl_usage axcl_usage = get_axcl_npu_stats()
stats["axengine"] = axcl_usage
if stats: if stats:
all_stats["npu_usages"] = stats all_stats["npu_usages"] = stats
frigate/test/test_config.py
+160 -37
View File
@@ -11,6 +11,8 @@ from ruamel.yaml.constructor import DuplicateKeyError
from frigate.config import BirdseyeModeEnum, FrigateConfig, RetainModeEnum from frigate.config import BirdseyeModeEnum, FrigateConfig, RetainModeEnum
from frigate.const import MODEL_CACHE_DIR from frigate.const import MODEL_CACHE_DIR
from frigate.detectors import DetectorTypeEnum from frigate.detectors import DetectorTypeEnum
from frigate.detectors.detector_config import SceneEnum
from frigate.detectors.device import build_detector_config, runner_names
from frigate.util.builtin import deep_merge from frigate.util.builtin import deep_merge
@@ -65,49 +67,171 @@ class TestConfig(unittest.TestCase):
def test_config_class(self): def test_config_class(self):
frigate_config = FrigateConfig(**self.minimal) frigate_config = FrigateConfig(**self.minimal)
assert "cpu" in frigate_config.detectors.keys() model = frigate_config.primary_model
assert frigate_config.detectors["cpu"].type == DetectorTypeEnum.cpu assert model.scene == SceneEnum.all
assert frigate_config.detectors["cpu"].model.width == 320 assert model.width == 320
assert frigate_config.devices_for_model(model)[0].detector == (
DetectorTypeEnum.cpu
)
@patch("frigate.detectors.detector_config.load_labels") @patch("frigate.detectors.detector_config.load_labels")
def test_detector_custom_model_path(self, mock_labels): def test_model_custom_path(self, mock_labels):
mock_labels.return_value = {} mock_labels.return_value = {}
config = { config = {
"detectors": { "models": [
"cpu": { # needs to be a file that will exist, doesn't matter what
"type": "cpu", {"path": "/etc/hosts", "width": 512, "devices": ["openvino:GPU"]},
"model_path": "/cpu_model.tflite", ],
}
frigate_config = FrigateConfig(**(deep_merge(config, self.minimal)))
model = frigate_config.primary_model
assert model.path == "/etc/hosts"
assert model.width == 512
detector_config = build_detector_config(
frigate_config.devices_for_model(model)[0], model
)
assert detector_config.type == DetectorTypeEnum.openvino
assert detector_config.device == "GPU"
assert detector_config.model.path == "/etc/hosts"
@patch("frigate.detectors.detector_config.load_labels")
def test_model_default_paths_per_detector(self, mock_labels):
mock_labels.return_value = {}
for devices, expected in (
(["cpu"], "/cpu_model.tflite"),
(["edgetpu:pci:0"], "/edgetpu_model.tflite"),
(["openvino:CPU"], "/openvino-model/ssdlite_mobilenet_v2.xml"),
):
config = {"models": [{"devices": devices}]}
frigate_config = FrigateConfig(**(deep_merge(config, self.minimal)))
assert frigate_config.primary_model.path == expected
@patch("frigate.detectors.detector_config.load_labels")
def test_camera_picks_model_by_scene(self, mock_labels):
mock_labels.return_value = {}
config = {
"models": [
{"scene": "outdoor", "devices": ["cpu"], "width": 320},
{"scene": "indoor", "devices": ["openvino:CPU"], "width": 300},
],
"cameras": {
"back": {
"detect": {"scene": "outdoor"},
"ffmpeg": {
"inputs": [
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]},
]
},
}, },
"edgetpu": { "front": {
"type": "edgetpu", "detect": {"scene": "indoor"},
"model_path": "/edgetpu_model.tflite", "ffmpeg": {
}, "inputs": [
"openvino": { {"path": "rtsp://10.0.0.2:554/video", "roles": ["detect"]},
"type": "openvino", ]
},
}, },
}, },
# needs to be a file that will exist, doesn't matter what
"model": {"path": "/etc/hosts", "width": 512},
} }
frigate_config = FrigateConfig(**(deep_merge(config, self.minimal))) frigate_config = FrigateConfig(**(deep_merge(config, self.minimal)))
assert "cpu" in frigate_config.detectors.keys() assert frigate_config.model_for_camera("back").scene == SceneEnum.outdoor
assert "edgetpu" in frigate_config.detectors.keys() assert frigate_config.model_for_camera("front").scene == SceneEnum.indoor
assert "openvino" in frigate_config.detectors.keys() assert frigate_config.model_for_camera("back").width == 320
assert frigate_config.model_for_camera("front").width == 300
assert frigate_config.detectors["cpu"].type == DetectorTypeEnum.cpu @patch("frigate.detectors.detector_config.load_labels")
assert frigate_config.detectors["edgetpu"].type == DetectorTypeEnum.edgetpu def test_camera_requires_a_scene_without_a_default(self, mock_labels):
assert frigate_config.detectors["openvino"].type == DetectorTypeEnum.openvino mock_labels.return_value = {}
config = {
"models": [
{"scene": "outdoor", "devices": ["cpu"]},
{"scene": "indoor", "devices": ["openvino:CPU"]},
],
}
assert frigate_config.detectors["cpu"].num_threads == 3 with self.assertRaises(ValidationError):
assert frigate_config.detectors["edgetpu"].device is None FrigateConfig(**(deep_merge(config, self.minimal)))
assert frigate_config.detectors["openvino"].device is None
assert frigate_config.model.path == "/etc/hosts" @patch("frigate.detectors.detector_config.load_labels")
assert frigate_config.detectors["cpu"].model.path == "/cpu_model.tflite" def test_camera_scene_must_match_a_model(self, mock_labels):
assert frigate_config.detectors["edgetpu"].model.path == "/edgetpu_model.tflite" mock_labels.return_value = {}
assert frigate_config.detectors["openvino"].model.path == "/etc/hosts" config = {
"models": [{"devices": ["cpu"]}],
"cameras": {
"back": {
"detect": {"scene": "outdoor"},
"ffmpeg": {
"inputs": [
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]},
]
},
},
},
}
with self.assertRaises(ValidationError):
FrigateConfig(**(deep_merge(config, self.minimal)))
@patch("frigate.detectors.detector_config.load_labels")
def test_models_must_use_unique_scenes(self, mock_labels):
mock_labels.return_value = {}
config = {
"models": [
{"scene": "outdoor", "devices": ["cpu"]},
{"scene": "outdoor", "devices": ["openvino:CPU"]},
],
}
with self.assertRaises(ValidationError):
FrigateConfig(**(deep_merge(config, self.minimal)))
@patch("frigate.detectors.detector_config.load_labels")
def test_model_devices_must_share_a_detector(self, mock_labels):
mock_labels.return_value = {}
config = {"models": [{"devices": ["cpu", "openvino:CPU"]}]}
with self.assertRaises(ValidationError):
FrigateConfig(**(deep_merge(config, self.minimal)))
@patch("frigate.detectors.detector_config.load_labels")
def test_model_requires_a_known_detector(self, mock_labels):
mock_labels.return_value = {}
config = {"models": [{"devices": ["not_a_detector:0"]}]}
with self.assertRaises(ValidationError):
FrigateConfig(**(deep_merge(config, self.minimal)))
@patch("frigate.detectors.detector_config.load_labels")
def test_model_requires_a_device(self, mock_labels):
mock_labels.return_value = {}
config = {"models": [{"devices": []}]}
with self.assertRaises(ValidationError):
FrigateConfig(**(deep_merge(config, self.minimal)))
@patch("frigate.detectors.detector_config.load_labels")
def test_shareable_devices_may_repeat(self, mock_labels):
mock_labels.return_value = {}
config = {"models": [{"devices": ["openvino:GPU", "openvino:GPU"]}]}
frigate_config = FrigateConfig(**(deep_merge(config, self.minimal)))
devices = frigate_config.devices_for_model(frigate_config.primary_model)
assert runner_names(devices) == ["openvino:GPU", "openvino:GPU#2"]
@patch("frigate.detectors.detector_config.load_labels")
def test_exclusive_devices_may_not_repeat(self, mock_labels):
mock_labels.return_value = {}
config = {"models": [{"devices": ["edgetpu:pci:0", "edgetpu:pci:0"]}]}
with self.assertRaises(ValidationError):
FrigateConfig(**(deep_merge(config, self.minimal)))
def test_invalid_mqtt_config(self): def test_invalid_mqtt_config(self):
config = { config = {
@@ -1131,7 +1255,7 @@ class TestConfig(unittest.TestCase):
def test_merge_labelmap(self): def test_merge_labelmap(self):
config = { config = {
"mqtt": {"host": "mqtt"}, "mqtt": {"host": "mqtt"},
"model": {"labelmap": {7: "truck"}}, "models": [{"labelmap": {7: "truck"}, "devices": ["cpu"]}],
"cameras": { "cameras": {
"back": { "back": {
"ffmpeg": { "ffmpeg": {
@@ -1152,7 +1276,7 @@ class TestConfig(unittest.TestCase):
} }
frigate_config = FrigateConfig(**config) frigate_config = FrigateConfig(**config)
assert frigate_config.model.merged_labelmap[7] == "truck" assert frigate_config.primary_model.merged_labelmap[7] == "truck"
def test_audio_labelmap_inheritance_is_separate_from_model_labelmap(self): def test_audio_labelmap_inheritance_is_separate_from_model_labelmap(self):
config = deep_merge( config = deep_merge(
@@ -1199,12 +1323,12 @@ class TestConfig(unittest.TestCase):
} }
frigate_config = FrigateConfig(**config) frigate_config = FrigateConfig(**config)
assert frigate_config.model.merged_labelmap[0] == "person" assert frigate_config.primary_model.merged_labelmap[0] == "person"
def test_default_labelmap(self): def test_default_labelmap(self):
config = { config = {
"mqtt": {"host": "mqtt"}, "mqtt": {"host": "mqtt"},
"model": {"width": 320, "height": 320}, "models": [{"width": 320, "height": 320, "devices": ["cpu"]}],
"cameras": { "cameras": {
"back": { "back": {
"ffmpeg": { "ffmpeg": {
@@ -1225,7 +1349,7 @@ class TestConfig(unittest.TestCase):
} }
frigate_config = FrigateConfig(**config) frigate_config = FrigateConfig(**config)
assert frigate_config.model.merged_labelmap[0] == "person" assert frigate_config.primary_model.merged_labelmap[0] == "person"
def test_plus_labelmap(self): def test_plus_labelmap(self):
with open(os.path.join(MODEL_CACHE_DIR, "test"), "w") as f: with open(os.path.join(MODEL_CACHE_DIR, "test"), "w") as f:
@@ -1235,8 +1359,7 @@ class TestConfig(unittest.TestCase):
config = { config = {
"mqtt": {"host": "mqtt"}, "mqtt": {"host": "mqtt"},
"detectors": {"cpu": {"type": "cpu"}}, "models": [{"path": "plus://test", "devices": ["cpu"]}],
"model": {"path": "plus://test"},
"cameras": { "cameras": {
"back": { "back": {
"ffmpeg": { "ffmpeg": {
@@ -1257,7 +1380,7 @@ class TestConfig(unittest.TestCase):
} }
frigate_config = FrigateConfig(**config) frigate_config = FrigateConfig(**config)
assert frigate_config.model.merged_labelmap[0] == "amazon" assert frigate_config.primary_model.merged_labelmap[0] == "amazon"
def test_fails_on_invalid_role(self): def test_fails_on_invalid_role(self):
config = { config = {
frigate/test/test_config_migration.py
+225
View File
@@ -0,0 +1,225 @@
"""Tests for migrating detectors and model into the models list."""
import logging
import os
import tempfile
import unittest
from unittest.mock import patch
from ruamel.yaml import YAML
from frigate.util.config import (
CURRENT_CONFIG_VERSION,
migrate_frigate_config,
migrate_models,
)
class TestMigrateModels(unittest.TestCase):
def test_single_cpu_detector(self):
migrated = migrate_models({"detectors": {"cpu": {"type": "cpu"}}})
self.assertEqual(migrated["models"], [{"scene": "all", "devices": ["cpu"]}])
self.assertNotIn("detectors", migrated)
def test_model_settings_are_carried_over(self):
migrated = migrate_models(
{
"detectors": {"coral": {"type": "edgetpu", "device": "pci:0"}},
"model": {"path": "plus://abc", "width": 320},
}
)
self.assertEqual(
migrated["models"],
[
{
"scene": "all",
"path": "plus://abc",
"width": 320,
"devices": ["edgetpu:pci:0"],
}
],
)
self.assertNotIn("model", migrated)
def test_multiple_corals_become_multiple_devices(self):
migrated = migrate_models(
{
"detectors": {
"coral1": {"type": "edgetpu", "device": "pci:0"},
"coral2": {"type": "edgetpu", "device": "pci:1"},
}
}
)
self.assertEqual(
migrated["models"][0]["devices"], ["edgetpu:pci:0", "edgetpu:pci:1"]
)
def test_several_detectors_on_one_device_stay_separate(self):
# a repeated device is now what running two inference processes on one
# piece of hardware looks like
migrated = migrate_models(
{
"detectors": {
"ov_0": {"type": "openvino", "device": "GPU"},
"ov_1": {"type": "openvino", "device": "GPU"},
}
}
)
self.assertEqual(
migrated["models"][0]["devices"], ["openvino:GPU", "openvino:GPU"]
)
def test_repeated_exclusive_devices_are_collapsed(self):
# two detectors both grabbing the first TPU was never really two TPUs
migrated = migrate_models(
{
"detectors": {
"coral_0": {"type": "edgetpu", "device": "usb"},
"coral_1": {"type": "edgetpu", "device": "usb"},
}
}
)
self.assertEqual(migrated["models"][0]["devices"], ["edgetpu:usb"])
def test_detectors_that_named_the_device_field_differently(self):
migrated = migrate_models(
{
"detectors": {
"rk": {"type": "rknn", "num_cores": 2},
}
}
)
self.assertEqual(migrated["models"][0]["devices"], ["rknn:2"])
def test_empty_edgetpu_device_is_kept(self):
# an empty device selects a native Coral, which is not the same as
# letting the delegate pick
migrated = migrate_models(
{"detectors": {"coral": {"type": "edgetpu", "device": ""}}}
)
self.assertEqual(migrated["models"][0]["devices"], ["edgetpu:"])
def test_model_path_overrides_the_model(self):
migrated = migrate_models(
{
"detectors": {
"coral": {"type": "edgetpu", "model_path": "/custom.tflite"}
},
"model": {"path": "/ignored.tflite", "width": 320},
}
)
self.assertEqual(migrated["models"][0]["path"], "/custom.tflite")
def test_no_detectors_falls_back_to_cpu(self):
migrated = migrate_models({"model": {"width": 320}})
self.assertEqual(migrated["models"][0]["devices"], ["cpu"])
def test_dropped_remote_detector_options_are_logged(self):
with self.assertLogs("frigate.util.config", level=logging.ERROR) as logs:
migrated = migrate_models(
{
"detectors": {
"ds": {
"type": "deepstack",
"api_url": "http://host:5000/v1/vision/detection",
"api_key": "secret",
}
}
}
)
self.assertEqual(
migrated["models"][0]["devices"],
["deepstack:http://host:5000/v1/vision/detection"],
)
self.assertTrue(any("api_key" in message for message in logs.output))
def test_mixed_detector_types_are_logged(self):
with self.assertLogs("frigate.util.config", level=logging.ERROR) as logs:
migrate_models(
{
"detectors": {
"ov": {"type": "openvino", "device": "GPU"},
"coral": {"type": "edgetpu", "device": "pci:0"},
}
}
)
self.assertTrue(any("more than one type" in message for message in logs.output))
def test_other_keys_are_untouched(self):
migrated = migrate_models(
{"mqtt": {"host": "mqtt"}, "detectors": {"cpu": {"type": "cpu"}}}
)
self.assertEqual(migrated["mqtt"], {"host": "mqtt"})
class TestMigrateConfigFile(unittest.TestCase):
"""The full file migration, which is gated on shape as well as version."""
def setUp(self):
self.temp_dir = tempfile.TemporaryDirectory()
self.addCleanup(self.temp_dir.cleanup)
self.config_file = os.path.join(self.temp_dir.name, "config.yml")
patcher = patch("frigate.util.config.CONFIG_DIR", self.temp_dir.name)
patcher.start()
self.addCleanup(patcher.stop)
def _migrate(self, config: str) -> dict:
with open(self.config_file, "w") as f:
f.write(config)
migrate_frigate_config(self.config_file)
with open(self.config_file) as f:
return YAML().load(f)
def test_migrates_a_config_already_stamped_with_the_current_version(self):
# 0.19 is unreleased, so a dev config can be current and still use
# the pre-models keys
migrated = self._migrate(
"mqtt:\n"
" enabled: false\n"
"detectors:\n"
" ov:\n"
" type: openvino\n"
" device: GPU\n"
"cameras: {}\n"
f"version: {CURRENT_CONFIG_VERSION}\n"
)
self.assertEqual(migrated["models"][0]["devices"], ["openvino:GPU"])
self.assertNotIn("detectors", migrated)
def test_a_migrated_config_is_left_alone(self):
migrated = self._migrate(
"mqtt:\n"
" enabled: false\n"
"models:\n"
" - scene: all\n"
" devices:\n"
" - openvino:GPU\n"
"cameras: {}\n"
f"version: {CURRENT_CONFIG_VERSION}\n"
)
self.assertEqual(
migrated["models"], [{"scene": "all", "devices": ["openvino:GPU"]}]
)
self.assertFalse(
os.path.exists(os.path.join(self.temp_dir.name, "backup_config.yaml"))
)
if __name__ == "__main__":
unittest.main(verbosity=2)
frigate/test/test_detector_device.py
+94
View File
@@ -0,0 +1,94 @@
"""Tests for parsing detection hardware device strings."""
import unittest
from frigate.detectors.detector_config import ModelConfig
from frigate.detectors.device import (
DeviceParseError,
build_detector_config,
parse_device,
runner_names,
)
class TestParseDevice(unittest.TestCase):
def test_bare_detector_has_no_device(self):
spec = parse_device("cpu")
self.assertEqual(spec.detector, "cpu")
self.assertIsNone(spec.device)
def test_device_is_everything_after_the_first_colon(self):
spec = parse_device("edgetpu:pci:0")
self.assertEqual(spec.detector, "edgetpu")
self.assertEqual(spec.device, "pci:0")
def test_trailing_colon_keeps_an_empty_device(self):
# an empty edgetpu device selects a native Coral
spec = parse_device("edgetpu:")
self.assertEqual(spec.detector, "edgetpu")
self.assertEqual(spec.device, "")
def test_unknown_detector_is_rejected(self):
with self.assertRaises(DeviceParseError):
parse_device("not_a_detector:0")
def test_device_that_the_detector_cannot_use_is_rejected(self):
# tensorrt takes a gpu index
with self.assertRaises(DeviceParseError):
parse_device("tensorrt:the-fast-one")
class TestBuildDetectorConfig(unittest.TestCase):
def _build(self, raw: str):
return build_detector_config(parse_device(raw), ModelConfig())
def test_device_lands_on_the_detector_field(self):
for raw, expected in (
("edgetpu:usb", "usb"),
("edgetpu:pci:1", "pci:1"),
("openvino:GPU.1", "GPU.1"),
("onnx:CPU", "CPU"),
("memryx:PCIe:0", "PCIe:0"),
):
with self.subTest(raw=raw):
self.assertEqual(self._build(raw).device, expected)
def test_detectors_that_name_the_field_something_else(self):
self.assertEqual(self._build("cpu:4").num_threads, 4)
self.assertEqual(self._build("rknn:2").num_cores, 2)
def test_device_is_coerced_to_the_detector_field_type(self):
self.assertEqual(self._build("tensorrt:1").device, 1)
def test_omitted_device_falls_back_to_the_detector_default(self):
self.assertEqual(self._build("cpu").num_threads, 3)
self.assertEqual(self._build("rknn").num_cores, 0)
self.assertEqual(self._build("openvino").device, "AUTO")
self.assertIsNone(self._build("edgetpu").device)
def test_the_model_is_attached(self):
model = ModelConfig(path="/cpu_model.tflite")
self.assertIs(build_detector_config(parse_device("cpu"), model).model, model)
class TestRunnerNames(unittest.TestCase):
def test_unique_devices_keep_their_name(self):
devices = [parse_device("edgetpu:pci:0"), parse_device("edgetpu:pci:1")]
self.assertEqual(runner_names(devices), ["edgetpu:pci:0", "edgetpu:pci:1"])
def test_repeated_devices_are_numbered(self):
devices = [parse_device("openvino:GPU")] * 3
self.assertEqual(
runner_names(devices),
["openvino:GPU", "openvino:GPU#2", "openvino:GPU#3"],
)
if __name__ == "__main__":
unittest.main(verbosity=2)
frigate/track/object_processing.py
+1 -1
View File
@@ -210,7 +210,7 @@ class TrackedObjectProcessor(threading.Thread):
if obj.obj_data.get("sub_label"): if obj.obj_data.get("sub_label"):
sub_label = obj.obj_data["sub_label"][0] sub_label = obj.obj_data["sub_label"][0]
if sub_label in self.config.model.all_attribute_logos: if sub_label in self.config.all_attribute_logos:
self.dispatcher.publish( self.dispatcher.publish(
f"{camera}/{sub_label}/snapshot", f"{camera}/{sub_label}/snapshot",
jpg_bytes, jpg_bytes,
frigate/util/config.py
+113 -1
View File
@@ -23,6 +23,25 @@ logger = logging.getLogger(__name__)
CURRENT_CONFIG_VERSION = "0.19-0" CURRENT_CONFIG_VERSION = "0.19-0"
DEFAULT_CONFIG_FILE = os.path.join(CONFIG_DIR, "config.yml") DEFAULT_CONFIG_FILE = os.path.join(CONFIG_DIR, "config.yml")
# the detector field that used to hold the device, for detectors that named it
# something other than "device"
DETECTOR_DEVICE_FIELDS = {
"cpu": "num_threads",
"rknn": "num_cores",
"deepstack": "api_url",
"degirum": "location",
"zmq": "endpoint",
}
# detector options that have no equivalent in a device string. The remote
# detectors that use them are being reworked, so they are dropped rather than
# carried over.
DROPPED_DETECTOR_OPTIONS = {
"deepstack": ["api_timeout", "api_key"],
"degirum": ["zoo", "token"],
"zmq": ["request_timeout_ms", "linger_ms"],
}
def resolve_ffmpeg_path(path: str, binary: str = "ffmpeg") -> str: def resolve_ffmpeg_path(path: str, binary: str = "ffmpeg") -> str:
"""Resolve an ffmpeg version alias or custom path to a binary path. """Resolve an ffmpeg version alias or custom path to a binary path.
@@ -87,7 +106,11 @@ def migrate_frigate_config(config_file: str):
previous_version = str(config.get("version", "0.13")) previous_version = str(config.get("version", "0.13"))
if previous_version == CURRENT_CONFIG_VERSION: # 0.19 is unreleased, so a config may already be stamped with the current
# version and still use the pre-models detectors and model keys
needs_models = "detectors" in config or "model" in config
if previous_version == CURRENT_CONFIG_VERSION and not needs_models:
logger.info("frigate config does not need migration...") logger.info("frigate config does not need migration...")
return return
@@ -155,6 +178,12 @@ def migrate_frigate_config(config_file: str):
yaml.dump(new_config, f) yaml.dump(new_config, f)
previous_version = "0.19-0" previous_version = "0.19-0"
if needs_models:
logger.info("Migrating frigate detectors and model to models...")
new_config = migrate_models(new_config)
with open(config_file, "w") as f:
yaml.dump(new_config, f)
logger.info("Finished frigate config migration...") logger.info("Finished frigate config migration...")
@@ -708,6 +737,89 @@ def migrate_019_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
return new_config return new_config
def migrate_models(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""Merge the detectors and model keys into a single models list.
Every config before this change ran one model across all of its detectors,
so this always produces exactly one model.
Args:
config: The loaded config
Returns:
The config with a models list in place of detectors and model
"""
# imported lazily so loading the detector plugins is not a cost of importing
# this module
from frigate.detectors.detector_types import config_types
new_config = config.copy()
detectors: dict[str, Any] = new_config.pop("detectors", None) or {}
model: dict[str, Any] = new_config.pop("model", None) or {}
devices: list[str] = []
model_path: str | None = None
for name, detector in detectors.items():
detector = detector or {}
detector_type = detector.get("type", "cpu")
device = detector.get(DETECTOR_DEVICE_FIELDS.get(detector_type, "device"))
device_string = detector_type if device is None else f"{detector_type}:{device}"
# repeating a device now means running an extra inference process on it,
# which is what several detectors on one device used to mean. Only
# collapse repeats of hardware that can serve a single process.
config_class = config_types.get(detector_type)
shareable = config_class.shareable if config_class else True
if shareable or device_string not in devices:
devices.append(device_string)
dropped = [
option
for option in DROPPED_DETECTOR_OPTIONS.get(detector_type, [])
if option in detector
]
if dropped:
logger.error(
"Detector '%s' had the %s options set, which are no longer supported and have been removed",
name,
", ".join(dropped),
)
detector_model_path = detector.get("model_path")
if detector_model_path:
if model_path is None:
model_path = detector_model_path
elif model_path != detector_model_path:
logger.warning(
"Detector '%s' set a different model_path than an earlier detector, using '%s' for the migrated model",
name,
model_path,
)
detector_types = {device.partition(":")[0] for device in devices}
if len(detector_types) > 1:
logger.error(
"Detectors of more than one type (%s) were configured. A model now runs on one detector type, so the migrated config will need to be corrected by hand",
", ".join(sorted(detector_types)),
)
entry: dict[str, Any] = {"scene": "all", **model}
if model_path:
entry["path"] = model_path
# a config with no detectors ran a single cpu detector
entry["devices"] = devices or ["cpu"]
new_config["models"] = [entry]
return new_config
def get_relative_coordinates( def get_relative_coordinates(
mask: str | list | None, mask: str | list | None,
frame_shape: tuple[int, int], frame_shape: tuple[int, int],
frigate/util/object_names.py
+3 -3
View File
@@ -47,10 +47,10 @@ def get_categorized_object_names(
""" """
tracked_objects = _get_tracked_objects(config, allowed_cameras) tracked_objects = _get_tracked_objects(config, allowed_cameras)
names: dict[str, set[str]] = {} names: dict[str, set[str]] = {}
logos = set(config.model.all_attribute_logos) logos = set(config.all_attribute_logos)
# 1. detector logo attributes, only for objects that are actually tracked # 1. detector logo attributes, only for objects that are actually tracked
for label, label_attributes in config.model.attributes_map.items(): for label, label_attributes in config.all_attributes_map.items():
if label not in tracked_objects: if label not in tracked_objects:
continue continue
@@ -126,7 +126,7 @@ def _objects_with_attribute(
""" """
objects = { objects = {
label label
for label, label_attributes in config.model.attributes_map.items() for label, label_attributes in config.all_attributes_map.items()
if attribute in label_attributes and label in tracked_objects if attribute in label_attributes and label in tracked_objects
} }
frigate/util/schema.py
+9 -38
View File
@@ -2,45 +2,16 @@
from typing import Any from typing import Any
from pydantic import BaseModel, TypeAdapter from pydantic import BaseModel
def get_config_schema(config_class: type[BaseModel]) -> dict[str, Any]: def get_config_schema(config_class: type[BaseModel]) -> dict[str, Any]:
"""Get the JSON schema for FrigateConfig.
Args:
config_class: The config model to describe
Returns:
The JSON schema
""" """
Returns the JSON schema for FrigateConfig with polymorphic detectors. return config_class.model_json_schema()
This utility patches the FrigateConfig schema to include the full polymorphic
definitions for detectors. By default, Pydantic's schema for Dict[str, BaseDetectorConfig]
only includes the base class fields. This function replaces it with a reference
to the DetectorConfig union, which includes all available detector subclasses.
"""
# Import here to ensure all detector plugins are loaded through the detectors module
from frigate.detectors import DetectorConfig
# Get the base schema for FrigateConfig
schema = config_class.model_json_schema()
# Get the schema for the polymorphic DetectorConfig union
detector_adapter: TypeAdapter = TypeAdapter(DetectorConfig)
detector_schema = detector_adapter.json_schema()
# Ensure $defs exists in FrigateConfig schema
if "$defs" not in schema:
schema["$defs"] = {}
# Merge $defs from DetectorConfig into FrigateConfig schema
# This includes the specific schemas for each detector plugin (OvDetectorConfig, etc.)
if "$defs" in detector_schema:
schema["$defs"].update(detector_schema["$defs"])
# Extract the union schema (oneOf/discriminator) and add it as a definition
detector_union_schema = {k: v for k, v in detector_schema.items() if k != "$defs"}
schema["$defs"]["DetectorConfig"] = detector_union_schema
# Update the 'detectors' property to use the polymorphic DetectorConfig definition
if "detectors" in schema.get("properties", {}):
schema["properties"]["detectors"]["additionalProperties"] = {
"$ref": "#/$defs/DetectorConfig"
}
return schema
generate_config_translations.py
+6 -85
View File
@@ -210,78 +210,6 @@ def generate_section_translation(config_class: type) -> dict[str, Any]:
return extract_translations_from_schema(schema) return extract_translations_from_schema(schema)
def get_detector_translations(
config_schema: dict[str, Any],
) -> tuple[dict[str, Any], dict[str, Any], set[str]]:
"""Build detector type translations with nested fields based on schema definitions.
Returns a tuple of (type_translations, shared_fields, nested_field_keys).
Shared fields (identical across all detector types) are returned separately
to avoid duplication in the output.
"""
defs = config_schema.get("$defs", {})
detector_schema = defs.get("DetectorConfig", {})
discriminator = detector_schema.get("discriminator", {})
mapping = discriminator.get("mapping", {})
# First pass: collect all nested fields per detector type
all_nested: dict[str, dict[str, Any]] = {}
type_meta: dict[str, dict[str, str]] = {}
for detector_type, ref in mapping.items():
if not isinstance(ref, str) or not ref.startswith("#/$defs/"):
continue
ref_name = ref.split("/")[-1]
ref_schema = defs.get(ref_name, {})
if not ref_schema:
continue
meta: dict[str, str] = {}
title = ref_schema.get("title")
description = ref_schema.get("description")
if title:
meta["label"] = title
if description:
meta["description"] = description
type_meta[detector_type] = meta
nested = extract_translations_from_schema(ref_schema, defs=defs)
all_nested[detector_type] = {
k: v for k, v in nested.items() if k not in ("label", "description")
}
# Find fields that are identical across all types that have them
shared_fields: dict[str, Any] = {}
if all_nested:
# Collect all field keys across all types
all_keys: set[str] = set()
for nested in all_nested.values():
all_keys.update(nested.keys())
for key in all_keys:
values = [nested[key] for nested in all_nested.values() if key in nested]
if len(values) == len(all_nested) and all(v == values[0] for v in values):
shared_fields[key] = values[0]
# Build per-type translations with only unique (non-shared) fields
type_translations: dict[str, Any] = {}
nested_field_keys: set[str] = set()
for detector_type, nested in all_nested.items():
type_entry: dict[str, Any] = {}
type_entry.update(type_meta.get(detector_type, {}))
unique_fields = {k: v for k, v in nested.items() if k not in shared_fields}
if unique_fields:
type_entry.update(unique_fields)
nested_field_keys.update(unique_fields.keys())
if type_entry:
type_translations[detector_type] = type_entry
return type_translations, shared_fields, nested_field_keys
def main(): def main():
"""Main function to generate config translations.""" """Main function to generate config translations."""
@@ -337,6 +265,12 @@ def main():
if args and len(args) > 1: if args and len(args) > 1:
field_type = args[1] # Get value type from Dict[key, value] field_type = args[1] # Get value type from Dict[key, value]
# Handle List[SomeModel] - extract the item type
if origin is list:
args = get_args(field_type)
if args:
field_type = args[0]
# Start with field's top-level metadata (label, description) # Start with field's top-level metadata (label, description)
section_data = get_field_translations(field_info) section_data = get_field_translations(field_info)
@@ -351,19 +285,6 @@ def main():
} }
section_data.update(nested_without_root) section_data.update(nested_without_root)
if field_name == "detectors":
detector_types, shared_fields, detector_field_keys = (
get_detector_translations(config_schema)
)
# Add shared fields at the base detectors level
section_data.update(shared_fields)
# Add per-type translations (only unique fields per type)
section_data.update(detector_types)
for key in detector_field_keys:
if key == "type":
continue
section_data.pop(key, None)
if field_name == "objects": if field_name == "objects":
# Produce a parallel `filters_attribute` block alongside `filters`, # Produce a parallel `filters_attribute` block alongside `filters`,
# with object-wording rewritten for attribute filters (face, # with object-wording rewritten for attribute filters (face,
testing-scripts/process_clip.py
+2 -2
View File
@@ -122,7 +122,7 @@ class ProcessClip:
self.camera_name, self.camera_name,
self.frame_queue, self.frame_queue,
self.frame_shape, self.frame_shape,
self.config.model, self.config.model_for_camera(self.camera_name),
self.camera_config.detect, self.camera_config.detect,
self.frame_manager, self.frame_manager,
motion_detector, motion_detector,
@@ -248,7 +248,7 @@ def process(path, label, output, debug_path):
json_config = { json_config = {
"mqtt": {"enabled": False}, "mqtt": {"enabled": False},
"detectors": {"coral": {"type": "edgetpu", "device": "usb"}}, "models": [{"devices": ["edgetpu:usb"]}],
"cameras": { "cameras": {
"camera": { "camera": {
"ffmpeg": { "ffmpeg": {
web/e2e/fixtures/mock-data/config-schema.json
+1 -1
View File
File diff suppressed because one or more lines are too long
web/e2e/fixtures/mock-data/config-snapshot.json
+1 -1
View File
File diff suppressed because one or more lines are too long
web/e2e/fixtures/mock-data/generate-mock-data.py
+6 -5
View File
@@ -102,11 +102,12 @@ def generate_config():
snapshot = config.model_dump() snapshot = config.model_dump()
# Runtime-computed fields not in the Pydantic dump # Runtime-computed fields not in the Pydantic dump
all_attrs = set() for model in snapshot.get("models", []):
for attrs in snapshot.get("model", {}).get("attributes_map", {}).values(): all_attrs = set()
all_attrs.update(attrs) for attrs in model.get("attributes_map", {}).values():
snapshot["model"]["all_attributes"] = sorted(all_attrs) all_attrs.update(attrs)
snapshot["model"]["colormap"] = {} model["all_attributes"] = sorted(all_attrs)
model["colormap"] = {}
return snapshot return snapshot
web/e2e/specs/settings/detectors-and-model.spec.ts
+4 -1
View File
@@ -6,7 +6,10 @@
import { test, expect } from "../../fixtures/frigate-test"; import { test, expect } from "../../fixtures/frigate-test";
test.describe("Detectors and model Settings @high", () => { // The settings page still reads the removed `detectors` and `model` config
// keys, so it cannot render against a `models` config. Re-enable these once
// the page is rebuilt around the models list.
test.describe.skip("Detectors and model Settings @high", () => {
test("page renders with detector and model cards", async ({ frigateApp }) => { test("page renders with detector and model cards", async ({ frigateApp }) => {
await frigateApp.goto("/settings?page=systemDetectorsAndModel"); await frigateApp.goto("/settings?page=systemDetectorsAndModel");
await frigateApp.page.waitForTimeout(2000); await frigateApp.page.waitForTimeout(2000);
web/public/locales/en/config/cameras.json
+4
View File
@@ -98,6 +98,10 @@
"label": "Detect width", "label": "Detect width",
"description": "Width (pixels) of frames used for the detect stream; leave empty to use the native stream resolution." "description": "Width (pixels) of frames used for the detect stream; leave empty to use the native stream resolution."
}, },
"scene": {
"label": "Detect scene",
"description": "The environment this camera looks at, used to pick which of the configured models runs on it. Defaults to the model with a scene of 'all'."
},
"fps": { "fps": {
"label": "Detect FPS", "label": "Detect FPS",
"description": "Desired frames per second to run detection on; lower values reduce CPU usage (recommended value is 5, only set higher - at most 10 - if tracking extremely fast moving objects)." "description": "Desired frames per second to run detection on; lower values reduce CPU usage (recommended value is 5, only set higher - at most 10 - if tracking extremely fast moving objects)."
web/public/locales/en/config/global.json
+13 -164
View File
@@ -275,172 +275,17 @@
"description": "Unit system for display (metric or imperial) used in the UI and MQTT." "description": "Unit system for display (metric or imperial) used in the UI and MQTT."
} }
}, },
"detectors": { "models": {
"label": "Detector hardware", "label": "Detection models",
"description": "Configuration for object detectors (CPU, GPU, ONNX backends) and any detector-specific model settings.", "description": "Object detection models and the hardware each one runs on. Cameras pick a model by matching their detect.scene against a model's scene.",
"type": { "scene": {
"label": "Type" "label": "Model scene",
"description": "The camera environment this model is used for. Cameras select a model by setting detect.scene to a matching value, and 'all' is used by any camera that does not set one."
}, },
"model": { "devices": {
"label": "Detector specific model configuration", "label": "Detection hardware",
"description": "Detector-specific model configuration options (path, input size, etc.).", "description": "Hardware this model runs on, as '<detector>' or '<detector>:<device>' (for example 'edgetpu:pci:0' or 'openvino:GPU'). Listing the same device more than once runs additional inference processes on it."
"path": {
"label": "Custom object detector model path",
"description": "Path to a custom detection model file (or plus://<model_id> for Frigate+ models)."
},
"labelmap_path": {
"label": "Label map for custom object detector",
"description": "Path to a labelmap file that maps numeric classes to string labels for the detector."
},
"width": {
"label": "Object detection model input width",
"description": "Width of the model input tensor in pixels."
},
"height": {
"label": "Object detection model input height",
"description": "Height of the model input tensor in pixels."
},
"labelmap": {
"label": "Labelmap customization",
"description": "Overrides or remapping entries to merge into the standard labelmap."
},
"attributes_map": {
"label": "Map of object labels to their attribute labels",
"description": "Mapping from object labels to attribute labels used to attach metadata (for example 'car' -> ['license_plate'])."
},
"input_tensor": {
"label": "Model Input Tensor Shape",
"description": "Tensor format expected by the model: 'nhwc' or 'nchw'."
},
"input_pixel_format": {
"label": "Model Input Pixel Color Format",
"description": "Pixel colorspace expected by the model: 'rgb', 'bgr', or 'yuv'."
},
"input_dtype": {
"label": "Model Input D Type",
"description": "Data type of the model input tensor (for example 'float32')."
},
"model_type": {
"label": "Object Detection Model Type",
"description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization."
}
}, },
"model_path": {
"label": "Detector specific model path",
"description": "File path to the detector model binary if required by the chosen detector."
},
"axengine": {
"label": "AXEngine NPU",
"description": "AXERA AX650N/AX8850N NPU detector running compiled .axmodel files via the AXEngine runtime."
},
"cpu": {
"label": "CPU",
"description": "CPU TFLite detector that runs TensorFlow Lite models on the host CPU without hardware acceleration. Not recommended.",
"num_threads": {
"label": "Number of detection threads",
"description": "The number of threads used for CPU-based inference."
}
},
"deepstack": {
"label": "DeepStack",
"description": "DeepStack/CodeProject.AI detector that sends images to a remote DeepStack HTTP API for inference. Not recommended.",
"api_url": {
"label": "DeepStack API URL",
"description": "The URL of the DeepStack API."
},
"api_timeout": {
"label": "DeepStack API timeout (in seconds)",
"description": "Maximum time allowed for a DeepStack API request."
},
"api_key": {
"label": "DeepStack API key (if required)",
"description": "Optional API key for authenticated DeepStack services."
}
},
"edgetpu": {
"label": "EdgeTPU",
"description": "EdgeTPU detector that runs TensorFlow Lite models compiled for Coral EdgeTPU using the EdgeTPU delegate.",
"device": {
"label": "Device Type",
"description": "The device to use for EdgeTPU inference (e.g. 'usb', 'pci')."
}
},
"hailo8l": {
"label": "Hailo-8/Hailo-8L",
"description": "Hailo-8/Hailo-8L detector using HEF models and the HailoRT SDK for inference on Hailo hardware.",
"device": {
"label": "Device Type",
"description": "The device to use for Hailo inference (e.g. 'PCIe', 'M.2')."
}
},
"memryx": {
"label": "MemryX",
"description": "MemryX MX3 detector that runs compiled DFP models on MemryX accelerators.",
"device": {
"label": "Device Path",
"description": "The device to use for MemryX inference (e.g. 'PCIe')."
}
},
"onnx": {
"label": "ONNX",
"description": "ONNX detector for running ONNX models; will use available acceleration backends (CUDA/ROCm/OpenVINO) when available.",
"device": {
"label": "Device Type",
"description": "The device to use for ONNX inference (e.g. 'AUTO', 'CPU', 'GPU')."
}
},
"openvino": {
"label": "OpenVINO",
"description": "OpenVINO detector for AMD and Intel CPUs, Intel GPUs and Intel VPU hardware.",
"device": {
"label": "Device Type",
"description": "The device to use for OpenVINO inference (e.g. 'CPU', 'GPU', 'NPU')."
}
},
"rknn": {
"label": "RKNN",
"description": "RKNN detector for Rockchip NPUs; runs compiled RKNN models on Rockchip hardware.",
"num_cores": {
"label": "Number of NPU cores to use.",
"description": "The number of NPU cores to use (0 for auto)."
}
},
"synaptics": {
"label": "Synaptics",
"description": "Synaptics NPU detector for models in .synap format using the Synap SDK on Synaptics hardware."
},
"teflon_tfl": {
"label": "Teflon",
"description": "Teflon delegate detector for TFLite using Mesa Teflon delegate library to accelerate inference on supported GPUs."
},
"tensorrt": {
"label": "TensorRT",
"description": "TensorRT detector for Nvidia Jetson devices using serialized TensorRT engines for accelerated inference.",
"device": {
"label": "GPU Device Index",
"description": "The GPU device index to use."
}
},
"zmq": {
"label": "ZMQ IPC",
"description": "ZMQ IPC detector that offloads inference to an external process via a ZeroMQ IPC endpoint.",
"endpoint": {
"label": "ZMQ IPC endpoint",
"description": "The ZMQ endpoint to connect to."
},
"request_timeout_ms": {
"label": "ZMQ request timeout in milliseconds",
"description": "Timeout for ZMQ requests in milliseconds."
},
"linger_ms": {
"label": "ZMQ socket linger in milliseconds",
"description": "Socket linger period in milliseconds."
}
}
},
"model": {
"label": "Detection model",
"description": "Settings to configure a custom object detection model and its input shape.",
"path": { "path": {
"label": "Custom object detector model path", "label": "Custom object detector model path",
"description": "Path to a custom detection model file (or plus://<model_id> for Frigate+ models)." "description": "Path to a custom detection model file (or plus://<model_id> for Frigate+ models)."
@@ -621,6 +466,10 @@
"label": "Detect width", "label": "Detect width",
"description": "Width (pixels) of frames used for the detect stream; leave empty to use the native stream resolution." "description": "Width (pixels) of frames used for the detect stream; leave empty to use the native stream resolution."
}, },
"scene": {
"label": "Detect scene",
"description": "The environment this camera looks at, used to pick which of the configured models runs on it. Defaults to the model with a scene of 'all'."
},
"fps": { "fps": {
"label": "Detect FPS", "label": "Detect FPS",
"description": "Desired frames per second to run detection on; lower values reduce CPU usage (recommended value is 5, only set higher - at most 10 - if tracking extremely fast moving objects)." "description": "Desired frames per second to run detection on; lower values reduce CPU usage (recommended value is 5, only set higher - at most 10 - if tracking extremely fast moving objects)."
web/src/components/card/SearchThumbnail.tsx
+3 -6
View File
@@ -13,6 +13,7 @@ import { cn } from "@/lib/utils";
import { TooltipPortal } from "@radix-ui/react-tooltip"; import { TooltipPortal } from "@radix-ui/react-tooltip";
import useContextMenu from "@/hooks/use-contextmenu"; import useContextMenu from "@/hooks/use-contextmenu";
import { getTranslatedLabel } from "@/utils/i18n"; import { getTranslatedLabel } from "@/utils/i18n";
import { isAttributeOfLabel } from "@/utils/modelUtil";
type SearchThumbnailProps = { type SearchThumbnailProps = {
searchResult: SearchResult; searchResult: SearchResult;
@@ -58,9 +59,7 @@ export default function SearchThumbnail({
} }
if ( if (
config.model.attributes_map[searchResult.label]?.includes( isAttributeOfLabel(config, searchResult.label, searchResult.sub_label)
searchResult.sub_label,
)
) { ) {
return searchResult.sub_label; return searchResult.sub_label;
} }
@@ -82,9 +81,7 @@ export default function SearchThumbnail({
} }
if ( if (
config.model.attributes_map[searchResult.label]?.includes( isAttributeOfLabel(config, searchResult.label, searchResult.sub_label)
searchResult.sub_label,
)
) { ) {
return ""; return "";
} }
web/src/components/classification/ClassificationModelEditDialog.tsx
+2 -1
View File
@@ -32,6 +32,7 @@ import {
} from "@/types/frigateConfig"; } from "@/types/frigateConfig";
import { ClassificationDatasetResponse } from "@/types/classification"; import { ClassificationDatasetResponse } from "@/types/classification";
import { getTranslatedLabel } from "@/utils/i18n"; import { getTranslatedLabel } from "@/utils/i18n";
import { isAttributeLabel } from "@/utils/modelUtil";
import { zodResolver } from "@hookform/resolvers/zod"; import { zodResolver } from "@hookform/resolvers/zod";
import axios from "axios"; import axios from "axios";
import { useCallback, useEffect, useMemo, useState } from "react"; import { useCallback, useEffect, useMemo, useState } from "react";
@@ -99,7 +100,7 @@ export default function ClassificationModelEditDialog({
} }
cameraConfig.objects.track.forEach((label) => { cameraConfig.objects.track.forEach((label) => {
if (!config.model.all_attributes.includes(label)) { if (!isAttributeLabel(config, label)) {
labels.add(label); labels.add(label);
} }
}); });
web/src/components/classification/wizard/Step1NameAndDefine.tsx
+2 -1
View File
@@ -27,6 +27,7 @@ import useSWR from "swr";
import { FrigateConfig } from "@/types/frigateConfig"; import { FrigateConfig } from "@/types/frigateConfig";
import { getTranslatedLabel } from "@/utils/i18n"; import { getTranslatedLabel } from "@/utils/i18n";
import { useDocDomain } from "@/hooks/use-doc-domain"; import { useDocDomain } from "@/hooks/use-doc-domain";
import { isAttributeLabel } from "@/utils/modelUtil";
import { import {
Popover, Popover,
PopoverContent, PopoverContent,
@@ -72,7 +73,7 @@ export default function Step1NameAndDefine({
} }
cameraConfig.objects.track.forEach((label) => { cameraConfig.objects.track.forEach((label) => {
if (!config.model.all_attributes.includes(label)) { if (!isAttributeLabel(config, label)) {
labels.add(label); labels.add(label);
} }
}); });
web/src/components/config-form/theme/widgets/ObjectLabelSwitchesWidget.tsx
+6 -13
View File
@@ -19,23 +19,16 @@ function collectLabelmapLabels(labelmap: unknown, labels: Set<string>) {
}); });
} }
// Read labelmap labels from the global model and detector models. // Read labelmap labels from every configured detection model.
function getLabelmapLabels(context: FormContext): string[] { function getLabelmapLabels(context: FormContext): string[] {
const labels = new Set<string>(); const labels = new Set<string>();
const fullConfig = context.fullConfig as FrigateConfig | undefined; const fullConfig = context.fullConfig as FrigateConfig | undefined;
if (fullConfig?.model) { fullConfig?.models?.forEach((model) => {
collectLabelmapLabels(fullConfig.model.labelmap, labels); if (model?.labelmap) {
} collectLabelmapLabels(model.labelmap, labels);
}
if (fullConfig?.detectors) { });
// detectors is a map of detector configs; each may include a model labelmap.
Object.values(fullConfig.detectors).forEach((detector) => {
if (detector?.model?.labelmap) {
collectLabelmapLabels(detector.model.labelmap, labels);
}
});
}
return [...labels]; return [...labels];
} }
web/src/components/filter/SearchFilterGroup.tsx
+2 -1
View File
@@ -26,6 +26,7 @@ import { CalendarRangeFilterButton } from "./CalendarFilterButton";
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { getTranslatedLabel } from "@/utils/i18n"; import { getTranslatedLabel } from "@/utils/i18n";
import { isAttributeLabel } from "@/utils/modelUtil";
import { useAllowedCameras } from "@/hooks/use-allowed-cameras"; import { useAllowedCameras } from "@/hooks/use-allowed-cameras";
type SearchFilterGroupProps = { type SearchFilterGroupProps = {
@@ -73,7 +74,7 @@ export default function SearchFilterGroup({
} }
cameraConfig.objects.track.forEach((label) => { cameraConfig.objects.track.forEach((label) => {
if (!config.model.all_attributes.includes(label)) { if (!isAttributeLabel(config, label)) {
labels.add(label); labels.add(label);
} }
}); });
web/src/components/overlay/ObjectTrackOverlay.tsx
+2 -1
View File
@@ -13,6 +13,7 @@ import { cn } from "@/lib/utils";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { Event } from "@/types/event"; import { Event } from "@/types/event";
import { resolveZoneName } from "@/hooks/use-zone-friendly-name"; import { resolveZoneName } from "@/hooks/use-zone-friendly-name";
import { getPrimaryModel } from "@/utils/modelUtil";
// Use a small tolerance (10ms) for browsers with seek precision by-design issues // Use a small tolerance (10ms) for browsers with seek precision by-design issues
const TOLERANCE = 0.01; const TOLERANCE = 0.01;
@@ -178,7 +179,7 @@ export default function ObjectTrackOverlay({
const getObjectColor = useCallback( const getObjectColor = useCallback(
(label: string, objectId: string) => { (label: string, objectId: string) => {
const objectColor = config?.model?.colormap[label]; const objectColor = getPrimaryModel(config)?.colormap?.[label];
if (objectColor) { if (objectColor) {
const reversed = [...objectColor].reverse(); const reversed = [...objectColor].reverse();
return `rgb(${reversed.join(",")})`; return `rgb(${reversed.join(",")})`;
web/src/pages/Replay.tsx
+2 -1
View File
@@ -49,6 +49,7 @@ import Logo from "@/components/Logo";
import { Separator } from "@/components/ui/separator"; import { Separator } from "@/components/ui/separator";
import { useDocDomain } from "@/hooks/use-doc-domain"; import { useDocDomain } from "@/hooks/use-doc-domain";
import DebugDrawingLayer from "@/components/overlay/DebugDrawingLayer"; import DebugDrawingLayer from "@/components/overlay/DebugDrawingLayer";
import { getPrimaryModel } from "@/utils/modelUtil";
import { IoMdArrowRoundBack } from "react-icons/io"; import { IoMdArrowRoundBack } from "react-icons/io";
type DebugReplayStatus = { type DebugReplayStatus = {
@@ -642,7 +643,7 @@ function ObjectList({ cameraConfig, objects, config }: ObjectListProps) {
if (!config) { if (!config) {
return; return;
} }
return config.model?.colormap; return getPrimaryModel(config)?.colormap;
}, [config]); }, [config]);
const getColorForObjectName = useCallback( const getColorForObjectName = useCallback(
web/src/pages/Settings.tsx
+5 -2
View File
@@ -115,6 +115,7 @@ import SaveAllPreviewPopover, {
type SaveAllPreviewItem, type SaveAllPreviewItem,
} from "@/components/overlay/detail/SaveAllPreviewPopover"; } from "@/components/overlay/detail/SaveAllPreviewPopover";
import { useRestart } from "@/api/ws"; import { useRestart } from "@/api/ws";
import { getPrimaryModel } from "@/utils/modelUtil";
import { import {
Tooltip, Tooltip,
TooltipContent, TooltipContent,
@@ -949,14 +950,16 @@ export default function Settings() {
const pendingKeySet = Object.keys( const pendingKeySet = Object.keys(
sanitizedDetectors as JsonObject, sanitizedDetectors as JsonObject,
).sort(); ).sort();
const savedKeySet = Object.keys(config.detectors ?? {}).sort(); const savedKeySet = [
...(getPrimaryModel(config)?.devices ?? []),
].sort();
detectorKeysChanged = detectorKeysChanged =
JSON.stringify(pendingKeySet) !== JSON.stringify(savedKeySet); JSON.stringify(pendingKeySet) !== JSON.stringify(savedKeySet);
} }
let modelTabChanged = false; let modelTabChanged = false;
if (sanitizedModel && typeof sanitizedModel === "object") { if (sanitizedModel && typeof sanitizedModel === "object") {
const newPath = (sanitizedModel as { path?: string }).path; const newPath = (sanitizedModel as { path?: string }).path;
const oldPath = config.model?.path; const oldPath = getPrimaryModel(config)?.path;
const newIsPlus = const newIsPlus =
typeof newPath === "string" && newPath.startsWith("plus://"); typeof newPath === "string" && newPath.startsWith("plus://");
const oldIsPlus = const oldIsPlus =
web/src/types/frigateConfig.ts
+28 -40
View File
@@ -66,6 +66,7 @@ export interface CameraConfig {
height: number; height: number;
max_disappeared: number; max_disappeared: number;
min_initialized: number; min_initialized: number;
scene: string | null;
stationary: { stationary: {
interval: number; interval: number;
max_frames: { max_frames: {
@@ -405,6 +406,32 @@ export type GenAIAgentConfig = {
runtime_options?: Record<string, unknown>; runtime_options?: Record<string, unknown>;
}; };
export type DetectionModelConfig = {
scene: string;
devices: string[];
height: number;
input_pixel_format: string;
input_tensor: string;
labelmap: Record<string, unknown>;
labelmap_path: string | null;
model_type: string;
path: string | null;
width: number;
colormap: { [key: string]: [number, number, number] };
attributes_map: { [key: string]: string[] };
all_attributes: string[];
plus?: {
name: string;
id: string;
trainDate: string;
baseModel: string;
isBaseModel: boolean;
supportedDetectors: string[];
width: number;
height: number;
} | null;
};
export interface FrigateConfig { export interface FrigateConfig {
version: string; version: string;
safe_mode: boolean; safe_mode: boolean;
@@ -468,23 +495,6 @@ export interface FrigateConfig {
width: number | null; width: number | null;
}; };
detectors: {
coral: {
device: string;
model: {
height: number;
input_pixel_format: string;
input_tensor: string;
labelmap: Record<string, string>;
labelmap_path: string | null;
model_type: string;
path: string;
width: number;
};
type: string;
};
};
environment_vars: Record<string, unknown>; environment_vars: Record<string, unknown>;
face_recognition: FaceRecognitionConfig; face_recognition: FaceRecognitionConfig;
@@ -524,29 +534,7 @@ export interface FrigateConfig {
logs: Record<string, string>; logs: Record<string, string>;
}; };
model: { models: DetectionModelConfig[];
height: number;
input_pixel_format: string;
input_tensor: string;
labelmap: Record<string, unknown>;
labelmap_path: string | null;
model_type: string;
path: string | null;
width: number;
colormap: { [key: string]: [number, number, number] };
attributes_map: { [key: string]: string[] };
all_attributes: string[];
plus?: {
name: string;
id: string;
trainDate: string;
baseModel: string;
isBaseModel: boolean;
supportedDetectors: string[];
width: number;
height: number;
} | null;
};
motion: Record<string, unknown> | null; motion: Record<string, unknown> | null;
web/src/utils/configUtil.ts
+2 -1
View File
@@ -493,6 +493,7 @@ export interface SectionSavePayload {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
import { resolveAndCleanSchema } from "@/lib/config-schema"; import { resolveAndCleanSchema } from "@/lib/config-schema";
import { getAllAttributes } from "@/utils/modelUtil";
type SchemaWithDefinitions = RJSFSchema & { type SchemaWithDefinitions = RJSFSchema & {
$defs?: Record<string, RJSFSchema>; $defs?: Record<string, RJSFSchema>;
@@ -796,7 +797,7 @@ export function getEffectiveAttributeLabels(
fullCameraConfig: CameraConfig | undefined, fullCameraConfig: CameraConfig | undefined,
level: "global" | "camera" | "replay" | undefined, level: "global" | "camera" | "replay" | undefined,
): string[] { ): string[] {
const all = fullConfig?.model?.all_attributes ?? []; const all = getAllAttributes(fullConfig);
if (level !== "global" && fullCameraConfig?.type === "lpr") { if (level !== "global" && fullCameraConfig?.type === "lpr") {
return all.filter((attr) => attr !== "license_plate"); return all.filter((attr) => attr !== "license_plate");
} }
web/src/utils/iconUtil.tsx
+4 -2
View File
@@ -56,8 +56,10 @@ export function getAttributeLabels(config?: FrigateConfig) {
const labels = new Set(); const labels = new Set();
Object.values(config.model.attributes_map).forEach((values) => config.models?.forEach((model) =>
values.forEach((label) => labels.add(label)), Object.values(model.attributes_map ?? {}).forEach((values) =>
values.forEach((label) => labels.add(label)),
),
); );
return [...labels]; return [...labels];
} }
web/src/utils/modelUtil.ts
+69
View File
@@ -0,0 +1,69 @@
import { DetectionModelConfig, FrigateConfig } from "@/types/frigateConfig";
/**
* The model a camera runs on, matched by the camera's detect scene.
*
* Falls back to the model for every scene, then to the only configured model,
* which is what the backend does when a camera does not name a scene.
*/
export function getModelForCamera(
config?: FrigateConfig,
camera?: string,
): DetectionModelConfig | undefined {
const models = config?.models;
if (!models?.length) {
return undefined;
}
const scene = camera ? config?.cameras?.[camera]?.detect?.scene : undefined;
if (scene) {
const match = models.find((model) => model.scene == scene);
if (match) {
return match;
}
}
return models.find((model) => model.scene == "all") ?? models[0];
}
/** The model used when the question is not about a specific camera. */
export function getPrimaryModel(
config?: FrigateConfig,
): DetectionModelConfig | undefined {
return getModelForCamera(config);
}
/** Every object attribute across all configured models. */
export function getAllAttributes(config?: FrigateConfig): string[] {
const attributes = new Set<string>();
config?.models?.forEach((model) =>
model.all_attributes?.forEach((attribute) => attributes.add(attribute)),
);
return [...attributes];
}
/** Whether a label is an attribute of any configured model. */
export function isAttributeLabel(
config: FrigateConfig | undefined,
label: string,
): boolean {
return !!config?.models?.some((model) =>
model.all_attributes?.includes(label),
);
}
/** Whether an attribute belongs to a parent label in any configured model. */
export function isAttributeOfLabel(
config: FrigateConfig | undefined,
label: string,
attribute: string,
): boolean {
return !!config?.models?.some((model) =>
model.attributes_map?.[label]?.includes(attribute),
);
}
web/src/views/settings/DetectorsAndModelSettingsView.tsx
+11 -7
View File
@@ -49,6 +49,7 @@ import {
import { ConfigSectionTemplate } from "@/components/config-form/sections"; import { ConfigSectionTemplate } from "@/components/config-form/sections";
import { ConfigMessageBanner } from "@/components/config-form/ConfigMessageBanner"; import { ConfigMessageBanner } from "@/components/config-form/ConfigMessageBanner";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { getPrimaryModel } from "@/utils/modelUtil";
import { import {
buildHiddenFieldContext, buildHiddenFieldContext,
getSectionConfig, getSectionConfig,
@@ -115,8 +116,9 @@ const STATUS_BAR_KEY = "detectors_and_model";
const EMPTY_PENDING: Record<string, ConfigSectionData> = {}; const EMPTY_PENDING: Record<string, ConfigSectionData> = {};
const deriveInitialState = (config: FrigateConfig): PageState => { const deriveInitialState = (config: FrigateConfig): PageState => {
const plusModelId = config.model?.plus?.id; const primaryModel = getPrimaryModel(config);
const modelPath = config.model?.path; const plusModelId = primaryModel?.plus?.id;
const modelPath = primaryModel?.path;
const plusEnabled = Boolean(config.plus?.enabled); const plusEnabled = Boolean(config.plus?.enabled);
// The reliable signal that a Plus model is currently active is the // The reliable signal that a Plus model is currently active is the
@@ -136,10 +138,12 @@ const deriveInitialState = (config: FrigateConfig): PageState => {
modelTab = "custom"; modelTab = "custom";
} }
const { plus: _plus, ...modelWithoutPlus } = (config.model ?? {}) as Record< const {
string, plus: _plus,
unknown scene: _scene,
>; devices: _devices,
...modelWithoutPlus
} = (primaryModel ?? {}) as Record<string, unknown>;
// If a Plus model is active, the resolved `model.path` is auto-derived from // If a Plus model is active, the resolved `model.path` is auto-derived from
// `plus.id` — drop it so the Custom tab starts clean and doesn't silently // `plus.id` — drop it so the Custom tab starts clean and doesn't silently
// re-save the same Plus model when the user thinks they switched modes. // re-save the same Plus model when the user thinks they switched modes.
@@ -148,7 +152,7 @@ const deriveInitialState = (config: FrigateConfig): PageState => {
} }
return { return {
detectors: (config.detectors ?? {}) as ConfigSectionData, detectors: { devices: primaryModel?.devices ?? [] } as ConfigSectionData,
modelTab, modelTab,
plusModelId: plusModelId ?? undefined, plusModelId: plusModelId ?? undefined,
customModel: modelWithoutPlus as ConfigSectionData, customModel: modelWithoutPlus as ConfigSectionData,
web/src/views/settings/FrigatePlusSettingsView.tsx
+3 -2
View File
@@ -17,6 +17,7 @@ import { CameraNameLabel } from "@/components/camera/FriendlyNameLabel";
import { FrigateConfig } from "@/types/frigateConfig"; import { FrigateConfig } from "@/types/frigateConfig";
import { isReplayCamera } from "@/utils/cameraUtil"; import { isReplayCamera } from "@/utils/cameraUtil";
import type { SettingsPageProps } from "@/views/settings/SingleSectionPage"; import type { SettingsPageProps } from "@/views/settings/SingleSectionPage";
import { getPrimaryModel } from "@/utils/modelUtil";
export default function FrigatePlusSettingsView(_props: SettingsPageProps) { export default function FrigatePlusSettingsView(_props: SettingsPageProps) {
const { t } = useTranslation("views/settings"); const { t } = useTranslation("views/settings");
@@ -51,7 +52,7 @@ export default function FrigatePlusSettingsView(_props: SettingsPageProps) {
description={ description={
<> <>
<p>{t("frigatePlus.apiKey.desc")}</p> <p>{t("frigatePlus.apiKey.desc")}</p>
{!config?.model.plus && ( {!getPrimaryModel(config)?.plus && (
<div className="mt-2 flex items-center text-primary-variant"> <div className="mt-2 flex items-center text-primary-variant">
<Link <Link
to="https://frigate.video/plus" to="https://frigate.video/plus"
@@ -85,7 +86,7 @@ export default function FrigatePlusSettingsView(_props: SettingsPageProps) {
{config?.plus?.enabled && ( {config?.plus?.enabled && (
<FrigatePlusCurrentModelSummary <FrigatePlusCurrentModelSummary
plusModel={config.model.plus} plusModel={getPrimaryModel(config)?.plus}
action={ action={
<Button <Button
size="sm" size="sm"
web/src/views/settings/ObjectSettingsView.tsx
+6 -6
View File
@@ -34,6 +34,7 @@ import { useCameraFriendlyName } from "@/hooks/use-camera-friendly-name";
import { AudioLevelGraph } from "@/components/audio/AudioLevelGraph"; import { AudioLevelGraph } from "@/components/audio/AudioLevelGraph";
import { useWs } from "@/api/ws"; import { useWs } from "@/api/ws";
import { cn } from "@/lib/utils"; import { cn } from "@/lib/utils";
import { getPrimaryModel } from "@/utils/modelUtil";
type ObjectSettingsViewProps = { type ObjectSettingsViewProps = {
selectedCamera?: string; selectedCamera?: string;
@@ -172,11 +173,10 @@ export default function ObjectSettingsView({
<div className="mb-5 space-y-3 text-sm text-muted-foreground"> <div className="mb-5 space-y-3 text-sm text-muted-foreground">
<p> <p>
{t("debug.detectorDesc", { {t("debug.detectorDesc", {
detectors: config detectors: (config?.models ?? [])
? Object.keys(config?.detectors) .flatMap((model) => model.devices ?? [])
.map((detector) => capitalizeFirstLetter(detector)) .map((device) => capitalizeFirstLetter(device))
.join(",") .join(","),
: "",
})} })}
</p> </p>
<p>{t("debug.desc")}</p> <p>{t("debug.desc")}</p>
@@ -380,7 +380,7 @@ function ObjectList({ cameraConfig, objects }: ObjectListProps) {
return; return;
} }
return config.model?.colormap; return getPrimaryModel(config)?.colormap;
}, [config]); }, [config]);
const getColorForObjectName = useCallback( const getColorForObjectName = useCallback(
web/src/views/settings/components/FrigatePlusCurrentModelSummary.tsx
+2 -2
View File
@@ -3,11 +3,11 @@ import {
SettingsGroupCard, SettingsGroupCard,
SplitCardRow, SplitCardRow,
} from "@/components/card/SettingsGroupCard"; } from "@/components/card/SettingsGroupCard";
import type { FrigateConfig } from "@/types/frigateConfig"; import type { DetectionModelConfig } from "@/types/frigateConfig";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
type FrigatePlusCurrentModelSummaryProps = { type FrigatePlusCurrentModelSummaryProps = {
plusModel: FrigateConfig["model"]["plus"]; plusModel: DetectionModelConfig["plus"];
action?: ReactNode; action?: ReactNode;
}; };