From 9054f412822605dfb0cab92959c9b8f75871a34a Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Fri, 7 Jun 2024 05:48:18 -0500 Subject: [PATCH] handle AUTO issue and update docs --- docker/main/rootfs/labelmap/coco.txt | 91 +++++++++++++++++++++ docs/docs/configuration/object_detectors.md | 27 +++--- frigate/detectors/plugins/openvino.py | 6 ++ 3 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 docker/main/rootfs/labelmap/coco.txt diff --git a/docker/main/rootfs/labelmap/coco.txt b/docker/main/rootfs/labelmap/coco.txt new file mode 100644 index 000000000..79fff1772 --- /dev/null +++ b/docker/main/rootfs/labelmap/coco.txt @@ -0,0 +1,91 @@ +0 person +1 bicycle +2 car +3 motorcycle +4 airplane +5 bus +6 train +7 car +8 boat +9 traffic light +10 fire hydrant +11 street sign +12 stop sign +13 parking meter +14 bench +15 bird +16 cat +17 dog +18 horse +19 sheep +20 cow +21 elephant +22 bear +23 zebra +24 giraffe +25 hat +26 backpack +27 umbrella +28 shoe +29 eye glasses +30 handbag +31 tie +32 suitcase +33 frisbee +34 skis +35 snowboard +36 sports ball +37 kite +38 baseball bat +39 baseball glove +40 skateboard +41 surfboard +42 tennis racket +43 bottle +44 plate +45 wine glass +46 cup +47 fork +48 knife +49 spoon +50 bowl +51 banana +52 apple +53 sandwich +54 orange +55 broccoli +56 carrot +57 hot dog +58 pizza +59 donut +60 cake +61 chair +62 couch +63 potted plant +64 bed +65 mirror +66 dining table +67 window +68 desk +69 toilet +70 door +71 tv +72 laptop +73 mouse +74 remote +75 keyboard +76 cell phone +77 microwave +78 oven +79 toaster +80 sink +81 refrigerator +82 blender +83 book +84 clock +85 vase +86 scissors +87 teddy bear +88 hair drier +89 toothbrush +90 hair brush \ No newline at end of file diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index f46ae8de5..55ef1e156 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -109,9 +109,9 @@ detectors: The OpenVINO detector type runs an OpenVINO IR model on AMD and Intel CPUs, Intel GPUs and Intel VPU hardware. To configure an OpenVINO detector, set the `"type"` attribute to `"openvino"`. -The OpenVINO device to be used is specified using the `"device"` attribute according to the naming conventions in the [Device Documentation](https://docs.openvino.ai/latest/openvino_docs_OV_UG_Working_with_devices.html). Other supported devices could be `AUTO`, `CPU`, `GPU`, `MYRIAD`, etc. If not specified, the default OpenVINO device will be selected by the `AUTO` plugin. +The OpenVINO device to be used is specified using the `"device"` attribute according to the naming conventions in the [Device Documentation](https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes.html). The most common devices are `CPU` and `GPU`. Currently, there is a known issue with using `AUTO`. For backwards compatibility, Frigate will attempt to use `GPU` if `AUTO` is set in your configuration. -OpenVINO is supported on 6th Gen Intel platforms (Skylake) and newer. It will also run on AMD CPUs despite having no official support for it. A supported Intel platform is required to use the `GPU` device with OpenVINO. The `MYRIAD` device may be run on any platform, including Arm devices. For detailed system requirements, see [OpenVINO System Requirements](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/system-requirements.html) +OpenVINO is supported on 6th Gen Intel platforms (Skylake) and newer. It will also run on AMD CPUs despite having no official support for it. A supported Intel platform is required to use the `GPU` device with OpenVINO. For detailed system requirements, see [OpenVINO System Requirements](https://docs.openvino.ai/2024/about-openvino/release-notes-openvino/system-requirements.html) ### Supported Models @@ -123,15 +123,14 @@ An OpenVINO model is provided in the container at `/openvino-model/ssdlite_mobil detectors: ov: type: openvino - device: AUTO - model: - path: /openvino-model/ssdlite_mobilenet_v2.xml + device: GPU model: width: 300 height: 300 input_tensor: nhwc input_pixel_format: bgr + path: /openvino-model/ssdlite_mobilenet_v2.xml labelmap_path: /openvino-model/coco_91cl_bkgr.txt ``` @@ -143,9 +142,7 @@ This detector also supports YOLOX. Frigate does not come with any YOLOX models p detectors: ov: type: openvino - device: AUTO - model: - path: /path/to/yolox_tiny.xml + device: GPU model: width: 416 @@ -153,6 +150,7 @@ model: input_tensor: nchw input_pixel_format: bgr model_type: yolox + path: /path/to/yolox_tiny.xml labelmap_path: /path/to/coco_80cl.txt ``` @@ -174,17 +172,20 @@ After placing the downloaded onnx model in your config folder, you can use the f detectors: ov: type: openvino - device: AUTO + device: GPU model: model_type: yolonas - path: /config/yolo_nas_s.onnx width: 320 # <--- should match whatever was set in notebook height: 320 # <--- should match whatever was set in notebook input_tensor: nchw input_pixel_format: bgr + path: /config/yolo_nas_s.onnx + labelmap_path: /labelmap/coco-80.txt ``` +Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. + ## NVidia TensorRT Detector Nvidia GPUs may be used for object detection using the TensorRT libraries. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt`. This detector is designed to work with Yolo models for object detection. @@ -359,9 +360,9 @@ model: # required The correct labelmap must be loaded for each model. If you use a custom model (see notes below), you must make sure to provide the correct labelmap. The table below lists the correct paths for the bundled models: -| `path` | `labelmap_path` | -| ------------------- | --------------------- | -| deci-fp16-yolonas_* | /labelmap/coco-80.txt | +| `path` | `labelmap_path` | +| --------------------- | --------------------- | +| deci-fp16-yolonas\_\* | /labelmap/coco-80.txt | ### Choosing a model diff --git a/frigate/detectors/plugins/openvino.py b/frigate/detectors/plugins/openvino.py index 52e33c946..4ac610d77 100644 --- a/frigate/detectors/plugins/openvino.py +++ b/frigate/detectors/plugins/openvino.py @@ -29,6 +29,12 @@ class OvDetector(DetectionApi): self.h = detector_config.model.height self.w = detector_config.model.width + if detector_config.device == "AUTO": + logger.warning( + "OpenVINO AUTO device type is not currently supported. Attempting to use GPU instead." + ) + detector_config.device = "GPU" + self.interpreter = self.ov_core.compile_model( model=detector_config.model.path, device_name=detector_config.device )