diff --git a/docker/rockchip/Dockerfile b/docker/rockchip/Dockerfile index 92ff171ba..9dd66f246 100644 --- a/docker/rockchip/Dockerfile +++ b/docker/rockchip/Dockerfile @@ -9,10 +9,11 @@ COPY docker/rockchip/requirements-wheels-rk.txt /requirements-wheels-rk.txt RUN sed -i "/https:\/\//d" /requirements-wheels.txt RUN pip3 wheel --wheel-dir=/rk-wheels -c /requirements-wheels.txt -r /requirements-wheels-rk.txt -FROM wget as rk-libs +FROM wget as rk-downloads RUN wget -qO librknnrt.so https://github.com/MarcA711/rknpu2/raw/master/runtime/RK3588/Linux/librknn_api/aarch64/librknnrt.so RUN wget -qO ffmpeg https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/latest/ffmpeg RUN wget -qO ffprobe https://github.com/MarcA711/Rockchip-FFmpeg-Builds/releases/download/latest/ffprobe +RUN wget -qO yolov8n-320x320.rknn https://github.com/MarcA711/rknn-models/releases/download/latest/yolov8n-320x320.rknn FROM deps AS rk-deps ARG TARGETARCH @@ -21,12 +22,12 @@ RUN --mount=type=bind,from=rk-wheels,source=/rk-wheels,target=/deps/rk-wheels \ WORKDIR /opt/frigate/ COPY --from=rootfs / / -COPY --from=rk-libs /rootfs/librknnrt.so /usr/lib/ -COPY docker/rockchip/yolov8n-320x320.rknn /models/ +COPY --from=rk-downloads /rootfs/librknnrt.so /usr/lib/ +COPY --from=rk-downloads /rootfs/yolov8n-320x320.rknn /models/ RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffmpeg RUN rm -rf /usr/lib/btbn-ffmpeg/bin/ffprobe -COPY --from=rk-libs /rootfs/ffmpeg /usr/lib/btbn-ffmpeg/bin/ -COPY --from=rk-libs /rootfs/ffprobe /usr/lib/btbn-ffmpeg/bin/ +COPY --from=rk-downloads /rootfs/ffmpeg /usr/lib/btbn-ffmpeg/bin/ +COPY --from=rk-downloads /rootfs/ffprobe /usr/lib/btbn-ffmpeg/bin/ RUN chmod +x /usr/lib/btbn-ffmpeg/bin/ffmpeg RUN chmod +x /usr/lib/btbn-ffmpeg/bin/ffprobe diff --git a/docker/rockchip/yolov8n-320x320.rknn b/docker/rockchip/yolov8n-320x320.rknn deleted file mode 100644 index 53ab27fc9..000000000 Binary files a/docker/rockchip/yolov8n-320x320.rknn and /dev/null differ diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 7233f04d0..c41794c3e 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -309,14 +309,23 @@ RKNN support is provided using the `-rk` suffix for the docker image. Moreover, ### Configuration This `config.yml` shows all relevant options to configure the detector and explains them. All values shown are the default values (except for one). Lines that are required at least to use the detector are labeled as required, all other lines are optional. + ```yaml detectors: # required rknn: # required type: rknn # required + # core mask for npu + core_mask: 0 + # yolov8 model in rknn format to use; allowed calues: n, s, m, l, x + yolov8_rknn_model: n + # minimal confidence for detection + min_score: 0.5 + # determines whether two overlapping boxes should be combined + nms_thresh: 0.45 model: # required # path to .rknn model file - path: /models/yolov8n-320x320.rknn + path: # width and height of detection frames width: 320 height: 320 @@ -325,4 +334,58 @@ model: # required input_pixel_format: bgr # required # shape of detection frame input_tensor: nhwc -``` \ No newline at end of file +``` + +Explanation for rknn specific options: +- **core mask** controls which cores of your NPU should be used. This option applies only to SoCs with a multicore NPU (at the time of writing this in only the RK3588/S). The easiest way is to pass the value as a binary number. To do so, use the prefix `0b` and write a `0` to disable a core and a `1` to enable a core, whereas the last digit coresponds to core0, the second last to core1, etc. Examples: + - `core_mask: 0b000` or just `core_mask: 0` let the NPU decide which cores should be used. Default and recommended value. + - `core_mask: 0b001` use only core0 + - `core_mask: 0b110` use core1 and core2 +- **yolov8_rknn_model** see section below. +- **min_score** sets the minimal detection confidence. Should have the same value as min_score in the object block. See also [Reducing false positives](/guides/false_positives.md). +- **nms_thresh** is the IoU threshold for Non-maximum Suppression (NMS). Enable "bounding boxes" in the debug viewer. + - *Decrease* if two overlapping objects (for example one person in front of another) are detected as one object. + - *Increase* if there are multiple boxes around one object. + +### Choosing a model + +There are 5 yolov8 models that differ in size and therefore load the NPU more or less. In ascending order, with the top one being the smallest and least computationally intensive model: + +| Model | Size in mb | +| ------- | ---------- | +| yolov8n | 9 | +| yolov8s | 25 | +| yolov8m | 54 | +| yolov8l | 90 | +| yolov8x | 136 | + +:::tip + +You can get the load of your NPU with the following command: + +```bash +$ cat /sys/kernel/debug/rknpu/load +>> NPU load: Core0: 0%, Core1: 0%, Core2: 0%, +``` + +::: + +- By default the rknn detector uses the yolov8**n** model (`yolov8_rknn_model: n`). This model comes with the image, so no further steps than those mentioned above are necessary. +- If you want to use are more precise model, you can set `yolov8_rknn_model:` to `s`, `m`, `l` or `x`. But additional steps are required: + 1. Mount the directory `/model/download/` to your system using one of the below methods. Of course, you can change to destination folder. + - If you start frigate with docker run, append this flag to your command: `-v /model/download:./data/rknn-models` + - If you use docker compose, append this to your `volumes` block: `/model/download:./data/rknn-models` + 2. Download the rknn model. + - If your server has an internet connection, it will download the model. + - Otherwise, you can download the model from [this Github repository](https://github.com/MarcA711/rknn-models/releases/tag/latest) on another device and place it in the `rknn-models` folder that you mounted to your system. +- Finally, you can also provide your own model. Note, that you will need to convert your model to the rknn format using `rknn-toolkit2` on a x86 machine. Afterwards, you can mount a directory to the image (docker run flag: `-v /model/custom:./data/my-rknn-models` or docker compose: add `/model/custom:./data/my-rknn-models` to the `volumes` block) and place your model file in that directory. Then you need to pass the path to your model using the `path` option of your `model` block like this: +```yaml +model: + path: /model/custom/my-rknn-model.rknn +``` + +:::caution + +The `path` option of the `model` block will overwrite the `yolov8_rknn_model` option of the `detectors` block. So if you want to use one of the provided yolov8 models, make sure to not specify the `path` option. + +::: diff --git a/frigate/detectors/plugins/rknn.py b/frigate/detectors/plugins/rknn.py index e40c1d4bb..f46d4fe9d 100644 --- a/frigate/detectors/plugins/rknn.py +++ b/frigate/detectors/plugins/rknn.py @@ -1,4 +1,6 @@ import logging +import os.path +import urllib.request from typing import Literal import cv2 @@ -25,7 +27,11 @@ DETECTOR_KEY = "rknn" class RknnDetectorConfig(BaseDetectorConfig): type: Literal[DETECTOR_KEY] - score_thresh: float = Field( + yolov8_rknn_model: Literal['n', 's', 'm', 'l', 'x'] = 'n' + core_mask: int = Field( + default=0, ge=0, le=7, title="Core mask for NPU." + ) + min_score: float = Field( default=0.5, ge=0, le=1, title="Minimal confidence for detection." ) nms_thresh: float = Field( @@ -37,20 +43,51 @@ class Rknn(DetectionApi): type_key = DETECTOR_KEY def __init__(self, config: RknnDetectorConfig): - self.height = config.model.height - self.width = config.model.width - self.score_thresh = config.score_thresh - self.nms_thresh = config.nms_thresh self.model_path = config.model.path or "/models/yolov8n-320x320.rknn" + if config.model.path != None: + self.model_path = config.model.path + else: + if config.yolov8_rknn_model == 'n': + self.model_path = "/models/yolov8n-320x320.rknn" + else: + # check if user mounted /models/download/ + if not os.path.isdir("/models/download/"): + logger.error("Make sure to mount the directory \"/models/download/\" to your system. Otherwise the file will be downloaded at every restart.") + raise Exception("Make sure to mount the directory \"/models/download/\" to your system. Otherwise the file will be downloaded at every restart.") + + + self.model_path = "/models/download/yolov8{}-320x320.rknn".format(config.yolov8_rknn_model) + if os.path.isfile(self.model_path) == False: + logger.info("Downloading yolov8{} model.".format(config.yolov8_rknn_model)) + urllib.request.urlretrieve("https://github.com/MarcA711/rknn-models/releases/download/latest/yolov8{}-320x320.rknn".format(config.yolov8_rknn_model), self.model_path) + + if (config.model.width != 320) or (config.model.height != 320): + logger.error("Make sure to set the model width and heigth to 320 in your config.yml.") + raise Exception("Make sure to set the model width and heigth to 320 in your config.yml.") + + if config.model.input_pixel_format != 'bgr': + logger.error("Make sure to set the model input_pixel_format to \"bgr\" in your config.yml.") + raise Exception("Make sure to set the model input_pixel_format to \"bgr\" in your config.yml.") + + if config.model.input_tensor != 'nhwc': + logger.error("Make sure to set the model input_tensor to \"nhwc\" in your config.yml.") + raise Exception("Make sure to set the model input_tensor to \"nhwc\" in your config.yml.") + + self.height = config.model.height + self.width = config.model.width + self.core_mask = config.core_mask + self.min_score = config.min_score + self.nms_thresh = config.nms_thresh + from rknnlite.api import RKNNLite self.rknn = RKNNLite(verbose=False) if self.rknn.load_rknn(self.model_path) != 0: logger.error("Error initializing rknn model.") - if self.rknn.init_runtime() != 0: - logger.error("Error initializing rknn runtime.") + if self.rknn.init_runtime(core_mask=self.core_mask) != 0: + logger.error("Error initializing rknn runtime. Do you run docker in privileged mode?") def __del__(self): self.rknn.release() @@ -86,9 +123,9 @@ class Rknn(DetectionApi): ) ) - # indices of rows with confidence > SCORE_THRESH with Non-maximum Suppression (NMS) + # indices of rows with confidence > min_score with Non-maximum Suppression (NMS) result_boxes = cv2.dnn.NMSBoxes( - boxes, scores, self.score_thresh, self.nms_thresh, 0.5 + boxes, scores, self.min_score, self.nms_thresh, 0.5 ) detections = np.zeros((20, 6), np.float32)