diff --git a/docker/rootfs/etc/ld.so.conf.d/vim3.conf b/docker/rootfs/etc/ld.so.conf.d/vim3.conf new file mode 100644 index 000000000..d93ad7891 --- /dev/null +++ b/docker/rootfs/etc/ld.so.conf.d/vim3.conf @@ -0,0 +1,2 @@ +#VIM3 objects +/lib/vim3 diff --git a/docker/rootfs/lib/vim3/libArchModelSw.so b/docker/rootfs/lib/vim3/libArchModelSw.so new file mode 100755 index 000000000..2c208df32 Binary files /dev/null and b/docker/rootfs/lib/vim3/libArchModelSw.so differ diff --git a/docker/rootfs/lib/vim3/libCLC.so b/docker/rootfs/lib/vim3/libCLC.so new file mode 100755 index 000000000..99c027b49 Binary files /dev/null and b/docker/rootfs/lib/vim3/libCLC.so differ diff --git a/docker/rootfs/lib/vim3/libGAL.so b/docker/rootfs/lib/vim3/libGAL.so new file mode 100755 index 000000000..af9bc364b Binary files /dev/null and b/docker/rootfs/lib/vim3/libGAL.so differ diff --git a/docker/rootfs/lib/vim3/libNNArchPerf.so b/docker/rootfs/lib/vim3/libNNArchPerf.so new file mode 100755 index 000000000..5b9cd4952 Binary files /dev/null and b/docker/rootfs/lib/vim3/libNNArchPerf.so differ diff --git a/docker/rootfs/lib/vim3/libNNVXCBinary.so b/docker/rootfs/lib/vim3/libNNVXCBinary.so new file mode 100755 index 000000000..60750f8b0 Binary files /dev/null and b/docker/rootfs/lib/vim3/libNNVXCBinary.so differ diff --git a/docker/rootfs/lib/vim3/libOpenVX.so b/docker/rootfs/lib/vim3/libOpenVX.so new file mode 100755 index 000000000..3cac7fe37 Binary files /dev/null and b/docker/rootfs/lib/vim3/libOpenVX.so differ diff --git a/docker/rootfs/lib/vim3/libOpenVXU.so b/docker/rootfs/lib/vim3/libOpenVXU.so new file mode 100755 index 000000000..64da570ea Binary files /dev/null and b/docker/rootfs/lib/vim3/libOpenVXU.so differ diff --git a/docker/rootfs/lib/vim3/libVSC.so b/docker/rootfs/lib/vim3/libVSC.so new file mode 100755 index 000000000..8a72dc14d Binary files /dev/null and b/docker/rootfs/lib/vim3/libVSC.so differ diff --git a/docker/rootfs/lib/vim3/libtim-vx.so b/docker/rootfs/lib/vim3/libtim-vx.so new file mode 100755 index 000000000..1200fb88b Binary files /dev/null and b/docker/rootfs/lib/vim3/libtim-vx.so differ diff --git a/docker/rootfs/lib/vim3/libvx_delegate.so b/docker/rootfs/lib/vim3/libvx_delegate.so new file mode 100755 index 000000000..5291ef053 Binary files /dev/null and b/docker/rootfs/lib/vim3/libvx_delegate.so differ diff --git a/frigate/detectors/plugins/vim3.py b/frigate/detectors/plugins/vim3.py new file mode 100644 index 000000000..47aaff541 --- /dev/null +++ b/frigate/detectors/plugins/vim3.py @@ -0,0 +1,79 @@ +import logging +import numpy as np + +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig +from typing import Literal +from pydantic import Extra, Field + +try: + from tflite_runtime.interpreter import Interpreter, load_delegate +except ModuleNotFoundError: + from tensorflow.lite.python.interpreter import Interpreter, load_delegate + + +logger = logging.getLogger(__name__) + +DETECTOR_KEY = "vim3" + + +class vim3DetectorConfig(BaseDetectorConfig): + type: Literal[DETECTOR_KEY] + device: str = Field(default=None, title="Device Type") + + +class vim3Tfl(DetectionApi): + type_key = DETECTOR_KEY + + def __init__(self, detector_config: vim3DetectorConfig): + device_config = {"device": "usb"} + if detector_config.device is not None: + device_config = {"device": detector_config.device} + + edge_tpu_delegate = None + + try: + logger.info(f"Attempting to register VIM3 TPU") + edge_tpu_delegate = load_delegate("libvx_delegate.so") + logger.info("TPU found") + self.interpreter = Interpreter( + model_path=detector_config.model.path or "/cpu_model.tflite", + experimental_delegates=[edge_tpu_delegate], + ) + except ValueError: + logger.error( + "No EdgeTPU was detected. If you do not have a Accelerator (VIM3) device yet, you must configure CPU detectors." + ) + raise + + self.interpreter.allocate_tensors() + + self.tensor_input_details = self.interpreter.get_input_details() + self.tensor_output_details = self.interpreter.get_output_details() + + def detect_raw(self, tensor_input): + self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) + self.interpreter.invoke() + + boxes = self.interpreter.tensor(self.tensor_output_details[0]["index"])()[0] + class_ids = self.interpreter.tensor(self.tensor_output_details[1]["index"])()[0] + scores = self.interpreter.tensor(self.tensor_output_details[2]["index"])()[0] + count = int( + self.interpreter.tensor(self.tensor_output_details[3]["index"])()[0] + ) + + detections = np.zeros((20, 6), np.float32) + + for i in range(count): + if scores[i] < 0.4 or i == 20: + break + detections[i] = [ + class_ids[i], + float(scores[i]), + boxes[i][0], + boxes[i][1], + boxes[i][2], + boxes[i][3], + ] + + return detections