Cleanup detection (#17785)

* Fix yolov9 NMS

* Improve batched yolo NMS

* Consolidate grids and strides calculation

* Use existing variable

* Remove

* Ensure init is called

frigate/detectors/plugins/onnx.py

@@ -31,6 +31,8 @@ class ONNXDetector(DetectionApi):
     type_key = DETECTOR_KEY
 
     def __init__(self, detector_config: ONNXDetectorConfig):
+        super().__init__(detector_config)
+
         try:
             import onnxruntime as ort
 
@@ -52,31 +54,13 @@ class ONNXDetector(DetectionApi):
             path, providers=providers, provider_options=options
         )
 
-        self.h = detector_config.model.height
-        self.w = detector_config.model.width
         self.onnx_model_type = detector_config.model.model_type
         self.onnx_model_px = detector_config.model.input_pixel_format
         self.onnx_model_shape = detector_config.model.input_tensor
         path = detector_config.model.path
 
         if self.onnx_model_type == ModelTypeEnum.yolox:
-            grids = []
-            expanded_strides = []
-
-            # decode and orient predictions
-            strides = [8, 16, 32]
-            hsizes = [self.h // stride for stride in strides]
-            wsizes = [self.w // stride for stride in strides]
-
-            for hsize, wsize, stride in zip(hsizes, wsizes, strides):
-                xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))
-                grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
-                grids.append(grid)
-                shape = grid.shape[:2]
-                expanded_strides.append(np.full((*shape, 1), stride))
-
-            self.grids = np.concatenate(grids, 1)
-            self.expanded_strides = np.concatenate(expanded_strides, 1)
+            self.calculate_grids_strides()
 
         logger.info(f"ONNX: {path} loaded")
 
@@ -86,10 +70,12 @@ class ONNXDetector(DetectionApi):
                 None,
                 {
                     "images": tensor_input,
-                    "orig_target_sizes": np.array([[self.h, self.w]], dtype=np.int64),
+                    "orig_target_sizes": np.array(
+                        [[self.height, self.width]], dtype=np.int64
+                    ),
                 },
             )
-            return post_process_dfine(tensor_output, self.w, self.h)
+            return post_process_dfine(tensor_output, self.width, self.height)
 
         model_input_name = self.model.get_inputs()[0].name
         tensor_output = self.model.run(None, {model_input_name: tensor_input})
@@ -111,17 +97,21 @@ class ONNXDetector(DetectionApi):
                 detections[i] = [
                     class_id,
                     confidence,
-                    y_min / self.h,
-                    x_min / self.w,
-                    y_max / self.h,
-                    x_max / self.w,
+                    y_min / self.height,
+                    x_min / self.width,
+                    y_max / self.height,
+                    x_max / self.width,
                 ]
             return detections
         elif self.onnx_model_type == ModelTypeEnum.yologeneric:
-            return post_process_yolo(tensor_output, self.w, self.h)
+            return post_process_yolo(tensor_output, self.width, self.height)
         elif self.onnx_model_type == ModelTypeEnum.yolox:
             return post_process_yolox(
-                tensor_output[0], self.w, self.h, self.grids, self.expanded_strides
+                tensor_output[0],
+                self.width,
+                self.height,
+                self.grids,
+                self.expanded_strides,
             )
         else:
             raise Exception(

frigate/detectors/plugins/openvino.py

@@ -38,6 +38,7 @@ class OvDetector(DetectionApi):
     ]
 
     def __init__(self, detector_config: OvDetectorConfig):
+        super().__init__(detector_config)
         self.ov_core = ov.Core()
         self.ov_model_type = detector_config.model.model_type
 
@@ -133,25 +134,7 @@ class OvDetector(DetectionApi):
                     break
             self.num_classes = tensor_shape[2] - 5
             logger.info(f"YOLOX model has {self.num_classes} classes")
-            self.set_strides_grids()
-
-    def set_strides_grids(self):
-        grids = []
-        expanded_strides = []
-
-        strides = [8, 16, 32]
-
-        hsize_list = [self.h // stride for stride in strides]
-        wsize_list = [self.w // stride for stride in strides]
-
-        for hsize, wsize, stride in zip(hsize_list, wsize_list, strides):
-            xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))
-            grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
-            grids.append(grid)
-            shape = grid.shape[:2]
-            expanded_strides.append(np.full((*shape, 1), stride))
-        self.grids = np.concatenate(grids, 1)
-        self.expanded_strides = np.concatenate(expanded_strides, 1)
+            self.calculate_grids_strides()
 
     ## Takes in class ID, confidence score, and array of [x, y, w, h] that describes detection position,
     ## returns an array that's easily passable back to Frigate.

frigate/detectors/plugins/rknn.py

@@ -4,6 +4,7 @@ import re
 import urllib.request
 from typing import Literal
 
+import numpy as np
 from pydantic import Field
 
 from frigate.const import MODEL_CACHE_DIR
@@ -150,6 +151,62 @@ class Rknn(DetectionApi):
                 'Make sure to set the model input_tensor to "nhwc" in your config.'
             )
 
+    def post_process_yolonas(self, output: list[np.ndarray]):
+        """
+        @param output: output of inference
+        expected shape: [np.array(1, N, 4), np.array(1, N, 80)]
+        where N depends on the input size e.g. N=2100 for 320x320 images
+
+        @return: best results: np.array(20, 6) where each row is
+        in this order (class_id, score, y1/height, x1/width, y2/height, x2/width)
+        """
+
+        N = output[0].shape[1]
+
+        boxes = output[0].reshape(N, 4)
+        scores = output[1].reshape(N, 80)
+
+        class_ids = np.argmax(scores, axis=1)
+        scores = scores[np.arange(N), class_ids]
+
+        args_best = np.argwhere(scores > self.thresh)[:, 0]
+
+        num_matches = len(args_best)
+        if num_matches == 0:
+            return np.zeros((20, 6), np.float32)
+        elif num_matches > 20:
+            args_best20 = np.argpartition(scores[args_best], -20)[-20:]
+            args_best = args_best[args_best20]
+
+        boxes = boxes[args_best]
+        class_ids = class_ids[args_best]
+        scores = scores[args_best]
+
+        boxes = np.transpose(
+            np.vstack(
+                (
+                    boxes[:, 1] / self.height,
+                    boxes[:, 0] / self.width,
+                    boxes[:, 3] / self.height,
+                    boxes[:, 2] / self.width,
+                )
+            )
+        )
+
+        results = np.hstack(
+            (class_ids[..., np.newaxis], scores[..., np.newaxis], boxes)
+        )
+
+        return np.resize(results, (20, 6))
+
+    def post_process(self, output):
+        if self.detector_config.model.model_type == ModelTypeEnum.yolonas:
+            return self.post_process_yolonas(output)
+        else:
+            raise ValueError(
+                f'Model type "{self.detector_config.model.model_type}" is currently not supported.'
+            )
+
     def detect_raw(self, tensor_input):
         output = self.rknn.inference(
             [