mirror of
https://github.com/blakeblackshear/frigate.git
synced 2025-12-06 05:24:11 +03:00
b8b07ee6e1
* [Init] Initial commit for Synaptics SL1680 NPU * add a rough detector which is testing with yolov8 tflite model. * [Feat] Add dependencies installation in docker build - Add runtime library and wheels installation in main/Dockerfile - Add model.synap(default model, transfer from mobilenet_224full80) in docker/synap1680 * [Update] Remove dependencies installation from main Dockerfile - remove deps installation from Dockerfile - add dependencies installation and split wheels, deps stage in synap1680 Dockerfile * Refactor synap detector to more closely match other implementations * [Update] Add model path configuration check * [Update] update ModelType to ssd * [Update] Remove unuse script - install_deps.sh has already been executing in deps download stage - Dockerfile.toolchain is for testing to extract runtime libraries from Synaptics toolchain * [Update] update Synaptics SL1680 setup description * [Update] remove install_synap1680 - The deps download and installation is existed in synap1680 * [Fix] update document content * [Update] Update detector from synap1680 to synaptics This update is in order to make the synaptics SL-series NPU detector more general. - Fix detector `os` module not import bug - Update detector type `synap1680` to `synaptics` - Update document description `SL1680` to `Synaptics` only - Update docker build content `synap1680` to `synaptics` * [Fix] Update configuration document * Update docs/docs/configuration/object_detectors.md Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com> * [Update] Update document content and detector default layout - Update object_detectors document - Update detector's default layout - Update default model name * [Update] Update object detector document content * [Fix] Fix InputTensorEnum not defined error - import InputTensorEnum from detector_config * [Update] Update detector script coding format * [Update] Update synaptics detector coding format * [Update] Add synaptics ci workflow * [Update] update synaptics runtime libs download path - Fork Synaptics astra sdk repo and put the runtime lib package on it - Frigate team can update this download path later --------- Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>
92 lines
3.2 KiB
Python
92 lines
3.2 KiB
Python
import logging
|
|
import os
|
|
|
|
import numpy as np
|
|
from synap import Network
|
|
from synap.postprocessor import Detector
|
|
from synap.preprocessor import Preprocessor
|
|
from synap.types import Layout, Shape
|
|
from typing_extensions import Literal
|
|
|
|
from frigate.detectors.detection_api import DetectionApi
|
|
from frigate.detectors.detector_config import (
|
|
BaseDetectorConfig,
|
|
InputTensorEnum,
|
|
ModelTypeEnum,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DETECTOR_KEY = "synaptics"
|
|
|
|
|
|
class SynapDetectorConfig(BaseDetectorConfig):
|
|
type: Literal[DETECTOR_KEY]
|
|
|
|
|
|
class SynapDetector(DetectionApi):
|
|
type_key = DETECTOR_KEY
|
|
|
|
def __init__(self, detector_config: SynapDetectorConfig):
|
|
try:
|
|
_, ext = os.path.splitext(detector_config.model.path)
|
|
if ext and ext != ".synap":
|
|
raise ValueError("Model path config for Synap1680 is wrong.")
|
|
|
|
synap_network = Network(detector_config.model.path)
|
|
logger.info(f"Synap NPU loaded model: {detector_config.model.path}")
|
|
except ValueError as ve:
|
|
logger.error(f"Config to Synap1680 was Failed: {ve}")
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Failed to init Synap NPU: {e}")
|
|
raise
|
|
|
|
self.width = detector_config.model.width
|
|
self.height = detector_config.model.height
|
|
self.model_type = detector_config.model.model_type
|
|
self.network = synap_network
|
|
self.network_input_details = self.network.inputs[0]
|
|
self.input_tensor_layout = detector_config.model.input_tensor
|
|
|
|
# Create Inference Engine
|
|
self.preprocessor = Preprocessor()
|
|
self.detector = Detector(score_threshold=0.4, iou_threshold=0.4)
|
|
|
|
def detect_raw(self, tensor_input: np.ndarray):
|
|
# It has only been testing for pre-converted mobilenet80 .tflite -> .synap model currently
|
|
layout = Layout.nhwc # default layout
|
|
detections = np.zeros((20, 6), np.float32)
|
|
|
|
if self.input_tensor_layout == InputTensorEnum.nhwc:
|
|
layout = Layout.nhwc
|
|
|
|
postprocess_data = self.preprocessor.assign(
|
|
self.network.inputs, tensor_input, Shape(tensor_input.shape), layout
|
|
)
|
|
output_tensor_obj = self.network.predict()
|
|
output = self.detector.process(output_tensor_obj, postprocess_data)
|
|
|
|
if self.model_type == ModelTypeEnum.ssd:
|
|
for i, item in enumerate(output.items):
|
|
if i == 20:
|
|
break
|
|
|
|
bb = item.bounding_box
|
|
# Convert corner coordinates to normalized [0,1] range
|
|
x1 = bb.origin.x / self.width # Top-left X
|
|
y1 = bb.origin.y / self.height # Top-left Y
|
|
x2 = (bb.origin.x + bb.size.x) / self.width # Bottom-right X
|
|
y2 = (bb.origin.y + bb.size.y) / self.height # Bottom-right Y
|
|
detections[i] = [
|
|
item.class_index,
|
|
float(item.confidence),
|
|
y1,
|
|
x1,
|
|
y2,
|
|
x2,
|
|
]
|
|
else:
|
|
logger.error(f"Unsupported model type: {self.model_type}")
|
|
return detections
|