[Init] Initial commit for Synaptics SL1680 NPU (#19680)

* [Init] Initial commit for Synaptics SL1680 NPU * add a rough detector which is testing with yolov8 tflite model. * [Feat] Add dependencies installation in docker build - Add runtime library and wheels installation in main/Dockerfile - Add model.synap(default model, transfer from mobilenet_224full80) in docker/synap1680 * [Update] Remove dependencies installation from main Dockerfile - remove deps installation from Dockerfile - add dependencies installation and split wheels, deps stage in synap1680 Dockerfile * Refactor synap detector to more closely match other implementations * [Update] Add model path configuration check * [Update] update ModelType to ssd * [Update] Remove unuse script - install_deps.sh has already been executing in deps download stage - Dockerfile.toolchain is for testing to extract runtime libraries from Synaptics toolchain * [Update] update Synaptics SL1680 setup description * [Update] remove install_synap1680 - The deps download and installation is existed in synap1680 * [Fix] update document content * [Update] Update detector from synap1680 to synaptics This update is in order to make the synaptics SL-series NPU detector more general. - Fix detector `os` module not import bug - Update detector type `synap1680` to `synaptics` - Update document description `SL1680` to `Synaptics` only - Update docker build content `synap1680` to `synaptics` * [Fix] Update configuration document * Update docs/docs/configuration/object_detectors.md Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com> * [Update] Update document content and detector default layout - Update object_detectors document - Update detector's default layout - Update default model name * [Update] Update object detector document content * [Fix] Fix InputTensorEnum not defined error - import InputTensorEnum from detector_config * [Update] Update detector script coding format * [Update] Update synaptics detector coding format * [Update] Add synaptics ci workflow * [Update] update synaptics runtime libs download path - Fork Synaptics astra sdk repo and put the runtime lib package on it - Frigate team can update this download path later --------- Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>
2025-12-06 05:24:11 +03:00 · 2025-09-26 20:07:12 +08:00 · 2025-09-26 20:07:12 +08:00 · b8b07ee6e1
commit b8b07ee6e1
parent 082867447b
9 changed files with 295 additions and 0 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -173,6 +173,31 @@ jobs:
          set: |
            rk.tags=${{ steps.setup.outputs.image-name }}-rk
            *.cache-from=type=gha
+  synaptics_build:
+    runs-on: ubuntu-22.04-arm
+    name: Synaptics Build
+    needs:
+      - arm64_build
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - name: Set up QEMU and Buildx
+        id: setup
+        uses: ./.github/actions/setup
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Synaptics build
+        uses: docker/bake-action@v6
+        with:
+          source: .
+          push: true
+          targets: synaptics
+          files: docker/synaptics/synaptics.hcl
+          set: |
+            synaptics.tags=${{ steps.setup.outputs.image-name }}-synaptics
+            *.cache-from=type=gha
  # The majority of users running arm64 are rpi users, so the rpi
  # build should be the primary arm64 image
  assemble_default_build:
--- a/docker/synaptics/Dockerfile
+++ b/docker/synaptics/Dockerfile
@ -0,0 +1,28 @@
+# syntax=docker/dockerfile:1.6
+
+# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Globally set pip break-system-packages option to avoid having to specify it every time
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
+
+FROM wheels AS synap1680-wheels
+ARG TARGETARCH
+
+# Install dependencies
+RUN wget -qO- "https://github.com/GaryHuang-ASUS/synaptics_astra_sdk/releases/download/v1.5.0/Synaptics-SL1680-v1.5.0-rt.tar" | tar -C / -xzf -
+RUN wget -P /wheels/ "https://github.com/synaptics-synap/synap-python/releases/download/v0.0.4-preview/synap_python-0.0.4-cp311-cp311-manylinux_2_35_aarch64.whl"
+
+FROM deps AS synap1680-deps
+ARG TARGETARCH
+ARG PIP_BREAK_SYSTEM_PACKAGES
+
+RUN --mount=type=bind,from=synap1680-wheels,source=/wheels,target=/deps/synap-wheels \
+pip3 install --no-deps -U /deps/synap-wheels/*.whl
+
+WORKDIR /opt/frigate/
+COPY --from=rootfs / /
+
+COPY --from=synap1680-wheels /rootfs/usr/local/lib/*.so /usr/lib
+
+ADD https://raw.githubusercontent.com/synaptics-astra/synap-release/v1.5.0/models/dolphin/object_detection/coco/model/mobilenet224_full80/model.synap /synaptics/mobilenet.synap
--- a/docker/synaptics/synaptics.hcl
+++ b/docker/synaptics/synaptics.hcl
@ -0,0 +1,27 @@
+target wheels {
+  dockerfile = "docker/main/Dockerfile"
+  platforms = ["linux/arm64"]
+  target = "wheels"
+}
+
+target deps {
+  dockerfile = "docker/main/Dockerfile"
+  platforms = ["linux/arm64"]
+  target = "deps"
+}
+
+target rootfs {
+  dockerfile = "docker/main/Dockerfile"
+  platforms = ["linux/arm64"]
+  target = "rootfs"
+}
+
+target synaptics {
+  dockerfile = "docker/synaptics/Dockerfile"
+  contexts = {
+    wheels = "target:wheels",
+    deps = "target:deps",
+    rootfs = "target:rootfs"
+  }
+  platforms = ["linux/arm64"]
+}
--- a/docker/synaptics/synaptics.mk
+++ b/docker/synaptics/synaptics.mk
@ -0,0 +1,15 @@
+BOARDS += synaptics
+
+local-synaptics: version
+	docker buildx bake --file=docker/synaptics/synaptics.hcl synaptics \
+		--set synaptics.tags=frigate:latest-synaptics \
+		--load
+
+build-synaptics: version
+	docker buildx bake --file=docker/synaptics/synaptics.hcl synaptics \
+		--set synaptics.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-synaptics
+
+push-synaptics: build-synaptics
+	docker buildx bake --file=docker/synaptics/synaptics.hcl synaptics \
+		--set synaptics.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-synaptics \
+		--push
--- a/docs/docs/configuration/hardware_acceleration_video.md
+++ b/docs/docs/configuration/hardware_acceleration_video.md
@ -427,3 +427,29 @@ cameras:
 ```

 :::
+
+## Synaptics
+
+Hardware accelerated video de-/encoding is supported on Synpatics SL-series SoC.
+
+### Prerequisites
+
+Make sure to follow the [Synaptics specific installation instructions](/frigate/installation#synaptics).
+
+### Configuration
+
+Add one of the following FFmpeg presets to your `config.yml` to enable hardware video processing:
+
+```yaml
+ffmpeg:
+  hwaccel_args: -c:v h264_v4l2m2m
+  input_args: preset-rtsp-restream
+output_args:
+  record: preset-record-generic-audio-aac
+```
+
+:::warning
+
+Make sure that your SoC supports hardware acceleration for your input stream and your input stream is h264 encoding. For example, if your camera streams with h264 encoding, your SoC must be able to de- and encode with it. If you are unsure whether your SoC meets the requirements, take a look at the datasheet.
+
+:::
--- a/docs/docs/configuration/object_detectors.md
+++ b/docs/docs/configuration/object_detectors.md
@ -43,6 +43,10 @@ Frigate supports multiple different detectors that work on different types of ha

 - [RKNN](#rockchip-platform): RKNN models can run on Rockchip devices with included NPUs.

+**Synaptics**
+
+- [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs.
+
 **For Testing**

 - [CPU Detector (not recommended for actual use](#cpu-detector-not-recommended): Use a CPU to run tflite model, this is not recommended and in most cases OpenVINO can be used in CPU mode with better results.
@ -1049,6 +1053,41 @@ model:
  height: 320 # MUST match the chosen model i.e yolov7-320 -> 320 yolov4-416 -> 416
 ```

+## Synaptics
+
+Hardware accelerated object detection is supported on the following SoCs:
+
+- SL1680
+
+This implementation uses the [Synaptics model conversion](https://synaptics-synap.github.io/doc/v/latest/docs/manual/introduction.html#offline-model-conversion), version v3.1.0.
+
+This implementation is based on sdk `v1.5.0`.
+
+See the [installation docs](../frigate/installation.md#synaptics) for information on configuring the SL-series NPU hardware.
+
+### Configuration
+
+When configuring the Synap detector, you have to specify the model: a local **path**.
+
+#### SSD Mobilenet
+
+A synap model is provided in the container at /mobilenet.synap and is used by this detector type by default. The model comes from [Synap-release Github](https://github.com/synaptics-astra/synap-release/tree/v1.5.0/models/dolphin/object_detection/coco/model/mobilenet224_full80).
+
+Use the model configuration shown below when using the synaptics detector with the default synap model:
+
+```yaml
+detectors:  # required
+  synap_npu:  # required
+    type: synaptics  # required
+
+model:  # required
+  path: /synaptics/mobilenet.synap  # required
+  width: 224  # required
+  height: 224  # required
+  tensor_format: nhwc  # default value (optional. If you change the model, it is required)
+  labelmap_path: /labelmap/coco-80.txt  # required
+```
+
 ## Rockchip platform

 Hardware accelerated object detection is supported on the following SoCs:
--- a/docs/docs/frigate/hardware.md
+++ b/docs/docs/frigate/hardware.md
@ -95,8 +95,21 @@ Frigate supports multiple different detectors that work on different types of ha
  - Runs best with tiny or small size models
  - Runs efficiently on low power hardware

+**Synaptics**
+
+- [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs to provide efficient object detection.
+
 :::

+### Synaptics
+
+- **Synaptics** Default model is **mobilenet**
+
+| Name             | Synaptics SL1680 Inference Time |
+| ---------------- | ------------------------------- |
+| ssd mobilenet    | ~ 25 ms                         |
+| yolov5m          | ~ 118 ms                        |
+
 ### Hailo-8

 Frigate supports both the Hailo-8 and Hailo-8L AI Acceleration Modules on compatible hardware platforms—including the Raspberry Pi 5 with the PCIe hat from the AI kit. The Hailo detector integration in Frigate automatically identifies your hardware type and selects the appropriate default model when a custom model isn’t provided.
--- a/docs/docs/frigate/installation.md
+++ b/docs/docs/frigate/installation.md
@ -256,6 +256,37 @@ or add these options to your `docker run` command:

 Next, you should configure [hardware object detection](/configuration/object_detectors#rockchip-platform) and [hardware video processing](/configuration/hardware_acceleration_video#rockchip-platform).

+### Synaptics
+
+- SL1680
+
+#### Setup
+
+Follow Frigate's default installation instructions, but use a docker image with `-synaptics` suffix for example `ghcr.io/blakeblackshear/frigate:stable-synaptics`.
+
+Next, you need to grant docker permissions to access your hardware:
+
+- During the configuration process, you should run docker in privileged mode to avoid any errors due to insufficient permissions. To do so, add `privileged: true` to your `docker-compose.yml` file or the `--privileged` flag to your docker run command.
+
+```yaml
+devices:
+  - /dev/synap
+  - /dev/video0
+  - /dev/video1
+```
+
+or add these options to your `docker run` command:
+
+```
+--device /dev/synap \
+--device /dev/video0 \
+--device /dev/video1 
+```
+
+#### Configuration
+
+Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics).
+
 ## Docker

 Running through Docker with Docker Compose is the recommended install method.
--- a/frigate/detectors/plugins/synaptics.py
+++ b/frigate/detectors/plugins/synaptics.py
@ -0,0 +1,91 @@
+import logging
+import os
+
+import numpy as np
+from synap import Network
+from synap.postprocessor import Detector
+from synap.preprocessor import Preprocessor
+from synap.types import Layout, Shape
+from typing_extensions import Literal
+
+from frigate.detectors.detection_api import DetectionApi
+from frigate.detectors.detector_config import (
+    BaseDetectorConfig,
+    InputTensorEnum,
+    ModelTypeEnum,
+)
+
+logger = logging.getLogger(__name__)
+
+DETECTOR_KEY = "synaptics"
+
+
+class SynapDetectorConfig(BaseDetectorConfig):
+    type: Literal[DETECTOR_KEY]
+
+
+class SynapDetector(DetectionApi):
+    type_key = DETECTOR_KEY
+
+    def __init__(self, detector_config: SynapDetectorConfig):
+        try:
+            _, ext = os.path.splitext(detector_config.model.path)
+            if ext and ext != ".synap":
+                raise ValueError("Model path config for Synap1680 is wrong.")
+
+            synap_network = Network(detector_config.model.path)
+            logger.info(f"Synap NPU loaded model: {detector_config.model.path}")
+        except ValueError as ve:
+            logger.error(f"Config to Synap1680 was Failed: {ve}")
+            raise
+        except Exception as e:
+            logger.error(f"Failed to init Synap NPU: {e}")
+            raise
+
+        self.width = detector_config.model.width
+        self.height = detector_config.model.height
+        self.model_type = detector_config.model.model_type
+        self.network = synap_network
+        self.network_input_details = self.network.inputs[0]
+        self.input_tensor_layout = detector_config.model.input_tensor
+
+        # Create Inference Engine
+        self.preprocessor = Preprocessor()
+        self.detector = Detector(score_threshold=0.4, iou_threshold=0.4)
+
+    def detect_raw(self, tensor_input: np.ndarray):
+        # It has only been testing for pre-converted mobilenet80 .tflite -> .synap model currently
+        layout = Layout.nhwc  # default layout
+        detections = np.zeros((20, 6), np.float32)
+
+        if self.input_tensor_layout == InputTensorEnum.nhwc:
+            layout = Layout.nhwc
+
+        postprocess_data = self.preprocessor.assign(
+            self.network.inputs, tensor_input, Shape(tensor_input.shape), layout
+        )
+        output_tensor_obj = self.network.predict()
+        output = self.detector.process(output_tensor_obj, postprocess_data)
+
+        if self.model_type == ModelTypeEnum.ssd:
+            for i, item in enumerate(output.items):
+                if i == 20:
+                    break
+
+                bb = item.bounding_box
+                # Convert corner coordinates to normalized [0,1] range
+                x1 = bb.origin.x / self.width  # Top-left X
+                y1 = bb.origin.y / self.height  # Top-left Y
+                x2 = (bb.origin.x + bb.size.x) / self.width  # Bottom-right X
+                y2 = (bb.origin.y + bb.size.y) / self.height  # Bottom-right Y
+                detections[i] = [
+                    item.class_index,
+                    float(item.confidence),
+                    y1,
+                    x1,
+                    y2,
+                    x2,
+                ]
+        else:
+            logger.error(f"Unsupported model type: {self.model_type}")
+        return detections