Compare commits

...

9 Commits

Author SHA1 Message Date
ivanshi1108
72e0a1df30
Merge 438df7d484 into 224cbdc2d6 2025-11-21 23:36:48 +00:00
Nicolas Mowen
224cbdc2d6
Miscellaneous Fixes (#20989)
Some checks failed
CI / AMD64 Build (push) Has been cancelled
CI / ARM Build (push) Has been cancelled
CI / Jetson Jetpack 6 (push) Has been cancelled
CI / AMD64 Extra Build (push) Has been cancelled
CI / ARM Extra Build (push) Has been cancelled
CI / Synaptics Build (push) Has been cancelled
CI / Assemble and push default build (push) Has been cancelled
* Include DB in safe mode config

Copy DB when going into safe mode to avoid creating a new one if a user has configured a separate location

* Fix documentation for example log module

* Set minimum duration for recording segments

Due to the inpoint logic, some recordings would get clipped on the end of the segment with a non-zero duration but not enough duration to include a frame. 100 ms is a safe value for any video that is 10fps or higher to have a frame

* Add docs to explain object assignment for classification

* Add warning for Intel GPU stats bug

Add warning with explanation on GPU stats page when all Intel GPU values are 0

* Update docs with creation instructions

* reset loading state when moving through events in tracking details

* disable pip on preview players

* Improve HLS handling for startPosition

The startPosition was incorrectly calculated assuming continuous recordings, when it needs to consider only some segments exist. This extracts that logic to a utility so all can use it.

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
2025-11-21 15:40:58 -06:00
Abinila Siva
3f9b153758
[MemryX] Update YOLOv9 post-processing (#20980)
* Update optimized YOLOv9 post-processing

* remove unused import
2025-11-21 14:24:17 -07:00
shizhicheng
438df7d484 The model inference time has been changed to the time displayed on the Frigate UI 2025-11-16 22:22:38 +08:00
shizhicheng
e27a94ae0b Fix logical errors caused by code formatting 2025-11-11 05:54:19 +00:00
shizhicheng
1dee548dbc Modifications to the YOLOv9 object detection model:
The model is now dynamically downloaded to the cache directory.
Post-processing is now done using Frigate's built-in `post_process_yolo`.
Configuration in the relevant documentation has been updated.
2025-11-11 05:42:28 +00:00
shizhicheng
91e17e12b7 Change the default detection model to YOLOv9 2025-11-09 13:21:17 +00:00
ivanshi1108
bb45483e9e
Modify AXERA section from hardware.md
Modify AXERA section and related content from hardware documentation.
2025-10-28 09:54:00 +08:00
shizhicheng
7b4eaf2d10 Initial commit for AXERA AI accelerators 2025-10-24 09:03:13 +00:00
23 changed files with 881 additions and 243 deletions

View File

@ -225,3 +225,29 @@ jobs:
sources: |
ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-amd64
ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-rpi
axera_build:
runs-on: ubuntu-22.04
name: AXERA Build
needs:
- amd64_build
- arm64_build
steps:
- name: Check out code
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Set up QEMU and Buildx
id: setup
uses: ./.github/actions/setup
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push Axera build
uses: docker/bake-action@v6
with:
source: .
push: true
targets: axcl
files: docker/axcl/axcl.hcl
set: |
axcl.tags=${{ steps.setup.outputs.image-name }}-axcl
*.cache-from=type=gha

55
docker/axcl/Dockerfile Normal file
View File

@ -0,0 +1,55 @@
# syntax=docker/dockerfile:1.6
# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
ARG DEBIAN_FRONTEND=noninteractive
# Globally set pip break-system-packages option to avoid having to specify it every time
ARG PIP_BREAK_SYSTEM_PACKAGES=1
FROM frigate AS frigate-axcl
ARG TARGETARCH
ARG PIP_BREAK_SYSTEM_PACKAGES
# Install axpyengine
RUN wget https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3.rc1/axengine-0.1.3-py3-none-any.whl -O /axengine-0.1.3-py3-none-any.whl
RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple/ /axengine-0.1.3-py3-none-any.whl \
&& rm /axengine-0.1.3-py3-none-any.whl
# Install axcl
RUN if [ "$TARGETARCH" = "amd64" ]; then \
echo "Installing x86_64 version of axcl"; \
wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb -O /axcl.deb; \
else \
echo "Installing aarch64 version of axcl"; \
wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb -O /axcl.deb; \
fi
RUN mkdir /unpack_axcl && \
dpkg-deb -x /axcl.deb /unpack_axcl && \
cp -R /unpack_axcl/usr/bin/axcl /usr/bin/ && \
cp -R /unpack_axcl/usr/lib/axcl /usr/lib/ && \
rm -rf /unpack_axcl /axcl.deb
# Install axcl ffmpeg
RUN mkdir -p /usr/lib/ffmpeg/axcl
RUN if [ "$TARGETARCH" = "amd64" ]; then \
wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffmpeg-x64 -O /usr/lib/ffmpeg/axcl/ffmpeg && \
wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffprobe-x64 -O /usr/lib/ffmpeg/axcl/ffprobe; \
else \
wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffmpeg-aarch64 -O /usr/lib/ffmpeg/axcl/ffmpeg && \
wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffprobe-aarch64 -O /usr/lib/ffmpeg/axcl/ffprobe; \
fi
RUN chmod +x /usr/lib/ffmpeg/axcl/ffmpeg /usr/lib/ffmpeg/axcl/ffprobe
# Set ldconfig path
RUN echo "/usr/lib/axcl" > /etc/ld.so.conf.d/ax.conf
# Set env
ENV PATH="$PATH:/usr/bin/axcl"
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/axcl"
ENTRYPOINT ["sh", "-c", "ldconfig && exec /init"]

13
docker/axcl/axcl.hcl Normal file
View File

@ -0,0 +1,13 @@
target frigate {
dockerfile = "docker/main/Dockerfile"
platforms = ["linux/amd64", "linux/arm64"]
target = "frigate"
}
target axcl {
dockerfile = "docker/axcl/Dockerfile"
contexts = {
frigate = "target:frigate",
}
platforms = ["linux/amd64", "linux/arm64"]
}

15
docker/axcl/axcl.mk Normal file
View File

@ -0,0 +1,15 @@
BOARDS += axcl
local-axcl: version
docker buildx bake --file=docker/axcl/axcl.hcl axcl \
--set axcl.tags=frigate:latest-axcl \
--load
build-axcl: version
docker buildx bake --file=docker/axcl/axcl.hcl axcl \
--set axcl.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-axcl
push-axcl: build-axcl
docker buildx bake --file=docker/axcl/axcl.hcl axcl \
--set axcl.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-axcl \
--push

View File

@ -0,0 +1,83 @@
#!/bin/bash
# Update package list and install dependencies
sudo apt-get update
sudo apt-get install -y build-essential cmake git wget pciutils kmod udev
# Check if gcc-12 is needed
current_gcc_version=$(gcc --version | head -n1 | awk '{print $NF}')
gcc_major_version=$(echo $current_gcc_version | cut -d'.' -f1)
if [[ $gcc_major_version -lt 12 ]]; then
echo "Current GCC version ($current_gcc_version) is lower than 12, installing gcc-12..."
sudo apt-get install -y gcc-12
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 12
echo "GCC-12 installed and set as default"
else
echo "Current GCC version ($current_gcc_version) is sufficient, skipping GCC installation"
fi
# Determine architecture
arch=$(uname -m)
download_url=""
if [[ $arch == "x86_64" ]]; then
download_url="https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb"
deb_file="axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb"
elif [[ $arch == "aarch64" ]]; then
download_url="https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb"
deb_file="axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb"
else
echo "Unsupported architecture: $arch"
exit 1
fi
# Download AXCL driver
echo "Downloading AXCL driver for $arch..."
wget "$download_url" -O "$deb_file"
if [ $? -ne 0 ]; then
echo "Failed to download AXCL driver"
exit 1
fi
# Install AXCL driver
echo "Installing AXCL driver..."
sudo dpkg -i "$deb_file"
if [ $? -ne 0 ]; then
echo "Failed to install AXCL driver, attempting to fix dependencies..."
sudo apt-get install -f -y
sudo dpkg -i "$deb_file"
if [ $? -ne 0 ]; then
echo "AXCL driver installation failed"
exit 1
fi
fi
# Update environment
echo "Updating environment..."
source /etc/profile
# Verify installation
echo "Verifying AXCL installation..."
if command -v axcl-smi &> /dev/null; then
echo "AXCL driver detected, checking AI accelerator status..."
axcl_output=$(axcl-smi 2>&1)
axcl_exit_code=$?
echo "$axcl_output"
if [ $axcl_exit_code -eq 0 ]; then
echo "AXCL driver installation completed successfully!"
else
echo "AXCL driver installed but no AI accelerator detected or communication failed."
echo "Please check if the AI accelerator is properly connected and powered on."
exit 1
fi
else
echo "axcl-smi command not found. AXCL driver installation may have failed."
exit 1
fi

View File

@ -25,7 +25,7 @@ Examples of available modules are:
- `frigate.app`
- `frigate.mqtt`
- `frigate.object_detection`
- `frigate.object_detection.base`
- `detector.<detector_name>`
- `watchdog.<camera_name>`
- `ffmpeg.<camera_name>.<sorted_roles>` NOTE: All FFmpeg logs are sent as `error` level.

View File

@ -35,6 +35,15 @@ For object classification:
- Ideal when multiple attributes can coexist independently.
- Example: Detecting if a `person` in a construction yard is wearing a helmet or not.
## Assignment Requirements
Sub labels and attributes are only assigned when both conditions are met:
1. **Threshold**: Each classification attempt must have a confidence score that meets or exceeds the configured `threshold` (default: `0.8`).
2. **Class Consensus**: After at least 3 classification attempts, 60% of attempts must agree on the same class label. If the consensus class is `none`, no assignment is made.
This two-step verification prevents false positives by requiring consistent predictions across multiple frames before assigning a sub label or attribute.
## Example use cases
### Sub label
@ -66,14 +75,18 @@ classification:
## Training the model
Creating and training the model is done within the Frigate UI using the `Classification` page.
Creating and training the model is done within the Frigate UI using the `Classification` page. The process consists of two steps:
### Getting Started
### Step 1: Name and Define
Enter a name for your model, select the object label to classify (e.g., `person`, `dog`, `car`), choose the classification type (sub label or attribute), and define your classes. Include a `none` class for objects that don't fit any specific category.
### Step 2: Assign Training Examples
The system will automatically generate example images from detected objects matching your selected label. You'll be guided through each class one at a time to select which images represent that class. Any images not assigned to a specific class will automatically be assigned to `none` when you complete the last class. Once all images are processed, training will begin automatically.
When choosing which objects to classify, start with a small number of visually distinct classes and ensure your training samples match camera viewpoints and distances typical for those objects.
// TODO add this section once UI is implemented. Explain process of selecting objects and curating training examples.
### Improving the Model
- **Problem framing**: Keep classes visually distinct and relevant to the chosen object types.

View File

@ -48,13 +48,23 @@ classification:
## Training the model
Creating and training the model is done within the Frigate UI using the `Classification` page.
Creating and training the model is done within the Frigate UI using the `Classification` page. The process consists of three steps:
### Getting Started
### Step 1: Name and Define
When choosing a portion of the camera frame for state classification, it is important to make the crop tight around the area of interest to avoid extra signals unrelated to what is being classified.
Enter a name for your model and define at least 2 classes (states) that represent mutually exclusive states. For example, `open` and `closed` for a door, or `on` and `off` for lights.
// TODO add this section once UI is implemented. Explain process of selecting a crop.
### Step 2: Select the Crop Area
Choose one or more cameras and draw a rectangle over the area of interest for each camera. The crop should be tight around the region you want to classify to avoid extra signals unrelated to what is being classified. You can drag and resize the rectangle to adjust the crop area.
### Step 3: Assign Training Examples
The system will automatically generate example images from your camera feeds. You'll be guided through each class one at a time to select which images represent that state.
**Important**: All images must be assigned to a state before training can begin. This includes images that may not be optimal, such as when people temporarily block the view, sun glare is present, or other distractions occur. Assign these images to the state that is actually present (based on what you know the state to be), not based on the distraction. This training helps the model correctly identify the state even when such conditions occur during inference.
Once all images are assigned, training will begin automatically.
### Improving the Model

View File

@ -47,6 +47,11 @@ Frigate supports multiple different detectors that work on different types of ha
- [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs.
**AXERA**
- [AXEngine](#axera): axmodels can run on AXERA AI acceleration.
**For Testing**
- [CPU Detector (not recommended for actual use](#cpu-detector-not-recommended): Use a CPU to run tflite model, this is not recommended and in most cases OpenVINO can be used in CPU mode with better results.
@ -962,7 +967,6 @@ model:
# path: /config/yolov9.zip
# The .zip file must contain:
# ├── yolov9.dfp (a file ending with .dfp)
# └── yolov9_post.onnx (optional; only if the model includes a cropped post-processing network)
```
#### YOLOX
@ -1169,6 +1173,41 @@ model: # required
labelmap_path: /labelmap/coco-80.txt # required
```
## AXERA
Hardware accelerated object detection is supported on the following SoCs:
- AX650N
- AX8850N
This implementation uses the [AXera Pulsar2 Toolchain](https://huggingface.co/AXERA-TECH/Pulsar2).
See the [installation docs](../frigate/installation.md#axera) for information on configuring the AXEngine hardware.
### Configuration
When configuring the AXEngine detector, you have to specify the model name.
#### yolov9
A yolov9 model is provided in the container at /axmodels and is used by this detector type by default.
Use the model configuration shown below when using the axengine detector with the default axmodel:
```yaml
detectors: # required
axengine: # required
type: axengine # required
model: # required
path: frigate-yolov9-tiny # required
model_type: yolo-generic # required
width: 320 # required
height: 320 # required
tensor_format: bgr # required
labelmap_path: /labelmap/coco-80.txt # required
```
## Rockchip platform
Hardware accelerated object detection is supported on the following SoCs:

View File

@ -110,6 +110,14 @@ Frigate supports multiple different detectors that work on different types of ha
| ssd mobilenet | ~ 25 ms |
| yolov5m | ~ 118 ms |
### AXERA
- **AXEngine** Default model is **yolov9**
| Name | AXERA AX650N/AX8850N Inference Time |
| ---------------- | ----------------------------------- |
| yolov9-tiny | ~ 4 ms |
### Hailo-8
Frigate supports both the Hailo-8 and Hailo-8L AI Acceleration Modules on compatible hardware platforms—including the Raspberry Pi 5 with the PCIe hat from the AI kit. The Hailo detector integration in Frigate automatically identifies your hardware type and selects the appropriate default model when a custom model isnt provided.

View File

@ -287,6 +287,40 @@ or add these options to your `docker run` command:
Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics).
### AXERA
AXERA accelerators are available in an M.2 form factor, compatible with both Raspberry Pi and Orange Pi. This form factor has also been successfully tested on x86 platforms, making it a versatile choice for various computing environments.
#### Installation
Using AXERA accelerators requires the installation of the AXCL driver. We provide a convenient Linux script to complete this installation.
Follow these steps for installation:
1. Copy or download [this script](https://github.com/ivanshi1108/assets/releases/download/v0.16.2/user_installation.sh).
2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh`
3. Run the script with `./user_installation.sh`
#### Setup
To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable`
Next, grant Docker permissions to access your hardware by adding the following lines to your `docker-compose.yml` file:
```yaml
devices:
- /dev/axcl_host
- /dev/ax_mmb_dev
- /dev/msg_userdev
```
If you are using `docker run`, add this option to your command `--device /dev/axcl_host --device /dev/ax_mmb_dev --device /dev/msg_userdev`
#### Configuration
Finally, configure [hardware object detection](/configuration/object_detectors#axera) to complete the setup.
## Docker
Running through Docker with Docker Compose is the recommended install method.

View File

@ -849,6 +849,7 @@ async def vod_ts(camera_name: str, start_ts: float, end_ts: float):
clips = []
durations = []
min_duration_ms = 100 # Minimum 100ms to ensure at least one video frame
max_duration_ms = MAX_SEGMENT_DURATION * 1000
recording: Recordings
@ -866,11 +867,11 @@ async def vod_ts(camera_name: str, start_ts: float, end_ts: float):
if recording.end_time > end_ts:
duration -= int((recording.end_time - end_ts) * 1000)
if duration <= 0:
# skip if the clip has no valid duration
if duration < min_duration_ms:
# skip if the clip has no valid duration (too short to contain frames)
continue
if 0 < duration < max_duration_ms:
if min_duration_ms <= duration < max_duration_ms:
clip["keyFrameDurations"] = [duration]
clips.append(clip)
durations.append(duration)

View File

@ -792,6 +792,10 @@ class FrigateConfig(FrigateBaseModel):
# copy over auth and proxy config in case auth needs to be enforced
safe_config["auth"] = config.get("auth", {})
safe_config["proxy"] = config.get("proxy", {})
# copy over database config for auth and so a new db is not created
safe_config["database"] = config.get("database", {})
return cls.parse_object(safe_config, **context)
# Validate and return the config dict.

View File

@ -0,0 +1,92 @@
import logging
import os.path
import re
import urllib.request
from typing import Literal
import cv2
import numpy as np
from pydantic import Field
from frigate.const import MODEL_CACHE_DIR
from frigate.detectors.detection_api import DetectionApi
from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
from frigate.util.model import post_process_yolo
import axengine as axe
from axengine import axclrt_provider_name, axengine_provider_name
logger = logging.getLogger(__name__)
DETECTOR_KEY = "axengine"
supported_models = {
ModelTypeEnum.yologeneric: "frigate-yolov9-.*$",
}
model_cache_dir = os.path.join(MODEL_CACHE_DIR, "axengine_cache/")
class AxengineDetectorConfig(BaseDetectorConfig):
type: Literal[DETECTOR_KEY]
class Axengine(DetectionApi):
type_key = DETECTOR_KEY
def __init__(self, config: AxengineDetectorConfig):
logger.info("__init__ axengine")
super().__init__(config)
self.height = config.model.height
self.width = config.model.width
model_path = config.model.path or "frigate-yolov9-tiny"
model_props = self.parse_model_input(model_path)
self.session = axe.InferenceSession(model_props["path"])
def __del__(self):
pass
def parse_model_input(self, model_path):
model_props = {}
model_props["preset"] = True
model_matched = False
for model_type, pattern in supported_models.items():
if re.match(pattern, model_path):
model_matched = True
model_props["model_type"] = model_type
if model_matched:
model_props["filename"] = model_path + f".axmodel"
model_props["path"] = model_cache_dir + model_props["filename"]
if not os.path.isfile(model_props["path"]):
self.download_model(model_props["filename"])
else:
supported_models_str = ", ".join(
model[1:-1] for model in supported_models
)
raise Exception(
f"Model {model_path} is unsupported. Provide your own model or choose one of the following: {supported_models_str}"
)
return model_props
def download_model(self, filename):
if not os.path.isdir(model_cache_dir):
os.mkdir(model_cache_dir)
GITHUB_ENDPOINT = os.environ.get("GITHUB_ENDPOINT", "https://github.com")
urllib.request.urlretrieve(
f"{GITHUB_ENDPOINT}/ivanshi1108/assets/releases/download/v0.16.2/{filename}",
model_cache_dir + filename,
)
def detect_raw(self, tensor_input):
results = None
results = self.session.run(None, {"images": tensor_input})
if self.detector_config.model.model_type == ModelTypeEnum.yologeneric:
return post_process_yolo(results, self.width, self.height)
else:
raise ValueError(
f'Model type "{self.detector_config.model.model_type}" is currently not supported.'
)

View File

@ -18,7 +18,6 @@ from frigate.detectors.detector_config import (
ModelTypeEnum,
)
from frigate.util.file import FileLock
from frigate.util.model import post_process_yolo
logger = logging.getLogger(__name__)
@ -178,13 +177,6 @@ class MemryXDetector(DetectionApi):
logger.error(f"Failed to initialize MemryX model: {e}")
raise
def load_yolo_constants(self):
base = f"{self.cache_dir}/{self.model_folder}"
# constants for yolov9 post-processing
self.const_A = np.load(f"{base}/_model_22_Constant_9_output_0.npy")
self.const_B = np.load(f"{base}/_model_22_Constant_10_output_0.npy")
self.const_C = np.load(f"{base}/_model_22_Constant_12_output_0.npy")
def check_and_prepare_model(self):
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir, exist_ok=True)
@ -236,7 +228,6 @@ class MemryXDetector(DetectionApi):
# Handle post model requirements by model type
if self.memx_model_type in [
ModelTypeEnum.yologeneric,
ModelTypeEnum.yolonas,
ModelTypeEnum.ssd,
]:
@ -245,7 +236,10 @@ class MemryXDetector(DetectionApi):
f"No *_post.onnx file found in custom model zip for {self.memx_model_type.name}."
)
self.memx_post_model = post_candidates[0]
elif self.memx_model_type == ModelTypeEnum.yolox:
elif self.memx_model_type in [
ModelTypeEnum.yolox,
ModelTypeEnum.yologeneric,
]:
# Explicitly ignore any post model even if present
self.memx_post_model = None
else:
@ -273,8 +267,6 @@ class MemryXDetector(DetectionApi):
logger.info("Using cached models.")
self.memx_model_path = dfp_path
self.memx_post_model = post_path
if self.memx_model_type == ModelTypeEnum.yologeneric:
self.load_yolo_constants()
return
# ---------- CASE 3: download MemryX model (no cache) ----------
@ -303,9 +295,6 @@ class MemryXDetector(DetectionApi):
else None
)
if self.memx_model_type == ModelTypeEnum.yologeneric:
self.load_yolo_constants()
finally:
if os.path.exists(zip_path):
try:
@ -600,127 +589,232 @@ class MemryXDetector(DetectionApi):
self.output_queue.put(final_detections)
def onnx_reshape_with_allowzero(
self, data: np.ndarray, shape: np.ndarray, allowzero: int = 0
def _generate_anchors(self, sizes=[80, 40, 20]):
"""Generate anchor points for YOLOv9 style processing"""
yscales = []
xscales = []
for s in sizes:
r = np.arange(s) + 0.5
yscales.append(np.repeat(r, s))
xscales.append(np.repeat(r[None, ...], s, axis=0).flatten())
yscales = np.concatenate(yscales)
xscales = np.concatenate(xscales)
anchors = np.stack([xscales, yscales], axis=1)
return anchors
def _generate_scales(self, sizes=[80, 40, 20]):
"""Generate scaling factors for each detection level"""
factors = [8, 16, 32]
s = np.concatenate([np.ones([int(s * s)]) * f for s, f in zip(sizes, factors)])
return s[:, None]
@staticmethod
def _softmax(x: np.ndarray, axis: int) -> np.ndarray:
"""Efficient softmax implementation"""
x = x - np.max(x, axis=axis, keepdims=True)
np.exp(x, out=x)
x /= np.sum(x, axis=axis, keepdims=True)
return x
def dfl(self, x: np.ndarray) -> np.ndarray:
"""Distribution Focal Loss decoding - YOLOv9 style"""
x = x.reshape(-1, 4, 16)
weights = np.arange(16, dtype=np.float32)
p = self._softmax(x, axis=2)
p = p * weights[None, None, :]
out = np.sum(p, axis=2, keepdims=False)
return out
def dist2bbox(
self, x: np.ndarray, anchors: np.ndarray, scales: np.ndarray
) -> np.ndarray:
shape = shape.astype(int)
input_shape = data.shape
output_shape = []
"""Convert distances to bounding boxes - YOLOv9 style"""
lt = x[:, :2]
rb = x[:, 2:]
for i, dim in enumerate(shape):
if dim == 0 and allowzero == 0:
output_shape.append(input_shape[i]) # Copy dimension from input
else:
output_shape.append(dim)
x1y1 = anchors - lt
x2y2 = anchors + rb
# Now let NumPy infer any -1 if needed
reshaped = np.reshape(data, output_shape)
wh = x2y2 - x1y1
c_xy = (x1y1 + x2y2) / 2
return reshaped
out = np.concatenate([c_xy, wh], axis=1)
out = out * scales
return out
def post_process_yolo_optimized(self, outputs):
"""
Custom YOLOv9 post-processing optimized for MemryX ONNX outputs.
Implements DFL decoding, confidence filtering, and NMS in pure NumPy.
"""
# YOLOv9 outputs: 6 outputs (lbox, lcls, mbox, mcls, sbox, scls)
conv_out1, conv_out2, conv_out3, conv_out4, conv_out5, conv_out6 = outputs
# Determine grid sizes based on input resolution
# YOLOv9 uses 3 detection heads with strides [8, 16, 32]
# Grid sizes = input_size / stride
sizes = [
self.memx_model_height
// 8, # Large objects (e.g., 80 for 640x640, 40 for 320x320)
self.memx_model_height
// 16, # Medium objects (e.g., 40 for 640x640, 20 for 320x320)
self.memx_model_height
// 32, # Small objects (e.g., 20 for 640x640, 10 for 320x320)
]
# Generate anchors and scales if not already done
if not hasattr(self, "anchors"):
self.anchors = self._generate_anchors(sizes)
self.scales = self._generate_scales(sizes)
# Process outputs in YOLOv9 format: reshape and moveaxis for ONNX format
lbox = np.moveaxis(conv_out1, 1, -1) # Large boxes
lcls = np.moveaxis(conv_out2, 1, -1) # Large classes
mbox = np.moveaxis(conv_out3, 1, -1) # Medium boxes
mcls = np.moveaxis(conv_out4, 1, -1) # Medium classes
sbox = np.moveaxis(conv_out5, 1, -1) # Small boxes
scls = np.moveaxis(conv_out6, 1, -1) # Small classes
# Determine number of classes dynamically from the class output shape
# lcls shape should be (batch, height, width, num_classes)
num_classes = lcls.shape[-1]
# Validate that all class outputs have the same number of classes
if not (mcls.shape[-1] == num_classes and scls.shape[-1] == num_classes):
raise ValueError(
f"Class output shapes mismatch: lcls={lcls.shape}, mcls={mcls.shape}, scls={scls.shape}"
)
# Concatenate boxes and classes
boxes = np.concatenate(
[
lbox.reshape(-1, 64), # 64 is for 4 bbox coords * 16 DFL bins
mbox.reshape(-1, 64),
sbox.reshape(-1, 64),
],
axis=0,
)
classes = np.concatenate(
[
lcls.reshape(-1, num_classes),
mcls.reshape(-1, num_classes),
scls.reshape(-1, num_classes),
],
axis=0,
)
# Apply sigmoid to classes
classes = self.sigmoid(classes)
# Apply DFL to box predictions
boxes = self.dfl(boxes)
# YOLOv9 postprocessing with confidence filtering and NMS
confidence_thres = 0.4
iou_thres = 0.6
# Find the class with the highest score for each detection
max_scores = np.max(classes, axis=1) # Maximum class score for each detection
class_ids = np.argmax(classes, axis=1) # Index of the best class
# Filter out detections with scores below the confidence threshold
valid_indices = np.where(max_scores >= confidence_thres)[0]
if len(valid_indices) == 0:
# Return empty detections array
final_detections = np.zeros((20, 6), np.float32)
return final_detections
# Select only valid detections
valid_boxes = boxes[valid_indices]
valid_class_ids = class_ids[valid_indices]
valid_scores = max_scores[valid_indices]
# Convert distances to actual bounding boxes using anchors and scales
valid_boxes = self.dist2bbox(
valid_boxes, self.anchors[valid_indices], self.scales[valid_indices]
)
# Convert bounding box coordinates from (x_center, y_center, w, h) to (x_min, y_min, x_max, y_max)
x_center, y_center, width, height = (
valid_boxes[:, 0],
valid_boxes[:, 1],
valid_boxes[:, 2],
valid_boxes[:, 3],
)
x_min = x_center - width / 2
y_min = y_center - height / 2
x_max = x_center + width / 2
y_max = y_center + height / 2
# Convert to format expected by cv2.dnn.NMSBoxes: [x, y, width, height]
boxes_for_nms = []
scores_for_nms = []
for i in range(len(valid_indices)):
# Ensure coordinates are within bounds and positive
x_min_clipped = max(0, x_min[i])
y_min_clipped = max(0, y_min[i])
x_max_clipped = min(self.memx_model_width, x_max[i])
y_max_clipped = min(self.memx_model_height, y_max[i])
width_clipped = x_max_clipped - x_min_clipped
height_clipped = y_max_clipped - y_min_clipped
if width_clipped > 0 and height_clipped > 0:
boxes_for_nms.append(
[x_min_clipped, y_min_clipped, width_clipped, height_clipped]
)
scores_for_nms.append(float(valid_scores[i]))
final_detections = np.zeros((20, 6), np.float32)
if len(boxes_for_nms) == 0:
return final_detections
# Apply NMS using OpenCV
indices = cv2.dnn.NMSBoxes(
boxes_for_nms, scores_for_nms, confidence_thres, iou_thres
)
if len(indices) > 0:
# Flatten indices if they are returned as a list of arrays
if isinstance(indices[0], list) or isinstance(indices[0], np.ndarray):
indices = [i[0] for i in indices]
# Limit to top 20 detections
indices = indices[:20]
# Convert to Frigate format: [class_id, confidence, y_min, x_min, y_max, x_max] (normalized)
for i, idx in enumerate(indices):
class_id = valid_class_ids[idx]
confidence = valid_scores[idx]
# Get the box coordinates
box = boxes_for_nms[idx]
x_min_norm = box[0] / self.memx_model_width
y_min_norm = box[1] / self.memx_model_height
x_max_norm = (box[0] + box[2]) / self.memx_model_width
y_max_norm = (box[1] + box[3]) / self.memx_model_height
final_detections[i] = [
class_id,
confidence,
y_min_norm, # Frigate expects y_min first
x_min_norm,
y_max_norm,
x_max_norm,
]
return final_detections
def process_output(self, *outputs):
"""Output callback function -- receives frames from the MX3 and triggers post-processing"""
if self.memx_model_type == ModelTypeEnum.yologeneric:
if not self.memx_post_model:
conv_out1 = outputs[0]
conv_out2 = outputs[1]
conv_out3 = outputs[2]
conv_out4 = outputs[3]
conv_out5 = outputs[4]
conv_out6 = outputs[5]
# Use complete YOLOv9-style postprocessing (includes NMS)
final_detections = self.post_process_yolo_optimized(outputs)
concat_1 = self.onnx_concat([conv_out1, conv_out2], axis=1)
concat_2 = self.onnx_concat([conv_out3, conv_out4], axis=1)
concat_3 = self.onnx_concat([conv_out5, conv_out6], axis=1)
shape = np.array([1, 144, -1], dtype=np.int64)
reshaped_1 = self.onnx_reshape_with_allowzero(
concat_1, shape, allowzero=0
)
reshaped_2 = self.onnx_reshape_with_allowzero(
concat_2, shape, allowzero=0
)
reshaped_3 = self.onnx_reshape_with_allowzero(
concat_3, shape, allowzero=0
)
concat_4 = self.onnx_concat([reshaped_1, reshaped_2, reshaped_3], 2)
axis = 1
split_sizes = [64, 80]
# Calculate indices at which to split
indices = np.cumsum(split_sizes)[
:-1
] # [64] — split before the second chunk
# Perform split along axis 1
split_0, split_1 = np.split(concat_4, indices, axis=axis)
num_boxes = 2100 if self.memx_model_height == 320 else 8400
shape1 = np.array([1, 4, 16, num_boxes])
reshape_4 = self.onnx_reshape_with_allowzero(
split_0, shape1, allowzero=0
)
transpose_1 = reshape_4.transpose(0, 2, 1, 3)
axis = 1 # As per ONNX softmax node
# Subtract max for numerical stability
x_max = np.max(transpose_1, axis=axis, keepdims=True)
x_exp = np.exp(transpose_1 - x_max)
x_sum = np.sum(x_exp, axis=axis, keepdims=True)
softmax_output = x_exp / x_sum
# Weight W from the ONNX initializer (1, 16, 1, 1) with values 0 to 15
W = np.arange(16, dtype=np.float32).reshape(
1, 16, 1, 1
) # (1, 16, 1, 1)
# Apply 1x1 convolution: this is a weighted sum over channels
conv_output = np.sum(
softmax_output * W, axis=1, keepdims=True
) # shape: (1, 1, 4, 8400)
shape2 = np.array([1, 4, num_boxes])
reshape_5 = self.onnx_reshape_with_allowzero(
conv_output, shape2, allowzero=0
)
# ONNX Slice — get first 2 channels: [0:2] along axis 1
slice_output1 = reshape_5[:, 0:2, :] # Result: (1, 2, 8400)
# Slice channels 2 to 4 → axis = 1
slice_output2 = reshape_5[:, 2:4, :]
# Perform Subtraction
sub_output = self.const_A - slice_output1 # Equivalent to ONNX Sub
# Perform the ONNX-style Add
add_output = self.const_B + slice_output2
sub1 = add_output - sub_output
add1 = sub_output + add_output
div_output = add1 / 2.0
concat_5 = self.onnx_concat([div_output, sub1], axis=1)
# Expand B to (1, 1, 8400) so it can broadcast across axis=1 (4 channels)
const_C_expanded = self.const_C[:, np.newaxis, :] # Shape: (1, 1, 8400)
# Perform ONNX-style element-wise multiplication
mul_output = concat_5 * const_C_expanded # Result: (1, 4, 8400)
sigmoid_output = self.sigmoid(split_1)
outputs = self.onnx_concat([mul_output, sigmoid_output], axis=1)
final_detections = post_process_yolo(
outputs, self.memx_model_width, self.memx_model_height
)
self.output_queue.put(final_detections)
elif self.memx_model_type == ModelTypeEnum.yolonas:

View File

@ -76,7 +76,12 @@
}
},
"npuUsage": "NPU Usage",
"npuMemory": "NPU Memory"
"npuMemory": "NPU Memory",
"intelGpuWarning": {
"title": "Intel GPU Stats Warning",
"message": "GPU stats unavailable",
"description": "This is a known bug in Intel's GPU stats reporting tools (intel_gpu_top) where it will break and repeatedly return a GPU usage of 0% even in cases where hardware acceleration and object detection are correctly running on the (i)GPU. This is not a Frigate bug. You can restart the host to temporarily fix the issue and confirm that the GPU is working correctly. This does not affect performance."
}
},
"otherProcesses": {
"title": "Other Processes",

View File

@ -56,6 +56,7 @@ export function TrackingDetails({
const apiHost = useApiHost();
const imgRef = useRef<HTMLImageElement | null>(null);
const [imgLoaded, setImgLoaded] = useState(false);
const [isVideoLoading, setIsVideoLoading] = useState(true);
const [displaySource, _setDisplaySource] = useState<"video" | "image">(
"video",
);
@ -70,6 +71,10 @@ export function TrackingDetails({
(event.start_time ?? 0) + annotationOffset / 1000 - REVIEW_PADDING,
);
useEffect(() => {
setIsVideoLoading(true);
}, [event.id]);
const { data: eventSequence } = useSWR<TrackingDetailsSequence[]>([
"timeline",
{
@ -527,6 +532,7 @@ export function TrackingDetails({
)}
>
{displaySource == "video" && (
<>
<HlsVideoPlayer
videoRef={videoRef}
containerRef={containerRef}
@ -539,10 +545,15 @@ export function TrackingDetails({
onTimeUpdate={handleTimeUpdate}
onSeekToTime={handleSeekToTime}
onUploadFrame={onUploadFrameToPlus}
onPlaying={() => setIsVideoLoading(false)}
isDetailMode={true}
camera={event.camera}
currentTimeOverride={currentTime}
/>
{isVideoLoading && (
<ActivityIndicator className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2" />
)}
</>
)}
{displaySource == "image" && (
<>

View File

@ -130,6 +130,8 @@ export default function HlsVideoPlayer({
return;
}
setLoadedMetadata(false);
const currentPlaybackRate = videoRef.current.playbackRate;
if (!useHlsCompat) {

View File

@ -309,6 +309,7 @@ function PreviewVideoPlayer({
playsInline
muted
disableRemotePlayback
disablePictureInPicture
onSeeked={onPreviewSeeked}
onLoadedData={() => {
if (firstLoad) {

View File

@ -2,7 +2,10 @@ import { Recording } from "@/types/record";
import { DynamicPlayback } from "@/types/playback";
import { PreviewController } from "../PreviewPlayer";
import { TimeRange, TrackingDetailsSequence } from "@/types/timeline";
import { calculateInpointOffset } from "@/utils/videoUtil";
import {
calculateInpointOffset,
calculateSeekPosition,
} from "@/utils/videoUtil";
type PlayerMode = "playback" | "scrubbing";
@ -72,39 +75,21 @@ export class DynamicVideoController {
return;
}
if (
this.recordings.length == 0 ||
time < this.recordings[0].start_time ||
time > this.recordings[this.recordings.length - 1].end_time
) {
this.setNoRecording(true);
return;
}
if (this.playerMode != "playback") {
this.playerMode = "playback";
}
let seekSeconds = 0;
(this.recordings || []).every((segment) => {
// if the next segment is past the desired time, stop calculating
if (segment.start_time > time) {
return false;
const seekSeconds = calculateSeekPosition(
time,
this.recordings,
this.inpointOffset,
);
if (seekSeconds === undefined) {
this.setNoRecording(true);
return;
}
if (segment.end_time < time) {
seekSeconds += segment.end_time - segment.start_time;
return true;
}
seekSeconds +=
segment.end_time - segment.start_time - (segment.end_time - time);
return true;
});
// adjust for HLS inpoint offset
seekSeconds -= this.inpointOffset;
if (seekSeconds != 0) {
this.playerController.currentTime = seekSeconds;

View File

@ -14,7 +14,10 @@ import { VideoResolutionType } from "@/types/live";
import axios from "axios";
import { cn } from "@/lib/utils";
import { useTranslation } from "react-i18next";
import { calculateInpointOffset } from "@/utils/videoUtil";
import {
calculateInpointOffset,
calculateSeekPosition,
} from "@/utils/videoUtil";
import { isFirefox } from "react-device-detect";
/**
@ -109,10 +112,10 @@ export default function DynamicVideoPlayer({
const [isLoading, setIsLoading] = useState(false);
const [isBuffering, setIsBuffering] = useState(false);
const [loadingTimeout, setLoadingTimeout] = useState<NodeJS.Timeout>();
const [source, setSource] = useState<HlsSource>({
playlist: `${apiHost}vod/${camera}/start/${timeRange.after}/end/${timeRange.before}/master.m3u8`,
startPosition: startTimestamp ? startTimestamp - timeRange.after : 0,
});
// Don't set source until recordings load - we need accurate startPosition
// to avoid hls.js clamping to video end when startPosition exceeds duration
const [source, setSource] = useState<HlsSource | undefined>(undefined);
// start at correct time
@ -184,7 +187,7 @@ export default function DynamicVideoPlayer({
);
useEffect(() => {
if (!controller || !recordings?.length) {
if (!recordings?.length) {
if (recordings?.length == 0) {
setNoRecording(true);
}
@ -192,10 +195,6 @@ export default function DynamicVideoPlayer({
return;
}
if (playerRef.current) {
playerRef.current.autoplay = !isScrubbing;
}
let startPosition = undefined;
if (startTimestamp) {
@ -203,14 +202,12 @@ export default function DynamicVideoPlayer({
recordingParams.after,
(recordings || [])[0],
);
const idealStartPosition = Math.max(
0,
startTimestamp - timeRange.after - inpointOffset,
);
if (idealStartPosition >= recordings[0].start_time - timeRange.after) {
startPosition = idealStartPosition;
}
startPosition = calculateSeekPosition(
startTimestamp,
recordings,
inpointOffset,
);
}
setSource({
@ -218,6 +215,18 @@ export default function DynamicVideoPlayer({
startPosition,
});
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [recordings]);
useEffect(() => {
if (!controller || !recordings?.length) {
return;
}
if (playerRef.current) {
playerRef.current.autoplay = !isScrubbing;
}
setLoadingTimeout(setTimeout(() => setIsLoading(true), 1000));
controller.newPlayback({
@ -225,7 +234,7 @@ export default function DynamicVideoPlayer({
timeRange,
});
// we only want this to change when recordings update
// we only want this to change when controller or recordings update
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [controller, recordings]);
@ -263,6 +272,7 @@ export default function DynamicVideoPlayer({
return (
<>
{source && (
<HlsVideoPlayer
videoRef={playerRef}
containerRef={containerRef}
@ -303,6 +313,7 @@ export default function DynamicVideoPlayer({
camera={contextCamera || camera}
currentTimeOverride={currentTime}
/>
)}
<PreviewPlayer
className={cn(
className,

View File

@ -24,3 +24,57 @@ export function calculateInpointOffset(
return 0;
}
/**
* Calculates the video player time (in seconds) for a given timestamp
* by iterating through recording segments and summing their durations.
* This accounts for the fact that the video is a concatenation of segments,
* not a single continuous stream.
*
* @param timestamp - The target timestamp to seek to
* @param recordings - Array of recording segments
* @param inpointOffset - HLS inpoint offset to subtract from the result
* @returns The calculated seek position in seconds, or undefined if timestamp is out of range
*/
export function calculateSeekPosition(
timestamp: number,
recordings: Recording[],
inpointOffset: number = 0,
): number | undefined {
if (!recordings || recordings.length === 0) {
return undefined;
}
// Check if timestamp is within the recordings range
if (
timestamp < recordings[0].start_time ||
timestamp > recordings[recordings.length - 1].end_time
) {
return undefined;
}
let seekSeconds = 0;
(recordings || []).every((segment) => {
// if the next segment is past the desired time, stop calculating
if (segment.start_time > timestamp) {
return false;
}
if (segment.end_time < timestamp) {
// Add the full duration of this segment
seekSeconds += segment.end_time - segment.start_time;
return true;
}
// We're in this segment - calculate position within it
seekSeconds +=
segment.end_time - segment.start_time - (segment.end_time - timestamp);
return true;
});
// Adjust for HLS inpoint offset
seekSeconds -= inpointOffset;
return seekSeconds >= 0 ? seekSeconds : undefined;
}

View File

@ -375,6 +375,50 @@ export default function GeneralMetrics({
return Object.keys(series).length > 0 ? Object.values(series) : undefined;
}, [statsHistory]);
// Check if Intel GPU has all 0% usage values (known bug)
const showIntelGpuWarning = useMemo(() => {
if (!statsHistory || statsHistory.length < 3) {
return false;
}
const gpuKeys = Object.keys(statsHistory[0]?.gpu_usages ?? {});
const hasIntelGpu = gpuKeys.some(
(key) => key === "intel-vaapi" || key === "intel-qsv",
);
if (!hasIntelGpu) {
return false;
}
// Check if all GPU usage values are 0% across all stats
let allZero = true;
let hasDataPoints = false;
for (const stats of statsHistory) {
if (!stats) {
continue;
}
Object.entries(stats.gpu_usages || {}).forEach(([key, gpuStats]) => {
if (key === "intel-vaapi" || key === "intel-qsv") {
if (gpuStats.gpu) {
hasDataPoints = true;
const gpuValue = parseFloat(gpuStats.gpu.slice(0, -1));
if (!isNaN(gpuValue) && gpuValue > 0) {
allZero = false;
}
}
}
});
if (!allZero) {
break;
}
}
return hasDataPoints && allZero;
}, [statsHistory]);
// npu stats
const npuSeries = useMemo(() => {
@ -639,8 +683,46 @@ export default function GeneralMetrics({
<>
{statsHistory.length != 0 ? (
<div className="rounded-lg bg-background_alt p-2.5 md:rounded-2xl">
<div className="mb-5">
<div className="mb-5 flex flex-row items-center justify-between">
{t("general.hardwareInfo.gpuUsage")}
{showIntelGpuWarning && (
<Popover>
<PopoverTrigger asChild>
<button
className="flex flex-row items-center gap-1.5 text-yellow-600 focus:outline-none dark:text-yellow-500"
aria-label={t(
"general.hardwareInfo.intelGpuWarning.title",
)}
>
<CiCircleAlert
className="size-5"
aria-label={t(
"general.hardwareInfo.intelGpuWarning.title",
)}
/>
<span className="text-sm">
{t(
"general.hardwareInfo.intelGpuWarning.message",
)}
</span>
</button>
</PopoverTrigger>
<PopoverContent className="w-80">
<div className="space-y-2">
<div className="font-semibold">
{t(
"general.hardwareInfo.intelGpuWarning.title",
)}
</div>
<div>
{t(
"general.hardwareInfo.intelGpuWarning.description",
)}
</div>
</div>
</PopoverContent>
</Popover>
)}
</div>
{gpuSeries.map((series) => (
<ThresholdBarGraph