Compare commits

..
Author SHA1 Message Date
dependabot[bot]andGitHub cffef45730 Update numpy requirement from ==1.26.* to ==2.5.* in /docker/main
Updates the requirements on [numpy](https://github.com/numpy/numpy) to permit the latest version.
- [Release notes](https://github.com/numpy/numpy/releases)
- [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst)
- [Commits](https://github.com/numpy/numpy/compare/v1.26.0.dev0...v2.5.3)

---
updated-dependencies:
- dependency-name: numpy
  dependency-version: 2.5.3
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-16 11:33:07 +00:00
307 changed files with 2647 additions and 15562 deletions
-1
View File
@@ -5,4 +5,3 @@
/docker/rockchip/ @MarcA711
/docker/rocm/ @harakas
/docker/hailo8l/ @spanner3003
/docker/deepx/ @sixfab
-131
View File
@@ -1,131 +0,0 @@
#!/bin/bash
# Installs the DEEPX NPU kernel driver and the DX-RT runtime on the Docker host,
# then enables the vendor's dxrt.service. A container cannot load kernel
# modules, so this runs outside the image; the driver creates the /dev/dxrt*
# nodes and the daemon multiplexes the NPU across host and container.
#
# Driver, runtime and firmware versions must agree or inference hangs instead
# of failing at startup. The set this script installs is pinned in
# driver_version, runtime_version and firmware_version below; move them
# together, never one at a time.
#
# DEEPX NPU support in Frigate is maintained by Sixfab (https://sixfab.com).
set -euo pipefail
driver_version="v2.6.0"
# the commit the tag resolves to, since DEEPX signs neither tags nor releases
# and this is compiled and installed as root. Update both together
driver_commit="7074748e7104f470b02f517583abba652b3f05fa"
firmware_version="v2.7.4"
sudo apt-get update
sudo apt-get install -y git build-essential "linux-headers-$(uname -r)" pciutils wget
if ! lspci -d 1ff4: | grep -q .; then
echo "No DEEPX device found on the PCIe bus (lspci -d 1ff4:)."
echo "Check that the module is seated correctly before continuing."
exit 1
fi
# fetch the pinned commit rather than cloning the tag, so a retag cannot swap
# in different source. The build directory is reused so a second run after a
# failure does not stop on the directory already being there
mkdir -p dx_rt_npu_linux_driver
cd dx_rt_npu_linux_driver
git init -q
git remote get-url origin > /dev/null 2>&1 ||
git remote add origin https://github.com/DEEPX-AI/dx_rt_npu_linux_driver.git
git fetch --depth 1 origin "${driver_commit}"
git checkout -q FETCH_HEAD
fetched_commit=$(git rev-parse HEAD)
if [[ "${fetched_commit}" != "${driver_commit}" ]]; then
echo "Fetched commit ${fetched_commit} does not match pinned driver_commit ${driver_commit}."
echo "Refusing to build unverified driver source."
exit 1
fi
cd modules
sudo ./build.sh -c install --reload
sudo depmod -A
# dx_dma is the PCIe transport, dxrt_driver the NPU driver on top of it
for module in dx_dma dxrt_driver; do
if ! sudo modprobe "${module}"; then
echo "Unable to load the ${module} kernel module, common reasons are:"
echo "- Secure Boot is enabled and is rejecting the unsigned module."
echo "- The running kernel does not match the installed linux-headers."
exit 1
fi
done
if ! compgen -G "/dev/dxrt*" > /dev/null; then
echo "Modules loaded but no /dev/dxrt* device node appeared."
echo "Run ./sanity_check.sh from the driver repo to diagnose."
exit 1
fi
runtime_version="v3.4.0"
declare -A runtime_sha256=(
[amd64]="736cfef009ce9e974ab1ab610d867239d19d72a426a53e367ddcbd53297b6e20"
[arm64]="eb6107f5f02f2ad76ae89f414e8b5f346f34fbc6f0888236136853a26be6f6a0"
)
runtime_release="${runtime_version#v}"
deb_arch=$(dpkg --print-architecture)
deb_file="/tmp/libdxrt-bin_${runtime_release}_${deb_arch}.deb"
wget -qO "${deb_file}" \
"https://raw.githubusercontent.com/DEEPX-AI/dx_rt/${runtime_version}/release/${runtime_release}/libdxrt-bin_${runtime_release}_${deb_arch}.deb"
expected_sha256="${runtime_sha256[${deb_arch}]:-}"
if [[ -z "${expected_sha256}" ]]; then
echo "No pinned SHA-256 for architecture ${deb_arch}; refusing to install."
exit 1
fi
if [[ "$(sha256sum "${deb_file}" | cut -d' ' -f1)" != "${expected_sha256}" ]]; then
echo "SHA-256 mismatch for ${deb_file}; refusing to install."
exit 1
fi
sudo dpkg -i "${deb_file}"
sudo ldconfig
rm -f "${deb_file}"
sudo cp /usr/share/libdxrt-bin/service/dxrt.service /etc/systemd/system/
# With an endpoint set, dxrtd binds that path only, so the socket goes in a
# directory Frigate can mount (kept across restarts so the mount stays valid)
# and a symlink at the default /tmp path keeps host tools that do not set the
# variable working through their own fallback.
sudo mkdir -p /etc/systemd/system/dxrt.service.d
sudo tee /etc/systemd/system/dxrt.service.d/frigate.conf > /dev/null <<'UNIT'
[Service]
RuntimeDirectory=dxrt
RuntimeDirectoryMode=0755
RuntimeDirectoryPreserve=yes
Environment=DXRT_DYNAMIC_IPC_ENDPOINT=/run/dxrt/dxrt_dynamic_ipc.sock
ExecStartPost=/bin/ln -sfn /run/dxrt/dxrt_dynamic_ipc.sock /tmp/dxrt_dynamic_ipc.sock
UNIT
sudo systemctl daemon-reload
sudo systemctl enable dxrt.service
sudo systemctl restart dxrt.service
if ! sudo systemctl is-active --quiet dxrt.service; then
echo "dxrt.service did not start. Check: sudo journalctl -u dxrt.service"
exit 1
fi
echo "DEEPX driver and runtime installation complete."
echo "Driver version: $(modinfo -F version dxrt_driver) (expected ${driver_version#v})"
echo "Runtime version: ${runtime_release}"
echo "Device node(s): $(echo /dev/dxrt*)"
echo
echo "This driver expects NPU firmware ${firmware_version}. Check it with:"
echo " dxrt-cli --status"
echo "Update the module if it does not match before starting Frigate."
+4 -4
View File
@@ -27,7 +27,7 @@ pydantic == 2.10.*
git+https://github.com/fbcotter/py3nvml#egg=py3nvml
pytz == 2025.*
pyzmq == 27.1.*
ruamel.yaml == 0.19.*
ruamel.yaml == 0.18.*
tzlocal == 5.2
requests == 2.33.*
types-requests == 2.32.*
@@ -37,13 +37,13 @@ ws4py == 0.5.*
unidecode == 1.4.*
titlecase == 2.4.*
# Image Manipulation
numpy == 1.26.*
numpy == 2.5.*
opencv-python-headless == 4.11.0.*
opencv-contrib-python == 4.11.0.*
scipy == 1.16.*
# OpenVino & ONNX
openvino == 2025.4.*
onnxruntime == 1.30.*
onnxruntime == 1.22.*
# Embeddings
transformers == 4.45.*
# Generative AI
@@ -58,7 +58,7 @@ pyclipper == 1.4.*
shapely == 2.0.*
rapidfuzz==3.12.*
# HailoRT
argcomplete==3.7.*
argcomplete==2.0.*
contextlib2==0.6.*
future==0.18.*
netaddr==1.3.*
@@ -37,7 +37,6 @@ device_globs=(
"/dev/nvmap"
"/dev/nvidia*"
"/dev/memx*"
"/dev/dxrt*"
)
IFS=',' read -ra extra_globs <<< "${DEVICE_ACL_PATHS:-}"
-59
View File
@@ -967,65 +967,6 @@ memryx:
# The .zip file must contain:
# ├── ssdlite_mobilenet.dfp (a file ending with .dfp)
# └── ssdlite_mobilenet_post.onnx (optional; only if the model includes a cropped post-processing network)
deepx:
title: DEEPX NPU
models:
- key: yolo
label: YOLO
recommended: true
download: No model is bundled with Frigate. Download a pre-compiled YOLO `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) or compile your own with DX-COM, then bind-mount it into the container and point the model's `path` at it. The recommended model is `yolox-s_640x640_ppu.dxnn`. Its Post-Processing Unit (PPU) compile moves candidate selection onto the NPU, which makes it the fastest ModelZoo model measured through Frigate (about 13 ms on a DX-M1). The output layout is read from the compiled model, so anchor-based, anchor-free, NMS-in-head and PPU models (anchor-based or anchor-free) all need no extra configuration; prefer a `PPU` variant whenever the ModelZoo offers one. PPU models must be compiled with DX-COM 2.4.0 or later, which writes the head layout Frigate reads into the file.
ui: |-
Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure:
| Field | Value |
| ---------------------------------------- | ---------------------------------------------------- |
| **Custom object detector model path** | `/config/model_cache/deepx/yolox-s_640x640_ppu.dxnn` |
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
| **Object detection model input width** | `640` |
| **Object detection model input height** | `640` |
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
| **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolo-generic` |
Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden.
yaml: |-
models:
- devices:
- deepx:PCIe:0
path: /config/model_cache/deepx/yolox-s_640x640_ppu.dxnn
labelmap_path: /labelmap/coco-80.txt
model_type: yolo-generic
width: 640
height: 640
- key: yolox
label: YOLOX
recommended: false
download: No model is bundled with Frigate. Download a pre-compiled YOLOX `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point the model's `path` at it. The `_ppu` variant is faster and also works with the `yolo-generic` model type; the plain export needs `yolox` so its raw head is decoded.
ui: |-
Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure:
| Field | Value |
| ---------------------------------------- | ---------------------------------------- |
| **Custom object detector model path** | `/config/model_cache/deepx/yolox-s_640x640.dxnn`|
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
| **Object detection model input width** | `640` |
| **Object detection model input height** | `640` |
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
| **Model Input D Type** | `int` (Frigate's default value) |
| **Object Detection Model Type** | `yolox` |
The width and height must match the resolution the `.dxnn` file was compiled for.
yaml: |-
models:
- devices:
- deepx:PCIe:0
path: /config/model_cache/deepx/yolox-s_640x640.dxnn
labelmap_path: /labelmap/coco-80.txt
model_type: yolox
width: 640
height: 640
tensorrt:
title: TensorRT
models:
+4 -12
View File
@@ -800,7 +800,7 @@ lpr:
# to Google or OpenAI's LLMs to generate descriptions. GenAI features can be configured at
# the camera level to enhance privacy for indoor cameras.
# NOTE: genai is a map of named providers. Each key is a name you choose for the provider,
# and each role (chat, descriptions, embeddings, transcribe) may be assigned to exactly one provider.
# and each role (chat, descriptions, embeddings) may be assigned to exactly one provider.
genai:
# Required: name of the provider (chosen by you, used to reference it elsewhere)
my_provider:
@@ -813,13 +813,11 @@ genai:
# Required: The model to use with the provider.
model: gemini-1.5-flash
# Optional: Roles this provider handles (default: shown below)
# Each role (chat, descriptions, embeddings, transcribe) must be assigned to exactly
# one provider.
# Each role (chat, descriptions, embeddings) must be assigned to exactly one provider.
roles:
- chat
- descriptions
- embeddings
- transcribe
# Optional additional args to pass to the GenAI Provider (default: None)
provider_options:
keep_alive: -1
@@ -832,19 +830,13 @@ genai:
audio_transcription:
# Optional: Enable live and speech event audio transcription (default: shown below)
enabled: False
# Optional: The transcription backend (default: shown below)
# Either 'whisper' for Frigate's built-in local models, or the name of a genai
# provider that has 'transcribe' in its roles. device and model_size are ignored
# when a genai provider is named.
model: whisper
# Optional: The device to run the models on for live transcription. (default: shown below)
device: CPU
# Optional: Set the model size used for live transcription. (default: shown below)
model_size: small
# Optional: Set the language used for transcription translation. (default: shown below)
# Use 'auto' to let the model detect the language, or a language code from
# https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
language: auto
# List of language codes: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
language: en
# Optional: Configuration for classification models
classification:
+7 -79
View File
@@ -204,7 +204,7 @@ Frequently-heard labels like `speech` can generate a lot of events, and each eve
### Audio Transcription
Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`, and can alternatively offload transcription to a [GenAI provider](#genai-provider). The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`. The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
:::info
@@ -224,7 +224,6 @@ To enable transcription, configure it globally and optionally disable for specif
**Global:** Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
- Set **Enable audio transcription** to on
- Set **Audio transcription model or GenAI provider name** to `whisper` for Frigate's built-in local models, or to the name of a GenAI provider
- Set **Transcription device** to the desired device
- Set **Model size** to the desired size
@@ -236,7 +235,6 @@ To enable transcription, configure it globally and optionally disable for specif
```yaml
audio_transcription:
enabled: True
model: whisper
device: ...
model_size: ...
```
@@ -265,88 +263,20 @@ The optional config parameters that can be set at the global level include:
- **`enabled`**: Enable or disable the audio transcription feature.
- Default: `False`
- It is recommended to only configure the features at the global level, and enable it at the individual camera level.
- **`model`**: The transcription backend.
- Default: `whisper`
- `whisper` uses Frigate's built-in local models, described by `device` and `model_size` below.
- Any other value must name a key in your `genai` config whose entry has `transcribe` in its `roles`. See [GenAI Provider](#genai-provider).
- **`device`**: Device to use to run transcription and translation models.
- Default: `CPU`
- This can be `CPU` or `GPU`. The `sherpa-onnx` models are lightweight and run on the CPU only. The `whisper` models can run on GPU but are only supported on CUDA hardware.
- Ignored when `model` names a GenAI provider.
- **`model_size`**: The size of the model used for live transcription.
- Default: `small`
- This can be `small` or `large`. The `small` setting uses `sherpa-onnx` models that are fast, lightweight, and always run on the CPU but are not as accurate as the `whisper` model.
- This config option applies to **live transcription only**. With `model: whisper`, recorded `speech` events always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
- Ignored when `model` names a GenAI provider.
- **`language`**: Defines the language used to transcribe and translate `speech` audio events (and live audio only if using the `large` model or a GenAI provider).
- Default: `auto`
- `auto` lets the model detect the language itself, which most models do well. Set an explicit language only if detection is picking the wrong one.
- Otherwise you must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
- This config option applies to **live transcription only**. Recorded `speech` events will always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
- **`language`**: Defines the language used by `whisper` to translate `speech` audio events (and live audio only if using the `large` model).
- Default: `en`
- You must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
- Transcriptions for `speech` events are translated.
- Live audio is translated only if you are using the `large` model. The `small` `sherpa-onnx` model is English-only.
The only field that is valid at the camera level is `enabled`. In particular `model` is global only: the transcription backend is a process-wide resource shared by every camera.
#### GenAI Provider
Frigate can send audio to a GenAI provider for transcription when that provider has the `transcribe` role. This is useful if you already run a GenAI provider, or if you do not have the CPU/GPU headroom for a local whisper model. Supported providers are **OpenAI**, **Azure OpenAI**, **Gemini**, and **llama.cpp** with an audio-capable model (a dedicated ASR model such as Qwen3-ASR, or a general multimodal model that accepts audio). Ollama is not supported as it has no audio input.
To use a GenAI provider for audio transcription:
1. Configure a GenAI provider with `transcribe` in its `roles`.
2. Set the audio transcription model to that GenAI config key (e.g. `whisper_cloud`).
<ConfigTabs>
<TabItem value="ui">
Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
| Field | Description |
| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
| **Audio transcription model or GenAI provider name** | Set to the GenAI config key (e.g. `whisper_cloud`) to use a configured GenAI provider for transcription |
The GenAI provider must also be configured with the `transcribe` role under <NavPath path="Settings > Enrichments > Generative AI" />.
</TabItem>
<TabItem value="yaml">
```yaml
genai:
whisper_cloud:
provider: openai
api_key: your-api-key
model: gpt-transcribe
roles:
- transcribe
audio_transcription:
enabled: True
model: whisper_cloud
language: en
```
</TabItem>
</ConfigTabs>
:::warning
**Give `transcribe` its own `genai` entry.** A `genai` entry has a single `model` string that is shared by every role it holds, so `roles: [descriptions, transcribe]` would send the same model name to both the chat endpoint and the transcription endpoint. Transcription models and chat models are almost never the same model, so define a dedicated entry as shown above.
:::
:::warning
**Live transcription against a metered provider is billed continuously.** In live mode Frigate uploads an overlapping ~2 second window of audio roughly once per second, per camera, for as long as audio stays above that camera's `audio.min_volume`. Windows below that threshold are never uploaded, which is what keeps a quiet camera near zero requests, but a camera pointed at a busy street will keep sending.
Three things keep this opt-in: `transcribe` is not one of the default roles, live transcription is off by default, and the volume gate suppresses silence. Transcription of recorded `speech` events is unaffected - it remains a manual, one-request-per-event action.
:::
`device` and `model_size` have no effect on this path and no local model is ever downloaded.
`language` defaults to `auto`, which sends no language hint and lets the model detect it. Most audio models detect language well, so leave it on `auto` unless detection is picking the wrong one.
When set explicitly, it is sent as the transcription endpoint's native `language` parameter for OpenAI, Azure, and llama.cpp, and as part of the prompt for Gemini. This matters for dedicated ASR models such as Qwen3-ASR: they read the prompt as contextual biasing rather than as an instruction, so a language named in the prompt is ignored, while the endpoint parameter is honored.
The only field that is valid at the camera level is `enabled`.
#### Live transcription
@@ -362,8 +292,6 @@ Results can be error-prone due to a number of factors, including:
For speech sources close to the camera with minimal background noise, use the `small` model.
A [GenAI provider](#genai-provider) is generally the most accurate option for live transcription, at the cost of a network round trip per window. That round trip has to stay under about a second to keep up with the audio; if it does not, Frigate drops the oldest buffered audio rather than letting the backlog grow.
If you have CUDA hardware, you can experiment with the `large` `whisper` model on GPU. Performance is not quite as fast as the `sherpa-onnx` `small` model, but live transcription is far more accurate. Using the `large` model with CPU will likely be too slow for real-time transcription.
#### Transcription and translation of `speech` audio events
@@ -380,7 +308,7 @@ Only one `speech` event may be transcribed at a time. Frigate does not automatic
:::
With `model: whisper`, recorded `speech` events always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient. With a [GenAI provider](#genai-provider), the recorded clip is sent to the provider instead and no local model is used.
Recorded `speech` events will always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient.
#### FAQ
+11 -13
View File
@@ -17,19 +17,17 @@ Hardware acceleration arguments tell FFmpeg to decode your camera's video stream
See [the hardware acceleration docs](/configuration/hardware_acceleration_video.md) for details on setting up hardware acceleration for your GPU / iGPU, then select the preset that matches your hardware.
| Preset (YAML config) | UI Label | Usage | Notes |
| ------------------------- | ----------------------- | --------------------------------------------- | --------------------------------------------------------------- |
| preset-rpi-64-h264 | Raspberry Pi (H.264) | 64-bit Raspberry Pi, H.264 stream | |
| preset-rpi-64-h265 | Raspberry Pi (H.265) | 64-bit Raspberry Pi, H.265 stream | |
| preset-apple-silicon-h264 | Apple Silicon (H.264) | Apple Silicon Mac under lighter, H.264 stream | Needs the `lighter.sh/video` device |
| preset-apple-silicon-h265 | Apple Silicon (H.265) | Apple Silicon Mac under lighter, H.265 stream | Needs the `lighter.sh/video` device |
| preset-vaapi | VAAPI (Intel/AMD GPU) | Intel or AMD GPU via VAAPI | Check the hwaccel docs to ensure the correct driver is selected |
| preset-intel-qsv-h264 | Intel QuickSync (H.264) | Intel QuickSync, H.264 stream | If you have issues, use the VAAPI preset instead |
| preset-intel-qsv-h265 | Intel QuickSync (H.265) | Intel QuickSync, H.265 stream | If you have issues, use the VAAPI preset instead |
| preset-nvidia | NVIDIA GPU | NVIDIA GPU | |
| preset-jetson-h264 | NVIDIA Jetson (H.264) | NVIDIA Jetson, H.264 stream | |
| preset-jetson-h265 | NVIDIA Jetson (H.265) | NVIDIA Jetson, H.265 stream | |
| preset-rkmpp | Rockchip RKMPP | Rockchip MPP | Use an image with the `-rk` suffix and run in privileged mode |
| Preset (YAML config) | UI Label | Usage | Notes |
| --------------------- | ----------------------- | --------------------------------- | --------------------------------------------------------------- |
| preset-rpi-64-h264 | Raspberry Pi (H.264) | 64-bit Raspberry Pi, H.264 stream | |
| preset-rpi-64-h265 | Raspberry Pi (H.265) | 64-bit Raspberry Pi, H.265 stream | |
| preset-vaapi | VAAPI (Intel/AMD GPU) | Intel or AMD GPU via VAAPI | Check the hwaccel docs to ensure the correct driver is selected |
| preset-intel-qsv-h264 | Intel QuickSync (H.264) | Intel QuickSync, H.264 stream | If you have issues, use the VAAPI preset instead |
| preset-intel-qsv-h265 | Intel QuickSync (H.265) | Intel QuickSync, H.265 stream | If you have issues, use the VAAPI preset instead |
| preset-nvidia | NVIDIA GPU | NVIDIA GPU | |
| preset-jetson-h264 | NVIDIA Jetson (H.264) | NVIDIA Jetson, H.264 stream | |
| preset-jetson-h265 | NVIDIA Jetson (H.265) | NVIDIA Jetson, H.265 stream | |
| preset-rkmpp | Rockchip RKMPP | Rockchip MPP | Use an image with the `-rk` suffix and run in privileged mode |
<ConfigTabs>
<TabItem value="ui">
+6 -17
View File
@@ -43,7 +43,7 @@ genai:
The examples on this page all use `my_provider`, but the name is arbitrary and is only used to reference the provider elsewhere in the config (for example, `semantic_search.model`).
Each provider handles one or more **roles**: `chat`, `descriptions`, `embeddings`, and `transcribe`. A provider handles the first three by default; `transcribe` must always be listed explicitly, and is not available on Ollama, which has no audio input. Each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
Each provider handles one or more **roles**: `chat`, `descriptions`, and `embeddings`. A provider handles all three by default, and each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
If the provider you choose requires an API key, you may either directly paste it in your configuration, or store it in an environment variable prefixed with `FRIGATE_`.
@@ -63,11 +63,11 @@ Running Generative AI models on CPU is not recommended, as high inference times
You must use a vision-capable model with Frigate. The following models are recommended for local deployment of the `descriptions` and `chat` roles:
| Model | Review [frame mode](/configuration/genai/genai_review#frame-mode) | Notes |
| ------------------- | --------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `qwen3-vl` | `frames` | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object. Follows a sequence of frames on its own. |
| `qwen3.6`/`qwen3.8` | `frames` | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
| `gemma4` | `annotated_frames` | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action. Loses track of activity that repeats or reverses, so it benefits from annotated frames. |
| Model | Notes |
| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `qwen3-vl` | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object. |
| `qwen3.6`/`qwen3.8` | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
| `gemma4` | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action. |
#### Embedding models
@@ -77,17 +77,6 @@ The `embeddings` role needs a different kind of model. Text queries are matched
| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `qwen3-vl-embedding` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Must be served by llama.cpp started with `--embeddings` and `--mmproj`. |
#### Transcription models
The `transcribe` role needs a model that accepts audio input. A text-only or vision-only model cannot serve this role. The following are recommended for local deployment of the `transcribe` role:
| Model | Notes |
| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `qwen3-asr` | Dedicated speech recognition model covering 30 languages, and the better choice for transcription quality. It only transcribes, so it cannot be shared with the `descriptions` or `chat` roles. |
| `gemma4` | General multimodal model that accepts audio as well as images, so one served model can cover `transcribe` alongside the other roles. Transcript quality is below `qwen3-asr`, particularly on noisy audio. |
Both must be served by llama.cpp started with the matching audio `--mmproj`. llama.cpp only reports audio support when an audio projector is loaded. Without it Frigate sees the model as text-only and the `transcribe` role is unavailable in the UI. Frigate transcribes through the server's `/v1/audio/transcriptions` route, which llama.cpp serves for any audio-capable model.
:::info
Each model is available in multiple parameter sizes (3b, 4b, 8b, etc.). Larger sizes are more capable of complex tasks and understanding of situations, but requires more memory and computational resources. It is recommended to try multiple models and experiment to see which performs best.
@@ -192,43 +192,6 @@ review:
</TabItem>
</ConfigTabs>
### Frame Mode
Review items are sent to the model as a sequence of still frames. Some models follow that sequence well on their own; others lose track of activity that repeats or reverses, and describe a single trip when the subject actually made several. The `frame_mode` option controls how those frames are presented.
- `frames` (default): the prompt followed by the frames, exactly as earlier versions of Frigate sent them.
- `annotated_frames`: each frame is preceded by its frame number and elapsed time, along with notes describing what the object tracker recorded at that moment, such as an object being first detected, starting to move, turning around, stopping, or no longer being detected.
The notes come from tracking data rather than from the images, so they describe activity the model may not have picked up on its own. In testing with a person carrying three waste bins to the curb one at a time, `gemma4` described a single trip on every attempt with `frames`, and consistently described multiple trips with `annotated_frames`. Models that already handle these sequences well, such as the `qwen3-vl` family, gain little and should stay on `frames`.
Annotated mode also caps the number of frames, since the notes already establish the order of events and extra near-duplicate frames tend to crowd out the middle of a clip. Longer review items are sampled more sparsely as a result, and typically use fewer tokens than `frames` mode for the same item.
:::note
Annotated mode needs tracking data for the review item. If none is available, Frigate falls back to sending plain frames for that item.
:::
<ConfigTabs>
<TabItem value="ui">
Navigate to <NavPath path="Settings > Global configuration > Review" />.
- Set **GenAI config > Frame mode** to the desired mode (e.g., `annotated_frames`)
</TabItem>
<TabItem value="yaml">
```yaml {4}
review:
genai:
enabled: true
frame_mode: annotated_frames
```
</TabItem>
</ConfigTabs>
### Response Style
Different models respond to the built-in prompt with very different writing styles: some produce natural narration while others sound short and mechanical. The `response_style` option selects a writing style preset that rewords the prompt's instructions for the user-facing fields (the title, short summary, and scene description). Presets replace those instructions rather than adding extra ones, so the model never receives competing style directions.
@@ -43,10 +43,6 @@ Frigate supports presets for optimal hardware accelerated video decoding:
- [RKNN](#rockchip-platform): Frigate can utilize the media engine in RockChip SOCs to accelerate video decoding.
**Apple Silicon Mac** <CommunityBadge />
- [lighter](#apple-silicon-mac-lighter): Frigate can utilize the media engine in Apple Silicon Macs to accelerate video decoding, when running under the lighter container runtime.
**Other Hardware**
Depending on your system, these presets may not be compatible, and you may need to use manual hwaccel args to take advantage of your hardware. More information on hardware accelerated decoding for ffmpeg can be found here: https://trac.ffmpeg.org/wiki/HWAccelIntro
@@ -537,35 +533,3 @@ output_args:
Make sure that your SoC supports hardware acceleration for your input stream and your input stream is h264 encoding. For example, if your camera streams with h264 encoding, your SoC must be able to de- and encode with it. If you are unsure whether your SoC meets the requirements, take a look at the datasheet.
:::
## Apple Silicon Mac (lighter)
[lighter](https://github.com/fieldwork-ai/lighter) is an open-source container runtime for macOS. It gives a container the Mac's media engine as a standard V4L2 decoder, backed by VideoToolbox, so Frigate decodes H.264 and H.265 streams in hardware with the ffmpeg it already ships. It works on M1 and newer Macs with lighter 0.9.2 or newer.
Give the container the video device. With Docker Compose:
```yaml {4-5}
services:
frigate:
...
devices:
- lighter.sh/video=all
```
Or with `docker run`, add `--device lighter.sh/video=all`.
Then set the preset for the codec your cameras stream. The decoder is specific to the codec, so if your cameras mix H.264 and H.265, set the preset for the most common codec globally and override it on the other cameras:
```yaml
ffmpeg:
hwaccel_args: preset-apple-silicon-h264
cameras:
garage: # an H.265 camera
ffmpeg:
hwaccel_args: preset-apple-silicon-h265
```
The presets decode on the media engine and encode the Birdseye restream and timelapses there too. Scaling to the detect resolution runs on the CPU, as ffmpeg's V4L2 decoders cannot scale.
lighter can also run object detection on the Mac's Neural Engine; see [Apple Neural Engine (lighter)](object_detectors.md#apple-neural-engine-lighter).
-31
View File
@@ -28,24 +28,6 @@ WebRTC may use an external STUN server for NAT traversal. MSE and HLS streaming
:::
### Selecting a streaming technology
Frigate [defaults to MSE](#why-does-frigate-prefer-mse-over-webrtc-for-live-view) for restreamed cameras by design. To use WebRTC, select it explicitly from a camera's single-camera Live view settings (the settings menu in the camera's Live view header on desktop, or the settings drawer on mobile). Three related controls work together:
- **Stream**: _what_ to play. This lists the [streams you've configured](#setting-streams-for-live-ui) (for example `Main Stream` and `Sub Stream`).
- **Force low-bandwidth mode**: a switch that always plays Frigate's built-in low-bandwidth feed (the stream assigned the `detect` role, using JSMpeg) instead of the selected stream. It works anywhere without go2rtc and is useful on slow or metered connections. While it is enabled, the stream and streaming technology selectors are disabled; your stream and technology choices are restored when you turn it off.
- **Streaming Technology**: _how_ to play the selected stream, listing **MSE** and **WebRTC**. It is only shown for a restreamed stream.
- The choices are saved **per device, per camera** in your browser's local storage.
- **WebRTC is only selectable when it can actually work for that stream.** When it can't, the option is shown disabled with the reason inline, and a more detailed reason (the failing codecs, or why the connectivity check failed) is logged to your browser's console. Common reasons:
- **Not configured**: no `candidates` or `ice_servers` are set under `go2rtc.webrtc` (see [WebRTC extra configuration](#webrtc-extra-configuration)).
- **Could not connect**: e.g. port `8555` isn't reachable, or a STUN/TURN server is misconfigured. Frigate runs a one-time WebRTC connectivity check when the Live view opens; the option may briefly show as "checking" while it runs.
- **Unsupported video codec**: the stream's video codec can't be played over WebRTC in your browser, most commonly H.265/HEVC in Firefox or Edge.
- **Unsupported audio codec**: WebRTC needs opus or G.711 audio, so a stream whose playback audio is only AAC (without an added opus/G.711 track) can't carry audio over WebRTC. See [Audio Support](#audio-support) for how to add one.
- **Unsupported browser**: the browser doesn't support WebRTC.
When WebRTC isn't available, Frigate automatically uses MSE (or falls back to JSMpeg), so live view keeps working regardless of the selection.
### Camera Settings Recommendations
If you are using go2rtc, you should adjust the following settings in your camera's firmware for the best experience with Live view:
@@ -175,17 +157,6 @@ WebRTC works by creating a TCP or UDP connection on port `8555`. However, it req
- stun:8555
```
- The web UI uses the STUN and TURN servers in `ice_servers` and falls back to Google's public STUN server when none are set:
```yaml title="config.yml"
go2rtc:
webrtc:
ice_servers:
- urls: [turn:turn.example.com:3478]
username: frigate
credential: password
```
- For access through Tailscale, the Frigate system's Tailscale IP must be added as a WebRTC candidate. Tailscale IPs all start with `100.`, and are reserved within the `100.64.0.0/10` CIDR block.
- Note that some browsers may not support H.265 (HEVC). You can check your browser's current version for H.265 compatibility [here](https://github.com/AlexxIT/go2rtc?tab=readme-ov-file#codecs-madness).
@@ -235,8 +206,6 @@ For devices that support two way talk, Frigate can be configured to use the feat
- Ensure you access Frigate via https (may require [opening port 8971](/frigate/installation/#ports)).
- For the Home Assistant Frigate card, [follow the docs](http://card.camera/#/usage/2-way-audio) for the correct source.
The two-way talk control in the single-camera Live view is only enabled when WebRTC is available; if WebRTC isn't configured or can't connect, the control is shown disabled.
To use the Reolink Doorbell with two way talk, you should use the [recommended Reolink configuration](/configuration/camera_specific#reolink-cameras)
As a starting point to check compatibility for your camera, view the list of cameras supported for two-way talk on the [go2rtc repository](https://github.com/AlexxIT/go2rtc?tab=readme-ov-file#two-way-audio). For cameras in the category `ONVIF Profile T`, you can use the [ONVIF Conformant Products Database](https://www.onvif.org/conformant-products/)'s FeatureList to check for the presence of `AudioOutput`. A camera that supports `ONVIF Profile T` _usually_ supports this, but due to inconsistent support, a camera that explicitly lists this feature may still not work. If no entry for your camera exists on the database, it is recommended not to buy it or to consult with the manufacturer's support on the feature availability.
+1 -79
View File
@@ -24,7 +24,6 @@ Frigate supports multiple different detectors that work on different types of ha
- [Coral EdgeTPU](#edge-tpu-detector): The Google Coral EdgeTPU is available in USB, Mini PCIe, and m.2 formats allowing for a wide range of compatibility with devices.
- [Hailo](#hailo): The Hailo-8, Hailo-8L and Hailo-8R AI Acceleration modules are available in m.2 format with a HAT for RPi devices, offering a wide range of compatibility with devices.
- <CommunityBadge /> [MemryX](#memryx-mx3): The MX3 Acceleration module is available in m.2 format, offering broad compatibility across various platforms.
- <CommunityBadge /> [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, offering broad compatibility across various platforms.
**AMD**
@@ -34,7 +33,6 @@ Frigate supports multiple different detectors that work on different types of ha
**Apple Silicon**
- [Apple Silicon](#apple-silicon-detector): Apple Silicon can run on M1 and newer Apple Silicon devices.
- <CommunityBadge /> [ONNX](#apple-neural-engine-lighter): the ONNX detector runs on the Neural Engine of M1 and newer Macs when Frigate runs under the lighter container runtime.
**Intel**
@@ -485,7 +483,7 @@ See [ONNX supported models](#onnx) for supported models, there are some caveats:
## ONNX
ONNX is an open format for building machine learning models, Frigate supports running ONNX models on CPU, OpenVINO, ROCm, TensorRT, and a Mac's Neural Engine. On startup Frigate will automatically try to use a GPU if one is available.
ONNX is an open format for building machine learning models, Frigate supports running ONNX models on CPU, OpenVINO, ROCm, and TensorRT. On startup Frigate will automatically try to use a GPU if one is available.
:::info
@@ -501,9 +499,6 @@ If the correct build is used for your GPU then the GPU will be detected and used
- Nvidia GPUs will automatically be detected and used with the ONNX detector in the `-tensorrt` Frigate image.
- Jetson devices will automatically be detected and used with the ONNX detector in the `-tensorrt-jp6` Frigate image.
- **Apple Silicon Mac** <CommunityBadge />
- The Neural Engine will automatically be detected and used with the ONNX detector when Frigate runs under lighter with its Neural Engine device. See [Apple Neural Engine (lighter)](#apple-neural-engine-lighter).
:::
:::tip
@@ -519,22 +514,6 @@ models:
:::
### Apple Neural Engine (lighter) {#apple-neural-engine-lighter}
[lighter](https://github.com/fieldwork-ai/lighter) is an open-source container runtime for macOS. A container started with its `lighter.sh/ane` device gets an ONNX Runtime execution provider that runs models on the Mac's Neural Engine, and the ONNX detector uses it automatically, with the same models and configuration as on any other hardware. It works on M1 and newer Macs with lighter 0.9.2 or newer.
Give the Frigate container the Neural Engine device. With Docker Compose:
```yaml
services:
frigate:
image: ghcr.io/blakeblackshear/frigate:stable-standard-arm64
devices:
- lighter.sh/ane=all
```
Or with `docker run`, add `--device lighter.sh/ane=all`. Frigate then reports the Neural Engine under **Settings > System > Detection models**, and the ONNX detector's model loads on it. lighter can also decode camera streams on the Mac's media engine; see [Video Decoding](hardware_acceleration_video.md#apple-silicon-mac-lighter).
### Configuration {#configuration-onnx}
<ModelConfigDropdown detectorTitle="ONNX" models={objectDetectorsModels.onnx.models} />
@@ -633,63 +612,6 @@ For detailed instructions on compiling models, refer to the [MemryX Compiler](ht
---
## DEEPX NPU
This detector is available for use with the DEEPX NPU, both the DX-M1 M.2 module and the DX-M1M on the Sixfab AI HAT+ for the Raspberry Pi 5. The configuration below applies unchanged to either form factor. DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
See the [installation docs](../frigate/installation.md#deepx-npu) for information on installing the DEEPX kernel driver and runtime on the host and passing the NPU through to the container.
To run a model on a DEEPX NPU, list a `deepx` device on that model.
:::info
The DX-RT Python bindings are not part of the Frigate image. They are downloaded and installed into `/config/.local` the first time a DEEPX device is configured, verified against pinned checksums, and updated automatically when a Frigate release pins a new version. If the container has no internet access, see [Detector runtimes](/frigate/network_requirements#detector-runtimes) for how to provide the files yourself.
:::
### Configuration {#configuration-deepx}
<ModelConfigDropdown detectorTitle="DEEPX" models={objectDetectorsModels.deepx.models} />
Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format. Two model types are supported:
- `yolo-generic` for YOLO object detection models, the recommended default. The detector reads the model's output layout from the compiled file, so anchor-based, anchor-free and NMS-in-head models all work with the same configuration, as do models compiled with DEEPX's Post-Processing Unit (PPU) support.
- `yolox` for YOLOX models compiled without PPU support, whose raw head needs Frigate's YOLOX decoder. A YOLOX model compiled with PPU support works under either `yolox` or `yolo-generic`.
The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of YOLO object detection models. Download the `.dxnn`, bind-mount it into the container, and point the model's `path` at it. Alternatively, compile your own model with the DX-COM compiler. The recommended starting point is `yolox-s_640x640_ppu.dxnn`, the fastest ModelZoo model measured through Frigate:
```yaml
models:
- devices:
- deepx:PCIe:0
path: /config/model_cache/deepx/yolox-s_640x640_ppu.dxnn
labelmap_path: /labelmap/coco-80.txt
model_type: yolo-generic
width: 640
height: 640
```
For PPU models, use a `.dxnn` compiled with DX-COM 2.4.0 or later. Frigate reads the PPU head layout the compiler writes into the file and refuses to load a PPU model without it.
`model_type` must be set to `yolo-generic` or `yolox` to match the model; `yolo-generic` is the recommended default unless the model is a raw YOLOX export. Frigate defaults it to `ssd`, which this detector does not support, so the detector refuses to start on a model that leaves it unset.
`width` and `height` must match the resolution the model was compiled for. Quantization parameters are baked into the `.dxnn` file at compile time, so no normalization is applied on the host and Frigate's default `input_tensor`, `input_pixel_format`, and `input_dtype` values do not need to be overridden.
A DEEPX device is `PCIe:<index>`, as reported on the detector settings page. The NPU daemon multiplexes across processes, so the same device may be listed more than once to run additional inference processes against it:
```yaml
models:
- devices:
- deepx:PCIe:0
- deepx:PCIe:0
```
#### Label maps
The object detection models in the DEEPX ModelZoo are trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. Frigate's default label map uses an extended 91-class COCO scheme, and leaving it in place will cause detections to be reported as the wrong object type. For `yolo-generic` models the label map is also what the detector uses to tell the output layout, so a label map with the wrong number of classes is reported as an error at startup.
---
## NVidia TensorRT Detector
Nvidia Jetson devices may be used for object detection using the TensorRT libraries. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt-jp6` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt-jp6`. This detector is designed to work with Yolo models for object detection.
+1 -42
View File
@@ -65,11 +65,6 @@ Frigate supports multiple different detectors that work on different types of ha
- [Supports many model architectures](../../configuration/object_detectors#memryx-mx3)
- Runs best with tiny, small, or medium-size models
- <CommunityBadge /> [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, allowing for a wide range of compatibility with devices.
- [Supports YOLO model architectures](../../configuration/object_detectors#deepx-npu)
- Runs best with tiny or small size models
- Runs efficiently on low power hardware
**AMD**
- [ROCm](#rocm---amd-gpu): ROCm can run on AMD Discrete GPUs to provide efficient object detection
@@ -78,10 +73,6 @@ Frigate supports multiple different detectors that work on different types of ha
**Apple Silicon**
- [ONNX via lighter](#apple-silicon): The ONNX detector runs on the Neural Engine of M1 and newer Macs when Frigate runs in the lighter container runtime
- [Supports the same model architectures as the ONNX detector](../../configuration/object_detectors#apple-neural-engine-lighter)
- Runs inside the Frigate container, with no separate detector process to set up
- The recommended way to run Frigate on a Mac
- [Apple Silicon](#apple-silicon): Apple Silicon is usable on all M1 and newer Apple Silicon devices to provide efficient and fast object detection
- [Supports primarily ssdlite and mobilenet model architectures](../../configuration/object_detectors#apple-silicon-detector)
- Runs well with any size models including large
@@ -215,13 +206,7 @@ Inference is done with the `onnx` detector type. Speeds will vary greatly depend
### Apple Silicon
Frigate on a Mac is best run in the [lighter](https://github.com/fieldwork-ai/lighter) container runtime, where the [ONNX detector](../configuration/object_detectors.md#apple-neural-engine-lighter) runs on the Neural Engine of M1 and newer Macs from inside the Frigate container. There is no separate detector process to install or keep running, and the same container can decode video on the Mac's media engine.
| Name | YOLOv9 Inference Time | YOLO-NAS Inference Time | RF-DETR Inference Time |
| ---- | -------------------------------------- | ----------------------- | ---------------------- |
| M1 | t-320: 3.3 ms s-320: 7 ms s-640: 13 ms | 320: 6.6 ms | Nano-320: 38 ms |
Alternatively, with the [Apple Silicon](../configuration/object_detectors.md#apple-silicon-detector) detector Frigate can take advantage of the NPU in M1 and newer Apple Silicon.
With the [Apple Silicon](../configuration/object_detectors.md#apple-silicon-detector) detector Frigate can take advantage of the NPU in M1 and newer Apple Silicon.
:::warning
@@ -272,32 +257,6 @@ The MX3 is a pipelined architecture, where the maximum frames per second support
Inference speeds may vary depending on the host platform. The above data was measured on an **Intel 13700 CPU**. Platforms like Raspberry Pi, Orange Pi, and other ARM-based SBCs have different levels of processing capability, which may limit total FPS.
### DEEPX NPU
Frigate supports the DEEPX NPU in both of its form factors: the **DX-M1** M.2 module, which works on x86 (Intel/AMD) and ARM-based SBCs such as the Raspberry Pi 5, and the **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart) for the Raspberry Pi 5. Both use the same driver and runtime, so the configuration is identical for either one. DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
The DEEPX driver and runtime run on the Docker host rather than inside the Frigate container and must be installed before the NPU can be used. See the [installation docs](installation.md#deepx-npu) for the setup steps and [the detector docs](/configuration/object_detectors#deepx-npu) for the configuration.
Frigate does not bundle a model for this detector. Models use DEEPX's `.dxnn` format, and pre-compiled YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo). Prefer a model with a `_ppu` suffix whenever one is available for the architecture you want: these run part of the post-processing on the NPU itself and are considerably faster, roughly 2.5x for the same architecture and input size. **YOLOX-S with PPU is the recommended starting point.**
Inference times for a few recommended models, measured through Frigate's own stats on a DX-M1:
| Model | Input Size | DX-M1 Inference Time |
| ----------------- | ---------- | -------------------- |
| YOLOX-S (PPU) | 640 | ~ 13 ms |
| YOLOv9-t (PPU) | 640 | ~ 18 ms |
| YOLOv4 (PPU) | 512 | ~ 20 ms |
| YOLOX-S | 640 | ~ 34 ms |
| YOLOv9-s | 640 | ~ 39 ms |
Other ModelZoo YOLO variants are also supported but have not been measured. Inference speeds vary with the host platform, so a slower host such as a Raspberry Pi 5 will report higher times than those above.
:::note
A few ModelZoo models can not be used with Frigate: SSD models (they are trained on Pascal VOC, so their labels do not match Frigate's), DAMO-YOLO models, face and pose models, and the PPU builds of YOLOv7.
:::
### Nvidia Jetson
Jetson devices are supported via the TensorRT or ONNX detectors when running Jetpack 6. It will [make use of the Jetson's hardware media engine](/configuration/hardware_acceleration_video#nvidia-jetson) when configured with the [appropriate presets](/configuration/ffmpeg_presets#hwaccel-presets), and will make use of the Jetson's GPU and DLA for object detection when configured with the [TensorRT detector](/configuration/object_detectors#nvidia-tensorrt-detector).
-93
View File
@@ -381,99 +381,6 @@ If you can't use Docker Compose, you can run the container with something simila
Finally, configure [hardware object detection](/configuration/object_detectors#memryx-mx3) to complete the setup.
### DEEPX NPU
The DEEPX NPU is available in two form factors, and Frigate supports both:
- **DX-M1** in the M.2 2280 form factor (like an NVMe SSD), for x86 (Intel/AMD) PCs, the Raspberry Pi 5, and other ARM SBCs with an exposed PCIe M.2 slot.
- **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart), a HAT+ board that connects to the Raspberry Pi 5 over PCIe Gen 3 x1.
Both present the NPU through the same PCIe driver and DX-RT runtime, so the setup below and the detector configuration are identical for either one. Nothing needs to change when moving between them.
DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
#### Versions
A DEEPX install has several separately versioned pieces, and they all have to agree. The driver, the runtime, and the daemon live on the Docker host; Frigate itself carries only the Python bindings, which it downloads on first start:
| Component | Version | Installed on | Installed by |
| -------------- | -------- | ------------ | ------------------------- |
| Kernel driver | `v2.6.0` | Host | `user_installation.sh` |
| DX-RT runtime | `v3.4.0` | Host | `user_installation.sh` |
| NPU firmware | `v2.7.4` | The module | Flashed from the host |
| DX-RT bindings | `v3.4.0` | Frigate | Downloaded at first start |
:::warning
A version mismatch does not produce a startup error. It typically shows up as inference requests that are accepted but never return a result, so detections simply stop appearing while Frigate looks healthy. If that happens after a Frigate upgrade, check every version in the table before anything else.
:::
The installation script installs the DX-RT runtime on the host and enables `dxrt.service`, so the daemon starts at boot and any other program on the host can share the NPU with Frigate. Check the firmware version with `dxrt-cli --status` and update the module if it does not match the table above.
#### Installation
The DEEPX kernel driver must be installed on the host rather than in the container, because containers share the host kernel and cannot load kernel modules. Installing it creates the `/dev/dxrt*` device nodes that are passed through to Frigate. The same script installs the DX-RT runtime and enables `dxrt.service`, the daemon that owns the NPU and hands work to it on behalf of Frigate and anything else on the host.
1. Copy or download [this script](https://github.com/blakeblackshear/frigate/blob/dev/docker/deepx/user_installation.sh).
2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh`
3. Run the script with `./user_installation.sh`
4. **Restart your computer** to complete driver installation.
Confirm the NPU is visible before continuing:
```bash
ls /dev/dxrt*
```
Then confirm the daemon is running and listening in `/run/dxrt`:
```bash
systemctl is-active dxrt.service
ls /run/dxrt/
```
#### Setup
To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable`
#### Docker configuration
Frigate needs the NPU device node and the directory holding the daemon's socket:
```yaml
services:
frigate:
devices:
- /dev/dxrt0:/dev/dxrt0
volumes:
- /run/dxrt:/run/dxrt
```
If you can't use Docker Compose, add `--device /dev/dxrt0:/dev/dxrt0 -v /run/dxrt:/run/dxrt` to your `docker run` command.
Add one `--device` per NPU, contiguously from `/dev/dxrt0`, since the client stops enumerating at the first gap.
The installation script configures `dxrt.service` to place its socket in `/run/dxrt` through a systemd drop-in. Mounting the directory rather than the socket file means the container sees the new socket after `dxrt.service` is restarted, rather than holding on to a deleted one.
`dxrtd` listens on an abstract socket as well, but that one does not cross into a container, so Frigate names the filesystem socket through `DXRT_DYNAMIC_IPC_ENDPOINT` on your behalf. Set that variable on the container yourself only if the daemon listens somewhere else, which means you also set it for `dxrtd` through its own systemd drop-in. The script writes `/etc/systemd/system/dxrt.service.d/frigate.conf` for exactly that, and has `dxrt.service` link the socket to `/tmp/dxrt_dynamic_ipc.sock` when it starts, so the host's own `dxrt-cli` and `dxtop` keep finding it at the default path they fall back to.
:::note
The DX-RT client exits when `dxrt.service` stops, so restart the Frigate container after restarting `dxrt.service`.
:::
The device node is needed as well as the socket, because the client opens the NPU directly even though the daemon arbitrates access. Without it, inference fails with `Device not found`.
`/dev/shm` does not need sharing.
The DX-RT python bindings are not shipped in the Frigate image. Frigate downloads them on first start when a DEEPX detector is configured, and caches them under `/config`.
#### Configuration
Finally, configure [hardware object detection](/configuration/object_detectors#deepx-npu) to complete the setup.
### Rockchip platform
Make sure that you use a linux distribution that comes with the rockchip BSP kernel 5.10 or 6.1 and necessary drivers (especially rkvdec2 and rknpu). To check, enter the following commands:
+1 -1
View File
@@ -147,7 +147,7 @@ If an [MQTT broker](/integrations/mqtt) is configured, Frigate maintains a conne
For [WebRTC live streaming](/configuration/live), Frigate uses STUN for NAT traversal:
- **go2rtc** defaults to a local STUN listener (`stun:8555`), no internet required.
- **The web UI** uses the servers in `go2rtc.webrtc.ice_servers` for its WebRTC player and for the WebRTC connectivity check it runs when the Live view loads. If none are set, it uses Google's public STUN server (`stun:stun.l.google.com:19302`), which requires internet access from the browser. Set `ice_servers` to a STUN or TURN server on your network to avoid this.
- **The web UI's WebRTC player** includes a fallback to Google's public STUN server (`stun:stun.l.google.com:19302`), which requires internet.
## Home Assistant Supervisor
+1 -1
View File
@@ -184,6 +184,6 @@ Filters and masks only hide the incorrect result - they don't teach Frigate what
### Where do I see problems Frigate has detected?
Open System > Health. The Notices list keeps a record of problems Frigate has found. Acknowledge an entry to hide it until the problem happens again, or mute it to hide it for good. Hidden entries stay listed under Show hidden in the filter. Ongoing conditions, such as an offline camera or recordings deleted before their retention period, appear in the status bar for admins until they clear, and the status bar links to the Notices list while it has entries showing. On mobile, tap the warning icon in the bottom navigation bar to see them.
Open System > Health. The Notices list keeps a record of problems Frigate has found, and you can dismiss any entry to acknowledge it. Ongoing conditions, such as an offline camera or recordings deleted before their retention period, appear in the status bar for admins until they clear, and the status bar links to the Notices list while it has undismissed entries. On mobile, tap the warning icon in the bottom navigation bar to see them.
The Hardware section below the notices shows whether the detection hardware, hardware acceleration, and enrichment devices in your config were found and are being used, so a GPU that silently fell back to the CPU shows up as a warning. Run stream checks to probe every camera's streams for the same problems the camera wizard reports.
+4 -4
View File
@@ -11375,15 +11375,15 @@
}
},
"node_modules/image-size": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/image-size/-/image-size-2.0.4.tgz",
"integrity": "sha512-QRUkFFsRV/6fuESxb9Vkq+a0LkSrgKXuc2NEqfikiXxxN/G3tjWt5EVUlMaImRBZRZK/jRBEbYvpPYZL8t08Zw==",
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/image-size/-/image-size-2.0.2.tgz",
"integrity": "sha512-IRqXKlaXwgSMAMtpNzZa1ZAe8m+Sa1770Dhk8VkSsP9LS+iHD62Zd8FQKs8fbPiagBE7BzoFX23cxFnwshpV6w==",
"license": "MIT",
"bin": {
"image-size": "bin/image-size.js"
},
"engines": {
"node": ">=18"
"node": ">=16.x"
}
},
"node_modules/immer": {
+16 -82
View File
@@ -4174,20 +4174,19 @@ paths:
Get notices, most severe first.
Args:
include_hidden: Also return acknowledged and muted notices, for the
hidden list
include_dismissed: Also return dismissed notices, for the history view
Returns:
The notices
operationId: get_notices_notices_get
parameters:
- name: include_hidden
- name: include_dismissed
in: query
required: false
schema:
type: boolean
default: false
title: Include Hidden
title: Include Dismissed
responses:
'200':
description: Successful Response
@@ -4222,16 +4221,16 @@ paths:
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/muted_checks:
/notices/dismissed_checks:
get:
tags:
- Notices
summary: Get Muted Checks
summary: Get Dismissed Checks
description: |-
**Access:** Admin role required.
Get the muted config and stream check rows, newest first.
operationId: get_muted_checks_notices_muted_checks_get
Get the dismissed config and stream check rows, newest first.
operationId: get_dismissed_checks_notices_dismissed_checks_get
responses:
'200':
description: Successful Response
@@ -4241,16 +4240,16 @@ paths:
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/hidden:
/notices/dismissed:
delete:
tags:
- Notices
summary: Unhide All Notices
summary: Purge Dismissed
description: |-
**Access:** Admin role required.
Show every acknowledged and muted notice and check row again.
operationId: unhide_all_notices_notices_hidden_delete
Delete every dismissed notice and check row so each can show again.
operationId: purge_dismissed_notices_dismissed_delete
responses:
'200':
description: Successful Response
@@ -4260,83 +4259,18 @@ paths:
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/{notice_id}/acknowledge:
/notices/{notice_id}/dismiss:
post:
tags:
- Notices
summary: Acknowledge Notice
summary: Dismiss Notice
description: |-
**Access:** Admin role required.
Hide a notice until it happens again.
Hide a notice or a config or stream check row.
Config and stream check rows and the update notice never repeat, so they
can only be muted.
operationId: acknowledge_notice_notices__notice_id__acknowledge_post
parameters:
- name: notice_id
in: path
required: true
schema:
type: string
title: Notice Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/{notice_id}/mute:
post:
tags:
- Notices
summary: Mute Notice
description: |-
**Access:** Admin role required.
Hide a notice or a config or stream check row for good.
operationId: mute_notice_notices__notice_id__mute_post
parameters:
- name: notice_id
in: path
required: true
schema:
type: string
title: Notice Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/{notice_id}/hidden:
delete:
tags:
- Notices
summary: Unhide Notice
description: |-
**Access:** Admin role required.
Show an acknowledged or muted notice or check row again.
operationId: unhide_notice_notices__notice_id__hidden_delete
It stays hidden if the same problem happens again.
operationId: dismiss_notice_notices__notice_id__dismiss_post
parameters:
- name: notice_id
in: path
-4
View File
@@ -99,10 +99,6 @@ def main() -> None:
print("*** End Config Validation Errors ***")
print("*************************************************************")
# force a non-zero exit code for config failures
if args.validate_config:
sys.exit(1)
# attempt to start Frigate in recovery mode
try:
config = FrigateConfig.load(install=True, safe_load=True)
-2
View File
@@ -429,8 +429,6 @@ def ffmpeg_presets():
hwaccel_presets = [
"preset-rpi-64-h264",
"preset-rpi-64-h265",
"preset-apple-silicon-h264",
"preset-apple-silicon-h265",
"preset-jetson-h264",
"preset-jetson-h265",
"preset-rkmpp",
+61 -82
View File
@@ -258,15 +258,15 @@ async def latest_frame(
frame = request.app.camera_error_image
height = int(params.height or str(frame.shape[0]))
width = int(height * frame.shape[1] / frame.shape[0])
if frame is None:
return JSONResponse(
content={"success": False, "message": "Unable to get valid frame"},
status_code=500,
)
height = int(params.height or str(frame.shape[0]))
width = int(height * frame.shape[1] / frame.shape[0])
if height < 1 or width < 1:
return JSONResponse(
content="Invalid height / width requested :: {} / {}".format(
@@ -886,8 +886,9 @@ async def vod_event(
# If the recordings are not found and the event started more than 5 minutes ago, set has_clip to false
if (
event.start_time < datetime.now().timestamp() - 300
and isinstance(vod_response, JSONResponse)
and vod_response.status_code == 404
and type(vod_response) is tuple
and len(vod_response) == 2
and vod_response[1] == 404
):
Event.update(has_clip=False).where(Event.id == event_id).execute()
@@ -955,80 +956,64 @@ async def event_snapshot(
event_complete = False
jpg_bytes = None
frame_time = 0
try:
event = Event.get(Event.id == event_id, Event.end_time != None)
await require_camera_access(event.camera, request=request)
except DoesNotExist:
event = None
if event is not None:
event_complete = True
await require_camera_access(event.camera, request=request)
if not event.has_snapshot:
return JSONResponse(
content={"success": False, "message": "Snapshot not available"},
status_code=404,
)
snapshot_settings = _resolve_snapshot_settings(
request.app.frigate_config.cameras[event.camera].snapshots, params
)
jpg_bytes, frame_time = get_event_snapshot_bytes(
event,
ext="jpg",
timestamp=snapshot_settings["timestamp"],
bounding_box=snapshot_settings["bounding_box"],
crop=snapshot_settings["crop"],
height=snapshot_settings["height"],
quality=snapshot_settings["quality"],
timestamp_style=request.app.frigate_config.cameras[
event.camera
].timestamp_style,
colormap=request.app.frigate_config.model_for_camera(event.camera).colormap,
)
except DoesNotExist:
# see if the object is currently being tracked
try:
snapshot_settings = _resolve_snapshot_settings(
request.app.frigate_config.cameras[event.camera].snapshots, params
)
jpg_bytes, frame_time = get_event_snapshot_bytes(
event,
ext="jpg",
timestamp=snapshot_settings["timestamp"],
bounding_box=snapshot_settings["bounding_box"],
crop=snapshot_settings["crop"],
height=snapshot_settings["height"],
quality=snapshot_settings["quality"],
timestamp_style=request.app.frigate_config.cameras[
event.camera
].timestamp_style,
colormap=request.app.frigate_config.model_for_camera(
event.camera
).colormap,
camera_states: list[CameraState] = (
request.app.detected_frames_processor.get_camera_states()
)
for camera_state in camera_states:
if event_id in camera_state.tracked_objects:
tracked_obj = camera_state.tracked_objects.get(event_id)
if tracked_obj is not None:
snapshot_settings = _resolve_snapshot_settings(
camera_state.camera_config.snapshots, params
)
jpg_bytes, frame_time = tracked_obj.get_img_bytes(
ext="jpg",
timestamp=snapshot_settings["timestamp"],
bounding_box=snapshot_settings["bounding_box"],
crop=snapshot_settings["crop"],
height=snapshot_settings["height"],
quality=snapshot_settings["quality"],
)
await require_camera_access(camera_state.name, request=request)
except Exception:
return JSONResponse(
content={"success": False, "message": "Unknown error occurred"},
content={"success": False, "message": "Ongoing event not found"},
status_code=404,
)
else:
# see if the object is currently being tracked
camera_states: list[CameraState] = (
request.app.detected_frames_processor.get_camera_states()
except Exception:
return JSONResponse(
content={"success": False, "message": "Unknown error occurred"},
status_code=404,
)
for camera_state in camera_states:
tracked_obj = camera_state.tracked_objects.get(event_id)
if tracked_obj is None:
continue
await require_camera_access(camera_state.name, request=request)
try:
snapshot_settings = _resolve_snapshot_settings(
camera_state.camera_config.snapshots, params
)
jpg_bytes, frame_time = tracked_obj.get_img_bytes(
ext="jpg",
timestamp=snapshot_settings["timestamp"],
bounding_box=snapshot_settings["bounding_box"],
crop=snapshot_settings["crop"],
height=snapshot_settings["height"],
quality=snapshot_settings["quality"],
)
except Exception:
return JSONResponse(
content={"success": False, "message": "Ongoing event not found"},
status_code=404,
)
break
if jpg_bytes is None:
return JSONResponse(
content={"success": False, "message": "Live frame not available"},
@@ -1077,25 +1062,19 @@ async def event_thumbnail(
if not thumbnail_bytes:
# see if the object is currently being tracked
camera_states = request.app.detected_frames_processor.get_camera_states()
for camera_state in camera_states:
tracked_obj = camera_state.tracked_objects.get(event_id)
if tracked_obj is None:
continue
await require_camera_access(camera_state.name, request=request)
try:
thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
except Exception:
return JSONResponse(
content={"success": False, "message": "Event not found"},
status_code=404,
)
break
try:
camera_states = request.app.detected_frames_processor.get_camera_states()
for camera_state in camera_states:
if event_id in camera_state.tracked_objects:
tracked_obj = camera_state.tracked_objects.get(event_id)
if tracked_obj is not None:
await require_camera_access(camera_state.name, request=request)
thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
except Exception:
return JSONResponse(
content={"success": False, "message": "Event not found"},
status_code=404,
)
if not thumbnail_bytes:
return JSONResponse(
+22 -50
View File
@@ -14,18 +14,17 @@ router = APIRouter(tags=[Tags.notices])
@router.get("/notices", dependencies=[Depends(require_role(["admin"]))])
def get_notices(request: Request, include_hidden: bool = False) -> JSONResponse:
def get_notices(request: Request, include_dismissed: bool = False) -> JSONResponse:
"""Get notices, most severe first.
Args:
include_hidden: Also return acknowledged and muted notices, for the
hidden list
include_dismissed: Also return dismissed notices, for the history view
Returns:
The notices
"""
return JSONResponse(
content=request.app.notice_registry.active(include_hidden=include_hidden)
content=request.app.notice_registry.active(include_dismissed=include_dismissed)
)
@@ -35,64 +34,37 @@ def get_notice_stats(request: Request) -> JSONResponse:
return JSONResponse(content=request.app.notice_registry.stats())
@router.get("/notices/muted_checks", dependencies=[Depends(require_role(["admin"]))])
def get_muted_checks(request: Request) -> JSONResponse:
"""Get the muted config and stream check rows, newest first."""
return JSONResponse(content=request.app.notice_registry.muted_checks())
@router.get(
"/notices/dismissed_checks", dependencies=[Depends(require_role(["admin"]))]
)
def get_dismissed_checks(request: Request) -> JSONResponse:
"""Get the dismissed config and stream check rows, newest first."""
return JSONResponse(content=request.app.notice_registry.dismissed_checks())
@router.delete("/notices/hidden", dependencies=[Depends(require_role(["admin"]))])
def unhide_all_notices(request: Request) -> JSONResponse:
"""Show every acknowledged and muted notice and check row again."""
request.app.notice_registry.unhide_all()
return JSONResponse(content={"success": True, "message": "Notices shown again"})
@router.delete("/notices/dismissed", dependencies=[Depends(require_role(["admin"]))])
def purge_dismissed(request: Request) -> JSONResponse:
"""Delete every dismissed notice and check row so each can show again."""
request.app.notice_registry.purge_dismissed()
return JSONResponse(
content={"success": True, "message": "Dismissed notices cleared"}
)
# model notice ids contain a slash, so the id is a path parameter
@router.post(
"/notices/{notice_id:path}/acknowledge",
"/notices/{notice_id:path}/dismiss",
dependencies=[Depends(require_role(["admin"]))],
)
def acknowledge_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide a notice until it happens again.
def dismiss_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide a notice or a config or stream check row.
Config and stream check rows and the update notice never repeat, so they
can only be muted.
It stays hidden if the same problem happens again.
"""
if not request.app.notice_registry.acknowledge(notice_id):
if not request.app.notice_registry.dismiss(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
return JSONResponse(content={"success": True, "message": "Notice acknowledged"})
@router.post(
"/notices/{notice_id:path}/mute",
dependencies=[Depends(require_role(["admin"]))],
)
def mute_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide a notice or a config or stream check row for good."""
if not request.app.notice_registry.mute(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
return JSONResponse(content={"success": True, "message": "Notice muted"})
@router.delete(
"/notices/{notice_id:path}/hidden",
dependencies=[Depends(require_role(["admin"]))],
)
def unhide_notice(request: Request, notice_id: str) -> JSONResponse:
"""Show an acknowledged or muted notice or check row again."""
if not request.app.notice_registry.unhide(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
return JSONResponse(content={"success": True, "message": "Notice shown again"})
return JSONResponse(content={"success": True, "message": "Notice dismissed"})
+2 -12
View File
@@ -483,7 +483,7 @@ class FrigateApp:
def start_audio_processor(self) -> None:
self.audio_process = AudioProcessor(
self.config, self.camera_metrics, self.embeddings_metrics, self.stop_event
self.config, self.camera_metrics, self.stop_event
)
self.audio_process.start()
self.processes["audio_detector"] = self.audio_process.pid or 0
@@ -556,20 +556,10 @@ class FrigateApp:
"output",
lambda: OutputProcess(self.config, self.stop_event),
),
(
"audio_process",
"audio_detector",
lambda: AudioProcessor(
self.config,
self.camera_metrics,
self.embeddings_metrics,
self.stop_event,
),
),
]
for attr, key, factory in specs:
if getattr(self, attr, None) is None:
if not hasattr(self, attr):
continue
def on_restart(
+3 -28
View File
@@ -1,7 +1,7 @@
from enum import Enum
from typing import Any, Self
from typing import Any
from pydantic import Field, model_validator
from pydantic import Field
from ..base import FrigateBaseModel
from ..env import EnvString
@@ -21,17 +21,6 @@ class GenAIRoleEnum(str, Enum):
chat = "chat"
descriptions = "descriptions"
embeddings = "embeddings"
transcribe = "transcribe"
# Providers that can accept audio input for the transcribe role. Ollama has no
# audio input support, so claiming the role there would fail at request time.
TRANSCRIBE_CAPABLE_PROVIDERS = {
GenAIProviderEnum.openai,
GenAIProviderEnum.azure_openai,
GenAIProviderEnum.gemini,
GenAIProviderEnum.llamacpp,
}
class GenAIConfig(FrigateBaseModel):
@@ -63,7 +52,7 @@ class GenAIConfig(FrigateBaseModel):
GenAIRoleEnum.chat,
],
title="Roles",
description="GenAI roles (chat, descriptions, embeddings, transcribe); one provider per role. Only chat, descriptions, and embeddings are granted by default; transcribe must be listed explicitly.",
description="GenAI roles (chat, descriptions, embeddings); one provider per role.",
)
provider_options: dict[str, Any] = Field(
default={},
@@ -77,17 +66,3 @@ class GenAIConfig(FrigateBaseModel):
description="Runtime options passed to the provider for each inference call.",
json_schema_extra={"additionalProperties": {}},
)
@model_validator(mode="after")
def validate_transcribe_provider(self) -> Self:
"""Reject the transcribe role on providers that cannot accept audio input."""
if (
GenAIRoleEnum.transcribe in self.roles
and self.provider not in TRANSCRIBE_CAPABLE_PROVIDERS
):
raise ValueError(
f"GenAI provider '{self.provider.value}' does not support audio input "
"and cannot be given the 'transcribe' role."
)
return self
-13
View File
@@ -9,7 +9,6 @@ __all__ = [
"DetectionsConfig",
"AlertsConfig",
"ImageSourceEnum",
"ReviewFrameModeEnum",
"ReviewResponseStyleEnum",
]
@@ -21,13 +20,6 @@ class ImageSourceEnum(str, Enum):
recordings = "recordings"
class ReviewFrameModeEnum(str, Enum):
"""How review frames are presented to the GenAI provider."""
frames = "frames"
annotated_frames = "annotated_frames"
class ReviewResponseStyleEnum(str, Enum):
"""Writing style presets for GenAI review descriptions."""
@@ -161,11 +153,6 @@ class GenAIReviewConfig(FrigateBaseModel):
description="Preferred language to request from the GenAI provider for generated responses.",
default=None,
)
frame_mode: ReviewFrameModeEnum = Field(
default=ReviewFrameModeEnum.frames,
title="Frame mode",
description="How frames are presented to the model. 'frames' sends the prompt followed by the frames, which suits models that track a sequence well on their own. 'annotated_frames' labels each frame and interleaves notes derived from object tracking, which helps models that lose track of activity that repeats or reverses.",
)
response_style: ReviewResponseStyleEnum = Field(
default=ReviewResponseStyleEnum.default,
title="Response style",
+2 -32
View File
@@ -5,7 +5,6 @@ from pydantic import ConfigDict, Field, field_validator
from .base import FrigateBaseModel
__all__ = [
"AudioTranscriptionModelEnum",
"CameraFaceRecognitionConfig",
"CameraLicensePlateRecognitionConfig",
"CameraAudioTranscriptionConfig",
@@ -21,10 +20,6 @@ class SemanticSearchModelEnum(str, Enum):
jinav2 = "jinav2"
class AudioTranscriptionModelEnum(str, Enum):
whisper = "whisper"
class EnrichmentsDeviceEnum(str, Enum):
GPU = "GPU"
CPU = "CPU"
@@ -58,35 +53,10 @@ class AudioTranscriptionConfig(FrigateBaseModel):
description="Enable or disable automatic audio transcription for all cameras; can be overridden per-camera.",
)
language: str = Field(
default="auto",
default="en",
title="Transcription language",
description="Language code used for transcription/translation (for example 'en' for English), or 'auto' to let the model detect it. See https://whisper-api.com/docs/languages/ for supported language codes.",
description="Language code used for transcription/translation (for example 'en' for English). See https://whisper-api.com/docs/languages/ for supported language codes.",
)
model: AudioTranscriptionModelEnum | str | None = Field(
default=AudioTranscriptionModelEnum.whisper,
title="Audio transcription model or GenAI provider name",
description="The transcription backend: 'whisper' for Frigate's built-in local models, or the name of a GenAI provider with the transcribe role.",
)
@field_validator("model", mode="before")
@classmethod
def coerce_model_enum(cls, v):
# An absent value ("model:" with nothing after it, or an explicit null)
# means unspecified, so fall back to the built-in backend. Left as None
# it would pass the GenAI-provider validation, which only inspects
# strings, and then be treated as a provider name that resolves to no
# client, turning transcription into a silent no-op.
if v is None or (isinstance(v, str) and not v.strip()):
return AudioTranscriptionModelEnum.whisper
if isinstance(v, str):
try:
return AudioTranscriptionModelEnum(v)
except ValueError:
return v
return v
device: EnrichmentsDeviceEnum = Field(
default=EnrichmentsDeviceEnum.CPU,
title="Transcription device",
+1 -31
View File
@@ -56,7 +56,6 @@ from .camera.timestamp import TimestampStyleConfig
from .camera_group import CameraGroupConfig
from .classification import (
AudioTranscriptionConfig,
AudioTranscriptionModelEnum,
ClassificationConfig,
FaceRecognitionConfig,
LicensePlateRecognitionConfig,
@@ -885,7 +884,7 @@ class FrigateConfig(FrigateBaseModel):
# set notifications state
self.notifications.enabled_in_config = self.notifications.enabled
# validate genai: each role (chat, descriptions, embeddings, transcribe) at most once
# validate genai: each role (chat, descriptions, embeddings) at most once
role_to_name: dict[GenAIRoleEnum, str] = {}
for name, genai_cfg in self.genai.items():
for role in genai_cfg.roles:
@@ -1246,35 +1245,6 @@ class FrigateConfig(FrigateBaseModel):
for model in self.models:
model.create_colormap(colored_labels)
# validate audio_transcription.model when it is a GenAI provider name.
# this runs here rather than beside the semantic_search check because the
# global->camera merge above is what resolves camera-level enablement.
transcription_active = self.audio_transcription.enabled or any(
camera.audio_transcription.enabled for camera in self.cameras.values()
)
if (
transcription_active
and isinstance(self.audio_transcription.model, str)
and not isinstance(
self.audio_transcription.model, AudioTranscriptionModelEnum
)
):
if self.audio_transcription.model not in self.genai:
raise ValueError(
f"audio_transcription.model '{self.audio_transcription.model}' is not a "
"valid GenAI config key. Must match a key in genai config."
)
if (
GenAIRoleEnum.transcribe
not in self.genai[self.audio_transcription.model].roles
):
raise ValueError(
f"GenAI provider '{self.audio_transcription.model}' must have "
"'transcribe' in its roles for audio transcription."
)
# Check audio transcription and audio detection requirements
if self.audio_transcription.enabled:
# If audio transcription is enabled globally, at least one camera must have audio detection enabled
@@ -10,7 +10,6 @@ from peewee import DoesNotExist
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.config.classification import AudioTranscriptionModelEnum
from frigate.const import (
CACHE_DIR,
MODEL_CACHE_DIR,
@@ -19,13 +18,8 @@ from frigate.const import (
)
from frigate.data_processing.types import PostProcessDataEnum
from frigate.embeddings.embeddings import Embeddings
from frigate.genai.manager import GenAIClientManager
from frigate.types import TrackedObjectUpdateTypesEnum
from frigate.util.audio import (
clean_transcript,
get_audio_from_recording,
resolve_language,
)
from frigate.util.audio import get_audio_from_recording
from ..types import DataProcessorMetrics
from .api import PostProcessorApi
@@ -40,25 +34,15 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
requestor: InterProcessRequestor,
embeddings: Embeddings,
metrics: DataProcessorMetrics,
genai_manager: GenAIClientManager | None = None,
):
super().__init__(config, metrics, None)
self.config = config
self.requestor = requestor
self.embeddings = embeddings
self.genai_manager = genai_manager
self.recognizer = None
self.transcription_lock = threading.Lock()
self.transcription_thread: threading.Thread | None = None
self.transcription_running = False
self._use_genai = not isinstance(
config.audio_transcription.model, AudioTranscriptionModelEnum
)
if self._use_genai:
# never build the local recognizer on the GenAI path; WhisperModel
# downloads several hundred MB on first use
return
# faster-whisper handles model downloading automatically
self.model_path = os.path.join(MODEL_CACHE_DIR, "whisper")
@@ -163,31 +147,6 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
logger.error(f"Error in audio transcription post-processing: {e}")
def __transcribe_audio(self, audio_data: bytes) -> str | None:
"""Transcribe WAV audio data with the configured backend."""
if self._use_genai:
return self.__transcribe_audio_genai(audio_data)
return self.__transcribe_audio_whisper(audio_data)
def __transcribe_audio_genai(self, audio_data: bytes) -> str | None:
"""Hand the WAV bytes to the GenAI provider holding the transcribe role."""
client = self.genai_manager.transcribe_client if self.genai_manager else None
if not client:
logger.error(
"audio_transcription.model is '%s' (GenAI provider) but no transcribe "
"client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
self.config.audio_transcription.model,
)
return None
text = client.transcribe(
audio_data,
language=resolve_language(self.config.audio_transcription.language),
)
return clean_transcript(text) or None
def __transcribe_audio_whisper(self, audio_data: bytes) -> str | None:
"""Transcribe WAV audio data using faster-whisper."""
if not self.recognizer:
logger.debug("Recognizer not initialized")
@@ -201,7 +160,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
segments, info = self.recognizer.transcribe(
temp_wav,
language=resolve_language(self.config.audio_transcription.language),
language=self.config.audio_transcription.language,
beam_size=5,
)
@@ -1,394 +0,0 @@
"""Frame annotations derived from object tracking data.
Builds short notes describing what changed during a review item, keyed to the
frames sampled from it. Everything here comes from tracked object data already
in the database (each event's `path_data` trajectory and the timeline's
stationary/active changes), so the notes can be stated to the model as fact
rather than as something it must perceive.
"""
import logging
import math
from collections.abc import Sequence
from typing import Any
from frigate.models import Event, Timeline
logger = logging.getLogger(__name__)
# Movement smaller than this (normalized frame units) between two path points
# is treated as the object holding still rather than travelling.
STILL_THRESHOLD = 0.02
# A heading change beyond this (dot product against the leg's own heading)
# counts as the object turning back rather than curving.
REVERSAL_DOT = -0.3
# A run of travel shorter than this (normalized frame units) is treated as
# milling about rather than going somewhere. Without it, a subject pacing in
# one spot produces a burst of contradictory "turns around" notes on a single
# frame.
MIN_LEG_DISTANCE = 0.08
# Movement that begins within this many seconds of detection is folded into
# the detection note, so each arrival reads as one event instead of several.
DETECT_MOVE_MERGE_SECONDS = 2.0
STATE_CHANGE_PHRASES = {
"stationary": "has stopped moving",
"active": "starts moving again",
}
Point = tuple[float, float, float]
Leg = tuple[int, int]
def describe_position(x: float, y: float) -> str:
"""Name a normalized frame position in plain terms."""
horizontal = "left" if x < 0.34 else ("right" if x > 0.66 else "center")
vertical = "top" if y < 0.34 else ("bottom" if y > 0.66 else "middle")
if horizontal == "center" and vertical == "middle":
return "the middle of the frame"
if horizontal == "center":
return f"the {vertical} of the frame"
if vertical == "middle":
return f"the {horizontal} of the frame"
return f"the {vertical} {horizontal} of the frame"
def describe_heading(dx: float, dy: float) -> str:
"""Name a direction of travel in frame terms.
y grows downward in normalized coordinates, so a falling y reads as moving
toward the top of the frame.
"""
parts = []
if abs(dy) > abs(dx) * 0.4:
parts.append("down" if dy > 0 else "up")
if abs(dx) > abs(dy) * 0.4:
parts.append("right" if dx > 0 else "left")
return " and ".join(parts) if parts else "in place"
def event_name(event: dict[str, Any]) -> str:
"""Name an object for the notes, e.g. 'a person' or 'waste bin "Compost"'.
Objects are never numbered or given track identifiers. Frigate opens a new
tracked object whenever a subject is re-detected, so the tracking data
cannot say whether two entries are the same subject, and the notes stay
ambiguous rather than implying either answer.
"""
label = str(event["label"]).replace("_", " ").replace("-verified", "")
sub_label = event.get("sub_label")
if sub_label:
return f'{label} "{sub_label}"'
article = "an" if label[:1].lower() in "aeiou" else "a"
return f"{article} {label}"
def path_legs(points: list[Point]) -> list[Leg]:
"""Split a trajectory into runs of travel in a consistent direction.
A leg ends when the subject starts moving back against the direction that
leg established, and only once the leg has covered MIN_LEG_DISTANCE, so
jitter around a standing subject does not register as a turn.
Returns (start, end) index pairs into `points`.
"""
legs: list[Leg] = []
start = 0
for i in range(1, len(points)):
lx = points[i][0] - points[start][0]
ly = points[i][1] - points[start][1]
leg_distance = (lx * lx + ly * ly) ** 0.5
if leg_distance < MIN_LEG_DISTANCE:
continue
sx = points[i][0] - points[i - 1][0]
sy = points[i][1] - points[i - 1][1]
step = (sx * sx + sy * sy) ** 0.5
if step < STILL_THRESHOLD:
continue
dot = (lx / leg_distance) * (sx / step) + (ly / leg_distance) * (sy / step)
if dot < REVERSAL_DOT:
legs.append((start, i - 1))
start = i - 1
if start < len(points) - 1:
legs.append((start, len(points) - 1))
return [
(a, b)
for a, b in legs
if ((points[b][0] - points[a][0]) ** 2 + (points[b][1] - points[a][1]) ** 2)
** 0.5
>= MIN_LEG_DISTANCE
]
def path_points(path_data: list[Any]) -> list[Point]:
"""Flatten path_data into (x, y, timestamp) tuples, or [] if malformed."""
try:
return [(p[0][0], p[0][1], p[1]) for p in path_data or []]
except (IndexError, TypeError):
logger.debug("Malformed path_data, skipping trajectory notes")
return []
def leg_start_time(points: list[Point], leg: Leg) -> float:
"""When a leg's movement actually began.
path_data always keeps an object's first two samples, so a leg can open
with points recorded long before the object moved. The first sample that
has left the leg's origin is the earliest evidence of movement.
"""
a, b = leg
x0, y0, t0 = points[a]
for x, y, t in points[a + 1 : b + 1]:
if math.hypot(x - x0, y - y0) >= STILL_THRESHOLD:
return t
return t0
def leg_heading(points: list[Point], leg: Leg) -> str:
a, b = leg
return describe_heading(points[b][0] - points[a][0], points[b][1] - points[a][1])
def leg_phrase(points: list[Point], leg: Leg, first: bool) -> str:
"""Describe the start of a leg, e.g. 'turns around at ... and heads left'."""
heading = leg_heading(points, leg)
place = describe_position(points[leg[0]][0], points[leg[0]][1])
if first:
return f"starts moving {heading} from {place}"
return f"turns around at {place} and heads {heading}"
def path_moments(path_data: list[Any]) -> list[tuple[float, str]]:
"""Key moments in one trajectory as (timestamp, phrase).
Emits one note per leg of travel. Where the last leg ends is left out:
path_data only records significant movement, so its final point cannot
distinguish an object coming to rest from one leaving the frame.
"""
points = path_points(path_data)
if len(points) < 2:
return []
return [
(leg_start_time(points, leg), leg_phrase(points, leg, index == 0))
for index, leg in enumerate(path_legs(points))
]
def build_timeline(
events: list[dict[str, Any]],
span_end: float,
state_changes: Sequence[dict[str, Any]] = (),
) -> list[tuple[float, str]]:
"""All annotated moments across every event, in time order.
Only changes are noted, since those are what sparse frames miss; an
object's state at the end of the clip is visible in the last frame.
`span_end` is the timestamp of the last sampled frame, and moments past it
describe nothing the model can see. A track ending means the object
stopped being detected, which may or may not mean it left the frame.
`state_changes` are timeline rows (timestamp, source_id, class_type); the
stationary and active ones become "has stopped moving" / "starts moving
again". Frigate only marks an object stationary after it has been still
for a while, which the past-tense wording reflects.
Each object keeps its own notes. Folding an object into the note of the
person moving it ("alongside ...") was tried and made models lose track of
where the object went.
"""
changes_by_event: dict[str, list[tuple[float, str]]] = {}
for change in state_changes:
phrase = STATE_CHANGE_PHRASES.get(change["class_type"])
if phrase:
changes_by_event.setdefault(change["source_id"], []).append(
(change["timestamp"], phrase)
)
timeline: list[tuple[float, str]] = []
for event in sorted(events, key=lambda e: e["start_time"]):
# Frame extraction can come up short at the end of a clip, leaving
# objects that only appear after the last frame we actually have.
if event["start_time"] > span_end:
continue
points = path_points(event.get("path_data") or [])
legs = path_legs(points) if len(points) >= 2 else []
name = event_name(event)
detected_at = event["start_time"]
where = describe_position(points[0][0], points[0][1]) if points else "the frame"
merges = bool(legs) and leg_start_time(points, legs[0]) - detected_at <= (
DETECT_MOVE_MERGE_SECONDS
)
remaining = list(enumerate(legs))
moments: list[tuple[float, str]] = []
if merges:
heading = leg_heading(points, legs[0])
timeline.append(
(detected_at, f"{name} first detected at {where}, moving {heading}")
)
remaining = remaining[1:]
else:
timeline.append((detected_at, f"{name} first detected at {where}"))
for index, leg in remaining:
moments.append(
(
leg_start_time(points, leg),
leg_phrase(points, leg, index == 0),
)
)
moments.extend(
(timestamp, phrase)
for timestamp, phrase in changes_by_event.get(event["id"], [])
if timestamp >= detected_at
)
for timestamp, phrase in moments:
if timestamp <= span_end:
timeline.append((timestamp, f"{name} {phrase}"))
if event["end_time"] and event["end_time"] <= span_end:
timeline.append((event["end_time"], f"{name} is no longer detected"))
return sorted(timeline, key=lambda m: m[0])
def annotations_by_frame(
timeline: list[tuple[float, str]], frame_times: list[float]
) -> dict[int, list[str]]:
"""Bucket timeline moments onto the frame that follows each one.
A moment is attached to the first frame at or after it happened, so the
note always precedes the image in which the change becomes visible.
"""
buckets: dict[int, list[str]] = {}
if not frame_times:
return buckets
for timestamp, phrase in timeline:
index = next(
(i for i, ft in enumerate(frame_times) if ft >= timestamp),
len(frame_times) - 1,
)
buckets.setdefault(index, []).append(phrase)
return buckets
def get_tracked_events(detection_ids: list[str]) -> list[dict[str, Any]]:
"""Load the tracked objects behind a review item's detections."""
if not detection_ids:
return []
rows = list(
Event.select(
Event.id,
Event.label,
Event.sub_label,
Event.start_time,
Event.end_time,
Event.data,
)
.where(Event.id << detection_ids)
.dicts()
.iterator()
)
return [
{
"id": row["id"],
"label": row["label"],
"sub_label": row["sub_label"],
"start_time": row["start_time"],
"end_time": row["end_time"],
"path_data": (row["data"] or {}).get("path_data") or [],
}
for row in rows
if row["start_time"] is not None
]
def get_state_changes(detection_ids: list[str]) -> list[dict[str, Any]]:
"""Stationary/active changes the timeline recorded for these objects."""
if not detection_ids:
return []
return list(
Timeline.select(Timeline.timestamp, Timeline.source_id, Timeline.class_type)
.where(
(Timeline.source_id << detection_ids)
& (Timeline.class_type << list(STATE_CHANGE_PHRASES))
)
.dicts()
.iterator()
)
def build_frame_captions(
detection_ids: list[str],
frame_times: list[float],
) -> list[str]:
"""A caption for each sampled frame, in frame order.
Every frame gets its index and elapsed time so the model can tell them
apart; frames where something changed also carry the tracker notes for
that moment. Returns an empty list when there is nothing to say, which
callers treat as a reason to fall back to sending plain frames.
"""
if not frame_times:
return []
events = get_tracked_events(detection_ids)
if not events:
logger.debug("No tracked events found for review item, skipping annotations")
return []
timeline = build_timeline(events, frame_times[-1], get_state_changes(detection_ids))
buckets = annotations_by_frame(timeline, frame_times)
if not buckets:
return []
total = len(frame_times)
origin = frame_times[0]
captions: list[str] = []
for index, timestamp in enumerate(frame_times):
lines = [f"Frame {index + 1} of {total} (+{timestamp - origin:.1f}s):"]
lines.extend(f"[tracker] {note}" for note in buckets.get(index, []))
captions.append("\n".join(lines))
return captions
@@ -19,11 +19,7 @@ from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.config.camera import CameraConfig
from frigate.config.camera.review import (
GenAIReviewConfig,
ImageSourceEnum,
ReviewFrameModeEnum,
)
from frigate.config.camera.review import GenAIReviewConfig, ImageSourceEnum
from frigate.const import (
ATTRIBUTE_LABEL_DISPLAY_MAP,
CACHE_DIR,
@@ -40,7 +36,6 @@ from frigate.util.image import get_image_from_recording
from ..post.api import PostProcessorApi
from ..types import DataProcessorMetrics
from .review_annotations import build_frame_captions
logger = logging.getLogger(__name__)
@@ -48,7 +43,6 @@ RECORDING_BUFFER_EXTENSION_PERCENT = 0.10
MIN_RECORDING_DURATION = 10
MAX_IMAGE_TOKENS = 24000
MAX_FRAMES_PER_SECOND = 1
MAX_ANNOTATED_FRAMES = 28
class ReviewDescriptionProcessor(PostProcessorApi):
@@ -73,7 +67,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
duration: float,
image_source: ImageSourceEnum = ImageSourceEnum.preview,
height: int = 480,
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
) -> int:
"""Calculate optimal number of frames based on event duration, context size,
image source, and resolution.
@@ -87,8 +80,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
- MAX_FRAMES_PER_SECOND x duration, to avoid drowning short events in
near-duplicate frames where the model latches onto the redundant middle
and skips the start/end action
- MAX_ANNOTATED_FRAMES in annotated mode, where the tracking notes
already carry the sequence
"""
client = self.genai_manager.description_client
@@ -134,10 +125,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
max_frames_by_tokens = int(image_token_budget / tokens_per_image)
max_frames_by_duration = int(duration * MAX_FRAMES_PER_SECOND)
max_frames = min(max_frames_by_tokens, max_frames_by_duration)
if frame_mode == ReviewFrameModeEnum.annotated_frames:
max_frames = min(max_frames, MAX_ANNOTATED_FRAMES)
return max(max_frames, 3)
def process_data(
@@ -179,7 +166,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
return
image_source = camera_config.review.genai.image_source
frame_mode = camera_config.review.genai.frame_mode
if image_source == ImageSourceEnum.recordings:
buffer_extension = get_recording_buffer_extension(
@@ -188,43 +174,40 @@ class ReviewDescriptionProcessor(PostProcessorApi):
final_data["start_time"] -= buffer_extension
final_data["end_time"] += buffer_extension
frames = self.get_recording_frames(
thumbs = self.get_recording_frames(
camera,
final_data["start_time"],
final_data["end_time"],
height=480, # Use 480p for good balance between quality and token usage
frame_mode=frame_mode,
)
if not frames:
if not thumbs:
# Fallback to preview frames if no recordings available
logger.warning(
f"No recording frames found for {camera}, falling back to preview frames"
)
frames = self.get_preview_frames_as_bytes(
thumbs = self.get_preview_frames_as_bytes(
camera,
final_data["start_time"],
final_data["end_time"],
final_data["thumb_path"],
id,
camera_config.review.genai.debug_save_thumbnails,
frame_mode,
)
elif camera_config.review.genai.debug_save_thumbnails:
self.save_debug_recording_frames(id, frames)
self.save_debug_recording_frames(id, thumbs)
else:
# Use preview frames
frames = self.get_preview_frames_as_bytes(
thumbs = self.get_preview_frames_as_bytes(
camera,
final_data["start_time"],
final_data["end_time"],
final_data["thumb_path"],
id,
camera_config.review.genai.debug_save_thumbnails,
frame_mode,
)
self.start_analysis(camera_config, final_data, frames)
self.start_analysis(camera_config, final_data, thumbs)
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
if topic == EmbeddingsRequestEnum.regenerate_review_description.value:
@@ -403,50 +386,31 @@ class ReviewDescriptionProcessor(PostProcessorApi):
buffer_extension = get_recording_buffer_extension(
final_data["end_time"] - final_data["start_time"]
)
frames = self.get_recording_frames(
thumbs = self.get_recording_frames(
str(review.camera),
final_data["start_time"] - buffer_extension,
final_data["end_time"] + buffer_extension,
height=480,
frame_mode=camera_config.review.genai.frame_mode,
)
if not frames:
if not thumbs:
logger.error(
"No recording frames are available for review item %s", review_id
)
return
if camera_config.review.genai.debug_save_thumbnails:
self.save_debug_recording_frames(review_id, frames)
self.save_debug_recording_frames(review_id, thumbs)
self.start_analysis(camera_config, final_data, frames)
self.start_analysis(camera_config, final_data, thumbs)
def start_analysis(
self,
camera_config: CameraConfig,
final_data: dict[str, Any],
frames: list[tuple[bytes, float]],
thumbs: list[bytes],
) -> None:
"""Kick off description generation for a review item in the background."""
thumbs = [frame for frame, _ in frames]
captions: list[str] = []
if (
camera_config.review.genai.frame_mode
== ReviewFrameModeEnum.annotated_frames
):
captions = build_frame_captions(
final_data["data"].get("detections") or [],
[timestamp for _, timestamp in frames],
)
if not captions:
logger.debug(
"No tracking annotations for review item %s, sending plain frames",
final_data["id"],
)
self.review_desc_dps.update()
threading.Thread(
target=run_analysis,
@@ -457,22 +421,19 @@ class ReviewDescriptionProcessor(PostProcessorApi):
camera_config,
final_data,
thumbs,
captions,
camera_config.review.genai,
sorted(self.config.all_labels),
self.config.all_attributes,
),
).start()
def save_debug_recording_frames(
self, review_id: str, frames: list[tuple[bytes, float]]
) -> None:
def save_debug_recording_frames(self, review_id: str, thumbs: list[bytes]) -> None:
"""Write the recording frames sent to the provider out for debugging."""
Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
parents=True, exist_ok=True
)
for idx, (frame_bytes, _) in enumerate(frames):
for idx, frame_bytes in enumerate(thumbs):
with open(
os.path.join(CLIPS_DIR, f"genai-requests/{review_id}/{idx}.jpg"),
"wb",
@@ -484,9 +445,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
camera: str,
start_time: float,
end_time: float,
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
) -> list[tuple[str, float]]:
"""Preview frame paths paired with the time each one was captured."""
) -> list[str]:
preview_dir = os.path.join(CACHE_DIR, "preview_frames")
file_start = f"preview_{camera}-"
start_file = f"{file_start}{start_time}.webp"
@@ -518,29 +477,18 @@ class ReviewDescriptionProcessor(PostProcessorApi):
frame_count = len(all_frames)
desired_frame_count = self.calculate_frame_count(
camera,
duration=end_time - start_time,
frame_mode=frame_mode,
camera, duration=end_time - start_time
)
def with_timestamp(path: str) -> tuple[str, float]:
# Preview frames are named preview_<camera>-<timestamp>.webp
stem = os.path.basename(path).removesuffix(".webp")
try:
return (path, float(stem.removeprefix(file_start)))
except ValueError:
return (path, start_time)
if frame_count <= desired_frame_count:
return [with_timestamp(f) for f in all_frames]
return all_frames
selected_frames = []
step_size = (frame_count - 1) / (desired_frame_count - 1)
for i in range(desired_frame_count):
index = round(i * step_size)
selected_frames.append(with_timestamp(all_frames[index]))
selected_frames.append(all_frames[index])
return selected_frames
@@ -550,12 +498,11 @@ class ReviewDescriptionProcessor(PostProcessorApi):
start_time: float,
end_time: float,
height: int = 480,
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
) -> list[tuple[bytes, float]]:
"""Get frames from recordings paired with the time each was captured."""
) -> list[bytes]:
"""Get frames from recordings at specified timestamps."""
duration = end_time - start_time
desired_frame_count = self.calculate_frame_count(
camera, duration, ImageSourceEnum.recordings, height, frame_mode
camera, duration, ImageSourceEnum.recordings, height
)
# Calculate evenly spaced timestamps throughout the duration
@@ -593,7 +540,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
except DoesNotExist:
return None
frames: list[tuple[bytes, float]] = []
frames = []
for timestamp in timestamps:
try:
@@ -606,7 +553,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
image_data = extract_frame_from_recording(rounded_timestamp)
if image_data:
frames.append((image_data, timestamp))
frames.append(image_data)
else:
logger.warning(
f"No recording found for {camera} at timestamp {timestamp}"
@@ -627,8 +574,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
thumb_path_fallback: str,
review_id: str,
save_debug: bool,
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
) -> list[tuple[bytes, float]]:
) -> list[bytes]:
"""Get preview frames and convert them to JPEG bytes.
Args:
@@ -640,14 +586,14 @@ class ReviewDescriptionProcessor(PostProcessorApi):
save_debug: Whether to save debug thumbnails
Returns:
List of (JPEG image bytes, capture timestamp) pairs
List of JPEG image bytes
"""
frame_paths = self.get_cache_frames(camera, start_time, end_time, frame_mode)
frame_paths = self.get_cache_frames(camera, start_time, end_time)
if not frame_paths:
frame_paths = [(thumb_path_fallback, start_time)]
frame_paths = [thumb_path_fallback]
thumbs: list[tuple[bytes, float]] = []
for idx, (thumb_path, timestamp) in enumerate(frame_paths):
thumbs = []
for idx, thumb_path in enumerate(frame_paths):
thumb_data = cv2.imread(thumb_path)
if thumb_data is None:
@@ -660,7 +606,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100]
)
if ret:
thumbs.append((jpg.tobytes(), timestamp))
thumbs.append(jpg.tobytes())
if save_debug:
Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
@@ -695,7 +641,6 @@ def run_analysis(
camera_config: CameraConfig,
final_data: dict[str, Any],
thumbs: list[bytes],
frame_captions: list[str],
genai_config: GenAIReviewConfig,
labelmap_objects: list[str],
attribute_labels: list[str],
@@ -752,7 +697,6 @@ def run_analysis(
genai_config.debug_save_thumbnails,
genai_config.activity_context_prompt,
genai_config.response_style,
frame_captions,
)
review_inference_speed.update(datetime.datetime.now().timestamp() - start)
@@ -1,19 +1,16 @@
"""Handle processing audio for speech transcription using sherpa-onnx with FFmpeg pipe."""
import collections
import logging
import os
import queue
import threading
import time
from typing import TYPE_CHECKING, Any
from typing import Any
import numpy as np
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import CameraConfig, FrigateConfig
from frigate.config.classification import AudioTranscriptionModelEnum
from frigate.const import AUDIO_DURATION, MODEL_CACHE_DIR
from frigate.const import MODEL_CACHE_DIR
from frigate.data_processing.common.audio_transcription.model import (
AudioTranscriptionModelRunner,
)
@@ -21,38 +18,12 @@ from frigate.data_processing.real_time.whisper_online import (
FasterWhisperASR,
OnlineASRProcessor,
)
from frigate.util.audio import (
clean_transcript,
pcm16_to_wav,
resolve_language,
stitch_transcripts,
)
from ..types import DataProcessorMetrics
from .api import RealTimeProcessorApi
if TYPE_CHECKING:
# importing frigate.genai eagerly would pull the provider SDKs into the
# audio process even when transcription runs on a local model
from frigate.genai.manager import GenAIClientManager
logger = logging.getLogger(__name__)
# Number of ~0.975s audio detector chunks per GenAI request. The window advances
# one chunk at a time, so two chunks means a 50% overlap: every word lands whole
# in at least one window, which whisper-family models need to avoid hallucinating
# on a clipped clip. The cadence is fixed by the audio detector's frame size, so
# this is a constant rather than a config knob.
GENAI_WINDOW_CHUNKS = 2
# Bound the queue at ~30s of audio so a slow or hung provider cannot grow it
# without limit. The producer is the ffmpeg read thread and must never block.
AUDIO_QUEUE_MAXSIZE = int(30 / AUDIO_DURATION)
# A backed-up queue drops a chunk per cycle, so warning on each one would spam
# the log once a second per camera for as long as the provider stays slow.
AUDIO_DROP_WARN_INTERVAL = 10.0
class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
def __init__(
@@ -60,10 +31,9 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
config: FrigateConfig,
camera_config: CameraConfig,
requestor: InterProcessRequestor,
model_runner: AudioTranscriptionModelRunner | None,
model_runner: AudioTranscriptionModelRunner,
metrics: DataProcessorMetrics,
stop_event: threading.Event,
genai_manager: "GenAIClientManager | None" = None,
):
super().__init__(config, metrics)
self.config = config
@@ -72,31 +42,11 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
self.stream: Any = None
self.whisper_model: FasterWhisperASR | None = None
self.model_runner = model_runner
self.genai_manager = genai_manager
self.transcription_segments: list[str] = []
self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue(
maxsize=AUDIO_QUEUE_MAXSIZE
)
self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue()
self.stop_event = stop_event
self._use_genai = not isinstance(
config.audio_transcription.model, AudioTranscriptionModelEnum
)
# sliding window of raw int16 chunks; the deque's maxlen is what evicts
# the oldest chunk and so produces the overlap
self._genai_window: collections.deque[np.ndarray] = collections.deque(
maxlen=GENAI_WINDOW_CHUNKS
)
self._genai_committed = ""
# set by the producer when it discards a chunk, so the consumer knows the
# audio it is about to receive is not contiguous with what it buffered
self._audio_dropped = threading.Event()
self._last_drop_warning = 0.0
def __build_recognizer(self) -> None:
if self._use_genai:
# nothing local to load; never import sherpa or FasterWhisperASR
return
try:
if self.config.audio_transcription.model_size == "large":
# Whisper models need to be per-process and can only run one stream at a time
@@ -114,7 +64,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
self.stream = OnlineASRProcessor(
asr=self.whisper_model,
)
elif self.model_runner is not None:
else:
logger.debug(f"Loading sherpa stream for {self.camera_config.name}")
self.stream = self.model_runner.model.create_stream()
logger.debug(
@@ -126,15 +76,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
)
def __process_audio_stream(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
# must precede both the model_runner guard (model_runner is None on this
# path) and the float32 normalization below (GenAI wants untouched int16)
if self._use_genai:
return self.__process_audio_genai(audio_data)
if self.model_runner is None:
logger.debug("Audio transcription (live) model runner not initialized")
return None
if (
self.model_runner.model is None
and self.config.audio_transcription.model_size == "small"
@@ -199,82 +140,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
logger.error(f"Error processing audio stream: {e}")
return None
def __process_audio_genai(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
"""Transcribe a sliding overlapped window through the GenAI provider."""
client = self.genai_manager.transcribe_client if self.genai_manager else None
if not client:
logger.error(
"audio_transcription.model is '%s' (GenAI provider) but no transcribe "
"client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
self.config.audio_transcription.model,
)
return None
if self._audio_dropped.is_set():
self._audio_dropped.clear()
# Chunks were discarded between what is buffered and this one, so
# concatenating them would splice non-adjacent audio into one window
# and destroy the overlap the stitcher depends on.
self._genai_window.clear()
if self._genai_committed:
# the transcript has a gap in it; close the utterance out rather
# than stitching across missing speech
return self.__end_genai_utterance()
self._genai_window.append(audio_data)
if len(self._genai_window) < GENAI_WINDOW_CHUNKS:
# wait for a full window so the first request is never a clipped clip
return None
window = np.concatenate(list(self._genai_window))
# Silence gate, using the same threshold audio detection uses. Gate the
# whole window rather than individual chunks; this is the primary cost
# and privacy brake and is what keeps a quiet camera near zero requests.
window_as_float = window.astype(np.float32)
rms = float(np.sqrt(np.mean(np.absolute(np.square(window_as_float)))))
if rms < self.camera_config.audio.min_volume:
logger.debug(
f"Window RMS {rms:.1f} below min_volume, skipping transcription"
)
return self.__end_genai_utterance()
text = client.transcribe(
pcm16_to_wav(window),
language=resolve_language(self.config.audio_transcription.language),
)
# cleaning has to come first: a silent window often comes back as the
# model's preamble alone, which is silence, not a word to commit
cleaned = clean_transcript(text)
if not cleaned:
return self.__end_genai_utterance()
self._genai_committed = stitch_transcripts(self._genai_committed, cleaned)
# no VAD on this path, so mirror the whisper branch's heuristic endpoint
is_endpoint = (
self._genai_committed.endswith((".", "!", "?"))
and len(self._genai_committed) > 300
)
logger.debug(f"GenAI transcription: '{self._genai_committed}'")
return self._genai_committed, is_endpoint
def __end_genai_utterance(self) -> tuple[str, bool] | None:
"""Close out the current utterance when a window carries no speech."""
if not self._genai_committed:
return None
return self._genai_committed, True
def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
pass
@@ -283,38 +148,8 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
logger.debug("No audio data provided for transcription")
return None
# enqueue audio data for processing in the thread. never block: the
# producer is the ffmpeg read thread that audio detection depends on,
# so on a backlog drop the oldest chunk instead.
try:
self.audio_queue.put_nowait((obj_data, audio))
except queue.Full:
try:
self.audio_queue.get_nowait()
self.audio_queue.task_done()
except queue.Empty:
pass
# the stream now has a hole in it, which the consumer has to know
# about before it splices the next chunk onto what it already holds
self._audio_dropped.set()
now = time.monotonic()
if now - self._last_drop_warning >= AUDIO_DROP_WARN_INTERVAL:
self._last_drop_warning = now
logger.warning(
"Audio transcription queue for %s is full, dropping audio. The "
"provider is not keeping up with the %.2fs chunk rate",
self.camera_config.name,
AUDIO_DURATION,
)
try:
self.audio_queue.put_nowait((obj_data, audio))
except queue.Full:
pass
# enqueue audio data for processing in the thread
self.audio_queue.put((obj_data, audio))
return None
def run(self) -> None:
@@ -370,14 +205,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
break
def reset(self) -> None:
if self._use_genai:
self._genai_committed = ""
# stale audio carried across an utterance boundary would be
# re-transcribed into the next one
self._genai_window.clear()
logger.debug("Stream reset")
return
if self.config.audio_transcription.model_size == "large":
# get final output from whisper
output = self.stream.finish()
@@ -391,7 +218,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
# reset whisper
self.stream.init()
self.transcription_segments = []
elif self.model_runner is not None:
else:
# reset sherpa
self.model_runner.model.reset(self.stream)
@@ -399,24 +226,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
def check_unload_model(self) -> None:
# regularly called in the loop in audio maintainer
if self._use_genai:
# no model to unload, but this is the hook that fires when
# live_enabled flips off. guard on emptiness: called ~1x/s per camera.
if self._genai_committed or self._genai_window:
logger.debug(
f"Clearing GenAI transcription state for {self.camera_config.name}"
)
self.clear_audio_queue()
self._genai_committed = ""
self._genai_window.clear()
self.requestor.send_data(
f"{self.camera_config.name}/audio/transcription",
"",
)
return
if (
self.config.audio_transcription.model_size == "large"
and self.whisper_model is not None
@@ -461,10 +270,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
self, topic: str, request_data: dict[str, Any]
) -> dict[str, Any] | None:
if topic == "clear_audio_recognizer":
if self._use_genai:
self.reset()
return {"message": "Audio transcription state cleared", "success": True}
self.stream = None
self.__build_recognizer()
return {"message": "Audio recognizer cleared and rebuilt", "success": True}
-51
View File
@@ -46,42 +46,8 @@ _PROVIDER_LABELS = {
"MIGraphXExecutionProvider": "MIGraphX",
"OpenVINOExecutionProvider": "OpenVINO",
"CPUExecutionProvider": "CPU",
"LighterANE": "Neural Engine",
}
# lighter (https://github.com/fieldwork-ai/lighter) places an ONNX Runtime plugin
# execution provider in a container started with --device lighter.sh/ane=all,
# which runs models on a Mac's Neural Engine; LIGHTER_ANE_EP names where it is
LIGHTER_ANE_EP_NAME = "LighterANE"
LIGHTER_ANE_LIBRARY = "/usr/lib/lighter/liblighter_ane_ep.so"
def get_lighter_ane_devices() -> list[Any]:
"""Get the Neural Engine devices lighter's provider offers, registering it once.
Returns:
The provider's ONNX Runtime devices, or an empty list without lighter's device
"""
library = os.environ.get("LIGHTER_ANE_EP", LIGHTER_ANE_LIBRARY)
if not os.path.exists(library):
return []
devices = [d for d in ort.get_ep_devices() if d.ep_name == LIGHTER_ANE_EP_NAME]
if not devices:
try:
ort.register_execution_provider_library(LIGHTER_ANE_EP_NAME, library)
except Exception as e:
logger.warning(
f"Failed to load the Neural Engine provider from {library}: {e}"
)
return []
devices = [d for d in ort.get_ep_devices() if d.ep_name == LIGHTER_ANE_EP_NAME]
return devices
def is_arm64_platform() -> bool:
"""Check if we're running on an ARM platform."""
@@ -723,23 +689,6 @@ def get_optimized_runner(
if rknn_path:
return _record_runner(model_path, model_type, RKNNModelRunner(rknn_path))
if device != "CPU" and (ane_devices := get_lighter_ane_devices()):
sess_options = get_ort_session_options(model_type) or ort.SessionOptions()
sess_options.add_provider_for_devices(ane_devices, {})
try:
session = ort.InferenceSession(model_path, sess_options=sess_options)
except Exception as e:
logger.warning(
f"Failed to load {model_path} on the Neural Engine, using the default providers: {e}"
)
else:
return _record_runner(
model_path,
model_type,
ONNXModelRunner(session, model_type=model_type),
)
providers, options = get_ort_providers(device == "CPU", device, **kwargs)
if providers[0] == "CPUExecutionProvider":
-26
View File
@@ -25,7 +25,6 @@ SYS_ROOT = "/sys"
DEV_ROOT = "/dev"
PROC_ROOT = "/proc"
ETC_ROOT = "/etc"
LIB_ROOT = "/usr/lib"
# a Coral reports as Global Unichip until its firmware is loaded, then as Google
CORAL_USB_IDS = {("1a6e", "089a"), ("18d1", "9302")}
@@ -274,16 +273,6 @@ def detect_memryx() -> DetectionHardware | None:
return _hardware("memryx", "memryx", "MemryX MX3", units)
def detect_deepx() -> DetectionHardware | None:
"""Find DEEPX NPUs by their device nodes."""
units = _dev_units("dxrt*", "deepx:PCIe:{index}", "PCIe")
if not units:
return None
return _hardware("deepx", "deepx", "DEEPX NPU", units)
def detect_rockchip() -> DetectionHardware | None:
"""Find a Rockchip NPU by reading the SoC from the device tree."""
compatible = _read(f"{PROC_ROOT}/device-tree/compatible")
@@ -318,19 +307,6 @@ def detect_synaptics() -> DetectionHardware | None:
return _hardware("synaptics", "synaptics", "Synaptics NPU", units)
def detect_lighter_ane() -> DetectionHardware | None:
"""Find a Mac's Neural Engine by the provider library lighter's device places."""
library = os.environ.get(
"LIGHTER_ANE_EP", f"{LIB_ROOT}/lighter/liblighter_ane_ep.so"
)
if not os.path.exists(library):
return None
# runs through onnx, whose session picks lighter's provider when it is present
units = [HardwareUnit(device="onnx", label="Neural Engine")]
return _hardware("onnx:lighter", "onnx", "Apple Neural Engine", units)
def detect_cpu() -> DetectionHardware:
"""The CPU, which is always available."""
units = [HardwareUnit(device="cpu", label="CPU")]
@@ -343,7 +319,6 @@ PROBES = (
detect_coral_usb,
detect_hailo,
detect_memryx,
detect_deepx,
detect_intel_npu,
detect_intel_gpu,
detect_nvidia_gpu,
@@ -352,7 +327,6 @@ PROBES = (
detect_rockchip,
detect_axengine,
detect_synaptics,
detect_lighter_ane,
detect_cpu,
)
File diff suppressed because it is too large Load Diff
+1 -2
View File
@@ -1,7 +1,7 @@
import json
import logging
import os
from typing import Any, ClassVar, Literal
from typing import Any, Literal
import numpy as np
import zmq
@@ -21,7 +21,6 @@ class ZmqDetectorConfig(BaseDetectorConfig):
model_config = ConfigDict(
title="ZMQ IPC",
)
device_spec_field: ClassVar[str] = "endpoint"
type: Literal[DETECTOR_KEY]
endpoint: str = Field(
+3 -7
View File
@@ -68,7 +68,7 @@ from frigate.events.types import (
RegenerateDescriptionEnum,
)
from frigate.genai import GenAIClientManager
from frigate.models import Event, Recordings, ReviewSegment, Timeline, Trigger
from frigate.models import Event, Recordings, ReviewSegment, Trigger
from frigate.types import TrackedObjectUpdateTypesEnum
from frigate.util.builtin import serialize
from frigate.util.file import get_event_thumbnail_bytes
@@ -139,7 +139,7 @@ class EmbeddingMaintainer(threading.Thread):
),
load_vec_extension=True,
)
models = [Event, Recordings, ReviewSegment, Timeline, Trigger]
models = [Event, Recordings, ReviewSegment, Trigger]
db.bind(models)
self.genai_manager = GenAIClientManager(config)
@@ -251,11 +251,7 @@ class EmbeddingMaintainer(threading.Thread):
):
self.post_processors.append(
AudioTranscriptionPostProcessor(
self.config,
self.requestor,
self.embeddings,
metrics,
self.genai_manager,
self.config, self.requestor, self.embeddings, metrics
)
)
+38 -71
View File
@@ -11,7 +11,6 @@ from typing import Any
import numpy as np
from frigate.camera import CameraMetrics
from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import CameraConfig, CameraInput, FrigateConfig
@@ -20,7 +19,6 @@ from frigate.config.camera.updater import (
CameraConfigUpdateEnum,
CameraConfigUpdateSubscriber,
)
from frigate.config.classification import AudioTranscriptionModelEnum
from frigate.const import (
AUDIO_DURATION,
AUDIO_FORMAT,
@@ -37,7 +35,6 @@ from frigate.data_processing.common.audio_transcription.model import (
from frigate.data_processing.real_time.audio_transcription import (
AudioTranscriptionRealTimeProcessor,
)
from frigate.data_processing.types import DataProcessorMetrics
from frigate.ffmpeg_presets import parse_preset_input
from frigate.log import LogPipe, suppress_stderr_during
from frigate.util.builtin import get_ffmpeg_arg_list, load_labels
@@ -88,7 +85,6 @@ class AudioProcessor(FrigateProcess):
self,
config: FrigateConfig,
camera_metrics: DictProxy,
embeddings_metrics: DataProcessorMetrics,
stop_event: MpEvent,
):
super().__init__(
@@ -96,38 +92,8 @@ class AudioProcessor(FrigateProcess):
)
self.camera_metrics = camera_metrics
self.embeddings_metrics = embeddings_metrics
self.config = config
def spawn_if_needed(self, camera: CameraConfig) -> None:
"""Start an audio maintainer for the camera once everything it needs
has arrived. Returning early leaves the camera for the next poll."""
name = camera.name
if name is None or name in self.audio_threads:
return
if not camera.enabled or not camera.audio.enabled:
return
# ffmpeg update may not have arrived yet
if not any("audio" in i.roles for i in camera.ffmpeg.inputs):
return
# the camera maintainer creates metrics on its own poll of the same
# add update and may not have gotten there yet
metrics = self.camera_metrics.get(name)
if metrics is None:
return
thread = AudioEventMaintainer(
camera,
self.config,
metrics,
self.embeddings_metrics,
self.transcription_model_runner,
self.stop_event, # type: ignore[arg-type]
self.genai_manager,
)
self.audio_threads[name] = thread
thread.start()
self.logger.info(f"Audio maintainer started for {name}")
def __stop_audio_thread(self, camera: str) -> None:
thread = self.audio_threads.pop(camera, None)
if thread is None:
@@ -146,31 +112,18 @@ class AudioProcessor(FrigateProcess):
threading.current_thread().name = "process:audio_manager"
self.transcription_model_runner: AudioTranscriptionModelRunner | None = None
self.genai_manager: Any = None
if any(
c.enabled_in_config and c.audio_transcription.enabled
for c in self.config.cameras.values()
):
if isinstance(
self.config.audio_transcription.model, AudioTranscriptionModelEnum
):
# AudioTranscriptionModelRunner.__init__ unconditionally fetches
# sherpa-onnx or whisper weights, so only build it on the local path
self.transcription_model_runner = AudioTranscriptionModelRunner(
self.transcription_model_runner: AudioTranscriptionModelRunner | None = (
AudioTranscriptionModelRunner(
self.config.audio_transcription.device or "AUTO",
self.config.audio_transcription.model_size,
)
else:
# imported here rather than at module scope: frigate.genai pulls in
# numpy, the provider SDKs, frigate.models, and the prompt builders,
# and this process runs at PROCESS_PRIORITY_HIGH. built after the
# fork because SDK clients hold sockets and TLS state that must not
# cross it; clients themselves stay lazy behind the role property.
from frigate.genai.manager import GenAIClientManager
self.genai_manager = GenAIClientManager(self.config)
)
else:
self.transcription_model_runner = None
config_subscriber = CameraConfigUpdateSubscriber(
self.config,
@@ -183,8 +136,28 @@ class AudioProcessor(FrigateProcess):
],
)
def spawn_if_needed(camera: CameraConfig) -> None:
name = camera.name
if name is None or name in self.audio_threads:
return
if not camera.enabled or not camera.audio.enabled:
return
# ffmpeg update may not have arrived yet; wait for next poll
if not any("audio" in i.roles for i in camera.ffmpeg.inputs):
return
thread = AudioEventMaintainer(
camera,
self.config,
self.camera_metrics,
self.transcription_model_runner,
self.stop_event, # type: ignore[arg-type]
)
self.audio_threads[name] = thread
thread.start()
self.logger.info(f"Audio maintainer started for {name}")
for camera in self.config.cameras.values():
self.spawn_if_needed(camera)
spawn_if_needed(camera)
self.logger.info(f"Audio processor started (pid: {self.pid})")
@@ -194,14 +167,15 @@ class AudioProcessor(FrigateProcess):
updated_topics = config_subscriber.check_for_updates()
# stop maintainers for removed cameras so their ffmpeg process is
# torn down
# torn down and they stop touching camera_metrics (which the camera
# maintainer has already popped for the removed camera)
for removed_camera in updated_topics.get(
CameraConfigUpdateEnum.remove.name, []
):
self.__stop_audio_thread(removed_camera)
for camera in self.config.cameras.values():
self.spawn_if_needed(camera)
spawn_if_needed(camera)
config_subscriber.stop()
@@ -223,21 +197,15 @@ class AudioEventMaintainer(threading.Thread):
self,
camera: CameraConfig,
config: FrigateConfig,
metrics: CameraMetrics,
embeddings_metrics: DataProcessorMetrics,
camera_metrics: DictProxy,
audio_transcription_model_runner: AudioTranscriptionModelRunner | None,
stop_event: threading.Event,
genai_manager: Any = None,
) -> None:
super().__init__(name=f"{camera.name}_audio_event_processor")
self.config = config
self.camera_config = camera
# hold the metrics object rather than indexing the manager dict per
# chunk, which costs an IPC round trip and breaks once the camera
# maintainer pops the entry on removal
self.metrics = metrics
self.embeddings_metrics = embeddings_metrics
self.camera_metrics = camera_metrics
self.stop_event = stop_event
# per-camera stop signal so a single maintainer can be torn down at
# runtime (e.g. on camera removal) without stopping the whole process
@@ -254,7 +222,6 @@ class AudioEventMaintainer(threading.Thread):
self.logpipe = LogPipe(f"ffmpeg.{self.camera_config.name}.audio")
self.audio_listener: subprocess.Popen[Any] | None = None
self.audio_transcription_model_runner = audio_transcription_model_runner
self.genai_manager = genai_manager
self.transcription_processor = None
self.transcription_thread = None
@@ -271,9 +238,9 @@ class AudioEventMaintainer(threading.Thread):
)
self.detection_publisher = DetectionPublisher(DetectionTypeEnum.audio.value)
if self.camera_config.audio_transcription.enabled and (
self.audio_transcription_model_runner is not None
or self.genai_manager is not None
if (
self.camera_config.audio_transcription.enabled
and self.audio_transcription_model_runner is not None
):
# init the transcription processor for this camera
self.transcription_processor = AudioTranscriptionRealTimeProcessor(
@@ -281,9 +248,8 @@ class AudioEventMaintainer(threading.Thread):
camera_config=self.camera_config,
requestor=self.requestor,
model_runner=self.audio_transcription_model_runner,
metrics=self.embeddings_metrics,
metrics=self.camera_metrics[self.camera_config.name],
stop_event=self.stop_event,
genai_manager=self.genai_manager,
)
self.transcription_thread = threading.Thread(
@@ -307,8 +273,8 @@ class AudioEventMaintainer(threading.Thread):
audio_as_float: np.ndarray = audio.astype(np.float32)
rms, dBFS = self.calculate_audio_levels(audio_as_float)
self.metrics.audio_rms.value = rms
self.metrics.audio_dBFS.value = dBFS
self.camera_metrics[self.camera_config.name].audio_rms.value = rms
self.camera_metrics[self.camera_config.name].audio_dBFS.value = dBFS
audio_detections: list[tuple[str, float]] = []
@@ -393,6 +359,7 @@ class AudioEventMaintainer(threading.Thread):
return
time.sleep(self.camera_config.ffmpeg.retry_interval)
self.logpipe.dump()
self.start_or_restart_ffmpeg()
if self.audio_listener is None or self.audio_listener.stdout is None:
-9
View File
@@ -84,8 +84,6 @@ _user_agent_args = [
PRESETS_HW_ACCEL_DECODE = {
"preset-rpi-64-h264": "-c:v:1 h264_v4l2m2m",
"preset-rpi-64-h265": "-c:v:1 hevc_v4l2m2m",
"preset-apple-silicon-h264": "-c:v h264_v4l2m2m",
"preset-apple-silicon-h265": "-c:v hevc_v4l2m2m",
FFMPEG_HWACCEL_VAAPI: "-hwaccel_flags allow_profile_mismatch -hwaccel vaapi -hwaccel_device {3} -hwaccel_output_format vaapi",
"preset-intel-qsv-h264": f"-hwaccel qsv -qsv_device {{3}} -hwaccel_output_format qsv -c:v h264_qsv{' -bsf:v dump_extra' if LIBAVFORMAT_VERSION_MAJOR >= 61 else ''}", # https://trac.ffmpeg.org/ticket/9766#comment:17
"preset-intel-qsv-h265": f"-load_plugin hevc_hw -hwaccel qsv -qsv_device {{3}} -hwaccel_output_format qsv{' -bsf:v dump_extra' if LIBAVFORMAT_VERSION_MAJOR >= 61 else ''}", # https://trac.ffmpeg.org/ticket/9766#comment:17
@@ -122,9 +120,6 @@ PRESETS_HW_ACCEL_DECODE["preset-rk-h265"] = PRESETS_HW_ACCEL_DECODE[
PRESETS_HW_ACCEL_SCALE = {
"preset-rpi-64-h264": "-r {0} -vf fps={0},scale={1}:{2}",
"preset-rpi-64-h265": "-r {0} -vf fps={0},scale={1}:{2}",
# ffmpeg's v4l2m2m decoders cannot scale, so frames are scaled on the CPU
"preset-apple-silicon-h264": "-r {0} -vf fps={0},scale={1}:{2}",
"preset-apple-silicon-h265": "-r {0} -vf fps={0},scale={1}:{2}",
FFMPEG_HWACCEL_VAAPI: "-r {0} -vf fps={0},scale_vaapi=w={1}:h={2},hwdownload,format=nv12",
"preset-intel-qsv-h264": "-r {0} -vf vpp_qsv=w={1}:h={2}:format=nv12,hwdownload,format=nv12,fps={0},format=yuv420p",
"preset-intel-qsv-h265": "-r {0} -vf vpp_qsv=w={1}:h={2}:format=nv12,hwdownload,format=nv12,fps={0},format=yuv420p",
@@ -155,8 +150,6 @@ PRESETS_HW_ACCEL_SCALE["preset-rk-h265"] = PRESETS_HW_ACCEL_SCALE[FFMPEG_HWACCEL
PRESETS_HW_ACCEL_ENCODE_BIRDSEYE = {
"preset-rpi-64-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m {2}",
"preset-rpi-64-h265": "{0} -hide_banner {1} -c:v hevc_v4l2m2m {2}",
"preset-apple-silicon-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m {2}",
"preset-apple-silicon-h265": "{0} -hide_banner {1} -c:v h264_v4l2m2m {2}",
# -vaapi_device is required in addition to -hwaccel_device: this is the only
# birdseye preset that uses hwupload, and ffmpeg 8 initializes filters before
# the decoder creates a device, so hwupload cannot see an -hwaccel_device one.
@@ -191,8 +184,6 @@ PRESETS_HW_ACCEL_ENCODE_BIRDSEYE["preset-rk-h264"] = PRESETS_HW_ACCEL_ENCODE_BIR
PRESETS_HW_ACCEL_ENCODE_TIMELAPSE = {
"preset-rpi-64-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m -pix_fmt yuv420p {2}",
"preset-rpi-64-h265": "{0} -hide_banner {1} -c:v hevc_v4l2m2m -pix_fmt yuv420p {2}",
"preset-apple-silicon-h264": "{0} -hide_banner {1} -c:v h264_v4l2m2m -pix_fmt yuv420p {2}",
"preset-apple-silicon-h265": "{0} -hide_banner {1} -c:v hevc_v4l2m2m -pix_fmt yuv420p {2}",
FFMPEG_HWACCEL_VAAPI: "{0} -hide_banner -hwaccel vaapi -hwaccel_output_format vaapi -hwaccel_device {3} {1} -c:v h264_vaapi {2}",
"preset-intel-qsv-h264": "{0} -hide_banner {1} -c:v h264_qsv -profile:v high -level:v 4.1 -async_depth:v 1 {2}",
"preset-intel-qsv-h265": "{0} -hide_banner {1} -c:v hevc_qsv -profile:v main -level:v 4.1 -async_depth:v 1 {2}",
+2 -89
View File
@@ -105,21 +105,8 @@ class GenAIClient:
debug_save: bool,
activity_context_prompt: str,
response_style: str = "default",
frame_captions: list[str] | None = None,
) -> ReviewMetadata | None:
"""Generate a description for the review item activity.
`frame_captions` holds one caption per thumbnail for the annotated
frame mode; each is sent directly before its frame.
"""
if frame_captions and len(frame_captions) != len(thumbnails):
logger.warning(
"Got %d frame captions for %d thumbnails, sending plain frames",
len(frame_captions),
len(thumbnails),
)
frame_captions = None
"""Generate a description for the review item activity."""
context_prompt = build_review_description_prompt(
review_data,
thumbnails,
@@ -127,7 +114,6 @@ class GenAIClient:
preferred_language,
activity_context_prompt,
response_style,
frame_captions,
)
logger.debug(
@@ -143,30 +129,9 @@ class GenAIClient:
) as f:
f.write(context_prompt)
if frame_captions:
# One file per frame, numbered to match the image it precedes
# (0.txt goes with 0.jpg), so the debug folder replays without
# having to re-derive the mapping.
for index, caption in enumerate(frame_captions):
with open(
os.path.join(
CLIPS_DIR,
"genai-requests",
review_data["id"],
f"{index}.txt",
),
"w",
) as f:
f.write(caption)
response_format = build_review_description_response_format(concerns)
response = self._send(
context_prompt,
thumbnails,
response_format,
image_captions=frame_captions,
)
response = self._send(context_prompt, thumbnails, response_format)
if debug_save and response:
with open(
@@ -304,7 +269,6 @@ class GenAIClient:
images: list[bytes],
response_format: dict | None = None,
enable_thinking: bool = False,
image_captions: list[str] | None = None,
) -> str | None:
"""Submit a request to the provider.
@@ -312,10 +276,6 @@ class GenAIClient:
``supports_toggleable_thinking``. Description-style callers leave it
at the default (off) since synthesis tasks don't benefit from
reasoning traces.
``image_captions`` carries one caption per image, to be placed
immediately before its image so the model can tell the frames apart.
Providers build their request order with ``interleave_images``.
"""
return None
@@ -338,11 +298,6 @@ class GenAIClient:
"""Whether the configured model can generate embeddings via embed()."""
return False
@property
def supports_transcription(self) -> bool:
"""Whether the configured model can transcribe audio via transcribe()."""
return False
def list_models(self) -> list[str]:
"""Return the list of model names available from this provider.
@@ -350,21 +305,6 @@ class GenAIClient:
"""
return []
def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
"""Return capability flags for each model the provider serves.
Only providers whose backend advertises capabilities per model can
populate this; llama.cpp reports input modalities for every model it
serves, so one request describes them all. An empty mapping means "no
per-model information available", and callers fall back to this
client's own capability properties, which describe only the configured
model. A model absent from a non-empty mapping means the same thing.
Returns:
Model name (including aliases) to its capability flags
"""
return {}
def get_context_size(self) -> int:
"""Get the context window size for this provider in tokens."""
return 4096
@@ -396,33 +336,6 @@ class GenAIClient:
)
return []
def transcribe(
self,
audio: bytes,
language: str | None = None,
mime_type: str = "audio/wav",
) -> str | None:
"""Transcribe speech audio to text.
Audio is passed as a self-describing blob rather than raw samples so
every provider receives a container it can declare, and WAV framing
lives in one place instead of in each plugin.
Args:
audio: The encoded audio payload (WAV bytes by default)
language: Optional ISO language hint for the provider
mime_type: Media type of ``audio``
Returns:
The transcript, or None when the provider cannot produce one
"""
logger.warning(
"%s does not support transcription. "
"This method should be overridden by the provider implementation.",
self.__class__.__name__,
)
return None
def chat_with_tools(
self,
messages: list[dict[str, Any]],
-12
View File
@@ -110,12 +110,6 @@ class GenAIClientManager:
name = self._role_map.get(GenAIRoleEnum.embeddings)
return self._get_client(name) if name else None
@property
def transcribe_client(self) -> "GenAIClient | None":
"""Client configured for the transcribe role."""
name = self._role_map.get(GenAIRoleEnum.transcribe)
return self._get_client(name) if name else None
def role_info(self) -> dict[str, dict[str, Any]]:
"""Return the model selected for each configured role and its context size.
@@ -150,11 +144,5 @@ class GenAIClientManager:
"roles": [r.value for r in genai_cfg.roles],
"supports_toggleable_thinking": client.supports_toggleable_thinking,
"supports_embeddings": client.supports_embeddings,
"supports_transcription": client.supports_transcription,
# Capabilities of the configured model are above; this maps every
# model the provider serves to its own, so the UI can react to a
# model selected but not yet saved. Empty when the provider
# cannot report capabilities without loading a model.
"model_capabilities": client.list_model_capabilities(),
}
return result
-13
View File
@@ -13,19 +13,6 @@ overrides what is genuinely Azure-specific:
- Context size: Azure does not expose a per-model ``max_model_len`` field
reliably, so we keep the historical 128K default rather than the
model-name heuristic used by OpenAI.
Transcription is inherited too: :class:`openai.AzureOpenAI` exposes the same
``audio.transcriptions.create``. Two Azure-specific caveats apply when using
the ``transcribe`` role:
- ``model`` must be the Azure *deployment* name, not the underlying model name.
- The ``api-version`` parsed from ``base_url`` must be 2024-06-01 or later;
earlier versions have no transcriptions route and the 404 surfaces only as a
generic provider error.
- Because ``model`` is a deployment name, the inherited check that picks
``languages`` over ``language`` for gpt-transcribe cannot fire unless the
deployment happens to be named after the model. Name the deployment
``gpt-transcribe`` to get the right field, or leave the language on ``auto``.
"""
import logging
+3 -65
View File
@@ -13,14 +13,9 @@ from google.genai.types import FunctionCallingConfigMode
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
from frigate.genai.utils import interleave_images
logger = logging.getLogger(__name__)
# Gemini requests carrying inline data are capped at ~20 MB total; stay well
# under it so the request fails as a log line rather than a 400.
GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024
def _decode_thought_signature(value: Any) -> bytes | None:
"""Decode a base64-encoded thought_signature carried across conversation turns."""
@@ -123,16 +118,11 @@ class GeminiClient(GenAIClient):
images: list[bytes],
response_format: dict | None = None,
enable_thinking: bool = False,
image_captions: list[str] | None = None,
) -> str | None:
"""Submit a request to Gemini."""
contents: list[Any] = [
part
if isinstance(part, str)
else types.Part.from_bytes(data=part, mime_type="image/jpeg")
for part in interleave_images(prompt, images, image_captions)
contents = [prompt] + [
types.Part.from_bytes(data=img, mime_type="image/jpeg") for img in images
]
try:
# Merge runtime_options into generation_config if provided
generation_config_dict: dict[str, Any] = {"candidate_count": 1}
@@ -146,7 +136,7 @@ class GeminiClient(GenAIClient):
response = self.provider.models.generate_content(
model=self.genai_config.model,
contents=contents,
contents=contents, # type: ignore[arg-type]
config=types.GenerateContentConfig(
**generation_config_dict,
),
@@ -167,58 +157,6 @@ class GeminiClient(GenAIClient):
return None
return description
@property
def supports_transcription(self) -> bool:
"""Gemini models accept inline audio parts."""
return True
def transcribe(
self,
audio: bytes,
language: str | None = None,
mime_type: str = "audio/wav",
) -> str | None:
"""Transcribe audio by sending it as an inline part alongside a prompt."""
if len(audio) > GEMINI_MAX_INLINE_BYTES:
logger.warning(
"Audio payload of %d bytes exceeds the Gemini inline limit; skipping transcription",
len(audio),
)
return None
prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
if language:
prompt += f" The speech is in language '{language}'."
try:
contents: list[Any] = [
prompt,
types.Part.from_bytes(data=audio, mime_type=mime_type),
]
response = self.provider.models.generate_content(
model=self.genai_config.model,
contents=contents,
config=types.GenerateContentConfig(candidate_count=1),
)
except errors.APIError as e:
logger.warning("Gemini returned an error: %s", str(e))
return None
except Exception as e:
logger.warning("An unexpected error occurred with Gemini: %s", str(e))
return None
try:
if response.text is None:
return None
transcript = response.text.strip()
except (ValueError, AttributeError):
# No transcript was generated
return None
return transcript or None
def list_models(self) -> list[str]:
"""Return available model names from Gemini."""
try:
+18 -183
View File
@@ -14,7 +14,7 @@ from PIL import Image
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
from frigate.genai.utils import parse_tool_calls_from_message
logger = logging.getLogger(__name__)
@@ -333,7 +333,6 @@ class LlamaCppClient(GenAIClient):
images: list[bytes],
response_format: dict | None = None,
enable_thinking: bool = False,
image_captions: list[str] | None = None,
) -> str | None:
"""Submit a request to llama.cpp server."""
if self.provider is None:
@@ -343,17 +342,18 @@ class LlamaCppClient(GenAIClient):
return None
try:
content: list[dict[str, Any]] = []
for part in interleave_images(prompt, images, image_captions):
if isinstance(part, str):
content.append({"type": "text", "text": part})
continue
encoded_image = base64.b64encode(part).decode("utf-8")
content = [
{
"type": "text",
"text": prompt,
}
]
for image in images:
encoded_image = base64.b64encode(image).decode("utf-8")
content.append(
{
"type": "image_url",
"image_url": {
"image_url": { # type: ignore[dict-item]
"url": f"data:image/jpeg;base64,{encoded_image}",
},
}
@@ -408,126 +408,6 @@ class LlamaCppClient(GenAIClient):
"""Whether the loaded model supports audio input."""
return self._supports_audio
@property
def supports_transcription(self) -> bool:
"""Audio-capable models can transcribe through chat completions."""
return self._supports_audio
def transcribe(
self,
audio: bytes,
language: str | None = None,
mime_type: str = "audio/wav",
) -> str | None:
"""Transcribe audio through the OpenAI-compatible transcriptions route.
llama.cpp serves /v1/audio/transcriptions for any audio-capable model,
not only a separately loaded whisper (ggml-org/llama.cpp#21863), so it
covers exactly the models supports_transcription detects. It takes the
language as a native multipart field, which is the only thing dedicated
ASR models honor: they read the chat prompt as contextual biasing, so
asking one there to use a language does nothing.
Falls back to chat completions when the server predates that route.
"""
if self.provider is None:
logger.warning(
"llama.cpp provider has not been initialized, audio will not be transcribed. Check your llama.cpp configuration."
)
return None
if not self._supports_audio:
logger.warning(
"llama.cpp model '%s' does not accept audio input",
self.genai_config.model,
)
return None
try:
data = {"model": self.genai_config.model, "response_format": "json"}
if language:
data["language"] = language
response = self._post(
f"{self.provider}/v1/audio/transcriptions",
files={"file": ("audio.wav", audio, mime_type)},
data=data,
timeout=self.timeout,
)
if response.status_code == 404:
logger.debug(
"llama.cpp server has no /v1/audio/transcriptions route, using chat completions"
)
return self._transcribe_via_chat(audio, language)
response.raise_for_status()
result = response.json()
text = result.get("text") if isinstance(result, dict) else None
return str(text).strip() or None if text else None
except Exception as e:
logger.warning("llama.cpp returned an error: %s", str(e))
return None
def _transcribe_via_chat(self, audio: bytes, language: str | None) -> str | None:
"""Transcribe through /v1/chat/completions, for servers without the
transcriptions route.
The _media_marker / multimodal_data convention is an /embeddings-only
protocol, so no marker-refresh retry is needed here.
"""
prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
if language:
prompt += f" The speech is in language '{language}'."
try:
encoded_audio = base64.b64encode(audio).decode("utf-8")
payload: dict[str, Any] = {
"model": self.genai_config.model,
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "input_audio",
"input_audio": {
"data": encoded_audio,
"format": "wav",
},
},
],
},
],
**self.provider_options,
}
response = self._post(
f"{self.provider}/v1/chat/completions",
json=payload,
timeout=self.timeout,
)
response.raise_for_status()
result = response.json()
if (
result is not None
and "choices" in result
and len(result["choices"]) > 0
):
choice = result["choices"][0]
if "message" in choice and choice["message"].get("content"):
return str(choice["message"]["content"].strip()) or None
return None
except Exception as e:
logger.warning("llama.cpp returned an error: %s", str(e))
return None
@property
def supports_tools(self) -> bool:
"""Whether the loaded model supports tool/function calling."""
@@ -537,73 +417,28 @@ class LlamaCppClient(GenAIClient):
def supports_toggleable_thinking(self) -> bool:
return self._supports_reasoning
def _fetch_models_data(self) -> list[dict[str, Any]]:
"""Return the raw /v1/models entries, or an empty list if unreachable."""
def list_models(self) -> list[str]:
"""Return available model IDs from the llama.cpp server."""
base_url = self.provider or (
self.genai_config.base_url.rstrip("/")
if self.genai_config.base_url
else None
)
if base_url is None:
return []
try:
response = self._get(f"{base_url}/v1/models", timeout=10)
response.raise_for_status()
data = response.json().get("data", [])
models = []
for m in response.json().get("data", []):
models.append(m.get("id", "unknown"))
for alias in m.get("aliases", []):
models.append(alias)
return sorted(models)
except Exception as e:
logger.warning("Failed to list llama.cpp models: %s", e)
return []
return data if isinstance(data, list) else []
def list_models(self) -> list[str]:
"""Return available model IDs from the llama.cpp server."""
models: set[str] = set()
# llama-server lists the id among the aliases when --alias is set
for m in self._fetch_models_data():
models.add(m.get("id", "unknown"))
models.update(m.get("aliases", []))
return sorted(models)
def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
"""Report input modalities for every model the server serves.
Since ggml-org/llama.cpp#22952 each /v1/models entry carries
architecture.input_modalities, so a single request describes every
model rather than just the configured one. That is what lets the UI
answer "can the model I just picked transcribe" before the config is
saved and a client for it exists.
Models whose entry predates that field are omitted rather than reported
as incapable, so an older server falls back to the /props probe instead
of silently losing capabilities it actually has.
"""
capabilities: dict[str, dict[str, bool]] = {}
for model in self._fetch_models_data():
architecture = model.get("architecture") or {}
modalities = architecture.get("input_modalities")
if not isinstance(modalities, list) or not modalities:
continue
flags = {
"supports_vision": "image" in modalities,
"supports_transcription": "audio" in modalities,
}
names = [model.get("id"), *(model.get("aliases") or [])]
for name in names:
if isinstance(name, str) and name:
capabilities[name] = flags
return capabilities
def get_context_size(self) -> int:
"""Get the context window size for llama.cpp.
+52 -88
View File
@@ -14,7 +14,7 @@ from ollama import ResponseError
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
from frigate.genai.utils import parse_tool_calls_from_message
logger = logging.getLogger(__name__)
@@ -50,28 +50,6 @@ def _extract_ollama_stats(response: Any) -> dict[str, Any] | None:
return stats or None
# Ollama replaces each occurrence of this marker in a message, in order, with
# the next image from the message's images list. Without markers it puts every
# image before the text.
IMAGE_PLACEHOLDER = "[img]"
def _flatten_parts(parts: list[str | bytes]) -> tuple[str, list[bytes] | None]:
"""Collapse ordered text and image parts into Ollama's (content, images)
shape, marking where each image goes so the order survives."""
text: list[str] = []
images: list[bytes] = []
for part in parts:
if isinstance(part, bytes):
text.append(IMAGE_PLACEHOLDER)
images.append(part)
elif part:
text.append(part)
return "\n".join(text), (images or None)
def _normalize_multimodal_content(
content: Any,
) -> tuple[str | None, list[bytes] | None]:
@@ -80,13 +58,13 @@ def _normalize_multimodal_content(
The chat API constructs user messages with content as a list of
``{"type": "text"}`` and ``{"type": "image_url"}`` parts when a tool
returns a live frame. Ollama's SDK requires content to be a string and
images to be passed in a separate field, so images are pulled out and
their positions marked with placeholders.
images to be passed in a separate field, so we extract each.
"""
if not isinstance(content, list):
return content, None
parts: list[str | bytes] = []
text_parts: list[str] = []
images: list[bytes] = []
for part in content:
if not isinstance(part, dict):
continue
@@ -94,20 +72,17 @@ def _normalize_multimodal_content(
if part_type == "text":
text = part.get("text")
if text:
parts.append(str(text))
text_parts.append(str(text))
elif part_type == "image_url":
url = (part.get("image_url") or {}).get("url", "")
if isinstance(url, str) and url.startswith("data:"):
try:
encoded = url.split(",", 1)[1]
parts.append(base64.b64decode(encoded, validate=True))
images.append(base64.b64decode(encoded, validate=True))
except (ValueError, IndexError, binascii.Error) as e:
logger.debug("Failed to decode multimodal image url: %s", e)
if not parts:
return None, None
return _flatten_parts(parts)
return ("\n".join(text_parts) if text_parts else None), (images or None)
@register_genai_provider(GenAIProviderEnum.ollama)
@@ -221,46 +196,58 @@ class OllamaClient(GenAIClient):
images: list[bytes],
response_format: dict | None = None,
enable_thinking: bool = False,
image_captions: list[str] | None = None,
) -> str | None:
"""Submit a request to Ollama through the chat API, the same path the
tool-calling chat uses, with image placeholders keeping any captions
next to their frames."""
"""Submit a request to Ollama"""
if self.provider is None:
logger.warning(
"Ollama provider has not been initialized, a description will not be generated. Check your Ollama configuration."
)
return None
content, message_images = _flatten_parts(
interleave_images(prompt, images, image_captions)
)
message: dict[str, Any] = {"role": "user", "content": content}
if message_images:
message["images"] = message_images
request_params = self._build_request_params(
[message], None, None, enable_thinking=enable_thinking
)
if response_format and response_format.get("type") == "json_schema":
schema = response_format.get("json_schema", {}).get("schema")
if schema:
request_params["format"] = self._clean_schema_for_ollama(schema)
logger.debug(
"Ollama chat request: model=%s, prompt_len=%s, image_count=%s, "
"has_format=%s, think=%s",
self.genai_config.model,
len(prompt),
len(images),
"format" in request_params,
request_params.get("think"),
)
try:
response = self.provider.chat(**request_params)
ollama_options = {
**self.provider_options,
**self.genai_config.runtime_options,
}
if response_format and response_format.get("type") == "json_schema":
schema = response_format.get("json_schema", {}).get("schema")
if schema:
ollama_options["format"] = self._clean_schema_for_ollama(schema)
if self.supports_toggleable_thinking:
ollama_options["think"] = enable_thinking
logger.debug(
"Ollama generate request: model=%s, prompt_len=%s, image_count=%s, "
"has_format=%s, options=%s",
self.genai_config.model,
len(prompt),
len(images) if images else 0,
"format" in ollama_options,
{k: v for k, v in ollama_options.items() if k != "format"},
)
result = self.provider.generate(
self.genai_config.model,
prompt,
images=images if images else None,
**ollama_options,
)
logger.debug(
"Ollama generate response: done=%s, done_reason=%s, eval_count=%s, "
"prompt_eval_count=%s, response_len=%s",
result.get("done"),
result.get("done_reason"),
result.get("eval_count"),
result.get("prompt_eval_count"),
len(result.get("response", "") or ""),
)
response_text = str(result["response"]).strip()
if not response_text:
logger.warning(
"Ollama returned a blank response for model %s (done_reason=%s, "
"eval_count=%s). Check model output, ensure thinking is disabled.",
self.genai_config.model,
result.get("done_reason"),
result.get("eval_count"),
)
return response_text
except (
TimeoutException,
ResponseError,
@@ -270,27 +257,6 @@ class OllamaClient(GenAIClient):
logger.warning("Ollama returned an error: %s", str(e))
return None
logger.debug(
"Ollama chat response: done=%s, done_reason=%s, eval_count=%s, "
"prompt_eval_count=%s",
response.get("done"),
response.get("done_reason"),
response.get("eval_count"),
response.get("prompt_eval_count"),
)
response_text = self._message_from_response(response)["content"] or ""
if not response_text:
logger.warning(
"Ollama returned a blank response for model %s (done_reason=%s, "
"eval_count=%s). Check model output, ensure thinking is disabled.",
self.genai_config.model,
response.get("done_reason"),
response.get("eval_count"),
)
return response_text
def list_models(self) -> list[str]:
"""Return available model names from the Ollama server."""
client = self.provider
@@ -340,8 +306,6 @@ class OllamaClient(GenAIClient):
}
if images:
msg_dict["images"] = images
elif msg.get("images"):
msg_dict["images"] = msg["images"]
if msg.get("tool_call_id"):
msg_dict["tool_call_id"] = msg["tool_call_id"]
if msg.get("name"):
+9 -61
View File
@@ -11,16 +11,9 @@ from openai import OpenAI
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
from frigate.genai.utils import interleave_images
logger = logging.getLogger(__name__)
# gpt-transcribe replaced the singular `language` field with a `languages` array
# and rejects a request that sends both. Older transcription models
# (gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1) still take the singular
# form. https://developers.openai.com/api/docs/guides/speech-to-text
_LANGUAGES_ARRAY_MODEL_PREFIX = "gpt-transcribe"
def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None:
"""Build a stats dict from an OpenAI-compatible usage object."""
@@ -70,21 +63,21 @@ class OpenAIClient(GenAIClient):
images: list[bytes],
response_format: dict | None = None,
enable_thinking: bool = False,
image_captions: list[str] | None = None,
) -> str | None:
"""Submit a request to OpenAI."""
messages_content: list[dict] = []
for part in interleave_images(prompt, images, image_captions):
if isinstance(part, str):
messages_content.append({"type": "text", "text": part})
continue
encoded = base64.b64encode(part).decode("utf-8")
encoded_images = [base64.b64encode(image).decode("utf-8") for image in images]
messages_content: list[dict] = [
{
"type": "text",
"text": prompt,
}
]
for image in encoded_images:
messages_content.append(
{
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{encoded}",
"url": f"data:image/jpeg;base64,{image}",
"detail": "low",
},
}
@@ -140,51 +133,6 @@ class OpenAIClient(GenAIClient):
logger.warning("OpenAI returned an error: %s", str(e))
return None
@property
def supports_transcription(self) -> bool:
"""OpenAI exposes /v1/audio/transcriptions for its speech models."""
return True
def transcribe(
self,
audio: bytes,
language: str | None = None,
mime_type: str = "audio/wav",
) -> str | None:
"""Transcribe audio via the OpenAI audio transcriptions endpoint."""
try:
# runtime_options are chat-completion parameters; the transcriptions
# endpoint rejects unknown fields, so they are deliberately not splatted
# in here the way _send() does.
request_params: dict[str, Any] = {
"model": self.genai_config.model,
"file": ("audio.wav", audio, mime_type),
"response_format": "text",
"timeout": self.timeout,
}
if language:
if (
self.genai_config.model.strip()
.lower()
.startswith(_LANGUAGES_ARRAY_MODEL_PREFIX)
):
# not a typed parameter on the SDK method, so it has to ride
# along in extra_body
request_params["extra_body"] = {"languages": [language]}
else:
request_params["language"] = language
result = self.provider.audio.transcriptions.create(**request_params)
except (TimeoutException, Exception) as e:
logger.warning("OpenAI returned an error: %s", str(e))
return None
# response_format="text" yields a bare string, but some compatible
# servers still return the object form
text = result if isinstance(result, str) else getattr(result, "text", None)
return text.strip() if text else None
def list_models(self) -> list[str]:
"""Return available model IDs from the OpenAI-compatible API."""
try:
+2 -15
View File
@@ -59,13 +59,6 @@ def get_review_field_guidelines(response_style: str = "default") -> dict[str, st
}
# Explains the per-frame labels and tracker notes used by the annotated frame
# mode. Neither the notes nor this guidance say whether repeated detections are
# the same subject, since the tracking data cannot tell.
FRAME_ANNOTATION_GUIDANCE = """- Each image below is immediately preceded by a text label giving its frame number and how many seconds into the sequence it was captured. Use these labels to track the order of events and the time between them.
- Some images below are preceded by notes from the camera's object tracker recording what changed at that point: an object being first detected, starting to move, reversing direction, stopping, or no longer being detected. These notes come from tracking data rather than from the images, and they are reliable. Use them to establish how many distinct activities occur and in what order, and describe every one of them."""
def build_review_description_prompt(
review_data: dict[str, Any],
thumbnails: list[bytes],
@@ -73,13 +66,8 @@ def build_review_description_prompt(
preferred_language: str | None,
activity_context_prompt: str,
response_style: str = "default",
frame_captions: list[str] | None = None,
) -> str:
"""Build the prompt for review activity description generation.
When `frame_captions` is set, each caption is sent directly before its
image, so the prompt explains that layout.
"""
"""Build the prompt for review activity description generation."""
def get_concern_prompt() -> str:
if concerns:
@@ -105,7 +93,6 @@ def build_review_description_prompt(
return "\n- (No objects detected)"
fields = get_review_field_guidelines(response_style)
frame_guidance = f"\n{FRAME_ANNOTATION_GUIDANCE}" if frame_captions else ""
return f"""
Your task is to analyze a sequence of images taken in chronological order from a security camera.
@@ -143,7 +130,7 @@ Respond with a JSON object matching the provided schema. Field-specific guidance
## Sequence Details
- Camera: {review_data["camera"]}
- Total frames: {len(thumbnails)} (Frame 1 = earliest, Frame {len(thumbnails)} = latest){frame_guidance}
- Total frames: {len(thumbnails)} (Frame 1 = earliest, Frame {len(thumbnails)} = latest)
- Activity started at {review_data["start"]} and lasted {review_data["duration"]} seconds
- Zones involved: {", ".join(review_data["zones"]) if review_data["zones"] else "None"}
-19
View File
@@ -7,25 +7,6 @@ from typing import Any
logger = logging.getLogger(__name__)
def interleave_images(
prompt: str, images: list[bytes], captions: list[str] | None = None
) -> list[str | bytes]:
"""The prompt, then each image preceded by its caption when one is given.
Providers map the text and image parts onto their own request format, so
every provider sends the same order.
"""
parts: list[str | bytes] = [prompt]
for index, image in enumerate(images):
if captions and index < len(captions):
parts.append(captions[index])
parts.append(image)
return parts
def parse_tool_calls_from_message(
message: dict[str, Any],
) -> list[dict[str, Any]] | None:
-8
View File
@@ -144,14 +144,6 @@ class LogPipe(threading.Thread):
self.pipeReader.close()
def dump(self) -> None:
if not self.deque:
return
self.logger.log(
self.level,
"The following ffmpeg logs include the last 100 lines prior to exit.",
)
while len(self.deque) > 0:
self.logger.log(self.level, self.deque.popleft())
+3 -8
View File
@@ -195,20 +195,15 @@ class Notice(Model):
first_seen = DateTimeField()
last_seen = DateTimeField()
count = IntegerField(default=1)
# hidden until the next occurrence
acknowledged_at = DateTimeField(null=True)
# hidden for good
muted_at = DateTimeField(null=True)
dismissed_at = DateTimeField(null=True)
class NoticeStats(Model):
kind = CharField(null=False, primary_key=True, max_length=50)
occurrences = IntegerField(default=0)
acknowledgements = IntegerField(default=0)
mutes = IntegerField(default=0)
dismissals = IntegerField(default=0)
first_seen = DateTimeField()
last_seen = DateTimeField()
# watermarks for a future analytics reporter; unused until then
reported_occurrences = IntegerField(default=0)
reported_acknowledgements = IntegerField(default=0)
reported_mutes = IntegerField(default=0)
reported_dismissals = IntegerField(default=0)
+1 -5
View File
@@ -188,12 +188,8 @@ class ImprovedMotionDetector(MotionDetector):
self.config.skip_motion_threshold is not None
and pct_motion > self.config.skip_motion_threshold
):
# recalibrate so we transition to the new background. the frame
# still has to be blended in here, otherwise the background stays
# frozen and every subsequent frame skips as well
# force a recalibration so we transition to the new background
self.calibrating = True
cv2.accumulateWeighted(resized_frame, self.avg_frame, 0.2)
self.motion_frame_count = 0
return []
# once the motion is less than 5% and the number of contours is < 4, assume its calibrated
+41 -108
View File
@@ -5,7 +5,7 @@ import threading
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from typing import Any, cast
from typing import Any
from frigate.const import REPLAY_CAMERA_PREFIX
from frigate.models import Notice, NoticeStats
@@ -77,8 +77,9 @@ class NoticeRegistry:
) -> None:
"""Insert a notice or count another occurrence of it.
Another occurrence shows an acknowledged notice again. A muted notice
stays hidden.
A dismissed notice stays dismissed when it is raised again, unless its
kind sets reopen_at_count. A kind that should come back after a
dismissal gives each episode its own scope.
"""
definition = NOTICE_KINDS.get(kind)
@@ -111,6 +112,7 @@ class NoticeRegistry:
first_seen=now,
last_seen=now,
count=1,
dismissed_at=None,
)
self._bump_occurrences(kind, 1, now)
@@ -173,7 +175,7 @@ class NoticeRegistry:
self._notify()
def resolve_camera(self, camera: str) -> None:
"""Drop the notices and check mutes of a camera being deleted."""
"""Drop the notices and check dismissals of a camera being deleted."""
camera_kinds = [
key
for key, definition in NOTICE_KINDS.items()
@@ -191,7 +193,7 @@ class NoticeRegistry:
)
# a stream id names its camera first; a config id ends with camera.<name>
for check in self.muted_checks():
for check in self.dismissed_checks():
check_id = check["id"]
if check_id.startswith(f"stream:{camera}:") or (
@@ -203,50 +205,26 @@ class NoticeRegistry:
if deleted:
self._notify()
def acknowledge(self, row_id: str) -> bool:
"""Hide a notice until it happens again.
Returns False for an unknown id or a kind that never repeats, such as
a check row or the update notice.
"""
def purge_dismissed(self) -> int:
"""Delete every dismissed row so each can show again. Returns how many."""
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
return int(
Notice.delete().where(Notice.dismissed_at.is_null(False)).execute()
)
if existing is None:
return False
definition = NOTICE_KINDS.get(existing.kind)
if definition is None or not definition.counts_repeats:
return False
if existing.acknowledged_at is not None or existing.muted_at is not None:
return True
Notice.update(acknowledged_at=datetime.now().timestamp()).where(
Notice.id == row_id
).execute()
NoticeStats.update(acknowledgements=NoticeStats.acknowledgements + 1).where(
NoticeStats.kind == existing.kind
).execute()
self._notify()
return True
def mute(self, row_id: str) -> bool:
def dismiss(self, row_id: str) -> bool:
"""Hide a notice or check row for good. Returns False for an unknown id."""
now = datetime.now().timestamp()
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
# a check row gets a notice row only once it is muted
# a check row gets a notice row only once it is dismissed
kind, _, scope = row_id.partition(":")
if kind not in CHECK_KINDS or not scope:
return False
now = datetime.now().timestamp()
Notice.create(
id=row_id,
kind=kind,
@@ -255,78 +233,40 @@ class NoticeRegistry:
first_seen=now,
last_seen=now,
count=1,
muted_at=now,
dismissed_at=now,
)
return True
if existing.muted_at is not None:
if existing.dismissed_at is not None:
return True
if existing.kind not in NOTICE_KINDS:
return False
Notice.update(acknowledged_at=None, muted_at=now).where(
Notice.update(dismissed_at=datetime.now().timestamp()).where(
Notice.id == row_id
).execute()
NoticeStats.update(mutes=NoticeStats.mutes + 1).where(
NoticeStats.update(dismissals=NoticeStats.dismissals + 1).where(
NoticeStats.kind == existing.kind
).execute()
self._notify()
return True
def unhide(self, row_id: str) -> bool:
"""Show an acknowledged or muted row again. Returns False for an unknown id."""
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
return False
if existing.kind in CHECK_KINDS:
Notice.delete_by_id(row_id)
return True
Notice.update(acknowledged_at=None, muted_at=None).where(
Notice.id == row_id
).execute()
self._notify()
return True
def unhide_all(self) -> None:
"""Show every acknowledged and muted row again."""
with self._lock:
for check in self.muted_checks():
Notice.delete_by_id(check["id"])
shown = (
Notice.update(acknowledged_at=None, muted_at=None)
.where(
Notice.acknowledged_at.is_null(False)
| Notice.muted_at.is_null(False)
)
.execute()
)
if shown:
self._notify()
def muted_checks(self) -> list[dict[str, Any]]:
"""Muted config and stream check rows, newest first."""
def dismissed_checks(self) -> list[dict[str, Any]]:
"""Dismissed config and stream check rows, newest first."""
rows = (
Notice.select()
.where(Notice.kind.in_(list(CHECK_KINDS)))
.order_by(Notice.muted_at.desc())
.order_by(Notice.dismissed_at.desc())
)
return [{"id": row.id, "muted_at": row.muted_at} for row in rows]
return [{"id": row.id, "dismissed_at": row.dismissed_at} for row in rows]
def active(self, include_hidden: bool = False) -> list[dict[str, Any]]:
def active(self, include_dismissed: bool = False) -> list[dict[str, Any]]:
"""Notices most severe first, then most recent first.
Args:
include_hidden: Also return acknowledged and muted notices, for the
hidden list
include_dismissed: Also return dismissed notices, for the history view
"""
rows = []
@@ -336,9 +276,7 @@ class NoticeRegistry:
if definition is None:
continue
hidden = row.acknowledged_at is not None or row.muted_at is not None
if hidden and not include_hidden:
if row.dismissed_at is not None and not include_dismissed:
continue
rows.append(
@@ -353,9 +291,7 @@ class NoticeRegistry:
"first_seen": row.first_seen,
"last_seen": row.last_seen,
"count": row.count,
"acknowledgeable": definition.counts_repeats,
"acknowledged_at": row.acknowledged_at,
"muted_at": row.muted_at,
"dismissed_at": row.dismissed_at,
}
)
@@ -373,13 +309,11 @@ class NoticeRegistry:
{
"kind": row.kind,
"occurrences": row.occurrences,
"acknowledgements": row.acknowledgements,
"mutes": row.mutes,
"dismissals": row.dismissals,
"first_seen": row.first_seen,
"last_seen": row.last_seen,
"reported_occurrences": row.reported_occurrences,
"reported_acknowledgements": row.reported_acknowledgements,
"reported_mutes": row.reported_mutes,
"reported_dismissals": row.reported_dismissals,
}
for row in NoticeStats.select()
if row.kind in NOTICE_KINDS
@@ -393,18 +327,18 @@ class NoticeRegistry:
last_seen: float,
params: dict[str, Any],
) -> None:
# called with the lock held; held repeats from before an acknowledgement
# still count but leave the notice hidden
still_acknowledged = row.acknowledged_at is not None and (
last_seen <= cast(float, row.acknowledged_at)
)
# called with the lock held
fields: dict[str, Any] = {
"count": row.count + count,
"last_seen": last_seen,
"params": params,
}
reopen_at = NOTICE_KINDS[kind].reopen_at_count
Notice.update(
count=row.count + count,
last_seen=last_seen,
params=params,
acknowledged_at=row.acknowledged_at if still_acknowledged else None,
).where(Notice.id == row.id).execute()
if reopen_at is not None and row.count < reopen_at <= row.count + count:
fields["dismissed_at"] = None
Notice.update(**fields).where(Notice.id == row.id).execute()
self._bump_occurrences(kind, count, last_seen)
def _prune(self, kind: str, keep: int) -> None:
@@ -428,8 +362,7 @@ class NoticeRegistry:
NoticeStats.create(
kind=kind,
occurrences=count,
acknowledgements=0,
mutes=0,
dismissals=0,
first_seen=now,
last_seen=now,
)
+6 -10
View File
@@ -28,9 +28,9 @@ class NoticeKind:
category: camera, detector, model, or system; a camera scope is a
camera name, and the UI shows it
link: app route or absolute URL for the row, filled in from params
counts_repeats: whether raising an existing notice counts another
occurrence, which also shows an acknowledged notice again
counts_repeats: whether raising an existing notice counts another occurrence
batch_repeats: whether repeats wait in memory for the next flush
reopen_at_count: count at which a dismissed notice shows again
keep_latest: rows of this kind to keep; a new row drops the oldest
reportable: whether a future analytics reporter may send this kind's counts
"""
@@ -41,6 +41,7 @@ class NoticeKind:
link: str | None = None
counts_repeats: bool = True
batch_repeats: bool = False
reopen_at_count: int | None = None
keep_latest: int | None = None
reportable: bool = True
@@ -66,12 +67,6 @@ _KINDS = (
"camera",
link="/system#cameras",
),
NoticeKind(
"ffmpeg_high_cpu", NoticeSeverity.warning, "camera", link="/system#cameras"
),
NoticeKind(
"detect_high_cpu", NoticeSeverity.warning, "camera", link="/system#cameras"
),
NoticeKind("shm_too_low", NoticeSeverity.warning, "system", link="/system#storage"),
# one row per user per burst; the login log lines carry the address
NoticeKind(
@@ -80,9 +75,10 @@ _KINDS = (
"system",
link="/logs",
batch_repeats=True,
reopen_at_count=5,
keep_latest=100,
),
# one row per release, so muting it lasts until the next release
# one row per release, so a dismissal lasts until the next release
NoticeKind(
"update_available",
NoticeSeverity.info,
@@ -96,7 +92,7 @@ _KINDS = (
NOTICE_KINDS: dict[str, NoticeKind] = {kind.key: kind for kind in _KINDS}
# the Health tab builds config and stream check rows in the browser, so a notice
# row of these kinds only records a mute; its other fields are placeholders
# row of these kinds only records a dismissal; its other fields are placeholders
CHECK_KINDS = frozenset({"config", "stream"})
-22
View File
@@ -1120,18 +1120,6 @@ class OnvifController:
f"Camera {camera_name} is still in ONVIF 'MOVING' status."
)
async def _shutdown(self) -> None:
"""Close the camera sessions and cancel the tasks running on the loop."""
for cam_name in list(self.cams):
await self._close_camera(cam_name)
tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
def close(self) -> None:
"""Gracefully shut down the ONVIF controller."""
if not hasattr(self, "loop") or self.loop.is_closed():
@@ -1139,16 +1127,6 @@ class OnvifController:
return
logger.info("Exiting ONVIF controller...")
# anything left open here is garbage collected during interpreter
# shutdown, where its warnings can no longer be logged cleanly
try:
asyncio.run_coroutine_threadsafe(self._shutdown(), self.loop).result(
timeout=5
)
except TimeoutError:
logger.debug("Timed out closing ONVIF sessions")
self.config_subscriber.stop()
def stop_and_cleanup():
+15 -61
View File
@@ -29,59 +29,38 @@ VERSION_REFRESH_S = 24 * 60 * 60
# a camera skipping at least this percent of its frames is falling behind
SKIPPED_DETECTIONS_PCT = 5
# lifetime CPU averages at or above these are high; they match
# CameraFfmpegThreshold.error and CameraDetectThreshold.error in the UI
FFMPEG_HIGH_CPU_PCT = 20
DETECT_HIGH_CPU_PCT = 40
# a camera stays over a threshold this long before it becomes a notice
EPISODE_HOLD_S = 60
# for at least this long before it becomes a notice
SKIPPED_DETECTIONS_HOLD_S = 60
# detectors warm up after a start; the status bar waits this long too
STARTUP_GRACE_S = 120
def cpu_average(cpu_usages: dict[str, Any], pid: int | None) -> float | None:
"""A process's CPU use averaged over its lifetime, or None if unknown."""
usage = cpu_usages.get(str(pid)) if pid else None
class SkippedDetectionsTracker:
"""Finds cameras whose skipped share stays high long enough for a notice."""
try:
return float(usage["cpu_average"]) if usage else None
except (KeyError, TypeError, ValueError):
return None
class EpisodeTracker:
"""Finds cameras whose value stays over a threshold long enough for a notice."""
def __init__(self, threshold: float) -> None:
self._threshold = threshold
def __init__(self) -> None:
self._since: dict[str, float] = {}
self._raised: set[str] = set()
def update(self, values: dict[str, float | None], now: float) -> list[str]:
"""Return the cameras whose episode qualified on this sample.
Args:
values: Each camera's current value, or None when it has none
now: Sample time in seconds
"""
def update(self, cameras: dict[str, dict[str, Any]], now: float) -> list[str]:
"""Return the cameras whose episode qualified on this sample."""
qualified: list[str] = []
# a removed camera that comes back starts a new episode
for camera in self._since.keys() - values.keys():
for camera in self._since.keys() - cameras.keys():
self._since.pop(camera)
self._raised.discard(camera)
for camera, value in values.items():
if value is None or value < self._threshold:
for camera, camera_stats in cameras.items():
if camera_stats["skipped_pct"] < SKIPPED_DETECTIONS_PCT:
self._since.pop(camera, None)
self._raised.discard(camera)
continue
since = self._since.setdefault(camera, now)
if camera not in self._raised and now - since >= EPISODE_HOLD_S:
if camera not in self._raised and now - since >= SKIPPED_DETECTIONS_HOLD_S:
self._raised.add(camera)
qualified.append(camera)
@@ -101,9 +80,7 @@ class StatsEmitter(threading.Thread):
self.stop_event = stop_event
self.hardware_stats = HardwareStats(config)
self.stats_history: list[dict[str, Any]] = []
self.skipped_detections = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
self.ffmpeg_cpu = EpisodeTracker(FFMPEG_HIGH_CPU_PCT)
self.detect_cpu = EpisodeTracker(DETECT_HIGH_CPU_PCT)
self.skipped_detections = SkippedDetectionsTracker()
# the shm notice's params as last sent, so only a change is written
self._shm_checked = False
@@ -250,38 +227,15 @@ class StatsEmitter(threading.Thread):
def _update_notices(self, stats: dict[str, Any], now: float) -> None:
"""Update notices based on current stats or time."""
cameras = stats["cameras"]
# absent when CPU collection timed out or failed on this tick
cpu_usages = stats.get("cpu_usages", {})
# skipped detections
if stats["service"]["uptime"] >= STARTUP_GRACE_S:
# skipped detections
skipped = {
camera: camera_stats["skipped_pct"]
for camera, camera_stats in cameras.items()
}
for camera in self.skipped_detections.update(skipped, now):
for camera in self.skipped_detections.update(stats["cameras"], now):
raise_notice(
"skipped_detections",
scope=camera,
params={"pct": skipped[camera]},
params={"pct": stats["cameras"][camera]["skipped_pct"]},
)
# high ffmpeg and detect CPU
for tracker, kind, pid_key in (
(self.ffmpeg_cpu, "ffmpeg_high_cpu", "ffmpeg_pid"),
(self.detect_cpu, "detect_high_cpu", "pid"),
):
averages = {
camera: cpu_average(cpu_usages, camera_stats.get(pid_key))
for camera, camera_stats in cameras.items()
}
for camera in tracker.update(averages, now):
raise_notice(kind, scope=camera, params={"cpu": averages[camera]})
# shm too small for the cameras
self._update_shm_notice(stats["service"]["storage"]["/dev/shm"])
@@ -1,193 +0,0 @@
"""Tests for renaming and deleting a zone through config_set's JSON body."""
import os
import tempfile
from unittest.mock import MagicMock, Mock, patch
import ruamel.yaml
from fastapi import Request
from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user
from frigate.api.fastapi_app import create_fastapi_app
from frigate.config import FrigateConfig
from frigate.config.camera.updater import CameraConfigUpdatePublisher
from frigate.models import Event, Recordings, ReviewSegment
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
class TestConfigSetZones(BaseTestHttp):
def setUp(self):
super().setUp(models=[Event, Recordings, ReviewSegment])
self.minimal_config = {
"mqtt": {"host": "mqtt"},
"profiles": {"armed": {"friendly_name": "Armed"}},
"snapshots": {"required_zones": ["driveway"]},
"cameras": {
"front": {
"ffmpeg": {
"inputs": [
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
]
},
"detect": {"height": 1080, "width": 1920, "fps": 5},
"zones": {
"driveway": {
"coordinates": "0,0,1,0,1,1",
"inertia": 5,
"filters": {"person": {"min_area": 5000}},
},
"porch": {"coordinates": "0,0,0.5,0,0.5,0.5"},
},
"review": {
"alerts": {
"labels": ["person"],
"required_zones": ["driveway"],
},
},
"mqtt": {"required_zones": ["driveway", "porch"]},
"profiles": {
"armed": {
"zones": {
"driveway": {
"coordinates": "0,0,1,0,1,1",
"objects": ["car"],
},
},
},
},
},
},
}
yaml = ruamel.yaml.YAML()
with tempfile.NamedTemporaryFile(
mode="w", suffix=".yml", delete=False
) as config_file:
yaml.dump(self.minimal_config, config_file)
self.config_path = config_file.name
self.addCleanup(os.unlink, self.config_path)
def _create_app(self):
publisher = Mock(spec=CameraConfigUpdatePublisher)
publisher.publisher = MagicMock()
app = create_fastapi_app(
FrigateConfig(**self.minimal_config),
self.db,
None,
None,
None,
None,
None,
None,
publisher,
None,
enforce_default_admin=False,
)
async def mock_get_current_user(request: Request):
return {
"username": request.headers.get("remote-user"),
"role": request.headers.get("remote-role"),
}
async def mock_get_allowed_cameras_for_filter(request: Request):
return ["front"]
app.dependency_overrides[get_current_user] = mock_get_current_user
app.dependency_overrides[get_allowed_cameras_for_filter] = (
mock_get_allowed_cameras_for_filter
)
return app
def _put(self, camera_data: dict):
with patch("frigate.api.app.find_config_file", return_value=self.config_path):
with AuthTestClient(self._create_app()) as client:
return client.put(
"/config/set",
json={
"config_data": {"cameras": {"front": camera_data}},
"requires_restart": 0,
"update_topic": "config/cameras/front/zones",
},
)
def _front(self) -> dict:
with open(self.config_path) as f:
return ruamel.yaml.YAML().load(f)["cameras"]["front"]
def test_rename_moves_zone_references_in_one_request(self):
"""The new base zone exists when validation checks the moved override."""
resp = self._put(
{
"zones": {
"driveway": None,
"front_drive": {
"coordinates": "0,0,1,0,1,1",
"enabled": True,
# as /api/config returns them, with defaults filled in
"filters": {
"person": {
"min_area": 5000,
"max_area": 24000000,
"min_ratio": 0.0,
"max_ratio": 24000000.0,
"threshold": 0.7,
"min_score": 0.5,
"mask": {},
}
},
"inertia": 5,
},
},
"review": {"alerts": {"required_zones": ["front_drive"]}},
# inherited from the global snapshots config, so no camera key
"snapshots": {"required_zones": ["front_drive"]},
"mqtt": {"required_zones": ["front_drive", "porch"]},
"profiles": {
"armed": {
"zones": {
"driveway": None,
"front_drive": {
"coordinates": "0,0,1,0,1,1",
"objects": ["car"],
},
},
},
},
}
)
self.assertEqual(resp.status_code, 200, resp.json())
front = self._front()
self.assertEqual(list(front["zones"]), ["porch", "front_drive"])
self.assertEqual(front["zones"]["front_drive"]["inertia"], 5)
self.assertEqual(
front["zones"]["front_drive"]["filters"]["person"]["min_area"], 5000
)
self.assertEqual(front["review"]["alerts"]["labels"], ["person"])
self.assertEqual(front["review"]["alerts"]["required_zones"], ["front_drive"])
self.assertEqual(front["snapshots"]["required_zones"], ["front_drive"])
self.assertEqual(front["mqtt"]["required_zones"], ["front_drive", "porch"])
self.assertEqual(
dict(front["profiles"]["armed"]["zones"]),
{"front_drive": {"coordinates": "0,0,1,0,1,1", "objects": ["car"]}},
)
def test_delete_empties_lists_without_dropping_their_section(self):
resp = self._put(
{
"zones": {"driveway": None},
"review": {"alerts": {"required_zones": []}},
"snapshots": {"required_zones": []},
"mqtt": {"required_zones": ["porch"]},
"profiles": {"armed": {"zones": {"driveway": None}}},
}
)
self.assertEqual(resp.status_code, 200, resp.json())
front = self._front()
self.assertEqual(list(front["zones"]), ["porch"])
self.assertEqual(front["review"]["alerts"]["labels"], ["person"])
self.assertEqual(front["review"]["alerts"]["required_zones"], [])
self.assertEqual(front["mqtt"]["required_zones"], ["porch"])
self.assertNotIn("driveway", front["profiles"]["armed"]["zones"] or {})
+46 -62
View File
@@ -53,108 +53,92 @@ class TestHttpNotices(BaseTestHttp):
self.assertEqual(response.json()[0]["kind"], "detector_stuck")
self.assertEqual(response.json()[0]["occurrences"], 1)
def test_acknowledge_hides_and_the_hidden_list_keeps_it(self):
def test_dismiss_hides_and_history_keeps_it(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
with self.client() as client:
response = client.post("/notices/detector_stuck:ov/acknowledge")
response = client.post("/notices/detector_stuck:ov/dismiss")
listed = client.get("/notices").json()
hidden = client.get("/notices", params={"include_hidden": True}).json()
history = client.get("/notices", params={"include_dismissed": True}).json()
self.assertEqual(response.status_code, 200)
self.assertEqual(listed, [])
self.assertEqual([n["id"] for n in hidden], ["detector_stuck:ov"])
self.assertIsNotNone(hidden[0]["acknowledged_at"])
self.assertEqual([n["id"] for n in history], ["detector_stuck:ov"])
self.assertIsNotNone(history[0]["dismissed_at"])
def test_acknowledging_a_check_is_404(self):
with self.client() as client:
response = client.post(f"/notices/{CONFIG_CHECK}/acknowledge")
self.assertEqual(response.status_code, 404)
def test_mute_an_id_with_a_slash(self):
def test_dismiss_an_id_with_a_slash(self):
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
with self.client() as client:
response = client.post(
"/notices/model_download_failed:yolo/model.onnx/mute"
"/notices/model_download_failed:yolo/model.onnx/dismiss"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(self.registry.active(), [])
def test_unknown_ids_are_404(self):
def test_dismiss_unknown_is_404(self):
with self.client() as client:
responses = [
client.post("/notices/nope/acknowledge"),
client.post("/notices/nope/mute"),
client.delete("/notices/nope/hidden"),
]
response = client.post("/notices/nope/dismiss")
self.assertEqual([r.status_code for r in responses], [404, 404, 404])
self.assertEqual(response.status_code, 404)
def test_requires_admin(self):
viewer = {"remote-user": "viewer", "remote-role": "viewer"}
with TestClient(self.app) as client:
responses = [
client.get("/notices", headers=viewer),
client.get("/notices/muted_checks", headers=viewer),
client.post("/notices/detector_stuck:ov/acknowledge", headers=viewer),
client.post("/notices/detector_stuck:ov/mute", headers=viewer),
client.delete("/notices/detector_stuck:ov/hidden", headers=viewer),
client.delete("/notices/hidden", headers=viewer),
]
response = client.get(
"/notices", headers={"remote-user": "viewer", "remote-role": "viewer"}
)
self.assertEqual([r.status_code for r in responses], [403] * 6)
self.assertEqual(response.status_code, 403)
def test_muted_checks_are_listed_once(self):
def test_dismissed_checks_are_listed_once(self):
with self.client() as client:
first = client.post(f"/notices/{CONFIG_CHECK}/mute")
again = client.post(f"/notices/{CONFIG_CHECK}/mute")
listed = client.get("/notices/muted_checks").json()
hidden = client.get("/notices", params={"include_hidden": True}).json()
first = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
again = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
listed = client.get("/notices/dismissed_checks").json()
history = client.get("/notices", params={"include_dismissed": True}).json()
self.assertEqual(first.status_code, 200)
self.assertEqual(again.status_code, 200)
self.assertEqual([check["id"] for check in listed], [CONFIG_CHECK])
self.assertIsNotNone(listed[0]["muted_at"])
self.assertIsNotNone(listed[0]["dismissed_at"])
# the Health tab builds the row itself, so it never comes back as a notice
self.assertEqual(hidden, [])
self.assertEqual(history, [])
def test_unhide_one_row(self):
def test_dismissed_checks_require_admin(self):
with TestClient(self.app) as client:
response = client.get(
"/notices/dismissed_checks",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
self.assertEqual(response.status_code, 403)
def test_purge_dismissed_clears_notices_and_checks(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.mute("detector_stuck:ov")
self.registry.mute(CONFIG_CHECK)
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss(CONFIG_CHECK)
with self.client() as client:
notice = client.delete("/notices/detector_stuck:ov/hidden")
check = client.delete(f"/notices/{CONFIG_CHECK}/hidden")
listed = client.get("/notices").json()
checks = client.get("/notices/muted_checks").json()
self.assertEqual([notice.status_code, check.status_code], [200, 200])
self.assertEqual([n["id"] for n in listed], ["detector_stuck:ov"])
self.assertEqual(checks, [])
def test_unhide_all_shows_notices_and_drops_check_mutes(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.acknowledge("detector_stuck:ov")
self.registry.mute(CONFIG_CHECK)
with self.client() as client:
response = client.delete("/notices/hidden")
listed = client.get("/notices").json()
checks = client.get("/notices/muted_checks").json()
response = client.delete("/notices/dismissed")
history = client.get("/notices", params={"include_dismissed": True}).json()
checks = client.get("/notices/dismissed_checks").json()
self.assertEqual(response.status_code, 200)
self.assertEqual([n["id"] for n in listed], ["detector_stuck:ov"])
self.assertEqual(history, [])
self.assertEqual(checks, [])
def test_purge_dismissed_requires_admin(self):
with TestClient(self.app) as client:
response = client.delete(
"/notices/dismissed",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
self.assertEqual(response.status_code, 403)
-90
View File
@@ -1,90 +0,0 @@
"""Tests for audio maintainer spawning on runtime camera adds."""
import threading
import unittest
from unittest.mock import MagicMock, patch
from frigate.config import FrigateConfig
from frigate.events.audio import AudioProcessor
def _build_config() -> FrigateConfig:
return FrigateConfig(
**{
"mqtt": {"host": "mqtt"},
"cameras": {
"front_door": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "audio"],
}
]
},
"audio": {"enabled": True},
}
},
}
)
class TestAudioSpawnIfNeeded(unittest.TestCase):
def _processor(self, config: FrigateConfig, camera_metrics: dict) -> AudioProcessor:
processor = AudioProcessor.__new__(AudioProcessor)
processor.config = config
processor.camera_metrics = camera_metrics
processor.embeddings_metrics = MagicMock()
processor.audio_threads = {}
processor.transcription_model_runner = None
processor.genai_manager = None
processor.stop_event = threading.Event()
processor.logger = MagicMock()
return processor
def test_skips_camera_whose_metrics_have_not_been_created_yet(self):
"""The camera maintainer creates metrics on its own poll of the add
update, so the audio processor can see the camera first."""
config = _build_config()
processor = self._processor(config, {})
with patch("frigate.events.audio.AudioEventMaintainer") as maintainer:
processor.spawn_if_needed(config.cameras["front_door"])
maintainer.assert_not_called()
assert processor.audio_threads == {}
def test_spawns_once_metrics_exist_and_passes_the_metrics_object(self):
config = _build_config()
metrics = object()
processor = self._processor(config, {"front_door": metrics})
with patch("frigate.events.audio.AudioEventMaintainer") as maintainer:
processor.spawn_if_needed(config.cameras["front_door"])
assert maintainer.call_args.args[2] is metrics
assert "front_door" in processor.audio_threads
maintainer.return_value.start.assert_called_once()
def test_does_not_respawn_for_a_camera_already_running(self):
config = _build_config()
processor = self._processor(config, {"front_door": object()})
processor.audio_threads["front_door"] = MagicMock()
with patch("frigate.events.audio.AudioEventMaintainer") as maintainer:
processor.spawn_if_needed(config.cameras["front_door"])
maintainer.assert_not_called()
def test_skips_camera_whose_ffmpeg_update_has_not_arrived_yet(self):
"""The add update carries the camera before its ffmpeg inputs are
applied, so the audio role can be briefly missing."""
config = _build_config()
camera = config.cameras["front_door"]
camera.ffmpeg.inputs[0].roles = ["detect"]
processor = self._processor(config, {"front_door": object()})
with patch("frigate.events.audio.AudioEventMaintainer") as maintainer:
processor.spawn_if_needed(camera)
maintainer.assert_not_called()
@@ -1,286 +0,0 @@
"""Config validation and the historical path for the audio_transcription GenAI backend."""
import unittest
from copy import deepcopy
from unittest.mock import MagicMock, patch
from pydantic import ValidationError
from frigate.config import FrigateConfig
from frigate.config.camera.genai import GenAIConfig, GenAIRoleEnum
from frigate.config.classification import AudioTranscriptionModelEnum
from frigate.const import UPDATE_EVENT_DESCRIPTION
from frigate.data_processing.post.audio_transcription import (
AudioTranscriptionPostProcessor,
)
from frigate.data_processing.types import PostProcessDataEnum
class TestAudioTranscriptionGenAIConfig(unittest.TestCase):
def setUp(self):
self.base = {
"mqtt": {"host": "mqtt"},
"cameras": {
"back": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "audio"],
}
]
},
"detect": {"height": 1080, "width": 1920, "fps": 5},
"audio": {"enabled": True},
}
},
}
def _config(self, **overrides) -> dict:
config = deepcopy(self.base)
config.update(deepcopy(overrides))
return config
def _provider(self, roles: list[str]) -> dict:
return {
"whisper_cloud": {
"provider": "openai",
"model": "gpt-4o-transcribe",
"api_key": "k",
"roles": roles,
}
}
def test_default_model_is_whisper_enum(self):
config = FrigateConfig(**self._config())
self.assertEqual(
config.audio_transcription.model, AudioTranscriptionModelEnum.whisper
)
def test_whisper_string_coerces_to_enum(self):
config = FrigateConfig(
**self._config(audio_transcription={"enabled": True, "model": "whisper"})
)
self.assertIsInstance(
config.audio_transcription.model, AudioTranscriptionModelEnum
)
def test_provider_name_stays_a_string(self):
config = FrigateConfig(
**self._config(
genai=self._provider(["transcribe"]),
audio_transcription={"enabled": True, "model": "whisper_cloud"},
)
)
self.assertNotIsInstance(
config.audio_transcription.model, AudioTranscriptionModelEnum
)
self.assertEqual(config.audio_transcription.model, "whisper_cloud")
def test_unspecified_model_falls_back_to_whisper(self):
"""An empty value must not read as a GenAI provider that resolves to no client."""
for value in (None, "", " "):
with self.subTest(repr(value)):
config = FrigateConfig(
**self._config(
audio_transcription={"enabled": True, "model": value}
)
)
self.assertIs(
config.audio_transcription.model,
AudioTranscriptionModelEnum.whisper,
)
def test_missing_genai_key_raises(self):
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(
**self._config(
audio_transcription={"enabled": True, "model": "nope"},
)
)
self.assertIn("is not a valid GenAI config key", str(ctx.exception))
def test_provider_without_role_raises(self):
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(
**self._config(
genai=self._provider(["descriptions"]),
audio_transcription={"enabled": True, "model": "whisper_cloud"},
)
)
self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
def test_global_off_camera_on_still_validates(self):
"""Global-off/camera-on is a supported deployment and must not skip the check."""
config = self._config(
genai=self._provider(["descriptions"]),
audio_transcription={"enabled": False, "model": "whisper_cloud"},
)
config["cameras"]["back"]["audio_transcription"] = {"enabled": True}
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(**config)
self.assertIn("must have 'transcribe' in its roles", str(ctx.exception))
def test_disabled_transcription_skips_validation(self):
config = FrigateConfig(
**self._config(audio_transcription={"enabled": False, "model": "nope"})
)
self.assertEqual(config.audio_transcription.model, "nope")
def test_camera_level_model_is_rejected(self):
config = self._config()
config["cameras"]["back"]["audio_transcription"] = {
"enabled": True,
"model": "whisper",
}
with self.assertRaises(ValidationError):
FrigateConfig(**config)
def test_default_roles_do_not_include_transcribe(self):
"""Backward compatibility: existing providers must not silently claim it."""
genai = GenAIConfig(provider="openai", model="gpt-4o")
self.assertNotIn(GenAIRoleEnum.transcribe, genai.roles)
def test_two_providers_claiming_transcribe_raises(self):
genai = self._provider(["transcribe"])
genai["other"] = {
"provider": "gemini",
"model": "gemini-2.0-flash",
"api_key": "k",
"roles": ["transcribe"],
}
with self.assertRaises(ValidationError) as ctx:
FrigateConfig(
**self._config(
genai=genai,
audio_transcription={"enabled": True, "model": "whisper_cloud"},
)
)
self.assertIn("each role must have", str(ctx.exception))
def test_transcribe_rejected_on_provider_without_audio_input(self):
with self.assertRaises(ValidationError) as ctx:
GenAIConfig(provider="ollama", model="llava", roles=["transcribe"])
self.assertIn("does not support audio input", str(ctx.exception))
class TestAudioTranscriptionPostProcessorGenAI(unittest.TestCase):
"""The recorded-speech path must reach the provider and skip the local model."""
def setUp(self):
self.config = FrigateConfig(
**{
"mqtt": {"host": "mqtt"},
"genai": {
"whisper_cloud": {
"provider": "openai",
"model": "gpt-4o-transcribe",
"api_key": "k",
"roles": ["transcribe"],
}
},
"audio_transcription": {
"enabled": True,
"model": "whisper_cloud",
"language": "en",
},
"cameras": {
"back": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "audio"],
}
]
},
"detect": {"height": 1080, "width": 1920, "fps": 5},
"audio": {"enabled": True},
}
},
}
)
self.client = MagicMock()
self.client.transcribe.return_value = "recorded speech"
self.manager = MagicMock()
self.manager.transcribe_client = self.client
self.requestor = MagicMock()
self.processor = AudioTranscriptionPostProcessor(
self.config,
self.requestor,
MagicMock(),
MagicMock(),
self.manager,
)
def _process(self):
self.processor.process_data(
{
"event_id": "1234.5-abc",
"camera": "back",
"event": {
"id": "1234.5-abc",
"camera": "back",
"start_time": 100.0,
"end_time": 110.0,
"data": {},
},
},
PostProcessDataEnum.tracked_object,
)
def test_local_recognizer_is_never_built(self):
self.assertTrue(self.processor._use_genai)
self.assertIsNone(self.processor.recognizer)
def test_audio_bytes_and_language_reach_the_client(self):
with patch(
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
return_value=b"RIFF....WAVE",
):
self._process()
self.client.transcribe.assert_called_once()
self.assertEqual(self.client.transcribe.call_args.args[0], b"RIFF....WAVE")
self.assertEqual(self.client.transcribe.call_args.kwargs["language"], "en")
def test_transcript_is_published_as_the_description(self):
with patch(
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
return_value=b"RIFF....WAVE",
):
self._process()
topics = [call.args[0] for call in self.requestor.send_data.call_args_list]
self.assertIn(UPDATE_EVENT_DESCRIPTION, topics)
payload = next(
call.args[1]
for call in self.requestor.send_data.call_args_list
if call.args[0] == UPDATE_EVENT_DESCRIPTION
)
self.assertEqual(payload["description"], "recorded speech")
self.assertEqual(payload["id"], "1234.5-abc")
def test_missing_client_publishes_nothing(self):
self.manager.transcribe_client = None
with patch(
"frigate.data_processing.post.audio_transcription.get_audio_from_recording",
return_value=b"RIFF....WAVE",
):
self._process()
self.requestor.send_data.assert_not_called()
if __name__ == "__main__":
unittest.main()
@@ -1,288 +0,0 @@
"""Live GenAI transcription: sliding overlapped windows and their lifecycle."""
import io
import threading
import unittest
import wave
from unittest.mock import MagicMock, patch
import numpy as np
from frigate.config import FrigateConfig
from frigate.const import AUDIO_DURATION, AUDIO_SAMPLE_RATE
from frigate.data_processing.real_time.audio_transcription import (
GENAI_WINDOW_CHUNKS,
AudioTranscriptionRealTimeProcessor,
)
CHUNK_SAMPLES = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE))
def _chunk(amplitude: int) -> np.ndarray:
"""One audio-detector-sized chunk of int16 samples at a constant amplitude."""
return np.full(CHUNK_SAMPLES, amplitude, dtype=np.int16)
class TestLiveGenAITranscription(unittest.TestCase):
def setUp(self):
self.config = FrigateConfig(
**{
"mqtt": {"host": "mqtt"},
"genai": {
"whisper_cloud": {
"provider": "openai",
"model": "gpt-4o-transcribe",
"api_key": "k",
"roles": ["transcribe"],
}
},
"audio_transcription": {
"enabled": True,
"model": "whisper_cloud",
"language": "en",
},
"cameras": {
"back": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "audio"],
}
]
},
"detect": {"height": 1080, "width": 1920, "fps": 5},
"audio": {"enabled": True},
}
},
}
)
self.client = MagicMock()
self.client.transcribe.return_value = "hello"
self.manager = MagicMock()
self.manager.transcribe_client = self.client
self.requestor = MagicMock()
self.processor = AudioTranscriptionRealTimeProcessor(
config=self.config,
camera_config=self.config.cameras["back"],
requestor=self.requestor,
model_runner=None,
metrics=MagicMock(),
stop_event=threading.Event(),
genai_manager=self.manager,
)
def _feed(self, chunk: np.ndarray):
return (
self.processor._AudioTranscriptionRealTimeProcessor__process_audio_stream(
chunk
)
)
def _sent_wav(self, call_index: int) -> wave.Wave_read:
payload = self.client.transcribe.call_args_list[call_index].args[0]
return wave.open(io.BytesIO(payload), "rb")
def test_uses_genai_path(self):
self.assertTrue(self.processor._use_genai)
def test_first_chunk_does_not_transcribe(self):
self.assertIsNone(self._feed(_chunk(4000)))
self.client.transcribe.assert_not_called()
def test_full_window_transcribes_two_chunks(self):
self._feed(_chunk(4000))
result = self._feed(_chunk(4000))
self.assertEqual(result, ("hello", False))
self.client.transcribe.assert_called_once()
self.assertEqual(
self.client.transcribe.call_args.kwargs["language"],
"en",
)
with self._sent_wav(0) as wav:
self.assertEqual(wav.getnchannels(), 1)
self.assertEqual(wav.getsampwidth(), 2)
self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
def test_window_slides_with_overlap(self):
"""The third chunk's window is chunks 2+3, not 3 alone and not 1+2+3."""
self.client.transcribe.side_effect = ["one two", "two three"]
self._feed(_chunk(1000))
self._feed(_chunk(2000))
result = self._feed(_chunk(3000))
self.assertEqual(self.client.transcribe.call_count, 2)
with self._sent_wav(1) as wav:
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
self.assertEqual(samples[0], 2000)
self.assertEqual(samples[-1], 3000)
# the shared "two" appears once
self.assertEqual(result, ("one two three", False))
def test_silent_window_is_not_uploaded(self):
self._feed(_chunk(0))
self.assertIsNone(self._feed(_chunk(0)))
self.client.transcribe.assert_not_called()
def test_silent_window_ends_a_pending_utterance(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
# the gate covers the whole window, so it takes GENAI_WINDOW_CHUNKS
# silent chunks to push the last speech out of it
self._feed(_chunk(0))
self.assertEqual(self._feed(_chunk(0)), ("hello", True))
def test_empty_transcript_ends_a_pending_utterance(self):
self.client.transcribe.side_effect = ["hello", ""]
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
def test_empty_transcript_with_nothing_pending_returns_none(self):
self.client.transcribe.return_value = ""
self._feed(_chunk(4000))
self.assertIsNone(self._feed(_chunk(4000)))
def test_reset_clears_committed_text_and_window(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.processor.reset()
self.assertEqual(self.processor._genai_committed, "")
self.assertEqual(len(self.processor._genai_window), 0)
# a fresh window is required again before the next request
self.client.transcribe.reset_mock()
self._feed(_chunk(4000))
self.client.transcribe.assert_not_called()
def test_check_unload_model_clears_once_then_is_idempotent(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.processor.check_unload_model()
self.requestor.send_data.assert_called_once_with("back/audio/transcription", "")
self.assertEqual(self.processor._genai_committed, "")
self.assertEqual(len(self.processor._genai_window), 0)
self.processor.check_unload_model()
self.requestor.send_data.assert_called_once()
def test_build_recognizer_never_loads_a_local_model(self):
with patch(
"frigate.data_processing.real_time.audio_transcription.FasterWhisperASR"
) as whisper:
self.processor._AudioTranscriptionRealTimeProcessor__build_recognizer()
whisper.assert_not_called()
self.assertIsNone(self.processor.stream)
def test_clear_audio_recognizer_request_only_resets(self):
self._feed(_chunk(4000))
self._feed(_chunk(4000))
with patch.object(
self.processor,
"_AudioTranscriptionRealTimeProcessor__build_recognizer",
) as build:
result = self.processor.handle_request("clear_audio_recognizer", {})
build.assert_not_called()
self.assertTrue(result["success"])
self.assertEqual(self.processor._genai_committed, "")
def test_missing_client_logs_and_returns_none(self):
self.manager.transcribe_client = None
self.assertIsNone(self._feed(_chunk(4000)))
self.assertIsNone(self._feed(_chunk(4000)))
def test_dropped_audio_discards_the_buffered_window(self):
"""A gap in the stream must not be spliced into a single window.
Dropping a queued chunk leaves the next one non-adjacent to what is
buffered, so concatenating them would hand the provider audio with a
hole in it and break the 50% overlap the stitcher relies on.
"""
self._feed(_chunk(4000))
self.assertEqual(len(self.processor._genai_window), 1)
# the producer discards a chunk while the consumer is blocked
self.processor._audio_dropped.set()
self._feed(_chunk(5000))
# the buffered chunk was discarded, so this one starts a fresh window
self.assertEqual(len(self.processor._genai_window), 1)
self.client.transcribe.assert_not_called()
# and the window that does go out holds only contiguous audio
self._feed(_chunk(5000))
self.client.transcribe.assert_called_once()
with self._sent_wav(0) as wav:
samples = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
self.assertEqual(wav.getnframes(), CHUNK_SAMPLES * GENAI_WINDOW_CHUNKS)
self.assertTrue((samples == 5000).all(), "window spliced across the gap")
def test_dropped_audio_ends_a_pending_utterance(self):
"""Committed text cannot be stitched across missing speech."""
self._feed(_chunk(4000))
self._feed(_chunk(4000))
self.assertEqual(self.processor._genai_committed, "hello")
self.processor._audio_dropped.set()
self.assertEqual(self._feed(_chunk(4000)), ("hello", True))
self.assertEqual(len(self.processor._genai_window), 0)
def test_drop_flag_is_consumed_once(self):
self.processor._audio_dropped.set()
self._feed(_chunk(4000))
self.assertFalse(self.processor._audio_dropped.is_set())
def test_full_queue_flags_a_drop(self):
for i in range(self.processor.audio_queue.maxsize + 1):
self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
self.assertTrue(self.processor._audio_dropped.is_set())
def test_queue_is_bounded_and_drops_oldest(self):
maxsize = self.processor.audio_queue.maxsize
self.assertGreater(maxsize, 0)
for i in range(maxsize + 5):
self.processor.process_audio({"id": "back_audio"}, _chunk(i + 1))
self.assertEqual(self.processor.audio_queue.qsize(), maxsize)
# the newest chunk survived, the oldest did not
remaining = []
while not self.processor.audio_queue.empty():
remaining.append(self.processor.audio_queue.get_nowait()[1][0])
self.assertEqual(remaining[-1], maxsize + 5)
self.assertNotIn(1, remaining)
if __name__ == "__main__":
unittest.main()
-176
View File
@@ -1,176 +0,0 @@
"""Tests for the WAV helpers and transcript stitcher in frigate.util.audio."""
import io
import struct
import unittest
import wave
import numpy as np
from frigate.const import AUDIO_SAMPLE_RATE
from frigate.util.audio import fix_wav_header, pcm16_to_wav, stitch_transcripts
def _wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
buffer = io.BytesIO()
with wave.open(buffer, "wb") as out:
out.setnchannels(1)
out.setsampwidth(2)
out.setframerate(sample_rate)
out.writeframes(samples.tobytes())
return buffer.getvalue()
class TestPcm16ToWav(unittest.TestCase):
def test_round_trips_through_wave(self):
samples = np.arange(-1000, 1000, dtype=np.int16)
with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
self.assertEqual(wav.getnchannels(), 1)
self.assertEqual(wav.getsampwidth(), 2)
self.assertEqual(wav.getframerate(), AUDIO_SAMPLE_RATE)
self.assertEqual(wav.getnframes(), samples.size)
decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
np.testing.assert_array_equal(decoded, samples)
def test_casts_non_int16_input(self):
samples = np.array([0.0, 100.0, -100.0], dtype=np.float32)
with wave.open(io.BytesIO(pcm16_to_wav(samples)), "rb") as wav:
decoded = np.frombuffer(wav.readframes(wav.getnframes()), dtype=np.int16)
np.testing.assert_array_equal(decoded, np.array([0, 100, -100], np.int16))
def test_honors_sample_rate(self):
with wave.open(
io.BytesIO(pcm16_to_wav(np.zeros(4, np.int16), 8000)), "rb"
) as w:
self.assertEqual(w.getframerate(), 8000)
class TestFixWavHeader(unittest.TestCase):
def test_rewrites_placeholder_sizes(self):
samples = np.arange(64, dtype=np.int16)
data = bytearray(_wav(samples))
# ffmpeg piping to non-seekable stdout leaves both sizes unpatched
struct.pack_into("<I", data, 4, 0xFFFFFFFF)
data_offset = data.index(b"data")
struct.pack_into("<I", data, data_offset + 4, 0xFFFFFFFF)
fixed = fix_wav_header(bytes(data))
self.assertEqual(struct.unpack_from("<I", fixed, 4)[0], len(fixed) - 8)
self.assertEqual(
struct.unpack_from("<I", fixed, data_offset + 4)[0],
len(fixed) - (data_offset + 8),
)
with wave.open(io.BytesIO(fixed), "rb") as wav:
self.assertEqual(wav.getnframes(), samples.size)
def test_leaves_a_well_formed_header_alone(self):
data = _wav(np.arange(32, dtype=np.int16))
self.assertEqual(fix_wav_header(data), data)
def test_non_riff_payload_passes_through(self):
self.assertEqual(fix_wav_header(b"not a wav"), b"not a wav")
self.assertEqual(fix_wav_header(b""), b"")
class TestStitchTranscripts(unittest.TestCase):
def test_table(self):
cases = [
# (committed, incoming, expected, description)
(
"the quick brown",
"brown fox jumps",
"the quick brown fox jumps",
"one word",
),
(
"and then the quick brown",
"the quick brown fox",
"and then the quick brown fox",
"multi word",
),
(
"hello there",
"general kenobi",
"hello there general kenobi",
"no overlap",
),
("the quick brown fox", "brown fox", "the quick brown fox", "contained"),
("", "first words", "first words", "empty committed"),
("already here", "", "already here", "empty incoming"),
(
" spaced out ",
"out again",
"spaced out again",
"whitespace normalized",
),
]
for committed, incoming, expected, description in cases:
with self.subTest(description):
self.assertEqual(stitch_transcripts(committed, incoming), expected)
def test_overlap_found_mid_window(self):
"""The shared run is rarely at the start of the new window.
The provider re-transcribes the overlapping audio independently and
often renders its first word differently, so anchoring the match to the
start of the incoming window duplicates the whole phrase.
"""
self.assertEqual(
stitch_transcripts(
"this is just gonna be a fun time", "It's gonna be a fun time."
),
"this is just gonna be a fun time",
)
def test_overlap_longer_than_five_words(self):
"""The cap is bounded by window duration, not by the old 5-word n-gram."""
self.assertEqual(
stitch_transcripts(
"well anyway one two three four five six",
"one two three four five six seven",
),
"well anyway one two three four five six seven",
)
def test_repeated_phrase_keeps_its_second_utterance(self):
"""Preferring the earliest match is what protects a real repeat."""
self.assertEqual(
stitch_transcripts("a b c fun time", "fun time fun time"),
"a b c fun time fun time",
)
def test_window_wholly_repeating_the_tail_is_dropped(self):
"""The accepted trade-off: an entirely redundant window adds nothing."""
self.assertEqual(stitch_transcripts("go go go", "go go go"), "go go go")
def test_revises_a_mistranscribed_tail(self):
"""A wrong last word would otherwise block every alignment.
Those words came from the newest audio, which the next window re-covers,
so replacing them is better than duplicating the phrase behind them.
"""
self.assertEqual(
stitch_transcripts("Yeah. this is Jessica.", "This is just gonna be fun."),
"Yeah. this is just gonna be fun.",
)
def test_revision_needs_more_than_one_shared_word(self):
"""A revision deletes published text, so it takes real evidence."""
self.assertEqual(
stitch_transcripts("the cat sat on a mat", "a dog barked"),
"the cat sat on a mat a dog barked",
)
if __name__ == "__main__":
unittest.main()
-25
View File
@@ -36,31 +36,6 @@ class TestEventsPerSecond(unittest.TestCase):
clock[0] += 100.0
self.assertEqual(eps.eps(), 0.0)
def test_burst_after_start_is_not_divided_by_a_tiny_window(self) -> None:
eps = EventsPerSecond(last_n_seconds=10)
clock = [1000.0]
with patch("frigate.util.builtin.time.monotonic", side_effect=lambda: clock[0]):
eps.start()
# eleven buffered frames arrive within 100 ms of starting
for _ in range(11):
clock[0] += 0.01
eps.update()
# 11 events over less than a second is at most 11 per second
self.assertLessEqual(eps.eps(), 11.0)
def test_subsecond_window_keeps_its_rate(self) -> None:
eps = EventsPerSecond(last_n_seconds=0.5)
clock = [1000.0]
with patch("frigate.util.builtin.time.monotonic", side_effect=lambda: clock[0]):
eps.start()
# twenty events per second for two seconds
for _ in range(40):
clock[0] += 0.05
eps.update()
# read between events, so none sits exactly on the window edge
clock[0] += 0.01
self.assertAlmostEqual(eps.eps(), 20.0)
if __name__ == "__main__":
unittest.main()
-896
View File
@@ -1,896 +0,0 @@
"""Tests for the DEEPX detector."""
import json
import os
import struct
import sys
import tempfile
import unittest
from unittest.mock import MagicMock, patch
import numpy as np
from pydantic import ValidationError
from frigate.detectors.detector_config import ModelConfig, ModelTypeEnum
from frigate.detectors.device import (
DeviceParseError,
build_detector_config,
parse_device,
)
from frigate.detectors.plugins.deepx import (
DEEPX_MANIFEST,
DXRT_VERSION,
PPU_RECORD_SIZE,
DeepxDetector,
DeepxDetectorConfig,
PpuLayout,
YoloLayout,
class_count,
decode_raw_anchor,
decode_raw_nms_in_head,
infer_yolo_layout,
read_ppu_layout,
resolve_device,
validate_yolox_outputs,
)
def model_with_type(model_type) -> ModelConfig:
return ModelConfig(
model_type=model_type,
labelmap_path=None,
labelmap={79: "toothbrush"},
width=640,
height=640,
)
def build_ppu_record(box, score=0.9, label=0, grid=(7, 9, 2, 2)) -> np.ndarray:
record = np.zeros(PPU_RECORD_SIZE, dtype=np.uint8)
record[0:16] = np.array(box, dtype=np.float32).view(np.uint8)
record[16:20] = grid
record[20:24] = np.array([score], dtype=np.float32).view(np.uint8)
record[24:28] = np.array([label], dtype=np.uint32).view(np.uint8)
return record.reshape(1, 1, PPU_RECORD_SIZE)
# compile_config.ppu as DX-COM writes it for the two head kinds
ANCHOR_BASED_PPU = {"type": 0, "num_classes": 80, "activation": "Sigmoid"}
ANCHOR_FREE_PPU = {"type": 1, "num_classes": 80}
PPU_BBOX_NODE = "/head/Mul_2"
# (grid_w, grid_h, entries) per scale, finest first; the grids give the
# strides at a 640 input
THREE_SCALE_ANCHORS = [(80, 80, 3), (40, 40, 3), (20, 20, 3)]
TWO_SCALE_ANCHORS = [(40, 40, 3), (20, 20, 3)]
FOUR_SCALE_ANCHORS = [(160, 160, 3), (80, 80, 3), (40, 40, 3), (20, 20, 3)]
THREE_SCALE_FREE = [(80, 80, 1), (40, 40, 1), (20, 20, 1)]
ONE_SCALE_FREE = [(100, 84, 1)]
def proto_varint(value: int) -> bytes:
out = bytearray()
while True:
byte = value & 0x7F
value >>= 7
out.append(byte | (0x80 if value else 0))
if not value:
return bytes(out)
def proto_bytes(field: int, payload: bytes) -> bytes:
return proto_varint(field << 3 | 2) + proto_varint(len(payload)) + payload
def proto_number(field: int, value: int) -> bytes:
return proto_varint(field << 3) + proto_varint(value)
def onnx_node(op_type: str, name: str, inputs: list, outputs: list) -> bytes:
body = b"".join(proto_bytes(1, tensor.encode()) for tensor in inputs)
body += b"".join(proto_bytes(2, tensor.encode()) for tensor in outputs)
return body + proto_bytes(3, name.encode()) + proto_bytes(4, op_type.encode())
def onnx_box_graph(box_format: str) -> bytes:
if box_format == "broken":
return proto_varint(1 << 3 | 3)
unnamed = box_format == "unnamed"
def graph_node(op_type: str, name: str, inputs: list, outputs: list) -> bytes:
return onnx_node(op_type, "" if unnamed else name, inputs, outputs)
nodes = [
graph_node("Split", "dfl_split", ["dfl", "sizes"], ["lt", "rb"]),
graph_node("Sub", "corner_min", ["anchors", "lt"], ["x1y1"]),
graph_node("Add", "corner_max", ["rb", "anchors"], ["x2y2"]),
]
if box_format in ("centre", "unnamed"):
nodes += [
graph_node("Add", "corner_sum", ["x1y1", "x2y2"], ["sum"]),
graph_node("Mul", "corner_mean", ["sum", "half"], ["cxy"]),
graph_node("Sub", "corner_span", ["x2y2", "x1y1"], ["wh"]),
graph_node("Concat", "box_concat", ["cxy", "wh"], ["box"]),
]
elif box_format == "corner":
nodes.append(graph_node("Concat", "box_concat", ["x1y1", "x2y2"], ["box"]))
else:
nodes.append(graph_node("Concat", "box_concat", ["x1y1", "x1y1"], ["box"]))
box = "box"
if unnamed:
box = "box_flat"
nodes.append(graph_node("Reshape", "box_reshape", ["shape", "box"], [box]))
nodes.append(onnx_node("Mul", PPU_BBOX_NODE, [box, "strides"], ["bbox_out"]))
graph = b"".join(proto_bytes(1, node) for node in nodes)
graph += proto_bytes(5, b"weights")
return (
proto_number(1, 10) # ir_version
+ proto_bytes(2, b"onnx_frontend_compiler") # producer_name
+ proto_bytes(7, graph)
)
def write_dxnn(
directory, ppu, layers, name="model.dxnn", table=True, box_format=None
) -> str:
"""A minimal .dxnn as DX-RT's parsers read it: the container header, a
compile_config carrying `ppu`, and either the PPU tensor table with one
entry per (layer, anchor) as a v8 file has, or with `table` False only
the rmap_info listing of the PPU output tensors, as a v7 file has.
`layers` is (grid_w, grid_h, entries) per scale, finest first; an
anchor-free scale has one entry, and grid_h 1 means a flattened
(1, cells, channels) tensor. `box_format`, "centre" or "corner", adds
the compiled graph that says how the head writes its boxes, along with
the compile_config layer naming the node the PPU reads them from."""
if box_format is not None and "layer" not in ppu:
ppu = dict(ppu, layer=[{"bbox": PPU_BBOX_NODE, "cls_conf": "/head/Sigmoid"}])
compile_config = json.dumps({"compile_version": "2.4.0", "ppu": ppu}).encode()
graph = onnx_box_graph(box_format) if box_format is not None else b""
if table:
part = bytearray(struct.pack("<BBBB", 1, sum(n for _, _, n in layers), 0, 0))
for conv, (grid_w, grid_h, entries) in enumerate(layers):
for anchor in range(entries):
part += struct.pack(
"<HHfBBBBBBBB",
128,
80,
0.001,
conv,
anchor,
0,
1,
0,
grid_w,
grid_h,
0,
)
else:
outputs = []
for conv, (grid_w, grid_h, entries) in enumerate(layers):
for anchor in range(entries):
name_ = f"PPU_Transpose_Output_{conv}"
if entries > 1:
name_ += f"_anchor_{anchor}"
shape = [1, grid_w, 127] if grid_h == 1 else [1, grid_h, grid_w, 128]
outputs.append({"name": name_, "shape": shape, "layout": "PPU_YOLO"})
part = json.dumps({"inputs": [], "outputs": outputs}).encode()
data = {
"compile_config": {
"type": "str",
"offset": 0,
"size": len(compile_config),
},
"compiled_data": {
"M1A_4K": {
"npu_0": {
"rmap": {"type": "bytes", "offset": 0, "size": 0},
"ppu" if table else "rmap_info": {
"type": "bytes" if table else "str",
"offset": len(compile_config),
"size": len(part),
},
}
}
},
}
if graph:
data["vis_npu_models"] = {
"npu_0": {
"type": "bytes",
"offset": len(compile_config) + len(part),
"size": len(graph),
}
}
index = json.dumps(
{
"version": 8 if table else 7,
"signature": "DXNN",
"size": 8192,
"data": data,
}
).encode()
path = os.path.join(directory, name)
with open(path, "wb") as model:
model.write(b"DXNN" + struct.pack("<I", 8))
model.write(index.ljust(8192 - 8, b"\0"))
model.write(compile_config + part + graph)
return path
def layout_of(shapes, num_classes, ppu=False, dynamic_output=False) -> YoloLayout:
return infer_yolo_layout(shapes, num_classes, ppu, dynamic_output).layout
class TestDeepxModelFile(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
def layout(self, ppu, layers, **kwargs) -> PpuLayout | None:
return read_ppu_layout(write_dxnn(self.tmp.name, ppu, layers, **kwargs))
def test_the_head_kind_and_one_grid_per_scale_are_read(self):
cases = {
"anchor-based: three anchors per scale collapse to one grid each": (
(ANCHOR_BASED_PPU, THREE_SCALE_ANCHORS, {}),
PpuLayout(anchor_based=True, grids=((80, 80), (40, 40), (20, 20))),
),
"anchor-free, one scale flattened into a single tensor": (
(ANCHOR_FREE_PPU, ONE_SCALE_FREE, {"box_format": "centre"}),
PpuLayout(anchor_based=False, grids=((100, 84),), centre_boxes=True),
),
"a head kind the compiler does not name still yields the scales": (
({"type": 7}, [(80, 80, 3), (40, 40, 3)], {}),
PpuLayout(anchor_based=None, grids=((80, 80), (40, 40))),
),
"no table: the per-anchor split says anchor-based": (
({"num_classes": 80}, THREE_SCALE_ANCHORS, {"table": False}),
PpuLayout(anchor_based=True, grids=((80, 80), (40, 40), (20, 20))),
),
"no table: compile_config places the scales ahead of the names": (
(
{
"type": 1,
"outputs": {
f"PPU_Transpose_Output_{i}": {"conv_idx": 2 - i}
for i in range(3)
},
},
[(20, 20, 1), (40, 40, 1), (80, 80, 1)],
{"table": False},
),
PpuLayout(anchor_based=False, grids=((80, 80), (40, 40), (20, 20))),
),
"no table: every scale in one (1, cells, channels) tensor": (
(ANCHOR_FREE_PPU, [(8400, 1, 1)], {"table": False}),
PpuLayout(anchor_based=False, grids=((8400, 1),)),
),
}
for head, ((ppu, layers, kwargs), expected) in cases.items():
with self.subTest(head=head):
self.assertEqual(self.layout(ppu, layers, **kwargs), expected)
def test_the_box_format_comes_from_the_compiled_graph(self):
for box_format, centre in (
("centre", True),
("corner", False),
("unnamed", True),
):
with self.subTest(box_format=box_format):
self.assertEqual(
self.layout(ANCHOR_FREE_PPU, ONE_SCALE_FREE, box_format=box_format),
PpuLayout(
anchor_based=False, grids=((100, 84),), centre_boxes=centre
),
)
def test_a_box_format_the_graph_does_not_answer_is_left_open(self):
cases = {
"no compiled graph at all": (ANCHOR_FREE_PPU, ONE_SCALE_FREE, {}),
"a graph without the node compile_config names": (
dict(ANCHOR_FREE_PPU, layer=[{"bbox": "/head/Missing"}]),
ONE_SCALE_FREE,
{"box_format": "centre"},
),
"a graph that builds its boxes from neither shape": (
ANCHOR_FREE_PPU,
ONE_SCALE_FREE,
{"box_format": "neither"},
),
"a graph section the reader cannot make sense of": (
ANCHOR_FREE_PPU,
ONE_SCALE_FREE,
{"box_format": "broken"},
),
"a grid-decoded head, where the question does not arise": (
ANCHOR_FREE_PPU,
THREE_SCALE_FREE,
{"box_format": "centre"},
),
}
for graph, (ppu, layers, kwargs) in cases.items():
with self.subTest(graph=graph):
self.assertIsNone(self.layout(ppu, layers, **kwargs).centre_boxes)
def test_nothing_is_read_from_a_model_without_ppu_metadata(self):
self.assertIsNone(read_ppu_layout(os.path.join(self.tmp.name, "missing")))
path = write_dxnn(self.tmp.name, None, [(80, 80, 3)])
self.assertIsNone(read_ppu_layout(path))
with open(path, "wb") as model:
model.write(b"ONNX" + b"\0" * 100)
self.assertIsNone(read_ppu_layout(path))
path = write_dxnn(self.tmp.name, ANCHOR_BASED_PPU, [(80, 80, 3), (40, 40, 3)])
os.truncate(path, os.path.getsize(path) - 40)
self.assertIsNone(read_ppu_layout(path))
class TestDeepxLayoutInference(unittest.TestCase):
def test_a_shape_and_class_count_pick_one_layout(self):
cases = {
"anchor-free, four columns ahead of the classes": (
[(1, 84, 8400)],
80,
YoloLayout.anchor_free,
84,
),
"anchor-free, row-major": ([(1, 8400, 84)], 80, YoloLayout.anchor_free, 84),
"anchor-based, an objectness column as well": (
[(1, 25200, 85)],
80,
YoloLayout.anchor,
85,
),
"anchor-based, channel-major": (
[(1, 85, 25200)],
80,
YoloLayout.anchor,
85,
),
"NMS in the head, a fixed run of corner records": (
[(1, 300, 6)],
80,
YoloLayout.nms_in_head,
None,
),
# only the label map can tell these two apart
"85 columns with 81 classes is anchor-free": (
[(1, 8400, 85)],
81,
YoloLayout.anchor_free,
85,
),
"85 columns with 80 classes is anchor-based": (
[(1, 8400, 85)],
80,
YoloLayout.anchor,
85,
),
# 6 columns is all three layouts, told apart by the row count
"6 columns and thousands of rows, one class": (
[(1, 25200, 6)],
1,
YoloLayout.anchor,
6,
),
"6 columns and thousands of rows, two classes": (
[(1, 8400, 6)],
2,
YoloLayout.anchor_free,
6,
),
"6 columns and a few hundred rows, one class": (
[(1, 300, 6)],
1,
YoloLayout.nms_in_head,
None,
),
"7 columns with two classes is only anchor-based": (
[(1, 8400, 7)],
2,
YoloLayout.anchor,
7,
),
# a square output matches the same width on both axes, which must
# not read as two candidate layouts
"a square output is not ambiguous with itself": (
[(1, 85, 85)],
80,
YoloLayout.anchor,
85,
),
"three NCHW maps with 255 channels are feature maps": (
[(1, 255, 80, 80), (1, 255, 40, 40), (1, 255, 20, 20)],
80,
YoloLayout.multipart,
None,
),
}
for head, (shapes, num_classes, layout, columns) in cases.items():
with self.subTest(head=head):
output = infer_yolo_layout(shapes, num_classes, False, False)
self.assertIs(output.layout, layout)
if columns is not None:
self.assertEqual(output.columns, columns)
def test_the_runtime_flags_outrank_the_shapes(self):
self.assertIs(layout_of([(8400,)], 80, ppu=True), YoloLayout.ppu)
self.assertIs(
layout_of([(1, -1, 6)], 80, dynamic_output=True), YoloLayout.nms_in_head
)
def test_a_shape_no_layout_fits_is_refused_with_the_reason(self):
cases = {
"fits two layouts": ([(1, 84, 6)], 80),
"labelmap_path": ([(1, 8400, 84)], 91),
"no output tensor": ([], 80),
"255 channels": (
[(1, 80, 80, 255), (1, 40, 40, 255), (1, 20, 20, 255)],
80,
),
"feature maps": ([(1, 8400, 80), (1, 8400, 4)], 80),
"80-class": ([(1, 24, 80, 80), (1, 24, 40, 40), (1, 24, 20, 20)], 3),
}
for reason, (shapes, num_classes) in cases.items():
with (
self.subTest(reason=reason),
self.assertRaisesRegex(ValueError, reason),
):
layout_of(shapes, num_classes)
class TestDeepxOutputValidation(unittest.TestCase):
def test_the_raw_yolox_head_is_read_for_the_configured_input(self):
for shapes, size in (
([(1, 8400, 85)], 640),
([(1, 85, 8400)], 640),
# 52*52 + 26*26 + 13*13 cells at 416
([(1, 3549, 85)], 416),
):
with self.subTest(shapes=shapes, size=size):
self.assertEqual(validate_yolox_outputs(shapes, 80, size, size), 85)
def test_another_head_under_yolox_is_refused_with_the_reason(self):
cases = {
"width and height": ([(1, 8400, 85)], 80, 416),
"labelmap_path": ([(1, 8400, 85)], 91, 640),
"yolo-generic": ([(1, 8400, 80), (1, 8400, 4)], 80, 640),
}
for reason, (shapes, num_classes, size) in cases.items():
with (
self.subTest(reason=reason),
self.assertRaisesRegex(ValueError, reason),
):
validate_yolox_outputs(shapes, num_classes, size, size)
def test_the_label_map_bounds_the_class_count(self):
self.assertEqual(class_count({0: "person", 79: "toothbrush"}), 80)
with self.assertRaisesRegex(ValueError, "labelmap_path"):
class_count({})
class TestDeepxRawDecode(unittest.TestCase):
def anchor_rows(self, rows) -> list:
out = np.zeros((1, len(rows), 85), dtype=np.float32)
for i, (cx, cy, w, h, obj, label, score) in enumerate(rows):
out[0, i, 0:4] = [cx, cy, w, h]
out[0, i, 4] = obj
out[0, i, 5 + label] = score
return [out]
def test_an_anchor_based_head_becomes_normalized_corners(self):
outputs = self.anchor_rows([(320.0, 160.0, 64.0, 32.0, 0.8, 3, 0.5)])
detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45)
self.assertEqual(detections[0][0], 3)
self.assertAlmostEqual(detections[0][1], 0.4, places=5)
self.assertAlmostEqual(detections[0][2], 144 / 640, places=5)
self.assertAlmostEqual(detections[0][3], 288 / 640, places=5)
self.assertAlmostEqual(detections[0][4], 176 / 640, places=5)
self.assertAlmostEqual(detections[0][5], 352 / 640, places=5)
def test_a_channel_major_export_is_read_by_column_count(self):
outputs = self.anchor_rows([(320.0, 160.0, 64.0, 32.0, 1.0, 3, 0.9)])
detections = decode_raw_anchor(
[np.swapaxes(outputs[0], 1, 2)], 640, 640, 0.25, 0.45, columns=85
)
self.assertEqual(detections[0][0], 3)
self.assertAlmostEqual(detections[0][3], 288 / 640, places=5)
def test_rows_below_the_combined_threshold_are_dropped(self):
# 0.4 * 0.5 = 0.2, under the threshold both parts clear on their own
outputs = self.anchor_rows([(320.0, 320.0, 40.0, 80.0, 0.4, 3, 0.5)])
self.assertTrue(np.all(decode_raw_anchor(outputs, 640, 640, 0.25, 0.45) == 0))
def test_at_most_twenty_detections_are_returned(self):
rows = [(20.0 + 24 * i, 320.0, 16.0, 16.0, 1.0, i % 80, 0.9) for i in range(25)]
detections = decode_raw_anchor(self.anchor_rows(rows), 640, 640, 0.25, 0.45)
self.assertEqual(detections.shape, (20, 6))
self.assertEqual(int((detections[:, 1] > 0).sum()), 20)
def test_an_nms_in_head_output_is_read_without_running_nms(self):
out = np.array(
[
[
[100.0, 100.0, 200.0, 200.0, 0.9, 2.0],
[102.0, 102.0, 202.0, 202.0, 0.8, 2.0],
[300.0, 300.0, 400.0, 400.0, 0.1, 5.0],
]
],
dtype=np.float32,
)
detections = decode_raw_nms_in_head([out], 640, 640, 0.25)
self.assertEqual(detections[0][0], 2)
self.assertAlmostEqual(detections[0][1], 0.9, places=5)
self.assertAlmostEqual(detections[0][3], 100 / 640, places=5)
self.assertEqual(detections[1][0], 2)
self.assertAlmostEqual(detections[1][1], 0.8, places=5)
self.assertTrue(np.all(detections[2] == 0))
empty = np.zeros((1, 0, 6), dtype=np.float32)
self.assertTrue(np.all(decode_raw_nms_in_head([empty], 640, 640, 0.25) == 0))
class TestDeepxConfig(unittest.TestCase):
def test_a_device_string_resolves_to_an_npu_index(self):
for configured, index in (("PCIe:1", 1), ("2", 2), ("", 0)):
with self.subTest(device=configured):
self.assertEqual(resolve_device(configured), index)
def test_a_device_that_is_not_an_index_is_rejected(self):
"""The whole string has to be an index. Reading only the tail would
take the 1 out of "PCIe:0,PCIe:1" and bind to an NPU the config never
named, and several NPUs are configured as separate devices entries."""
for configured in (
"PCIe:the-fast-one",
"PCIe:0,PCIe:1",
"0,1",
"PCIe:0 PCIe:1",
"PCIe:-1",
"PCIe:",
):
with self.subTest(device=configured):
with self.assertRaises(ValueError):
resolve_device(configured)
with self.assertRaises(ValidationError):
DeepxDetectorConfig(type="deepx", device=configured)
def test_a_bad_device_is_refused_where_the_config_is_parsed(self):
"""parse_device builds the detector config to surface a bad device at
startup, so the comma-separated form fails there rather than binding a
detector process to the wrong NPU."""
self.assertEqual(parse_device("deepx:PCIe:1").device, "PCIe:1")
for raw in ("deepx:PCIe:0,PCIe:1", "deepx:the-fast-one"):
with self.subTest(raw=raw), self.assertRaises(DeviceParseError):
parse_device(raw)
def test_a_device_string_builds_this_detector_config(self):
"""Frigate turns a `deepx:PCIe:0` entry into the detector config with
the model already attached, which is the path app.py takes; a bare
constructor call does not exercise it."""
config = build_detector_config(
parse_device("deepx:PCIe:0"), model_with_type(ModelTypeEnum.yologeneric)
)
self.assertIsInstance(config, DeepxDetectorConfig)
self.assertEqual(config.device, "PCIe:0")
self.assertEqual(config.model.model_type, ModelTypeEnum.yologeneric)
def test_the_runtime_manifest_pins_its_wheels_to_the_version(self):
"""A PyPI path carries a per-file digest, so bumping DXRT_VERSION has
to rewrite the whole URL; a stale one installs the old wheel and fails
the sha256 on every user's first start."""
self.assertEqual(DEEPX_MANIFEST.version, DXRT_VERSION)
for artifact in DEEPX_MANIFEST.artifacts:
with self.subTest(url=artifact.url):
self.assertIn(f"dx_engine-{DXRT_VERSION}-", artifact.url)
def test_a_model_less_config_still_validates(self):
config = DeepxDetectorConfig(type="deepx")
self.assertIsNone(config.model)
class DeepxDetectorTestCase(unittest.TestCase):
def detector(
self,
model_type=ModelTypeEnum.yologeneric,
outputs_info=None,
ppu=False,
dynamic=False,
model_path="/nonexistent/model.dxnn",
) -> DeepxDetector:
dx_engine = MagicMock()
dx_engine.Configuration.ITEM.SERVICE = object()
session = dx_engine.InferenceEngine.return_value
session.get_output_tensors_info.return_value = (
[{"shape": [8400]}] if ppu else outputs_info or []
)
session.is_ppu.return_value = ppu
session.has_dynamic_output.return_value = dynamic
config = DeepxDetectorConfig(type="deepx")
config.model = model_with_type(model_type)
config.model.path = model_path
with (
# the detector writes the endpoint into the environment, which the
# rest of the suite shares when it runs in one process
patch.dict(os.environ),
patch.dict(sys.modules, {"dx_engine": dx_engine}),
patch.object(DeepxDetector, "activate_dependencies"),
patch("os.path.isfile", return_value=True),
):
return DeepxDetector(config)
def ppu_detector(
self, ppu, layers, model_type=ModelTypeEnum.yologeneric, box_format=None
) -> DeepxDetector:
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
return self.detector(
model_type,
ppu=True,
model_path=write_dxnn(tmp.name, ppu, layers, box_format=box_format),
)
def detect(self, detector, outputs) -> np.ndarray:
detector.session.run.return_value = outputs
return detector.detect_raw(np.zeros((1, 640, 640, 3), np.uint8))
class TestDeepxModelType(DeepxDetectorTestCase):
def test_a_model_type_with_no_decoder_is_rejected(self):
for model_type in (ModelTypeEnum.ssd, ModelTypeEnum.dfine):
with (
self.subTest(model_type=model_type),
self.assertRaisesRegex(ValueError, model_type.value),
):
self.detector(model_type)
def test_supported_model_types_are_accepted(self):
for model_type, outputs_info in (
(ModelTypeEnum.yologeneric, [{"shape": [1, 84, 8400]}]),
(ModelTypeEnum.yolox, [{"shape": [1, 8400, 85]}]),
):
with self.subTest(model_type=model_type):
detector = self.detector(model_type, outputs_info)
self.assertEqual(detector.model_type, model_type)
class TestDeepxDetectorLoad(DeepxDetectorTestCase):
def test_the_layout_is_settled_at_load(self):
detector = self.detector(ModelTypeEnum.yologeneric, [{"shape": [1, 84, 8400]}])
self.assertIs(detector.output.layout, YoloLayout.anchor_free)
self.assertEqual(detector.output.columns, 84)
detector = self.detector(ModelTypeEnum.yolox, [{"shape": [1, 8400, 85]}])
self.assertIs(detector.output.layout, YoloLayout.yolox)
for model_type in (ModelTypeEnum.yologeneric, ModelTypeEnum.yolox):
with self.subTest(model_type=model_type):
detector = self.ppu_detector(
ANCHOR_FREE_PPU, THREE_SCALE_FREE, model_type=model_type
)
self.assertIs(detector.output.layout, YoloLayout.ppu)
self.assertEqual(detector.ppu_layout.scale_count, 3)
def test_a_model_the_detector_cannot_decode_is_refused_at_load(self):
cases = {
"Cannot decode DEEPX model": lambda: self.detector(
ModelTypeEnum.yologeneric, [{"shape": [1, 8400, 7]}]
),
"DX-COM 2.4.0": lambda: self.detector(ModelTypeEnum.yologeneric, ppu=True),
"face and pose": lambda: self.ppu_detector({"type": 2}, [(80, 80, 1)]),
"centre and size or as two corners": lambda: self.ppu_detector(
ANCHOR_FREE_PPU, ONE_SCALE_FREE
),
}
for reason, load in cases.items():
with (
self.subTest(reason=reason),
self.assertRaisesRegex(ValueError, reason),
):
load()
class TestDeepxDetectRaw(DeepxDetectorTestCase):
def test_a_ppu_record_decodes_by_the_head_in_the_model(self):
cases = {
"anchor-based, layer 2 of 3 at stride 32, the 373x326 anchor": (
(ANCHOR_BASED_PPU, THREE_SCALE_ANCHORS, None),
build_ppu_record((0.6, 0.4, 0.3, 0.7), label=5),
(5, (0.0, 0.380094, 0.864187, 0.589906)),
),
"anchor-based, layer 0 of 3 at stride 8, the 16x30 anchor": (
(ANCHOR_BASED_PPU, THREE_SCALE_ANCHORS, None),
build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(7, 9, 1, 0)),
(0, (None, 0.116750, None, None)),
),
"anchor-based, two scales: layer 0 is stride 16, not stride 8": (
(ANCHOR_BASED_PPU, TWO_SCALE_ANCHORS, None),
build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(7, 9, 1, 0)),
(0, (0.141156, 0.236031, 0.223844, 0.248969)),
),
"anchor-free, three scales: cell (10, 9) of stride 32": (
(ANCHOR_FREE_PPU, THREE_SCALE_FREE, None),
build_ppu_record((1.2, 0.5, 1.0, 0.5), grid=(9, 10, 0, 2), label=7),
(7, (0.433782, 0.492043, 0.516218, 0.627957)),
),
"anchor-free, one scale: the box fields are already pixels": (
(ANCHOR_FREE_PPU, ONE_SCALE_FREE, "centre"),
build_ppu_record((320.0, 160.0, 64.0, 32.0), label=3),
(3, (144 / 640, 288 / 640, 176 / 640, 352 / 640)),
),
"anchor-free, one scale: a sub-pixel box stays sub-pixel": (
(ANCHOR_FREE_PPU, ONE_SCALE_FREE, "centre"),
build_ppu_record((0.6, 0.4, 0.3, 0.7)),
(0, (None, 0.45 / 640, None, None)),
),
"a corner-format head reads the record as two corners": (
(ANCHOR_FREE_PPU, ONE_SCALE_FREE, "corner"),
build_ppu_record((100.0, 50.0, 300.0, 250.0), label=2),
(2, (50 / 640, 100 / 640, 250 / 640, 300 / 640)),
),
"a centre-format head reads it as a centre and size": (
(ANCHOR_FREE_PPU, ONE_SCALE_FREE, "centre"),
build_ppu_record((100.0, 50.0, 300.0, 250.0), label=2),
(2, (0.0, 0.0, 175 / 640, 250 / 640)),
),
}
for head, ((ppu, layers, box_format), record, (label, box)) in cases.items():
with self.subTest(head=head):
detector = self.ppu_detector(ppu, layers, box_format=box_format)
detections = self.detect(detector, [record])
self.assertEqual(detections[0][0], label)
for i, expected in enumerate(box, start=2):
if expected is not None:
self.assertAlmostEqual(detections[0][i], expected, places=5)
def test_the_strides_come_from_the_grids_in_the_model(self):
detector = self.ppu_detector(
ANCHOR_FREE_PPU, [(40, 40, 1), (20, 20, 1), (10, 10, 1)]
)
detections = self.detect(
detector,
[build_ppu_record((1.2, 0.5, 1.0, 0.5), grid=(9, 10, 0, 0), label=7)],
)
# layer 0 is stride 16: centre (11.2, 9.5) * 16, size e * 16 x sqrt(e) * 16
self.assertEqual(detections[0][0], 7)
self.assertAlmostEqual(detections[0][2], (152 - np.exp(0.5) * 8) / 640, 5)
self.assertAlmostEqual(detections[0][3], (179.2 - np.exp(1.0) * 8) / 640, 5)
def test_the_head_stays_what_the_model_said_across_frames(self):
detector = self.ppu_detector(ANCHOR_BASED_PPU, THREE_SCALE_ANCHORS)
first = self.detect(
detector, [build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(7, 9, 1, 0))]
)
# layer 0 of 3: stride 8, anchor 16x30
self.assertAlmostEqual(first[0][3], 0.116750, places=5)
second = self.detect(
detector, [build_ppu_record((0.5, 0.5, 0.4, 0.4), grid=(3, 4, 0, 1))]
)
# layer 1 of 3: stride 16, anchor 30x61, not layer 1 of 2
self.assertAlmostEqual(second[0][3], 0.0975, places=5)
detector = self.ppu_detector(
ANCHOR_FREE_PPU, ONE_SCALE_FREE, box_format="centre"
)
record = [build_ppu_record((100.0, 50.0, 300.0, 250.0), label=2)]
# centre (100, 50), size 300 x 250: the right edge lands at 250
for frame in range(2):
with self.subTest(frame=frame):
self.assertAlmostEqual(
self.detect(detector, record)[0][5], 250 / 640, places=5
)
def test_records_the_head_cannot_place_come_back_empty(self):
cases = {
"a level or box the anchor table does not carry": (
THREE_SCALE_ANCHORS,
[build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(7, 9, 2, 5))],
),
"a scale count Frigate has no anchor table for": (
FOUR_SCALE_ANCHORS,
[build_ppu_record((0.6, 0.4, 0.3, 0.7))],
),
"a record below the score threshold": (
THREE_SCALE_ANCHORS,
[build_ppu_record((0.6, 0.4, 0.3, 0.7), score=0.1)],
),
"an unexpected record width": (
THREE_SCALE_ANCHORS,
[np.zeros((1, 3, 16), dtype=np.uint8)],
),
**{
f"no records at all, shaped {shape}": (
THREE_SCALE_ANCHORS,
[np.zeros(shape, dtype=np.uint8)],
)
for shape in ((1, 0, PPU_RECORD_SIZE), (0, PPU_RECORD_SIZE), (0,))
},
}
for output, (layers, outputs) in cases.items():
with self.subTest(output=output):
detector = self.ppu_detector(ANCHOR_BASED_PPU, layers)
self.assertTrue(np.all(self.detect(detector, outputs) == 0))
def test_a_yolox_raw_head_is_decoded_through_the_grid(self):
detector = self.detector(ModelTypeEnum.yolox, [{"shape": [1, 8400, 85]}])
# cell (x 10, y 5) of the stride-8 grid is row 5 * 80 + 10
tensor = np.zeros((1, 8400, 85), np.float32)
tensor[0, 410, :5] = [0.5, 0.5, np.log(4.0), np.log(2.0), 0.9]
tensor[0, 410, 5 + 7] = 0.8
detections = self.detect(detector, [tensor])
# centre (84, 44), size 32 x 16
self.assertEqual(detections[0][0], 7)
self.assertAlmostEqual(detections[0][1], 0.72, places=5)
self.assertAlmostEqual(detections[0][2], 36 / 640, places=5)
self.assertAlmostEqual(detections[0][3], 68 / 640, places=5)
self.assertAlmostEqual(detections[0][4], 52 / 640, places=5)
self.assertAlmostEqual(detections[0][5], 100 / 640, places=5)
detector = self.detector(ModelTypeEnum.yolox, [{"shape": [1, 85, 8400]}])
detections = self.detect(detector, [np.swapaxes(tensor, 1, 2)])
self.assertAlmostEqual(detections[0][5], 100 / 640, places=5)
def test_a_raw_head_comes_back_as_frigates_detection_rows(self):
detector = self.detector(ModelTypeEnum.yologeneric, [{"shape": [1, 84, 8400]}])
output = np.zeros((1, 84, 8400), dtype=np.float32)
output[0, 0:4, 0] = [320.0, 160.0, 64.0, 32.0]
output[0, 4 + 2, 0] = 0.9
detections = self.detect(detector, [output])
self.assertEqual(detections.shape, (20, 6))
self.assertEqual(detections[0][0], 2)
self.assertAlmostEqual(detections[0][1], 0.9, places=5)
self.assertAlmostEqual(detections[0][3], 288 / 640, places=5)
-5
View File
@@ -59,10 +59,6 @@ class TestBuildDetectorConfig(unittest.TestCase):
def test_detectors_that_name_the_field_something_else(self):
self.assertEqual(self._build("cpu:4").num_threads, 4)
self.assertEqual(self._build("rknn:2").num_cores, 2)
self.assertEqual(
self._build("zmq:tcp://host.docker.internal:5555").endpoint,
"tcp://host.docker.internal:5555",
)
def test_device_is_coerced_to_the_detector_field_type(self):
self.assertEqual(self._build("tensorrt:1").device, 1)
@@ -70,7 +66,6 @@ class TestBuildDetectorConfig(unittest.TestCase):
def test_omitted_device_falls_back_to_the_detector_default(self):
self.assertEqual(self._build("cpu").num_threads, 3)
self.assertEqual(self._build("rknn").num_cores, 0)
self.assertEqual(self._build("zmq").endpoint, "ipc:///tmp/cache/zmq_detector")
self.assertEqual(self._build("openvino").device, "AUTO")
self.assertIsNone(self._build("edgetpu").device)
+1 -44
View File
@@ -25,7 +25,7 @@ class HardwareProbeTestCase(unittest.TestCase):
self.root = tempfile.TemporaryDirectory()
self.addCleanup(self.root.cleanup)
for name in ("SYS_ROOT", "DEV_ROOT", "PROC_ROOT", "ETC_ROOT", "LIB_ROOT"):
for name in ("SYS_ROOT", "DEV_ROOT", "PROC_ROOT", "ETC_ROOT"):
sub = os.path.join(self.root.name, name.split("_")[0].lower())
os.makedirs(sub, exist_ok=True)
patcher = patch.object(hardware, name, sub)
@@ -167,29 +167,6 @@ class TestNvidia(HardwareProbeTestCase):
class TestAccelerators(HardwareProbeTestCase):
def test_the_neural_engine_is_found_by_lighters_provider_library(self):
write(os.path.join(self.lib_root, "lighter", "liblighter_ane_ep.so"))
with patch.dict(os.environ, clear=False) as env:
env.pop("LIGHTER_ANE_EP", None)
ane = self.probe()["onnx:lighter"]
self.assertEqual(ane.detector, "onnx")
self.assertEqual(ane.units[0].device, "onnx")
self.assertEqual(ane.units[0].label, "Neural Engine")
def test_the_neural_engine_is_found_where_lighter_ane_ep_points(self):
library = os.path.join(self.root.name, "elsewhere", "liblighter_ane_ep.so")
write(library)
with patch.dict(os.environ, {"LIGHTER_ANE_EP": library}):
self.assertIn("onnx:lighter", self.probe())
def test_no_neural_engine_is_reported_without_the_library(self):
with patch.dict(os.environ, clear=False) as env:
env.pop("LIGHTER_ANE_EP", None)
self.assertNotIn("onnx:lighter", self.probe())
def test_hailo_is_found_by_its_device_node(self):
write(os.path.join(self.dev_root, "hailo0"))
@@ -207,26 +184,6 @@ class TestAccelerators(HardwareProbeTestCase):
)
self.assertFalse(memryx.unlimited)
def test_each_deepx_node_is_a_unit(self):
write(os.path.join(self.dev_root, "dxrt0"))
write(os.path.join(self.dev_root, "dxrt1"))
deepx = self.probe()["deepx"]
self.assertEqual(
[unit.device for unit in deepx.units],
["deepx:PCIe:0", "deepx:PCIe:1"],
)
def test_a_deepx_npu_is_unlimited(self):
# the host daemon multiplexes, so one module takes several processes
write(os.path.join(self.dev_root, "dxrt0"))
self.assertTrue(self.probe()["deepx"].unlimited)
def test_no_deepx_is_reported_without_a_node(self):
self.assertNotIn("deepx", self.probe())
def test_a_supported_rockchip_soc_is_reported(self):
write(
os.path.join(self.proc_root, "device-tree", "compatible"),
-54
View File
@@ -1,54 +0,0 @@
"""Tests for get_dst_transitions."""
import datetime
import unittest
from frigate.util.time import get_dst_transitions
class TestDstTransitions(unittest.TestCase):
def test_dst_transition_splits_periods_at_the_transition(self):
start = datetime.datetime(2026, 3, 7, 12, tzinfo=datetime.UTC).timestamp()
end = start + 2 * 86400
spring = datetime.datetime(2026, 3, 8, 7, tzinfo=datetime.UTC).timestamp()
self.assertEqual(
get_dst_transitions("America/New_York", start, end),
[(start, spring, -18000), (spring, end, -14400)],
)
def test_dst_transition_is_not_reported_a_day_late(self):
# local midnight on the day of the change used to report the
# transition a full day after it actually happened
start = datetime.datetime(2024, 3, 10, 5, tzinfo=datetime.UTC).timestamp()
end = start + 3 * 86400
spring = datetime.datetime(2024, 3, 10, 7, tzinfo=datetime.UTC).timestamp()
self.assertEqual(
get_dst_transitions("America/New_York", start, end),
[(start, spring, -18000), (spring, end, -14400)],
)
def test_dst_transition_after_the_last_daily_probe_is_found(self):
start = datetime.datetime(2024, 11, 2, 12, tzinfo=datetime.UTC).timestamp()
end = datetime.datetime(2024, 11, 3, 10, tzinfo=datetime.UTC).timestamp()
fall = datetime.datetime(2024, 11, 3, 6, tzinfo=datetime.UTC).timestamp()
self.assertEqual(
get_dst_transitions("America/New_York", start, end),
[(start, fall, -14400), (fall, end, -18000)],
)
def test_no_transition_returns_a_single_period(self):
start = datetime.datetime(2026, 6, 1, tzinfo=datetime.UTC).timestamp()
end = start + 5 * 86400
self.assertEqual(
get_dst_transitions("America/New_York", start, end),
[(start, end, -14400)],
)
def test_invalid_zone_retains_utc_fallback(self):
self.assertEqual(
get_dst_transitions("Invalid/Timezone", 100, 200), [(100, 200, 0)]
)
if __name__ == "__main__":
unittest.main(verbosity=2)
-10
View File
@@ -51,16 +51,6 @@ class TestFfmpegPresets(unittest.TestCase):
" ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"])
)
def test_ffmpeg_hwaccel_apple_preset_decodes_every_stream(self):
self.default_ffmpeg["cameras"]["back"]["ffmpeg"]["hwaccel_args"] = (
"preset-apple-silicon-h265"
)
frigate_config = FrigateConfig(**self.default_ffmpeg)
cmd = " ".join(frigate_config.cameras["back"].ffmpeg_cmds[0]["cmd"])
assert "-c:v hevc_v4l2m2m" in cmd
assert "-c:v:1" not in cmd
assert "scale=1920:1080" in cmd
def test_ffmpeg_hwaccel_not_preset(self):
self.default_ffmpeg["cameras"]["back"]["ffmpeg"]["hwaccel_args"] = (
"-other-hwaccel args"
+3 -339
View File
@@ -360,70 +360,9 @@ class TestOllamaProvider(unittest.TestCase):
from frigate.genai.plugins.ollama import _normalize_multimodal_content
text, images = _normalize_multimodal_content(MULTIMODAL_MESSAGES[-1]["content"])
self.assertEqual(
text, "Here is the current live image from camera 'front'.\n[img]"
)
self.assertEqual(images, [b"\xff\xd8\xff\xd9"])
def test_normalize_keeps_text_and_image_order(self):
from frigate.genai.plugins.ollama import _normalize_multimodal_content
text, images = _normalize_multimodal_content(
[
{"type": "text", "text": "intro"},
{"type": "text", "text": "Frame 1"},
{"type": "image_url", "image_url": {"url": _IMAGE_DATA_URI}},
{"type": "text", "text": "Frame 2"},
{"type": "image_url", "image_url": {"url": _IMAGE_DATA_URI}},
]
)
self.assertEqual(text, "intro\nFrame 1\n[img]\nFrame 2\n[img]")
self.assertEqual(len(images), 2)
def test_send_uses_chat_with_captions_before_each_image(self):
client = self._client()
client.provider = MagicMock()
client.provider.chat.return_value = {
"message": {"content": '{"ok": true}'},
"done": True,
"done_reason": "stop",
}
client._supports_thinking_cache = False
result = client._send(
"prompt",
[b"a", b"b"],
{"type": "json_schema", "json_schema": {"schema": {"type": "object"}}},
image_captions=["Frame 1 of 2", "Frame 2 of 2"],
)
self.assertEqual(result, '{"ok": true}')
client.provider.generate.assert_not_called()
params = client.provider.chat.call_args.kwargs
self.assertEqual(
params["messages"],
[
{
"role": "user",
"content": "prompt\nFrame 1 of 2\n[img]\nFrame 2 of 2\n[img]",
"images": [b"a", b"b"],
}
],
)
self.assertEqual(params["format"], {"type": "object"})
self.assertNotIn("think", params)
def test_send_without_captions_puts_images_after_prompt(self):
client = self._client()
client.provider = MagicMock()
client.provider.chat.return_value = {"message": {"content": "ok"}, "done": True}
client._supports_thinking_cache = False
client._send("prompt", [b"a"])
message = client.provider.chat.call_args.kwargs["messages"][0]
self.assertEqual(message["content"], "prompt\n[img]")
self.assertEqual(message["images"], [b"a"])
self.assertIn("live image", text)
self.assertEqual(len(images), 1)
self.assertEqual(images[0], b"\xff\xd8\xff\xd9")
# ---------------------------------------------------------------------------
@@ -580,281 +519,6 @@ class TestLlamaCppProvider(unittest.TestCase):
client = self._validated_client(4096, {"context_size": 32768})
self.assertEqual(client.get_context_size(), 32768)
def test_list_models_dedupes_alias_matching_id(self):
client = self._client()
models_data = [
{"id": "qwen3-asr", "aliases": ["qwen3-asr"]},
{"id": "gemma", "aliases": ["gemma", "g4"]},
]
with patch.object(client, "_fetch_models_data", return_value=models_data):
self.assertEqual(client.list_models(), ["g4", "gemma", "qwen3-asr"])
# ---------------------------------------------------------------------------
# transcribe role
# ---------------------------------------------------------------------------
WAV_BYTES = b"RIFF$\x00\x00\x00WAVEfmt "
class TestOpenAITranscribe(unittest.TestCase):
def _client(self):
return _make_client(
"openai",
model="gpt-4o-transcribe",
api_key="k",
base_url="http://localhost:9999/v1",
runtime_options={"temperature": 0.7},
)
def test_supports_transcription(self):
self.assertTrue(self._client().supports_transcription)
def test_passes_file_tuple_and_language(self):
client = self._client()
create = MagicMock(return_value=" hello there ")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "hello there")
kwargs = create.call_args.kwargs
self.assertEqual(kwargs["model"], "gpt-4o-transcribe")
self.assertEqual(kwargs["file"], ("audio.wav", WAV_BYTES, "audio/wav"))
self.assertEqual(kwargs["language"], "en")
self.assertEqual(kwargs["response_format"], "text")
def test_does_not_forward_runtime_options(self):
"""runtime_options are chat parameters; /audio/transcriptions rejects them."""
client = self._client()
create = MagicMock(return_value="hi")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
client.transcribe(WAV_BYTES)
self.assertNotIn("temperature", create.call_args.kwargs)
def test_gpt_transcribe_uses_languages_array(self):
"""gpt-transcribe replaced `language` with a `languages` array."""
client = _make_client("openai", model="gpt-transcribe", api_key="k")
create = MagicMock(return_value="hi")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
client.transcribe(WAV_BYTES, language="en")
kwargs = create.call_args.kwargs
self.assertEqual(kwargs["extra_body"], {"languages": ["en"]})
# sending both fields is rejected by the API
self.assertNotIn("language", kwargs)
def test_older_models_use_singular_language(self):
for model in ("gpt-4o-transcribe", "whisper-1"):
with self.subTest(model):
client = _make_client("openai", model=model, api_key="k")
create = MagicMock(return_value="hi")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
client.transcribe(WAV_BYTES, language="en")
kwargs = create.call_args.kwargs
self.assertEqual(kwargs["language"], "en")
self.assertNotIn("extra_body", kwargs)
def test_object_response_form(self):
client = self._client()
create = MagicMock(return_value=SimpleNamespace(text="hi"))
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertEqual(client.transcribe(WAV_BYTES), "hi")
def test_error_returns_none(self):
client = self._client()
create = MagicMock(side_effect=RuntimeError("boom"))
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertIsNone(client.transcribe(WAV_BYTES))
class TestAzureOpenAITranscribe(unittest.TestCase):
def _client(self):
return _make_client(
"azure_openai",
model="my-deployment",
api_key="k",
base_url="https://example.openai.azure.com/?api-version=2024-06-01",
)
def test_routes_through_azure_client(self):
from openai import AzureOpenAI
client = self._client()
self.assertIsInstance(client.provider, AzureOpenAI)
self.assertTrue(client.supports_transcription)
def test_transcribe_inherited(self):
client = self._client()
create = MagicMock(return_value="azure text")
client.provider = SimpleNamespace(
audio=SimpleNamespace(transcriptions=SimpleNamespace(create=create))
)
self.assertEqual(client.transcribe(WAV_BYTES, language="fr"), "azure text")
self.assertEqual(create.call_args.kwargs["model"], "my-deployment")
class TestGeminiTranscribe(unittest.TestCase):
def _client(self):
return _make_client("gemini", model="gemini-2.0-flash", api_key="k")
def test_supports_transcription(self):
self.assertTrue(self._client().supports_transcription)
def test_sends_audio_part(self):
client = self._client()
generate = MagicMock(return_value=SimpleNamespace(text=" spoken words "))
client.provider = SimpleNamespace(
models=SimpleNamespace(generate_content=generate)
)
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "spoken words")
contents = generate.call_args.kwargs["contents"]
audio_parts = [
p for p in contents if getattr(p, "inline_data", None) is not None
]
self.assertEqual(len(audio_parts), 1)
self.assertEqual(audio_parts[0].inline_data.mime_type, "audio/wav")
self.assertEqual(audio_parts[0].inline_data.data, WAV_BYTES)
def test_oversized_payload_is_skipped(self):
from frigate.genai.plugins.gemini import GEMINI_MAX_INLINE_BYTES
client = self._client()
generate = MagicMock()
client.provider = SimpleNamespace(
models=SimpleNamespace(generate_content=generate)
)
self.assertIsNone(client.transcribe(b"\x00" * (GEMINI_MAX_INLINE_BYTES + 1)))
generate.assert_not_called()
class TestLlamaCppTranscribe(unittest.TestCase):
def _client(self, supports_audio: bool):
cfg = GenAIConfig(
provider="llamacpp",
model="m",
base_url="http://localhost:9999",
)
info = {
"context_size": 4096,
"supports_vision": False,
"supports_audio": supports_audio,
"supports_tools": False,
"supports_reasoning": False,
"media_marker": "<__media__>",
}
cls = PROVIDERS[GenAIProviderEnum.llamacpp]
with patch.object(cls, "_get_model_info", return_value=info):
return cls(cfg, timeout=5)
def test_supports_transcription_tracks_supports_audio(self):
self.assertTrue(self._client(True).supports_transcription)
self.assertFalse(self._client(False).supports_transcription)
@staticmethod
def _transcriptions_response(text: str = " transcript "):
response = MagicMock()
response.status_code = 200
response.json.return_value = {"text": text}
return response
@staticmethod
def _chat_response(content: str = " fallback transcript "):
response = MagicMock()
response.status_code = 200
response.json.return_value = {"choices": [{"message": {"content": content}}]}
return response
def test_posts_multipart_to_transcriptions(self):
client = self._client(True)
with patch.object(
client, "_post", return_value=self._transcriptions_response()
) as post:
self.assertEqual(client.transcribe(WAV_BYTES, language="en"), "transcript")
self.assertTrue(post.call_args.args[0].endswith("/v1/audio/transcriptions"))
self.assertEqual(
post.call_args.kwargs["files"]["file"],
("audio.wav", WAV_BYTES, "audio/wav"),
)
self.assertEqual(post.call_args.kwargs["data"]["language"], "en")
def test_omits_language_when_not_set(self):
"""An unset language is what lets the model detect one itself."""
client = self._client(True)
with patch.object(
client, "_post", return_value=self._transcriptions_response()
) as post:
client.transcribe(WAV_BYTES)
self.assertNotIn("language", post.call_args.kwargs["data"])
def test_falls_back_to_chat_completions_on_404(self):
"""Servers predating llama.cpp#21863 have no transcriptions route."""
client = self._client(True)
missing = MagicMock()
missing.status_code = 404
with patch.object(
client, "_post", side_effect=[missing, self._chat_response()]
) as post:
self.assertEqual(
client.transcribe(WAV_BYTES, language="en"), "fallback transcript"
)
urls = [call.args[0] for call in post.call_args_list]
self.assertTrue(urls[0].endswith("/v1/audio/transcriptions"))
self.assertTrue(urls[1].endswith("/v1/chat/completions"))
payload = post.call_args_list[1].kwargs["json"]
content = payload["messages"][0]["content"]
audio_parts = [p for p in content if p["type"] == "input_audio"]
self.assertEqual(len(audio_parts), 1)
self.assertEqual(audio_parts[0]["input_audio"]["format"], "wav")
self.assertEqual(
base64.b64decode(audio_parts[0]["input_audio"]["data"]), WAV_BYTES
)
def test_audio_unsupported_returns_none(self):
client = self._client(False)
with patch.object(client, "_post") as post:
self.assertIsNone(client.transcribe(WAV_BYTES))
post.assert_not_called()
class TestBaseClientTranscribe(unittest.TestCase):
"""Providers that don't implement the role must be inert, not broken."""
def test_ollama_reports_and_returns_nothing(self):
client = _make_client("ollama", model="llava", base_url="http://localhost:9999")
self.assertFalse(client.supports_transcription)
self.assertIsNone(client.transcribe(WAV_BYTES, language="en"))
if __name__ == "__main__":
unittest.main()
@@ -34,12 +34,6 @@ class HwaccelRecommendationTestCase(unittest.TestCase):
patcher.start()
self.addCleanup(patcher.stop)
self.sys_root = os.path.join(self.root.name, "sys")
os.makedirs(self.sys_root)
patcher = patch.object(hwaccel, "SYS_ROOT", self.sys_root)
patcher.start()
self.addCleanup(patcher.stop)
drm = patch.object(hwaccel, "enumerate_drm_devices", return_value={})
self.drm = drm.start()
self.addCleanup(drm.stop)
@@ -72,12 +66,6 @@ class HwaccelRecommendationTestCase(unittest.TestCase):
with open(os.path.join(self.proc_root, "cpuinfo"), "w") as f:
f.write(f"processor\t: 0\nmodel name\t: {model_name}\n")
def write_video_device(self, vendor: str) -> None:
device = os.path.join(self.sys_root, "class", "video4linux", "video0", "device")
os.makedirs(device)
with open(os.path.join(device, "vendor"), "w") as f:
f.write(f"{vendor}\n")
def write_device_tree(self) -> None:
os.makedirs(os.path.join(self.proc_root, "device-tree"), exist_ok=True)
with open(os.path.join(self.proc_root, "device-tree", "compatible"), "w") as f:
@@ -203,22 +191,6 @@ class TestAvailableFamilies(HwaccelRecommendationTestCase):
self.assertIn(recommended, [family.key for family in families])
class TestLighter(HwaccelRecommendationTestCase):
def test_lighters_media_engine_is_recommended(self):
self.write_video_device(hwaccel.LIGHTER_VIRTIO_VENDOR)
self.assertEqual(self.recommend(codecs={"h264"}), "apple-silicon")
self.assertEqual(
self.presets()["apple-silicon"],
{"h264": "preset-apple-silicon-h264", "h265": "preset-apple-silicon-h265"},
)
def test_another_virtio_media_device_is_not_lighters(self):
self.write_video_device("0x554d4551")
self.assertEqual(self.available(), [])
class TestCodecCoverage(HwaccelRecommendationTestCase):
def test_a_family_carries_a_preset_per_codec(self):
self.assertEqual(
-29
View File
@@ -73,35 +73,6 @@ class TestImprovedMotionDetector(unittest.TestCase):
"Motion boxes should be empty when scene change exceeds skip threshold",
)
def _bright_frame(self, offset: int) -> np.ndarray:
"""Produce a bright frame with a small dark object that moves."""
frame = np.full((self.frame_shape[0], self.frame_shape[1]), 200, dtype=np.uint8)
x = 10 + (offset * 5) % 60
frame[40:60, x : x + 15] = 20
return frame
def test_skip_motion_threshold_recovers_after_skip(self):
"""Skipped frames must still be blended into the background.
A bright scene differs from the zeroed background across the whole
frame, so the first frames are skipped. The background has to catch up
anyway, otherwise motion detection never returns.
"""
self.config.skip_motion_threshold = 0.5
self.config.improve_contrast = False
self.detector.config = self.config
self.detector.update_mask()
boxes = [len(self.detector.detect(self._bright_frame(i))) for i in range(40)]
self.assertEqual(boxes[0], 0, "First frame should exceed the skip threshold")
self.assertGreater(
self.detector.avg_frame.max(),
0,
"Background was never updated while frames were skipped",
)
self.assertTrue(any(boxes), "Motion detection never recovered after a skip")
def test_skip_motion_threshold_does_not_affect_calibration(self):
"""Even when skipping, the detector should go into calibrating state."""
self.config.skip_motion_threshold = 0.4
+58 -120
View File
@@ -16,7 +16,9 @@ from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
# kinds that switch on the lifecycle knobs, so the tests do not depend on the
# values the real catalog picks
BATCHED = NoticeKind("batched", NoticeSeverity.warning, "system", batch_repeats=True)
BATCHED = NoticeKind(
"batched", NoticeSeverity.warning, "system", batch_repeats=True, reopen_at_count=3
)
PRUNED = NoticeKind("pruned", NoticeSeverity.info, "system", keep_latest=2)
TEST_KINDS = {kind.key: kind for kind in (BATCHED, PRUNED)}
@@ -75,11 +77,6 @@ class RegistryTestCase(unittest.TestCase):
mock_datetime.now.return_value.timestamp.return_value = timestamp
self.registry.raise_notice(kind, **kwargs)
def _acknowledge_at(self, timestamp: float, row_id: str) -> None:
with patch("frigate.notices.registry.datetime") as mock_datetime:
mock_datetime.now.return_value.timestamp.return_value = timestamp
self.registry.acknowledge(row_id)
class TestNoticeRegistry(RegistryTestCase):
def test_raise_inserts_and_counts_one_occurrence(self):
@@ -125,98 +122,34 @@ class TestNoticeRegistry(RegistryTestCase):
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_not_called()
def test_acknowledged_notice_returns_on_a_re_raise(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
self.assertEqual(self.registry.active(), [])
self.registry.raise_notice("detector_stuck", scope="ov", params={})
notice = self.registry.active()[0]
self.assertEqual(notice["count"], 2)
self.assertIsNone(notice["acknowledged_at"])
def test_muted_notice_stays_hidden_on_a_re_raise(self):
def test_dismissal_survives_a_re_raise(self):
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertTrue(self.registry.mute("skipped_detections:front_door"))
self.assertTrue(self.registry.dismiss("skipped_detections:front_door"))
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertEqual(self.registry.active(), [])
hidden = self.registry.active(include_hidden=True)
self.assertEqual(hidden[0]["count"], 2)
self.assertIsNotNone(hidden[0]["muted_at"])
history = self.registry.active(include_dismissed=True)
self.assertEqual(history[0]["count"], 2)
self.assertIsNotNone(history[0]["dismissed_at"])
def test_muting_an_acknowledged_notice_replaces_the_acknowledgement(self):
def test_dismiss_counts_once(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.acknowledge("detector_stuck:ov")
self.assertTrue(self.registry.mute("detector_stuck:ov"))
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
notice = self.registry.active(include_hidden=True)[0]
self.assertIsNone(notice["acknowledged_at"])
self.assertIsNotNone(notice["muted_at"])
self.assertEqual(self.registry.stats()[0]["dismissals"], 1)
def test_acknowledging_a_muted_notice_keeps_it_muted(self):
def test_dismiss_unknown_id(self):
self.assertFalse(self.registry.dismiss("nope"))
def test_dismissed_hidden_unless_requested(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.mute("detector_stuck:ov")
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
notice = self.registry.active(include_hidden=True)[0]
self.assertIsNone(notice["acknowledged_at"])
self.assertIsNotNone(notice["muted_at"])
def test_acknowledge_and_mute_each_count_once(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("shm_too_low", params={})
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
self.assertTrue(self.registry.mute("shm_too_low"))
self.assertTrue(self.registry.mute("shm_too_low"))
stats = {s["kind"]: s for s in self.registry.stats()}
self.assertEqual(stats["detector_stuck"]["acknowledgements"], 1)
self.assertEqual(stats["detector_stuck"]["mutes"], 0)
self.assertEqual(stats["shm_too_low"]["mutes"], 1)
def test_unknown_ids_are_rejected(self):
self.assertFalse(self.registry.acknowledge("nope"))
self.assertFalse(self.registry.mute("nope"))
self.assertFalse(self.registry.unhide("nope"))
def test_kinds_that_never_repeat_cannot_be_acknowledged(self):
self.registry.raise_notice(
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
self.assertFalse(self.registry.acknowledge("update_available:0.19.1"))
self.assertFalse(
self.registry.acknowledge("config:detect:fps-greater-than-five:global")
)
self.assertEqual(self.registry.active()[0]["acknowledgeable"], False)
def test_hidden_notices_listed_only_when_requested(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.acknowledge("detector_stuck:ov")
self.registry.dismiss("detector_stuck:ov")
self.assertEqual(self.registry.active(), [])
self.assertEqual(len(self.registry.active(include_hidden=True)), 1)
def test_unhide_shows_a_notice_and_drops_a_check_mute(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.mute("detector_stuck:ov")
self.registry.mute("config:detect:fps-greater-than-five:global")
self.assertTrue(self.registry.unhide("detector_stuck:ov"))
self.assertTrue(
self.registry.unhide("config:detect:fps-greater-than-five:global")
)
notice = self.registry.active()[0]
self.assertIsNone(notice["muted_at"])
self.assertEqual(self.registry.muted_checks(), [])
self.assertEqual(len(self.registry.active(include_dismissed=True)), 1)
def test_resolve_deletes_and_keeps_stats(self):
self.registry.raise_notice(
@@ -227,7 +160,7 @@ class TestNoticeRegistry(RegistryTestCase):
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.assertEqual(self.registry.active(include_hidden=True), [])
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
@@ -304,47 +237,45 @@ class TestNoticeRegistry(RegistryTestCase):
ids, ["model_download_failed:front_door", "skipped_detections:garage"]
)
def test_resolve_camera_drops_that_cameras_check_mutes(self):
def test_resolve_camera_drops_that_cameras_check_dismissals(self):
for check_id in (
"stream:front_door:0:probe",
"config:detect:fps-greater-than-five:camera.front_door",
"config:detect:fps-greater-than-five:global",
"stream:garage:0:probe",
):
self.assertTrue(self.registry.mute(check_id))
self.assertTrue(self.registry.dismiss(check_id))
self.registry.resolve_camera("front_door")
ids = sorted(check["id"] for check in self.registry.muted_checks())
ids = sorted(check["id"] for check in self.registry.dismissed_checks())
self.assertEqual(
ids,
["config:detect:fps-greater-than-five:global", "stream:garage:0:probe"],
)
def test_camera_named_global_keeps_global_check_mutes(self):
self.registry.mute("config:detect:fps-greater-than-five:global")
self.registry.mute("config:detect:fps-greater-than-five:camera.global")
def test_camera_named_global_keeps_global_check_dismissals(self):
self.registry.dismiss("config:detect:fps-greater-than-five:global")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.global")
self.registry.resolve_camera("global")
ids = [check["id"] for check in self.registry.muted_checks()]
ids = [check["id"] for check in self.registry.dismissed_checks()]
self.assertEqual(ids, ["config:detect:fps-greater-than-five:global"])
def test_unhide_all_shows_every_notice_and_keeps_counts(self):
def test_purge_dismissed_keeps_active_notices_and_counts(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.acknowledge("detector_stuck:ov")
self.registry.mute("skipped_detections:garage")
self.registry.mute("config:detect:fps-greater-than-five:camera.garage")
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.garage")
self.registry.unhide_all()
self.assertEqual(self.registry.purge_dismissed(), 2)
ids = sorted(n["id"] for n in self.registry.active())
self.assertEqual(ids, ["detector_stuck:ov", "skipped_detections:garage"])
self.assertEqual(self.registry.muted_checks(), [])
stats = {s["kind"]: s for s in self.registry.stats()}
self.assertEqual(stats["detector_stuck"]["acknowledgements"], 1)
self.assertEqual(stats["skipped_detections"]["mutes"], 1)
ids = [n["id"] for n in self.registry.active(include_dismissed=True)]
self.assertEqual(ids, ["skipped_detections:garage"])
self.assertEqual(self.registry.dismissed_checks(), [])
dismissals = {s["kind"]: s["dismissals"] for s in self.registry.stats()}
self.assertEqual(dismissals["detector_stuck"], 1)
class TestApply(RegistryTestCase):
@@ -418,7 +349,7 @@ class TestLifecycleKnobs(RegistryTestCase):
self.registry.flush()
self.assertEqual(self.registry.active(include_hidden=True), [])
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.listener.assert_not_called()
def test_a_new_row_starts_without_stale_repeats(self):
@@ -431,26 +362,32 @@ class TestLifecycleKnobs(RegistryTestCase):
self.assertEqual(self.registry.active()[0]["count"], 1)
def test_an_acknowledged_notice_returns_when_a_later_repeat_flushes(self):
self._raise_at(1000.0, "batched")
self._acknowledge_at(1010.0, "batched")
def test_a_dismissed_notice_reopens_at_the_count(self):
self.registry.raise_notice("batched")
self.registry.dismiss("batched")
self._raise_at(1020.0, "batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active(), [])
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active()[0]["count"], 2)
notice = self.registry.active()[0]
self.assertEqual(notice["count"], BATCHED.reopen_at_count)
self.assertIsNone(notice["dismissed_at"])
def test_repeats_held_from_before_an_acknowledgement_stay_hidden(self):
self._raise_at(1000.0, "batched")
self._raise_at(1005.0, "batched")
self._acknowledge_at(1010.0, "batched")
def test_a_notice_dismissed_past_the_count_stays_dismissed(self):
for _ in range(3):
self.registry.raise_notice("batched")
self.registry.flush()
self.registry.dismiss("batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active(), [])
self.assertEqual(self.registry.active(include_hidden=True)[0]["count"], 2)
self.assertEqual(self.registry.active(include_dismissed=True)[0]["count"], 4)
def test_keep_latest_drops_the_oldest_rows(self):
for index, scope in enumerate(("a", "b", "c")):
@@ -473,13 +410,14 @@ class TestCatalogLifecycles(RegistryTestCase):
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["update_available:0.19.1"])
def test_an_acknowledged_failed_login_burst_returns_on_the_next_attempt(self):
def test_a_dismissed_failed_login_burst_reopens_at_five_attempts(self):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.acknowledge("failed_login:1000")
self.registry.dismiss("failed_login:1000")
self.registry.raise_notice("failed_login", scope="1000")
for _ in range(4):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.flush()
burst = self.registry.active()[0]
self.assertEqual(burst["count"], 2)
self.assertIsNone(burst["acknowledged_at"])
self.assertEqual(burst["count"], 5)
self.assertIsNone(burst["dismissed_at"])
-37
View File
@@ -15,8 +15,6 @@ Also covers the inverse direction: the ptz movement timestamps must not be writt
for a camera that has autotracking off, because nothing clears them back out.
"""
import asyncio
import threading
import unittest
from unittest.mock import AsyncMock, MagicMock
@@ -236,40 +234,5 @@ class TestManualRelativeMoveMetrics(unittest.IsolatedAsyncioTestCase):
)
class TestOnvifClose(unittest.TestCase):
"""close() must release everything on the loop, since whatever it leaves is
garbage collected during interpreter shutdown, where the resulting warnings
fail to log and fill the shutdown output with logging errors."""
def setUp(self) -> None:
self.controller = _make_controller(autotracking_enabled=False)
self.onvif = self.controller.cams[CAMERA]["onvif"]
self.onvif.close = AsyncMock()
self.controller.config_subscriber = MagicMock()
self.controller.loop = asyncio.new_event_loop()
self.controller.loop_thread = threading.Thread(
target=self.controller._run_event_loop, daemon=True
)
self.controller.loop_thread.start()
self.addCleanup(self.controller.loop.close)
def test_close_closes_camera_sessions(self) -> None:
self.controller.close()
self.onvif.close.assert_awaited_once()
def test_close_cancels_tasks_left_on_the_loop(self) -> None:
async def forever() -> None:
while True:
await asyncio.sleep(1)
poll = asyncio.run_coroutine_threadsafe(forever(), self.controller.loop)
self.controller.close()
self.assertTrue(poll.cancelled())
self.assertFalse(self.controller.loop_thread.is_alive())
if __name__ == "__main__":
unittest.main()
-55
View File
@@ -1,55 +0,0 @@
"""Tests for restarting frigate under s6."""
import signal
import unittest
from unittest.mock import MagicMock, patch
import psutil
from frigate.util.services import restart_frigate
class TestRestartFrigate(unittest.TestCase):
def _s6_process(self) -> MagicMock:
proc = MagicMock()
proc.name.return_value = "s6-svscan"
return proc
@patch("frigate.util.services.os.kill")
@patch("frigate.util.services.psutil.Process")
def test_terminates_s6_when_permitted(self, mock_process, mock_kill):
proc = self._s6_process()
mock_process.return_value = proc
restart_frigate()
proc.terminate.assert_called_once()
mock_kill.assert_not_called()
@patch("frigate.util.services.os.getpid", return_value=99)
@patch("frigate.util.services.os.kill")
@patch("frigate.util.services.psutil.Process")
def test_exits_self_when_s6_signal_is_denied(
self, mock_process, mock_kill, _mock_getpid
):
"""Running unprivileged, frigate cannot signal root's s6-svscan."""
proc = self._s6_process()
proc.terminate.side_effect = psutil.AccessDenied(pid=1, name="s6-svscan")
mock_process.return_value = proc
restart_frigate()
mock_kill.assert_called_once_with(99, signal.SIGINT)
@patch("frigate.util.services.os.getpid", return_value=99)
@patch("frigate.util.services.os.kill")
@patch("frigate.util.services.psutil.Process")
def test_exits_self_without_s6(self, mock_process, mock_kill, _mock_getpid):
proc = MagicMock()
proc.name.return_value = "init"
mock_process.return_value = proc
restart_frigate()
proc.terminate.assert_not_called()
mock_kill.assert_called_once_with(99, signal.SIGINT)
-316
View File
@@ -1,316 +0,0 @@
"""Tests for tracker-derived review frame annotations."""
import unittest
from frigate.data_processing.post.review_annotations import (
annotations_by_frame,
build_timeline,
describe_heading,
describe_position,
event_name,
path_legs,
path_moments,
)
def straight_path(
start: tuple[float, float],
end: tuple[float, float],
steps: int,
t0: float,
) -> list:
"""A path_data-shaped trajectory travelling in a straight line."""
return [
[
[
start[0] + (end[0] - start[0]) * i / (steps - 1),
start[1] + (end[1] - start[1]) * i / (steps - 1),
],
t0 + i,
]
for i in range(steps)
]
class TestDescribers(unittest.TestCase):
def test_describes_frame_corners(self):
self.assertEqual(describe_position(0.9, 0.9), "the bottom right of the frame")
self.assertEqual(describe_position(0.1, 0.1), "the top left of the frame")
self.assertEqual(describe_position(0.5, 0.5), "the middle of the frame")
def test_heading_treats_falling_y_as_up(self):
self.assertEqual(describe_heading(0.0, -0.5), "up")
self.assertEqual(describe_heading(0.0, 0.5), "down")
self.assertEqual(describe_heading(-0.5, 0.0), "left")
def test_heading_combines_axes(self):
self.assertEqual(describe_heading(-0.5, -0.5), "up and left")
class TestPathLegs(unittest.TestCase):
def test_straight_travel_is_one_leg(self):
points = [(0.9 - 0.05 * i, 0.6, float(i)) for i in range(10)]
self.assertEqual(len(path_legs(points)), 1)
def test_out_and_back_is_two_legs(self):
out = [(0.9 - 0.05 * i, 0.6, float(i)) for i in range(8)]
back = [(0.55 + 0.05 * i, 0.6, 8.0 + i) for i in range(8)]
self.assertEqual(len(path_legs(out + back)), 2)
def test_jitter_in_place_produces_no_legs(self):
# A subject standing still wobbles by a couple of percent; without the
# minimum leg distance this became a burst of contradictory turns.
points = [(0.5 + 0.01 * (i % 2), 0.5, float(i)) for i in range(20)]
self.assertEqual(path_legs(points), [])
def test_single_point_path_has_no_moments(self):
self.assertEqual(path_moments([[[0.5, 0.5], 1.0]]), [])
def test_malformed_path_is_ignored(self):
self.assertEqual(path_moments([[0.5, 1.0], [0.6, 2.0]]), [])
class TestEventNames(unittest.TestCase):
def test_unnamed_objects_use_an_indefinite_article(self):
self.assertEqual(event_name({"label": "person"}), "a person")
self.assertEqual(event_name({"label": "animal"}), "an animal")
def test_sub_labeled_objects_use_their_name(self):
event = {"label": "waste_bin", "sub_label": "Compost"}
self.assertEqual(event_name(event), 'waste bin "Compost"')
def test_repeated_objects_are_never_numbered(self):
# Whether these are the same subject is unknown, so the notes must not
# imply either answer.
events = [
{
"id": f"1789481994.68448{i}-abcdef",
"label": "person",
"sub_label": None,
"start_time": float(i * 10),
"end_time": float(i * 10 + 50),
"zones": [],
"path_data": straight_path((0.9, 0.6), (0.4, 0.3), 10, i * 10.0),
}
for i in range(3)
]
phrases = [p for _, p in build_timeline(events, span_end=100.0)]
self.assertTrue(all(p.startswith("a person ") for p in phrases))
self.assertFalse(any("#" in p for p in phrases))
class TestTimeline(unittest.TestCase):
def setUp(self):
self.event = {
"id": "1789481994.684479-lpyc2z",
"label": "person",
"sub_label": None,
"start_time": 0.0,
"end_time": 20.0,
"zones": ["front_yard"],
"path_data": straight_path((0.9, 0.6), (0.4, 0.3), 10, 1.0),
}
def test_timeline_is_ordered_and_bounded(self):
timeline = build_timeline([self.event], span_end=30.0)
times = [t for t, _ in timeline]
self.assertEqual(times, sorted(times))
self.assertTrue(all(t <= 30.0 for t in times))
def test_track_identifiers_never_appear(self):
timeline = build_timeline([self.event], span_end=30.0)
joined = " ".join(phrase for _, phrase in timeline)
self.assertNotIn("track", joined.lower())
self.assertNotIn(self.event["id"], joined)
def test_object_still_tracked_at_end_gets_no_closing_note(self):
# Its state at the end is visible in the last frame, so nothing is said.
timeline = build_timeline([self.event], span_end=15.0)
self.assertEqual(
[p for _, p in timeline],
["a person first detected at the right of the frame, moving up and left"],
)
def test_object_ending_inside_the_clip_is_no_longer_detected(self):
timeline = build_timeline([self.event], span_end=40.0)
phrases = [p for _, p in timeline]
self.assertTrue(any("no longer detected" in p for p in phrases))
def test_moments_past_the_last_frame_are_dropped(self):
# The subject keeps moving after the final sampled frame; those notes
# describe nothing the model can see.
late = dict(self.event)
out = straight_path((0.9, 0.6), (0.4, 0.6), 8, 100.0)
back = straight_path((0.4, 0.6), (0.9, 0.6), 8, 108.0)
late["path_data"] = out + back
late["start_time"] = 100.0
timeline = build_timeline([late], span_end=105.0)
self.assertTrue(any("moving left" in p for _, p in timeline))
self.assertFalse(any("turns around" in p for _, p in timeline))
def track(event_id, label, start, path, sub_label=None, end=None):
return {
"id": event_id,
"label": label,
"sub_label": sub_label,
"start_time": start,
"end_time": end if end is not None else start + 500.0,
"zones": [],
"path_data": path,
}
class TestArrival(unittest.TestCase):
def test_objects_arriving_together_keep_separate_notes(self):
person = track(
"1789481994.684479-lpyc2z",
"person",
0.0,
straight_path((0.9, 0.63), (0.58, 0.34), 10, 0.1),
)
bin_ = track(
"1789481995.063395-vlzd7q",
"waste_bin",
0.3,
straight_path((0.91, 0.64), (0.6, 0.33), 10, 0.4),
sub_label="Compost",
)
phrases = [p for _, p in build_timeline([person, bin_], 100.0)]
self.assertEqual(
phrases,
[
"a person first detected at the right of the frame, moving up and left",
'waste bin "Compost" first detected at the right of the frame, '
"moving up and left",
],
)
def test_object_detected_well_before_it_moves_gets_separate_notes(self):
# path_data always keeps the first two samples, so a bin sitting in
# the yard opens its leg long before it is picked up.
path = [[[0.91, 0.64], 0.4]] + straight_path(
(0.91, 0.64), (0.6, 0.33), 10, 20.4
)
bin_ = track(
"1789481995.063395-vlzd7q", "waste_bin", 0.3, path, sub_label="Compost"
)
timeline = build_timeline([bin_], 100.0)
self.assertEqual(
timeline[0],
(0.3, 'waste bin "Compost" first detected at the right of the frame'),
)
self.assertAlmostEqual(timeline[1][0], 21.4)
self.assertEqual(
timeline[1][1],
'waste bin "Compost" starts moving up and left from the right of the frame',
)
class TestStateChanges(unittest.TestCase):
def test_stationary_and_active_rows_become_notes(self):
bin_ = track(
"1789481995.063395-vlzd7q",
"waste_bin",
0.3,
straight_path((0.91, 0.64), (0.6, 0.33), 10, 0.4),
sub_label="Compost",
)
changes = [
{"timestamp": 20.0, "source_id": bin_["id"], "class_type": "stationary"},
{"timestamp": 30.0, "source_id": bin_["id"], "class_type": "active"},
{"timestamp": 35.0, "source_id": bin_["id"], "class_type": "entered_zone"},
{
"timestamp": 40.0,
"source_id": "someone-else",
"class_type": "stationary",
},
]
timeline = build_timeline([bin_], 100.0, state_changes=changes)
self.assertEqual(
timeline[1:],
[
(20.0, 'waste bin "Compost" has stopped moving'),
(30.0, 'waste bin "Compost" starts moving again'),
],
)
def test_state_changes_after_the_last_frame_are_dropped(self):
bin_ = track(
"1789481995.063395-vlzd7q",
"waste_bin",
0.3,
straight_path((0.91, 0.64), (0.6, 0.33), 10, 0.4),
sub_label="Compost",
)
changes = [
{"timestamp": 200.0, "source_id": bin_["id"], "class_type": "stationary"}
]
timeline = build_timeline([bin_], 100.0, state_changes=changes)
self.assertFalse(any("stopped" in p for _, p in timeline))
class TestNoAssumedState(unittest.TestCase):
def test_no_note_for_where_movement_ends(self):
# Ending a leftward walk still in the right third was read as a turn
# back to the right, and "stops moving" was read as standing still.
moments = path_moments(straight_path((0.95, 0.6), (0.7, 0.4), 6, 0.0))
self.assertEqual(
[p for _, p in moments],
["starts moving up and left from the right of the frame"],
)
def test_notes_never_claim_an_object_is_stationary(self):
# A track still open at the last frame says nothing about motion.
event = {
"id": "1789482056.695307-3uhf47",
"label": "person",
"sub_label": None,
"start_time": 0.0,
"end_time": 500.0,
"zones": ["front_yard"],
"path_data": straight_path((0.9, 0.6), (0.4, 0.3), 10, 1.0),
}
joined = " ".join(p for _, p in build_timeline([event], span_end=20.0))
self.assertNotIn("stationary", joined)
self.assertNotIn("stops", joined)
self.assertNotIn("leaves", joined)
self.assertNotIn("still", joined)
class TestLateEvents(unittest.TestCase):
def test_objects_first_detected_after_the_last_frame_are_skipped(self):
# Without this the arrival lands on the final frame, which was
# captured before the object appeared.
late = track(
"1789481999.000000-latear",
"person",
50.0,
straight_path((0.9, 0.6), (0.4, 0.3), 10, 50.1),
)
self.assertEqual(build_timeline([late], span_end=40.0), [])
self.assertEqual(
annotations_by_frame(build_timeline([late], 40.0), [0.0, 40.0]), {}
)
class TestFrameBucketing(unittest.TestCase):
def test_moment_attaches_to_the_following_frame(self):
frame_times = [0.0, 10.0, 20.0, 30.0]
buckets = annotations_by_frame([(12.0, "something happened")], frame_times)
self.assertEqual(buckets, {2: ["something happened"]})
def test_moment_on_a_frame_boundary_uses_that_frame(self):
buckets = annotations_by_frame([(10.0, "x")], [0.0, 10.0, 20.0])
self.assertEqual(buckets, {1: ["x"]})
def test_moment_after_the_last_frame_falls_on_the_last_frame(self):
buckets = annotations_by_frame([(99.0, "x")], [0.0, 10.0])
self.assertEqual(buckets, {1: ["x"]})
def test_no_frames_yields_no_buckets(self):
self.assertEqual(annotations_by_frame([(1.0, "x")], []), {})
if __name__ == "__main__":
unittest.main()
-147
View File
@@ -1,7 +1,5 @@
"""Tests for the device each model runner reports after loading."""
import os
import tempfile
import threading
import unittest
from unittest.mock import MagicMock, patch
@@ -40,7 +38,6 @@ class TestRunnerDeviceName(unittest.TestCase):
self._onnx(["OpenVINOExecutionProvider"]).device_name, "OpenVINO"
)
self.assertEqual(self._onnx(["CPUExecutionProvider"]).device_name, "CPU")
self.assertEqual(self._onnx(["LighterANE"]).device_name, "Neural Engine")
self.assertEqual(self._onnx(["ROCMExecutionProvider"]).device_name, "ROCM")
self.assertEqual(self._onnx([]).device_name, "CPU")
@@ -132,147 +129,3 @@ class TestLoadedDeviceSnapshot(unittest.TestCase):
finally:
stop.set()
thread.join(timeout=5)
class TestLighterANE(unittest.TestCase):
"""lighter's plugin provider, picked up by the ONNX session setup."""
def setUp(self):
loaded_devices.clear()
self.root = tempfile.TemporaryDirectory()
self.addCleanup(self.root.cleanup)
self.library = os.path.join(self.root.name, "liblighter_ane_ep.so")
env = patch.dict(os.environ, {"LIGHTER_ANE_EP": self.library})
env.start()
self.addCleanup(env.stop)
def _device(self) -> MagicMock:
device = MagicMock()
device.ep_name = "LighterANE"
return device
def test_no_devices_without_the_library(self):
with patch.object(detection_runners.ort, "get_ep_devices") as get_ep_devices:
self.assertEqual(detection_runners.get_lighter_ane_devices(), [])
get_ep_devices.assert_not_called()
def test_the_provider_is_registered_once(self):
open(self.library, "w").close()
device = self._device()
other = MagicMock()
other.ep_name = "CPUExecutionProvider"
with (
patch.object(
detection_runners.ort,
"get_ep_devices",
side_effect=[[other], [other, device], [other, device]],
),
patch.object(
detection_runners.ort, "register_execution_provider_library"
) as register,
):
self.assertEqual(detection_runners.get_lighter_ane_devices(), [device])
self.assertEqual(detection_runners.get_lighter_ane_devices(), [device])
register.assert_called_once_with("LighterANE", self.library)
def test_a_provider_that_will_not_load_is_skipped(self):
open(self.library, "w").close()
with (
patch.object(detection_runners.ort, "get_ep_devices", return_value=[]),
patch.object(
detection_runners.ort,
"register_execution_provider_library",
side_effect=RuntimeError("not a provider"),
),
):
self.assertEqual(detection_runners.get_lighter_ane_devices(), [])
def test_the_session_runs_on_the_neural_engine(self):
device = self._device()
session = MagicMock()
session.get_providers.return_value = ["LighterANE", "CPUExecutionProvider"]
options = MagicMock()
with (
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
patch.object(
detection_runners, "get_lighter_ane_devices", return_value=[device]
),
patch.object(
detection_runners, "get_ort_session_options", return_value=options
),
patch.object(
detection_runners.ort, "InferenceSession", return_value=session
) as inference_session,
):
runner = get_optimized_runner("/models/yolo.onnx", "AUTO", "yolo-generic")
options.add_provider_for_devices.assert_called_once_with([device], {})
inference_session.assert_called_once_with(
"/models/yolo.onnx", sess_options=options
)
self.assertIsInstance(runner, ONNXModelRunner)
self.assertEqual(
loaded_devices["/models/yolo.onnx"], ("yolo-generic", "Neural Engine")
)
def test_a_model_the_neural_engine_cannot_load_uses_the_default_providers(self):
session = MagicMock()
session.get_providers.return_value = ["CPUExecutionProvider"]
with (
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
patch.object(
detection_runners, "get_lighter_ane_devices", return_value=[MagicMock()]
),
patch.object(
detection_runners,
"get_ort_providers",
return_value=(["CPUExecutionProvider"], [{}]),
),
patch.object(
detection_runners, "is_openvino_gpu_npu_available", return_value=False
),
patch.object(
detection_runners.ort,
"InferenceSession",
side_effect=[RuntimeError("unsupported"), session],
) as inference_session,
patch.object(
detection_runners, "get_ort_session_options", return_value=MagicMock()
),
self.assertLogs(detection_runners.logger, level="WARNING"),
):
runner = get_optimized_runner("/models/jina.onnx", "AUTO", "jina-v2")
self.assertEqual(inference_session.call_count, 2)
self.assertIsInstance(runner, ONNXModelRunner)
self.assertEqual(loaded_devices["/models/jina.onnx"], ("jina-v2", "CPU"))
def test_a_cpu_model_stays_on_the_cpu(self):
session = MagicMock()
session.get_providers.return_value = ["CPUExecutionProvider"]
with (
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
patch.object(
detection_runners, "get_lighter_ane_devices", return_value=[MagicMock()]
) as ane,
patch.object(
detection_runners,
"get_ort_providers",
return_value=(["CPUExecutionProvider"], [{}]),
),
patch.object(
detection_runners.ort, "InferenceSession", return_value=session
),
patch.object(
detection_runners, "get_ort_session_options", return_value=None
),
):
get_optimized_runner("/models/arcface.onnx", "CPU", "arcface")
ane.assert_not_called()
+18 -86
View File
@@ -1,14 +1,10 @@
"""Tests for the per-camera episode notices: skipped detections and high CPU."""
"""Tests for the skipped detection rate and the notice it can raise."""
import unittest
from unittest.mock import call, patch
from unittest.mock import patch
from frigate.stats import emitter
from frigate.stats.emitter import (
SKIPPED_DETECTIONS_PCT,
EpisodeTracker,
cpu_average,
)
from frigate.stats.emitter import SkippedDetectionsTracker
from frigate.stats.util import skipped_percent
@@ -23,18 +19,18 @@ class TestSkippedPercent(unittest.TestCase):
self.assertEqual(skipped_percent(2.0, 5.0, False), 0.0)
class TestEpisodeTracker(unittest.TestCase):
class TestSkippedDetectionsTracker(unittest.TestCase):
def _cameras(self, pct: float) -> dict:
return {"front_door": pct}
return {"front_door": {"skipped_pct": pct}}
def test_below_threshold_never_qualifies(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker = SkippedDetectionsTracker()
for tick in range(10):
self.assertEqual(tracker.update(self._cameras(4.9), tick * 15.0), [])
def test_qualifies_once_after_the_hold(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker = SkippedDetectionsTracker()
results = [
tracker.update(self._cameras(10.0), tick * 15.0) for tick in range(8)
@@ -46,7 +42,7 @@ class TestEpisodeTracker(unittest.TestCase):
self.assertEqual(results[5:], [[], [], []])
def test_a_dip_restarts_the_hold(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker = SkippedDetectionsTracker()
for now, pct in ((0.0, 10.0), (15.0, 10.0), (30.0, 10.0), (45.0, 2.0)):
self.assertEqual(tracker.update(self._cameras(pct), now), [])
@@ -55,7 +51,7 @@ class TestEpisodeTracker(unittest.TestCase):
self.assertEqual(tracker.update(self._cameras(10.0), 120.0), ["front_door"])
def test_recovery_then_relapse_is_a_new_episode(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker = SkippedDetectionsTracker()
tracker.update(self._cameras(10.0), 0.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), ["front_door"])
@@ -65,15 +61,17 @@ class TestEpisodeTracker(unittest.TestCase):
self.assertEqual(tracker.update(self._cameras(10.0), 150.0), ["front_door"])
def test_cameras_are_tracked_separately(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker.update({"a": 10.0, "b": 0.0}, 0.0)
tracker = SkippedDetectionsTracker()
tracker.update({"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 0.0}}, 0.0)
qualified = tracker.update({"a": 10.0, "b": 10.0}, 60.0)
qualified = tracker.update(
{"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 10.0}}, 60.0
)
self.assertEqual(qualified, ["a"])
def test_a_removed_camera_starts_a_new_episode(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker = SkippedDetectionsTracker()
tracker.update(self._cameras(10.0), 0.0)
tracker.update({}, 15.0)
tracker.update(self._cameras(10.0), 30.0)
@@ -81,25 +79,6 @@ class TestEpisodeTracker(unittest.TestCase):
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
self.assertEqual(tracker.update(self._cameras(10.0), 90.0), ["front_door"])
def test_a_missing_value_ends_the_episode(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker.update(self._cameras(10.0), 0.0)
tracker.update({"front_door": None}, 30.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
class TestCpuAverage(unittest.TestCase):
def test_reads_the_lifetime_average(self):
usages = {"42": {"cpu": "80.0", "cpu_average": "25.0"}}
self.assertEqual(cpu_average(usages, 42), 25.0)
def test_unknown_process_has_no_average(self):
self.assertIsNone(cpu_average({}, 42))
self.assertIsNone(cpu_average({"42": {"cpu_average": "25.0"}}, None))
self.assertIsNone(cpu_average({"42": {"cpu_average": ""}}, 42))
class TestEmitterNotices(unittest.TestCase):
def setUp(self):
@@ -115,25 +94,15 @@ class TestEmitterNotices(unittest.TestCase):
def _emitter(self) -> emitter.StatsEmitter:
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
stats_emitter.skipped_detections = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
stats_emitter.ffmpeg_cpu = EpisodeTracker(emitter.FFMPEG_HIGH_CPU_PCT)
stats_emitter.detect_cpu = EpisodeTracker(emitter.DETECT_HIGH_CPU_PCT)
stats_emitter.skipped_detections = SkippedDetectionsTracker()
stats_emitter._shm_checked = False
stats_emitter._shm_params = None
return stats_emitter
def _stats(
self, uptime: int, pct: float, ffmpeg_cpu: str = "5.0", detect_cpu: str = "5.0"
) -> dict:
def _stats(self, uptime: int, pct: float) -> dict:
return {
"service": {"uptime": uptime, "storage": {"/dev/shm": {}}},
"cameras": {
"front_door": {"skipped_pct": pct, "ffmpeg_pid": 10, "pid": 11}
},
"cpu_usages": {
"10": {"cpu_average": ffmpeg_cpu},
"11": {"cpu_average": detect_cpu},
},
"cameras": {"front_door": {"skipped_pct": pct}},
}
def test_qualified_camera_raises_a_notice(self):
@@ -146,43 +115,6 @@ class TestEmitterNotices(unittest.TestCase):
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
def test_high_cpu_raises_a_notice_per_process(self):
stats_emitter = self._emitter()
for uptime, now in ((300, 0.0), (360, 60.0)):
stats_emitter._update_notices(
self._stats(uptime, 0.0, ffmpeg_cpu="25.0", detect_cpu="45.0"), now
)
self.assertEqual(
self.raise_notice.call_args_list,
[
call("ffmpeg_high_cpu", scope="front_door", params={"cpu": 25.0}),
call("detect_high_cpu", scope="front_door", params={"cpu": 45.0}),
],
)
def test_cpu_below_each_threshold_raises_nothing(self):
stats_emitter = self._emitter()
for uptime, now in ((300, 0.0), (360, 60.0)):
stats_emitter._update_notices(
self._stats(uptime, 0.0, ffmpeg_cpu="19.0", detect_cpu="39.0"), now
)
self.raise_notice.assert_not_called()
def test_missing_cpu_stats_raise_nothing(self):
stats_emitter = self._emitter()
for uptime, now in ((300, 0.0), (360, 60.0)):
stats = self._stats(uptime, 0.0)
del stats["cpu_usages"]
stats_emitter._update_notices(stats, now)
self.raise_notice.assert_not_called()
self.assertEqual(self.flush_notices.call_count, 2)
def test_startup_window_is_ignored(self):
stats_emitter = self._emitter()
+3 -1
View File
@@ -324,7 +324,9 @@ class NorfairTracker(ObjectTracker):
):
tracker = self.get_tracker(obj["label"])
tracker.tracked_objects = [
o for o in tracker.tracked_objects if str(o.global_id) != track_id
o
for o in tracker.tracked_objects
if str(o.global_id) != track_id and o.hit_counter < 0
]
del self.track_id_map[track_id]
+2 -263
View File
@@ -1,61 +1,16 @@
"""Utilities for creating and manipulating audio."""
import io
import logging
import os
import re
import string
import struct
import subprocess as sp
import wave
import numpy as np
from pathvalidate import sanitize_filename
from frigate.const import (
AUDIO_SAMPLE_RATE,
CACHE_DIR,
STREAM_TYPE_MAIN,
STREAM_TYPE_SUB,
)
from frigate.const import CACHE_DIR, STREAM_TYPE_MAIN, STREAM_TYPE_SUB
from frigate.models import Recordings
logger = logging.getLogger(__name__)
# Ceiling on the run of words the stitcher will treat as an overlap between two
# consecutive windows. This is an audio-duration bound, not a linguistic one: a
# window holds GENAI_WINDOW_CHUNKS * AUDIO_DURATION seconds of speech, so at a
# fast talker's pace it tops out around this many words, and a whole window can
# legitimately be redundant. The vendored whisper_streaming HypothesisBuffer
# caps at 5, but there the n-gram is only a tie-break on top of word-level
# timestamps; here it is the entire alignment, so 5 truncates real overlaps.
# Sentinel meaning "let the model work out the language". The vendored
# whisper_streaming code already uses this spelling, so it is the established
# convention for the audio_transcription.language field.
AUTO_LANGUAGE = "auto"
MAX_STITCH_NGRAM = 16
# How many trailing committed words the stitcher may discard to find an
# alignment. Those words came from the newest audio, which the next window
# re-covers, so when the provider got one of them wrong it blocks every
# alignment and the whole phrase duplicates. Set to 0 to make committed text
# strictly append-only.
MAX_STITCH_REVISE = 3
# A revision deletes text that was already published, so it has to clear a
# higher bar than a plain append: a single coincidentally shared word is not
# enough evidence to throw committed words away.
MIN_STITCH_REVISE_RUN = 2
# ASR models often wrap their output in control markup. Qwen3-ASR, for example,
# answers "language English<asr_text>Yeah, that works." A structural opening tag
# marks where the transcript starts, so anything before the last one is metadata.
# Closing tags (</x>) and pipe-delimited special tokens (<|endoftext|>) are
# excluded: those mark where the text ends, so text before them must be kept.
_OPENING_TAG = re.compile(r"<(?![/|])[^<>]*>")
_ANY_TAG = re.compile(r"<[^<>]*>")
def _get_recordings_for_range(
camera_name: str, start_ts: float, end_ts: float, stream_type: str
@@ -162,9 +117,7 @@ def get_audio_from_recording(
logger.debug(
f"Successfully extracted audio for {camera_name} from {start_ts} to {end_ts}"
)
# ffmpeg writes to a pipe, so it cannot seek back to patch the chunk
# sizes it reserved; repair them before any strict consumer sees them
return fix_wav_header(process.stdout)
return process.stdout
else:
logger.error(f"Failed to extract audio: {process.stderr.decode()}")
return None
@@ -176,217 +129,3 @@ def get_audio_from_recording(
os.unlink(file_path)
except OSError:
pass
def fix_wav_header(data: bytes) -> bytes:
"""Recompute the RIFF and data chunk sizes in a WAV header.
ffmpeg writing to a non-seekable pipe cannot go back and patch the sizes it
reserved, so it leaves 0xFFFFFFFF placeholders. PyAV-based demuxers ignore
them, but strict validators may reject the file or read zero frames.
Args:
data: The complete WAV payload
Returns:
The payload with both sizes corrected, or unchanged if it is not a
parseable RIFF/WAVE stream
"""
if len(data) < 12 or data[0:4] != b"RIFF" or data[8:12] != b"WAVE":
return data
out = bytearray(data)
# RIFF size covers everything after the 8-byte RIFF header
struct.pack_into("<I", out, 4, len(out) - 8)
# walk the chunk list to find "data"; every chunk is padded to even length
pos = 12
while pos + 8 <= len(out):
chunk_id = bytes(out[pos : pos + 4])
(chunk_size,) = struct.unpack_from("<I", out, pos + 4)
if chunk_id == b"data":
struct.pack_into("<I", out, pos + 4, len(out) - (pos + 8))
return bytes(out)
if chunk_size == 0xFFFFFFFF:
# an unpatched size before the data chunk leaves nothing to walk
break
pos += 8 + chunk_size + (chunk_size % 2)
return bytes(out)
def pcm16_to_wav(samples: np.ndarray, sample_rate: int = AUDIO_SAMPLE_RATE) -> bytes:
"""Wrap mono int16 PCM samples in a WAV container.
Args:
samples: The audio samples; converted to int16 if they are not already
sample_rate: Sample rate to declare in the header
Returns:
WAV bytes suitable for upload to a GenAI provider
"""
if samples.dtype != np.int16:
samples = samples.astype(np.int16)
buffer = io.BytesIO()
with wave.open(buffer, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(sample_rate)
wav.writeframes(samples.tobytes())
return buffer.getvalue()
def stitch_transcripts(committed: str, incoming: str) -> str:
"""Append *incoming* to *committed*, dropping the speech they share.
Consecutive overlapped transcription windows re-transcribe the same audio at
their seam, so the tail of one and the newest one name the same words. Find
the longest run that is a suffix of *committed* and occurs anywhere in
*incoming*, then keep only what follows that run.
Searching all of *incoming* rather than just its start is what makes this
work in practice. The provider re-transcribes the shared audio independently
and often gets its first word or two different ("just gonna" one window,
"It's gonna" the next), which leaves the real overlap sitting in the middle
of *incoming*. A prefix-anchored match sees no overlap at all there and
duplicates the entire phrase.
Text-level rather than timestamp-level because only some providers return
word timings, and this has to work across all of them.
Args:
committed: The transcript accumulated so far
incoming: The newest window's transcript
Returns:
The combined transcript
"""
incoming_words = incoming.split()
if not incoming_words:
return committed
committed_words = committed.split()
if not committed_words:
return " ".join(incoming_words)
committed_keys = [_overlap_key(word) for word in committed_words]
incoming_keys = [_overlap_key(word) for word in incoming_words]
length, consumed = _find_overlap(committed_keys, incoming_keys)
if length:
return " ".join(committed_words + incoming_words[consumed:])
# Nothing aligns. Retry against a shortened committed tail: a single word the
# provider got wrong at the end of the previous window otherwise blocks every
# alignment, and the entire re-transcribed phrase duplicates behind it.
best: tuple[int, int, int] | None = None
for drop in range(1, min(MAX_STITCH_REVISE, len(committed_keys) - 1) + 1):
length, consumed = _find_overlap(committed_keys[:-drop], incoming_keys)
if length < MIN_STITCH_REVISE_RUN:
continue
# longest run wins; ties go to the smallest revision
if best is None or length > best[0]:
best = (length, drop, consumed)
if best is None:
return " ".join(committed_words + incoming_words)
_, drop, consumed = best
return " ".join(committed_words[:-drop] + incoming_words[consumed:])
def _find_overlap(
committed_keys: list[str], incoming_keys: list[str]
) -> tuple[int, int]:
"""Locate the speech *incoming* shares with the end of *committed*.
Returns the length of the longest run that is a suffix of *committed_keys*
and occurs anywhere in *incoming_keys*, along with the index just past that
run in *incoming_keys*. Returns ``(0, 0)`` when nothing matches.
Prefers the longest run so a real overlap is not cut short, and within one
length the earliest position, so a phrase genuinely spoken twice keeps its
second utterance.
"""
max_run = min(MAX_STITCH_NGRAM, len(committed_keys), len(incoming_keys))
for length in range(max_run, 0, -1):
tail = committed_keys[-length:]
for start in range(len(incoming_keys) - length + 1):
if incoming_keys[start : start + length] == tail:
return length, start + length
return 0, 0
def clean_transcript(text: str | None) -> str:
"""Strip provider control markup and any preamble from a raw transcript.
A window with no speech often still comes back as the preamble alone
("language English<asr_text>"), which must reduce to an empty string so
callers treat it as silence rather than committing it as spoken words.
Args:
text: The provider's raw response
Returns:
The transcript with markup removed and whitespace collapsed
"""
if not text:
return ""
# everything up to and including the last opening tag is metadata
openings = list(_OPENING_TAG.finditer(text))
if openings:
text = text[openings[-1].end() :]
# drop closing tags and special tokens wherever they landed
text = _ANY_TAG.sub(" ", text)
return " ".join(text.split())
def _overlap_key(word: str) -> str:
"""Comparison key for overlap matching.
Providers re-transcribe the shared audio at a window seam independently, so
the same word routinely comes back capitalized differently or with different
edge punctuation ("work." vs "Work"). Those differences must not defeat the
match, but the original spelling is what gets kept in the output.
"""
key = word.strip(string.punctuation).casefold()
# a token that is nothing but punctuation would otherwise match any other
return key or word
def resolve_language(language: str | None) -> str | None:
"""Turn a configured language into an explicit code, or None for auto-detect.
Args:
language: The configured value, possibly AUTO_LANGUAGE
Returns:
An ISO language code, or None when the backend should detect it
"""
if not language or language == AUTO_LANGUAGE:
return None
return language
+4 -7
View File
@@ -56,13 +56,10 @@ class EventsPerSecond:
self._start = now
# compute the (approximate) events in the last n seconds
self.expire_timestamps(now)
# rate over at least one second (or the whole window, if shorter),
# so a burst of events right after start() is not divided by a
# tiny window
seconds = max(
min(now - self._start, self._last_n_seconds),
min(1.0, self._last_n_seconds),
)
seconds = min(now - self._start, self._last_n_seconds)
# avoid divide by zero
if seconds == 0:
seconds = 1
return len(self._timestamps) / seconds
# remove aged out timestamps
-53
View File
@@ -829,57 +829,6 @@ def rename_hailo_detector(
return new_config
def _camera_enables_transcription(camera: dict[str, Any]) -> bool:
"""Whether a camera or one of its profiles turns audio transcription on."""
sections = [camera.get("audio_transcription")]
profiles = camera.get("profiles")
if isinstance(profiles, dict):
for profile in profiles.values():
if isinstance(profile, dict):
sections.append(profile.get("audio_transcription"))
return any(
isinstance(section, dict) and section.get("enabled") for section in sections
)
def _migrate_transcription_language(config: dict[str, Any]) -> None:
"""Pin English for configs written before the language default became auto.
audio_transcription.language used to default to "en", so a config that
turned transcription on without naming a language was transcribing English.
The default is now "auto" (let the model detect), which is better for new
users but would silently change behavior for existing ones, so write the old
value explicitly for anyone actually using the feature.
"""
transcription = config.get("audio_transcription")
if isinstance(transcription, dict) and "language" in transcription:
# named a language already, so nothing was relying on the default
return
enabled = isinstance(transcription, dict) and bool(transcription.get("enabled"))
if not enabled:
enabled = any(
_camera_enables_transcription(camera)
for camera in config.get("cameras", {}).values()
if isinstance(camera, dict)
)
if not enabled:
return
if not isinstance(transcription, dict):
# a camera enabled it without a global section, which still picked up
# the global default
transcription = {}
config["audio_transcription"] = transcription
transcription["language"] = "en"
def migrate_019_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""Handle migrating Frigate config to 0.19-0."""
new_config = rename_hailo_detector(config)
@@ -896,8 +845,6 @@ def migrate_019_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
new_config["cameras"][name] = camera_config
_migrate_transcription_language(new_config)
new_config["version"] = "0.19-0"
return new_config
-3
View File
@@ -49,9 +49,6 @@ def start_or_restart_ffmpeg(
if ffmpeg_process is not None:
stop_ffmpeg(ffmpeg_process, logger)
# flush after the stop so the logs cover ffmpeg's output up to exit
logpipe.dump()
if frame_size is None:
process = sp.Popen(
ffmpeg_cmd,
+1 -33
View File
@@ -9,7 +9,6 @@ and resolve it per camera against that camera's detect stream.
"""
import logging
import os
import re
from pydantic import BaseModel, Field
@@ -24,20 +23,14 @@ from frigate.util.services import enumerate_drm_devices
logger = logging.getLogger(__name__)
# roots the /proc and /sys reads use, so tests can point them at a fixture tree
# root the /proc reads use, so tests can point them at a fixture tree
PROC_ROOT = "/proc"
SYS_ROOT = "/sys"
ANY_CODEC = "any"
# a Raspberry Pi has no detection hardware of its own, so it gets a key here
RASPBERRY_PI = "raspberrypi"
# lighter (https://github.com/fieldwork-ai/lighter) decodes on a Mac's media
# engine through virtio-media devices, which carry its virtio vendor id "LGHT"
LIGHTER_MEDIA = "lighter"
LIGHTER_VIRTIO_VENDOR = "0x4c474854"
# ffprobe names h265 streams hevc
CODEC_ALIASES = {"hevc": "h265"}
@@ -59,7 +52,6 @@ DECODE_HARDWARE = (
"rknn",
"openvino:GPU",
"onnx:amd",
LIGHTER_MEDIA,
RASPBERRY_PI,
)
@@ -106,10 +98,6 @@ FAMILY_RPI = HwaccelFamily(
key="rpi",
presets={"h264": "preset-rpi-64-h264", "h265": "preset-rpi-64-h265"},
)
FAMILY_APPLE = HwaccelFamily(
key="apple-silicon",
presets={"h264": "preset-apple-silicon-h264", "h265": "preset-apple-silicon-h265"},
)
def _read(path: str) -> str | None:
@@ -151,20 +139,6 @@ def _is_raspberry_pi() -> bool:
return "raspberrypi" in compatible
def _has_lighter_media() -> bool:
video = f"{SYS_ROOT}/class/video4linux"
try:
devices = os.listdir(video)
except OSError:
return False
return any(
_read(f"{video}/{device}/device/vendor") == LIGHTER_VIRTIO_VENDOR
for device in devices
)
def _intel_families(generation: int | None) -> list[HwaccelFamily]:
"""vaapi drives every Intel GPU, qsv only those from gen8 on."""
if generation is not None and generation < INTEL_QSV_SUPPORTED_GEN:
@@ -190,9 +164,6 @@ def _families(key: str, generation: int | None) -> list[HwaccelFamily]:
if key == "onnx:amd":
return [FAMILY_VAAPI]
if key == LIGHTER_MEDIA:
return [FAMILY_APPLE]
if key == RASPBERRY_PI:
return [FAMILY_RPI]
@@ -222,9 +193,6 @@ def _decode_hardware(detector_key: str | None) -> list[str]:
"""
present = {found.key for found in hardware_prober.probe()}
if _has_lighter_media():
present.add(LIGHTER_MEDIA)
if _is_raspberry_pi():
present.add(RASPBERRY_PI)
+3 -10
View File
@@ -35,19 +35,12 @@ logger = logging.getLogger(__name__)
def restart_frigate():
proc = psutil.Process(1)
# if this is running via s6, sigterm pid 1
if proc.name() == "s6-svscan":
try:
proc.terminate()
return
except psutil.AccessDenied:
# frigate runs unprivileged, so it cannot signal root's s6-svscan.
# exiting this process instead runs frigate/finish, which halts s6
logger.debug("Not permitted to signal s6-svscan, exiting instead")
proc.terminate()
# otherwise, just try and exit frigate
os.kill(os.getpid(), signal.SIGINT)
else:
os.kill(os.getpid(), signal.SIGINT)
def print_stack(sig, frame):
+19 -40
View File
@@ -2,7 +2,6 @@
import datetime
import logging
import math
from zoneinfo import ZoneInfoNotFoundError
import pytz
@@ -44,33 +43,9 @@ def is_current_hour(timestamp: int) -> bool:
return timestamp < start_of_next_hour
def _utc_offset(tz: datetime.tzinfo, timestamp: float) -> float:
dt = datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)
return dt.astimezone(tz).utcoffset().total_seconds()
def _find_transition(
tz: datetime.tzinfo, lo: float, hi: float, lo_offset: float
) -> float:
"""Bisect (lo, hi] to the second where the UTC offset first differs from lo_offset."""
# whole seconds, so the midpoint always advances (a fractional bound can
# otherwise leave the midpoint sitting on lo) and lands on the transition
low = math.floor(lo)
high = math.ceil(hi)
while high - low > 1:
mid = (low + high) // 2
if _utc_offset(tz, mid) == lo_offset:
low = mid
else:
high = mid
return float(high)
def get_dst_transitions(
tz_name: str, start_time: float, end_time: float
) -> list[tuple[float, float, float]]:
) -> list[tuple[float, float]]:
"""
Find DST transition points and return time periods with consistent offsets.
@@ -91,24 +66,28 @@ def get_dst_transitions(
periods = []
current = start_time
# Get initial offset
dt = datetime.datetime.utcfromtimestamp(current).replace(tzinfo=pytz.UTC)
local_dt = dt.astimezone(tz)
prev_offset = local_dt.utcoffset().total_seconds()
period_start = start_time
prev_offset = _utc_offset(tz, current)
# Probe at most a day ahead, capped at end_time so a transition after the
# last full day is still seen instead of silently kept in the last period.
while current < end_time:
next_probe = min(current + 86400, end_time)
next_offset = _utc_offset(tz, next_probe)
# Check each day for offset changes
while current <= end_time:
dt = datetime.datetime.utcfromtimestamp(current).replace(tzinfo=pytz.UTC)
local_dt = dt.astimezone(tz)
current_offset = local_dt.utcoffset().total_seconds()
if next_offset != prev_offset:
transition = _find_transition(tz, current, next_probe, prev_offset)
periods.append((period_start, transition, prev_offset))
period_start = transition
prev_offset = _utc_offset(tz, transition)
current = transition
else:
current = next_probe
if current_offset != prev_offset:
# Found a transition - close previous period
periods.append((period_start, current, prev_offset))
period_start = current
prev_offset = current_offset
current += 86400 # Check daily
# Add final period
periods.append((period_start, end_time, prev_offset))
return periods
+4
View File
@@ -319,6 +319,9 @@ class CameraWatchdog(threading.Thread):
f"Capture thread for {self.config.name} did not exit in time"
)
self.logger.error(
"The following ffmpeg logs include the last 100 lines prior to exit."
)
self.logpipe.dump()
self.logger.info("Restarting ffmpeg...")
self.start_ffmpeg_detect()
@@ -540,6 +543,7 @@ class CameraWatchdog(threading.Thread):
f"{self.config.name}/status/{role.value}", "offline"
)
p["logpipe"].dump()
p["process"] = start_or_restart_ffmpeg(
p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"]
)
+3 -6
View File
@@ -18,21 +18,18 @@ def migrate(migrator, database, fake=False, **kwargs):
'"first_seen" DATETIME NOT NULL, '
'"last_seen" DATETIME NOT NULL, '
'"count" INTEGER NOT NULL, '
'"acknowledged_at" DATETIME, '
'"muted_at" DATETIME)'
'"dismissed_at" DATETIME)'
)
migrator.sql('CREATE INDEX IF NOT EXISTS "notice_kind" ON "notice" ("kind")')
migrator.sql(
'CREATE TABLE IF NOT EXISTS "noticestats" ('
'"kind" VARCHAR(50) NOT NULL PRIMARY KEY, '
'"occurrences" INTEGER NOT NULL, '
'"acknowledgements" INTEGER NOT NULL, '
'"mutes" INTEGER NOT NULL, '
'"dismissals" INTEGER NOT NULL, '
'"first_seen" DATETIME NOT NULL, '
'"last_seen" DATETIME NOT NULL, '
'"reported_occurrences" INTEGER NOT NULL DEFAULT 0, '
'"reported_acknowledgements" INTEGER NOT NULL DEFAULT 0, '
'"reported_mutes" INTEGER NOT NULL DEFAULT 0)'
'"reported_dismissals" INTEGER NOT NULL DEFAULT 0)'
)
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+3 -3
View File
@@ -49,7 +49,7 @@ export interface ApiMockOverrides {
};
users?: { username: string; role: string }[];
notices?: unknown[];
mutedChecks?: unknown[];
dismissedChecks?: unknown[];
/** camera name to the ffprobe entries returned for `paths=camera:<name>` */
ffprobe?: Record<string, unknown[]>;
}
@@ -238,8 +238,8 @@ export class ApiMocker {
await this.page.route("**/api/notices", (route) =>
route.fulfill({ json: overrides?.notices ?? [] }),
);
await this.page.route("**/api/notices/muted_checks", (route) =>
route.fulfill({ json: overrides?.mutedChecks ?? [] }),
await this.page.route("**/api/notices/dismissed_checks", (route) =>
route.fulfill({ json: overrides?.dismissedChecks ?? [] }),
);
// Users. GET lists them; POST/PUT (create, password) just succeed, so
-11
View File
@@ -56,14 +56,3 @@ export async function waitForErrorMarker(
.poll(() => hasErrorMarkers(page), { timeout: timeoutMs })
.toBe(true);
}
/**
* Append a trailing comment so the editor content differs from the
* loaded config. The save buttons are disabled until then.
*/
export async function makeMonacoEdit(page: Page): Promise<void> {
const editor = page.locator(".monaco-editor").first();
await editor.click();
await page.keyboard.press("ControlOrMeta+End");
await page.keyboard.type("# edited by e2e", { delay: 0 });
}
-5
View File
@@ -11,7 +11,6 @@ import { installWsFrameCapture, waitForWsFrame } from "../helpers/ws-frames";
import { grantClipboardPermissions, readClipboard } from "../helpers/clipboard";
import {
getMonacoVisibleText,
makeMonacoEdit,
replaceMonacoValue,
waitForErrorMarker,
} from "../helpers/monaco";
@@ -72,7 +71,6 @@ test.describe("Config Editor — Save @medium", () => {
await expect(frigateApp.page.locator(".monaco-editor").first()).toBeVisible(
{ timeout: 15_000 },
);
await makeMonacoEdit(frigateApp.page);
await frigateApp.page.getByLabel("Save Only").click();
await expect
.poll(() => capture.capturedUrl(), { timeout: 5_000 })
@@ -94,7 +92,6 @@ test.describe("Config Editor — Save @medium", () => {
await expect(frigateApp.page.locator(".monaco-editor").first()).toBeVisible(
{ timeout: 15_000 },
);
await makeMonacoEdit(frigateApp.page);
await frigateApp.page.getByLabel("Save Only").click();
await expect(frigateApp.page.getByText(/Invalid field/i)).toBeVisible({
timeout: 5_000,
@@ -120,7 +117,6 @@ test.describe("Config Editor — Save and Restart @medium", () => {
{ timeout: 15_000 },
);
await makeMonacoEdit(frigateApp.page);
await frigateApp.page.getByLabel("Save & Restart").click();
const dialog = frigateApp.page.getByRole("alertdialog");
await expect(dialog).toBeVisible({ timeout: 5_000 });
@@ -144,7 +140,6 @@ test.describe("Config Editor — Save and Restart @medium", () => {
{ timeout: 15_000 },
);
await makeMonacoEdit(frigateApp.page);
await frigateApp.page.getByLabel("Save & Restart").click();
const dialog = frigateApp.page.getByRole("alertdialog");
await expect(dialog).toBeVisible({ timeout: 5_000 });
@@ -1,450 +0,0 @@
/**
* WebRTC streaming-technology availability gating.
*
* The connectivity probe needs a live go2rtc, so this covers the
* statically-determinable gate: with no webrtc candidates/ice_servers
* configured, the WebRTC option must be disabled in the stream-technology
* selector (label "Streaming Technology").
*/
import type { Page } from "@playwright/test";
import { test, expect } from "../fixtures/frigate-test";
import { LivePage } from "../pages/live.page";
// the mocked profile is admin, so useUserPersistence keys are namespaced
const STREAMING_KEY = "streaming-settings:admin";
async function writeIdb(page: Page, entries: Record<string, unknown>) {
await page.evaluate(async (data) => {
await new Promise<void>((resolve, reject) => {
const request = indexedDB.open("keyval-store", 1);
request.onupgradeneeded = () =>
request.result.createObjectStore("keyval");
request.onerror = () => reject(request.error);
request.onsuccess = () => {
const tx = request.result.transaction("keyval", "readwrite");
const store = tx.objectStore("keyval");
Object.entries(data).forEach(([key, value]) => store.put(value, key));
tx.oncomplete = () => resolve();
tx.onerror = () => reject(tx.error);
};
});
}, entries);
}
async function readIdb(page: Page, key: string) {
return page.evaluate(async (target) => {
return new Promise((resolve, reject) => {
const request = indexedDB.open("keyval-store", 1);
request.onupgradeneeded = () =>
request.result.createObjectStore("keyval");
request.onerror = () => reject(request.error);
request.onsuccess = () => {
const tx = request.result.transaction("keyval", "readonly");
const get = tx.objectStore("keyval").get(target);
get.onsuccess = () => resolve(get.result ?? null);
get.onerror = () => reject(get.error);
};
});
}, key);
}
test.describe("WebRTC availability gating @critical", () => {
test("desktop: WebRTC option is disabled when no candidates or ice_servers", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Desktop dropdown only");
await frigateApp.installDefaults({
config: {
go2rtc: {
streams: { front_door: ["rtsp://127.0.0.1:8554/front_door"] },
webrtc: { candidates: [], ice_servers: [] },
},
},
});
// The single-camera view fetches go2rtc stream metadata once restreamed.
// The default mock returns {} which lacks `producers` and crashes the
// capability parser, so return a minimal valid metadata payload.
await frigateApp.page.route("**/api/go2rtc/streams/front_door**", (route) =>
route.fulfill({
json: {
producers: [{ medias: ["video, recvonly, H264"] }],
consumers: [],
},
}),
);
await frigateApp.goto("/#front_door");
const live = new LivePage(frigateApp.page, true);
await expect(live.backButton).toBeVisible({ timeout: 10_000 });
// Open the desktop camera-settings dropdown (the FaCog gear is the last
// button-like trigger in the single-camera header).
const gearButtons = frigateApp.page.locator("button:has(svg)");
await gearButtons.last().click();
const menu = frigateApp.page
.locator('[role="menu"], [data-radix-menu-content]')
.first();
await expect(menu).toBeVisible({ timeout: 3_000 });
// Open the "Streaming Technology" select. Anchor on its label, then click
// the combobox trigger that follows it within the same field container.
const technologyTrigger = menu
.locator('div:has(> label[for="streaming-mode"]) [role="combobox"]')
.first();
await expect(technologyTrigger).toBeVisible({ timeout: 3_000 });
await technologyTrigger.click();
// The Radix select content is portaled to the document body; the WebRTC
// option must be present and disabled (aria-disabled="true").
const webrtcOption = frigateApp.page.getByRole("option", {
name: /WebRTC/,
});
await expect(webrtcOption).toBeVisible({ timeout: 3_000 });
await expect(webrtcOption).toHaveAttribute("aria-disabled", "true");
// Sanity check: a non-gated option (MSE) is enabled, proving the locator
// distinguishes enabled from disabled options.
const mseOption = frigateApp.page.getByRole("option", { name: /MSE/ });
await expect(mseOption).not.toHaveAttribute("aria-disabled", "true");
});
test("the unavailable reason is logged to the console once", async ({
frigateApp,
}) => {
// Availability is consumed by several components at once, so the emitter
// dedupes; the count asserts that dedupe, not just the message.
const warnings: string[] = [];
frigateApp.page.on("console", (msg) => {
if (msg.type() === "warning" && msg.text().includes("WebRTC unavailable"))
warnings.push(msg.text());
});
await frigateApp.installDefaults({
config: {
go2rtc: {
streams: { front_door: ["rtsp://127.0.0.1:8554/front_door"] },
webrtc: { candidates: [], ice_servers: [] },
},
},
});
await frigateApp.page.route("**/api/go2rtc/streams/front_door**", (route) =>
route.fulfill({
json: {
producers: [{ medias: ["video, recvonly, H264"] }],
consumers: [],
},
}),
);
await frigateApp.goto("/#front_door");
await expect
.poll(() => warnings.length, { timeout: 10_000 })
.toBeGreaterThan(0);
expect(warnings).toHaveLength(1);
expect(warnings[0]).toContain("WebRTC unavailable 'not-configured'");
expect(warnings[0]).toContain("go2rtc.webrtc");
expect(warnings[0]).toContain("docs.frigate.video");
});
test("desktop: the WebRTC option reports the pending connectivity check", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Desktop dropdown only");
// Hold the signaling socket open so the probe stays pending. Left alone it
// fails fast against the preview server and resolves to unreachable, which
// is the state the first test already covers.
await frigateApp.page.routeWebSocket("**/live/webrtc/api/ws**", () => {
// never answer the offer
});
// Candidates configured, so the gate reaches the probe rather than
// stopping at not-configured.
await frigateApp.installDefaults({
config: {
go2rtc: {
streams: { front_door: ["rtsp://127.0.0.1:8554/front_door"] },
webrtc: { candidates: ["192.168.1.10:8555"], ice_servers: [] },
},
},
});
await frigateApp.page.route("**/api/go2rtc/streams/front_door**", (route) =>
route.fulfill({
json: {
producers: [{ medias: ["video, recvonly, H264"] }],
consumers: [],
},
}),
);
await frigateApp.goto("/#front_door");
const live = new LivePage(frigateApp.page, true);
await expect(live.backButton).toBeVisible({ timeout: 10_000 });
const gearButtons = frigateApp.page.locator("button:has(svg)");
await gearButtons.last().click();
const menu = frigateApp.page
.locator('[role="menu"], [data-radix-menu-content]')
.first();
await expect(menu).toBeVisible({ timeout: 3_000 });
const technologyTrigger = menu
.locator('div:has(> label[for="streaming-mode"]) [role="combobox"]')
.first();
await expect(technologyTrigger).toBeVisible({ timeout: 3_000 });
await technologyTrigger.click();
// Unselectable while the probe runs, but with the reason stated rather
// than a bare greyed-out row.
const webrtcOption = frigateApp.page.getByRole("option", {
name: /WebRTC/,
});
await expect(webrtcOption).toBeVisible({ timeout: 3_000 });
await expect(webrtcOption).toHaveAttribute("aria-disabled", "true");
await expect(webrtcOption).toContainText(/Checking WebRTC availability/i);
});
test("desktop: JSMpeg is no longer offered in the technology selector", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Desktop dropdown only");
await frigateApp.installDefaults({
config: {
go2rtc: {
streams: { front_door: ["rtsp://127.0.0.1:8554/front_door"] },
webrtc: { candidates: [], ice_servers: [] },
},
},
});
await frigateApp.page.route("**/api/go2rtc/streams/front_door**", (route) =>
route.fulfill({
json: {
producers: [{ medias: ["video, recvonly, H264"] }],
consumers: [],
},
}),
);
await frigateApp.goto("/#front_door");
const live = new LivePage(frigateApp.page, true);
await expect(live.backButton).toBeVisible({ timeout: 10_000 });
const gearButtons = frigateApp.page.locator("button:has(svg)");
await gearButtons.last().click();
const menu = frigateApp.page
.locator('[role="menu"], [data-radix-menu-content]')
.first();
await expect(menu).toBeVisible({ timeout: 3_000 });
// Technology selector offers MSE/WebRTC but NOT JSMpeg (replaced by the
// "Force low-bandwidth mode" switch).
const technologyTrigger = menu
.locator('div:has(> label[for="streaming-mode"]) [role="combobox"]')
.first();
await expect(technologyTrigger).toBeVisible({ timeout: 3_000 });
await technologyTrigger.click();
await expect(
frigateApp.page.getByRole("option", { name: /MSE/ }),
).toBeVisible({ timeout: 3_000 });
await expect(
frigateApp.page.getByRole("option", { name: /JSMpeg/ }),
).toHaveCount(0);
});
test("desktop: force low-bandwidth switch disables the technology and stream selectors", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Desktop dropdown only");
await frigateApp.installDefaults({
config: {
go2rtc: {
streams: { front_door: ["rtsp://127.0.0.1:8554/front_door"] },
webrtc: { candidates: [], ice_servers: [] },
},
},
});
await frigateApp.page.route("**/api/go2rtc/streams/front_door**", (route) =>
route.fulfill({
json: {
producers: [{ medias: ["video, recvonly, H264"] }],
consumers: [],
},
}),
);
await frigateApp.goto("/#front_door");
const live = new LivePage(frigateApp.page, true);
await expect(live.backButton).toBeVisible({ timeout: 10_000 });
const gearButtons = frigateApp.page.locator("button:has(svg)");
await gearButtons.last().click();
const menu = frigateApp.page
.locator('[role="menu"], [data-radix-menu-content]')
.first();
await expect(menu).toBeVisible({ timeout: 3_000 });
// Both selectors are present and enabled to begin with (the switch is off).
await expect(menu.locator('label[for="streaming-mode"]')).toHaveCount(1);
const technologyTrigger = menu
.locator('div:has(> label[for="streaming-mode"]) [role="combobox"]')
.first();
const streamTrigger = menu
.locator('div:has(> label[for="streaming-method"]) [role="combobox"]')
.first();
await expect(technologyTrigger).toBeVisible({ timeout: 3_000 });
await expect(technologyTrigger).toBeEnabled();
await expect(streamTrigger).toBeVisible({ timeout: 3_000 });
await expect(streamTrigger).toBeEnabled();
// Enabling the switch keeps both selectors visible but disables them (the
// low-bandwidth feed ignores the chosen stream and technology).
const lowBandwidthSwitch = menu.getByRole("switch", {
name: "Force low-bandwidth mode",
});
await expect(lowBandwidthSwitch).toBeVisible({ timeout: 3_000 });
await lowBandwidthSwitch.click();
await expect(menu.locator('label[for="streaming-mode"]')).toHaveCount(1);
await expect(technologyTrigger).toBeDisabled();
await expect(streamTrigger).toBeDisabled();
// Toggling it back off re-enables both.
await lowBandwidthSwitch.click();
await expect(technologyTrigger).toBeEnabled();
await expect(streamTrigger).toBeEnabled();
});
test("desktop: saving group streaming settings keeps an unavailable WebRTC choice", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Desktop context menu only");
await frigateApp.installDefaults({
config: {
go2rtc: {
streams: { front_door: ["rtsp://127.0.0.1:8554/front_door"] },
webrtc: { candidates: [], ice_servers: [] },
},
},
});
await frigateApp.page.route("**/api/go2rtc/streams/front_door**", (route) =>
route.fulfill({
json: {
producers: [{ medias: ["video, recvonly, H264"] }],
consumers: [],
},
}),
);
const saved = {
streamName: "front_door",
streamType: "smart",
playerMode: "webrtc",
compatibilityMode: false,
playAudio: false,
volume: 1,
};
await frigateApp.goto("/");
await writeIdb(frigateApp.page, {
[STREAMING_KEY]: { outdoor: { front_door: saved } },
});
await frigateApp.goto("/?group=outdoor");
// With no candidates the dialog resolves WebRTC to MSE for display, but
// saving must not persist that fallback over the user's choice.
const live = new LivePage(frigateApp.page, true);
const menu = await live.openContextMenuOn("front_door");
await expect(menu).toBeVisible({ timeout: 5_000 });
await menu.getByText("Streaming Settings").click();
const dialog = frigateApp.page.getByRole("dialog");
await expect(dialog).toBeVisible();
await dialog.getByRole("button", { name: "Save" }).click();
await expect(dialog).toBeHidden();
await expect
.poll(() => readIdb(frigateApp.page, STREAMING_KEY))
.toMatchObject({ outdoor: { front_door: { playerMode: "webrtc" } } });
});
});
test.describe("WebRTC availability gating @critical @mobile", () => {
test("mobile: WebRTC option is disabled when no candidates or ice_servers", async ({
frigateApp,
}) => {
test.skip(!frigateApp.isMobile, "Mobile drawer only");
await frigateApp.installDefaults({
config: {
go2rtc: {
streams: { front_door: ["rtsp://127.0.0.1:8554/front_door"] },
webrtc: { candidates: [], ice_servers: [] },
},
},
});
await frigateApp.page.route("**/api/go2rtc/streams/front_door**", (route) =>
route.fulfill({
json: {
producers: [{ medias: ["video, recvonly, H264"] }],
consumers: [],
},
}),
);
await frigateApp.goto("/#front_door");
// Open the mobile camera-settings drawer. The camera controls render a
// second dialog-popup trigger (the FaCog settings toggle) after the global
// header menu; wait for both to exist, then click the last one.
const dialogTriggers = frigateApp.page.locator(
'button[aria-haspopup="dialog"]',
);
await expect(dialogTriggers).toHaveCount(2, { timeout: 10_000 });
const settingsTrigger = dialogTriggers.last();
await settingsTrigger.scrollIntoViewIfNeeded();
await settingsTrigger.click();
// The drawer renders a "Streaming Technology" label above its select.
// Anchor on that label, then open the combobox in the same field block.
await expect(
frigateApp.page.getByText("Streaming Technology", { exact: true }),
).toBeVisible({ timeout: 3_000 });
const technologyTrigger = frigateApp.page
.locator(
'div:has(> div:text-is("Streaming Technology")) [role="combobox"]',
)
.first();
await expect(technologyTrigger).toBeVisible({ timeout: 3_000 });
await technologyTrigger.click();
const webrtcOption = frigateApp.page.getByRole("option", {
name: /WebRTC/,
});
await expect(webrtcOption).toBeVisible({ timeout: 3_000 });
await expect(webrtcOption).toHaveAttribute("aria-disabled", "true");
const mseOption = frigateApp.page.getByRole("option", { name: /MSE/ });
await expect(mseOption).not.toHaveAttribute("aria-disabled", "true");
});
});
-216
View File
@@ -1,216 +0,0 @@
/**
* Generative AI provider settings tests -- MEDIUM tier.
*
* A model name belongs to its provider, so switching provider clears the model
* field. The roles widget strips a role only for a model or provider picked in
* the form, never when capability data arrives for the saved entry, which would
* dirty the section on load and silently drop the role on the next save.
*/
import { readFileSync } from "node:fs";
import { resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { test, expect } from "../../fixtures/frigate-test";
import type { Page } from "@playwright/test";
import { configFactory } from "../../fixtures/mock-data/config";
const __dirname = dirname(fileURLToPath(import.meta.url));
const CONFIG_SCHEMA = JSON.parse(
readFileSync(
resolve(__dirname, "../../fixtures/mock-data/config-schema.json"),
"utf-8",
),
);
const ENTRY = "audio";
const SETTINGS_URL = "/settings?page=integrationGenerativeAi";
const UNSAVED = "You have unsaved changes";
const MODEL_PLACEHOLDER = "Select or enter a model…";
type Entry = {
provider: string;
model: string;
base_url?: string;
roles: string[];
};
type ProviderInfo = {
models: string[];
supports_transcription: boolean;
model_capabilities?: Record<string, { supports_transcription?: boolean }>;
};
async function installRoutes(page: Page, entry: Entry, info: ProviderInfo) {
const config = configFactory({ genai: { [ENTRY]: entry } });
await page.route("**/api/config/schema.json", (route) =>
route.fulfill({ json: CONFIG_SCHEMA }),
);
await page.route("**/api/config", (route) => {
if (route.request().method() === "GET") {
return route.fulfill({ json: config });
}
return route.fulfill({ json: { success: true } });
});
await page.route("**/api/config/raw_paths", (route) =>
route.fulfill({ json: { genai: { [ENTRY]: entry } } }),
);
await page.route("**/api/genai/models", (route) =>
route.fulfill({
json: {
[ENTRY]: {
roles: entry.roles,
supports_toggleable_thinking: false,
supports_embeddings: true,
model_capabilities: {},
...info,
},
},
}),
);
}
function roleSwitch(page: Page, role: string) {
return page.locator(`#root_${ENTRY}_roles-${role}`);
}
test.describe("genai provider settings @medium", () => {
test("a saved role the provider cannot confirm stays and is not dirty", async ({
frigateApp,
}) => {
// The server does not serve the saved model, so the backend reports every
// capability as false for the entry.
await installRoutes(
frigateApp.page,
{
provider: "llamacpp",
model: "stale-model",
base_url: "http://llama:8080",
roles: ["transcribe"],
},
{ models: ["qwen3-asr"], supports_transcription: false },
);
await frigateApp.goto(SETTINGS_URL);
await expect(roleSwitch(frigateApp.page, "transcribe")).toBeVisible();
await expect(roleSwitch(frigateApp.page, "transcribe")).toBeChecked();
// Give any stripping effect time to fire, then confirm the section stayed
// clean.
await frigateApp.page.waitForTimeout(1000);
await expect(frigateApp.page.getByText(UNSAVED)).toBeHidden();
await expect(roleSwitch(frigateApp.page, "transcribe")).toBeChecked();
});
test("switching provider clears the model", async ({ frigateApp }) => {
await installRoutes(
frigateApp.page,
{
provider: "openai",
model: "gpt-4o",
roles: ["descriptions"],
},
{ models: ["gpt-4o"], supports_transcription: true },
);
await frigateApp.goto(SETTINGS_URL);
const model = frigateApp.page.locator(`#root_${ENTRY}_model`);
await expect(model).toHaveText("gpt-4o");
await frigateApp.page.locator(`#root_${ENTRY}_provider`).click();
await frigateApp.page.getByRole("option", { name: "llamacpp" }).click();
await expect(model).toHaveText(MODEL_PLACEHOLDER);
await expect(frigateApp.page.getByText(UNSAVED)).toBeVisible();
});
test("switching back to the saved provider drops the other provider's model", async ({
frigateApp,
}) => {
await installRoutes(
frigateApp.page,
{
provider: "openai",
model: "gpt-4o",
roles: ["descriptions"],
},
{ models: ["gpt-4o", "qwen3"], supports_transcription: true },
);
await frigateApp.goto(SETTINGS_URL);
const model = frigateApp.page.locator(`#root_${ENTRY}_model`);
const provider = frigateApp.page.locator(`#root_${ENTRY}_provider`);
await expect(model).toHaveText("gpt-4o");
await provider.click();
await frigateApp.page.getByRole("option", { name: "llamacpp" }).click();
await model.click();
await frigateApp.page.getByRole("option", { name: "qwen3" }).click();
await expect(model).toHaveText("qwen3");
// the model picked for llamacpp must not carry over to openai, and the
// saved one isn't filled back in since the endpoint may have changed
await provider.click();
await frigateApp.page
.getByRole("option", { name: "openai", exact: true })
.click();
await expect(model).toHaveText(MODEL_PLACEHOLDER);
});
test("undo after switching provider brings back the saved model", async ({
frigateApp,
}) => {
await installRoutes(
frigateApp.page,
{
provider: "openai",
model: "gpt-4o",
roles: ["descriptions"],
},
{ models: ["gpt-4o"], supports_transcription: true },
);
await frigateApp.goto(SETTINGS_URL);
const model = frigateApp.page.locator(`#root_${ENTRY}_model`);
await frigateApp.page.locator(`#root_${ENTRY}_provider`).click();
await frigateApp.page.getByRole("option", { name: "llamacpp" }).click();
await expect(model).toHaveText(MODEL_PLACEHOLDER);
await frigateApp.page.getByRole("button", { name: "Undo" }).click();
await expect(model).toHaveText("gpt-4o");
await expect(frigateApp.page.getByText(UNSAVED)).toBeHidden();
});
test("picking a model that cannot transcribe strips the role", async ({
frigateApp,
}) => {
await installRoutes(
frigateApp.page,
{
provider: "llamacpp",
model: "qwen3-asr",
base_url: "http://llama:8080",
roles: ["transcribe"],
},
{
models: ["qwen3-asr", "text-only"],
supports_transcription: true,
model_capabilities: {
"qwen3-asr": { supports_transcription: true },
"text-only": { supports_transcription: false },
},
},
);
await frigateApp.goto(SETTINGS_URL);
await expect(roleSwitch(frigateApp.page, "transcribe")).toBeChecked();
await frigateApp.page.locator(`#root_${ENTRY}_model`).click();
await frigateApp.page.getByRole("option", { name: "text-only" }).click();
await expect(roleSwitch(frigateApp.page, "transcribe")).toBeHidden();
await expect(frigateApp.page.getByText(UNSAVED)).toBeVisible();
});
});
@@ -1,117 +0,0 @@
/**
* Runtime override tests -- MEDIUM tier.
*
* Live view, MQTT, and Home Assistant toggles change a camera's running config
* without touching yaml, and the change persists across restarts. The settings
* form edits yaml, so it keeps showing the saved value. Without a marker on the
* field and runtime-aware wording on dependent warnings, the two read as a
* contradiction: "audio detection is not enabled" next to a switch that is on.
*/
import { readFileSync } from "node:fs";
import { resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { test, expect } from "../../fixtures/frigate-test";
import type { Page } from "@playwright/test";
import { configFactory } from "../../fixtures/mock-data/config";
const __dirname = dirname(fileURLToPath(import.meta.url));
const CONFIG_SCHEMA = JSON.parse(
readFileSync(
resolve(__dirname, "../../fixtures/mock-data/config-schema.json"),
"utf-8",
),
);
const CAMERA = "front_door";
const TRANSCRIPTION_URL = `/settings?page=cameraAudioTranscription&camera=${CAMERA}`;
const AUDIO_URL = `/settings?page=cameraAudioEvents&camera=${CAMERA}`;
const CONFIG_DISABLED = /Audio detection is not enabled for this camera/;
const RUNTIME_DISABLED =
/Audio detection is enabled in your config, but it is currently turned off/;
type AudioState = { enabled: boolean; enabled_in_config: boolean };
async function installRoutes(page: Page, audio: AudioState) {
const config = configFactory({
cameras: {
[CAMERA]: {
audio,
// audio detection only runs on a stream carrying the audio role
ffmpeg: {
inputs: [
{
path: "rtsp://user:pass@host/front",
roles: ["record", "detect", "audio"],
},
],
},
audio_transcription: { enabled: true, enabled_in_config: true },
},
},
});
await page.route("**/api/config/raw_paths", (route) =>
route.fulfill({
json: { cameras: { [CAMERA]: { ffmpeg: { inputs: [] } } } },
}),
);
await page.route("**/api/config/schema.json", (route) =>
route.fulfill({ json: CONFIG_SCHEMA }),
);
await page.route("**/api/config", (route) => {
if (route.request().method() === "GET") {
return route.fulfill({ json: config });
}
return route.fulfill({ json: { success: true } });
});
}
test.describe("runtime overrides @medium", () => {
test("a runtime-only toggle gets its own wording and a field marker", async ({
frigateApp,
}) => {
await installRoutes(frigateApp.page, {
enabled: false,
enabled_in_config: true,
});
await frigateApp.goto(TRANSCRIPTION_URL);
await expect(frigateApp.page.getByText(RUNTIME_DISABLED)).toBeVisible();
await expect(frigateApp.page.getByText(CONFIG_DISABLED)).toBeHidden();
// The audio section still shows the saved value, so the switch stays on and
// the marker carries the live state.
await frigateApp.goto(AUDIO_URL);
await expect(
frigateApp.page.getByRole("switch", { name: "Enable audio detection" }),
).toHaveAttribute("data-state", "checked");
// the boolean layout renders a mobile and a desktop label block, so only
// one of the two badges is on screen
await expect(
frigateApp.page.getByText("Overridden (Live)").filter({ visible: true }),
).toHaveCount(1);
});
test("a config-disabled section keeps the original wording", async ({
frigateApp,
}) => {
await installRoutes(frigateApp.page, {
enabled: false,
enabled_in_config: false,
});
await frigateApp.goto(TRANSCRIPTION_URL);
await expect(frigateApp.page.getByText(CONFIG_DISABLED)).toBeVisible();
await expect(frigateApp.page.getByText(RUNTIME_DISABLED)).toBeHidden();
await frigateApp.goto(AUDIO_URL);
await expect(
frigateApp.page.getByRole("switch", { name: "Enable audio detection" }),
).toHaveAttribute("data-state", "unchecked");
await expect(
frigateApp.page.getByText("Overridden (Live)").filter({ visible: true }),
).toHaveCount(0);
});
});
@@ -18,20 +18,15 @@ const OUTDOOR_LAYOUT = [
{ i: "backyard", x: 6, y: 0, w: 6, h: 4 },
];
// a camera as an export from before streaming technology selection carries
// it, so tests can assert both what a pre-feature file writes and what the
// technology adds on top
const FRONT_DOOR_WITHOUT_TECHNOLOGY = {
streamName: "front_door",
streamType: "smart",
compatibilityMode: false,
playAudio: false,
volume: 1,
};
const STREAMING_SETTINGS = {
outdoor: {
front_door: { ...FRONT_DOOR_WITHOUT_TECHNOLOGY, playerMode: "webrtc" },
front_door: {
streamName: "front_door",
streamType: "smart",
compatibilityMode: false,
playAudio: false,
volume: 1,
},
},
};
@@ -345,7 +340,6 @@ test.describe("UI settings import/export @medium", () => {
garage: {
streamName: "garage",
streamType: "continuous",
playerMode: "jsmpeg",
compatibilityMode: true,
playAudio: true,
volume: 0.5,
@@ -381,77 +375,6 @@ test.describe("UI settings import/export @medium", () => {
},
});
});
test("drops only the streaming technology when its value is unrecognized", async ({
frigateApp,
}) => {
// a hand-edited file, or an export from a future Frigate that added a
// technology this build does not know: the camera's other settings still
// import rather than the whole file failing validation
await frigateApp.goto("/settings?page=uiSettings");
await chooseImportFile(
frigateApp.page,
importPayload({
sections: {
layouts: {},
streaming: {
outdoor: {
front_door: {
...FRONT_DOOR_WITHOUT_TECHNOLOGY,
playerMode: "quantum",
},
},
},
preferences: {},
},
}),
);
await expect(
frigateApp.page.getByText("Streaming settings (1 camera)"),
).toBeVisible();
await confirmImport(frigateApp.page);
expect(await readIdb(frigateApp.page, STREAMING_KEY)).toEqual({
outdoor: { front_door: FRONT_DOOR_WITHOUT_TECHNOLOGY },
});
});
test("clears a stored streaming technology when the file predates it", async ({
frigateApp,
}) => {
// import replaces whole camera objects, as it already does for streamName
// and volume, so a pre-feature export resets the technology rather than
// leaving the local choice in place
await frigateApp.goto("/settings?page=uiSettings");
await writeIdb(frigateApp.page, { [STREAMING_KEY]: STREAMING_SETTINGS });
await chooseImportFile(
frigateApp.page,
importPayload({
sections: {
layouts: {},
streaming: {
outdoor: { front_door: FRONT_DOOR_WITHOUT_TECHNOLOGY },
},
preferences: {},
},
}),
);
await expect(
frigateApp.page.getByText("Streaming settings (1 camera)"),
).toBeVisible();
await confirmImport(frigateApp.page);
expect(await readIdb(frigateApp.page, STREAMING_KEY)).toEqual({
outdoor: { front_door: FRONT_DOOR_WITHOUT_TECHNOLOGY },
});
});
test("rejects a file that is not valid JSON", async ({ frigateApp }) => {
await frigateApp.goto("/settings?page=uiSettings");
-251
View File
@@ -1,251 +0,0 @@
/**
* Zone rename and delete tests -- MEDIUM tier.
*
* A zone's name also lives in required_zones lists and profile overrides.
* These tests pin that renaming or deleting a zone sends one config/set JSON
* body that moves or drops every reference, with nothing in the query string,
* and flags a restart. Editing the name alone must not rename the zone.
*/
import { test, expect } from "../../fixtures/frigate-test";
import type { Page } from "@playwright/test";
import { configFactory } from "../../fixtures/mock-data/config";
const SETTINGS_URL = "/settings?page=masksAndZones&camera=front_door";
const COORDINATES = "0.1,0.1,0.5,0.1,0.5,0.5,0.1,0.5";
// /api/config returns zone filters with defaults filled in
const FILTERS = {
person: {
min_area: 5000,
max_area: 24000000,
min_ratio: 0,
max_ratio: 24000000,
threshold: 0.7,
min_score: 0.5,
mask: {},
},
};
type ConfigSetRequest = { url: string; body: Record<string, unknown> };
async function installRoutes(page: Page) {
const config = configFactory({
profiles: { armed: { friendly_name: "Armed" } },
cameras: {
front_door: {
zones: {
driveway: {
coordinates: COORDINATES,
enabled: true,
inertia: 3,
loitering_time: 0,
objects: [],
filters: FILTERS,
color: [128, 128, 0],
},
},
review: { alerts: { required_zones: ["driveway"] } },
snapshots: { required_zones: ["driveway"] },
mqtt: { required_zones: ["driveway"] },
profiles: {
armed: {
zones: { driveway: { coordinates: COORDINATES, inertia: 6 } },
review: { alerts: { required_zones: ["driveway"] } },
},
},
},
},
});
const requests: ConfigSetRequest[] = [];
await page.route("**/api/config", (route) => route.fulfill({ json: config }));
await page.route("**/api/config/set**", async (route) => {
requests.push({
url: route.request().url(),
body: route.request().postDataJSON(),
});
await route.fulfill({ json: { success: true } });
});
return requests;
}
async function openZoneAction(
page: Page,
isMobile: boolean,
action: "Edit" | "Delete",
) {
const row = page.locator("[data-index]", { hasText: "Driveway" });
if (isMobile) {
await row.locator("button[aria-haspopup='menu']").click();
await page.getByRole("menuitem", { name: action }).click();
return;
}
// Desktop shows the actions on hover
await row.hover();
await row.getByLabel(action, { exact: true }).click();
}
test.describe("zone rename and delete @medium @mobile", () => {
test("editing the name of a referenced zone keeps its id", async ({
frigateApp,
}) => {
const requests = await installRoutes(frigateApp.page);
await frigateApp.goto(SETTINGS_URL);
await openZoneAction(frigateApp.page, frigateApp.isMobile, "Edit");
await frigateApp.page
.getByRole("textbox", { name: "Name", exact: true })
.fill("Main Driveway");
await expect(
frigateApp.page.getByRole("textbox", { name: "ID", exact: true }),
).toHaveValue("driveway");
await frigateApp.page.getByRole("button", { name: "Save" }).click();
await expect.poll(() => requests.length).toBe(1);
expect(new URL(requests[0].url).search).toBe("");
expect(requests[0].body).toEqual({
requires_restart: 0,
update_topic: "config/cameras/front_door/zones",
config_data: {
cameras: {
front_door: {
zones: {
driveway: {
coordinates: COORDINATES,
enabled: true,
inertia: 3,
loitering_time: 0,
friendly_name: "Main Driveway",
},
},
},
},
},
});
});
test("turning speed estimation on and off sends no distances delete", async ({
frigateApp,
}) => {
// The zone has no distances in the YAML, and config/set fails to delete
// a missing key
const requests = await installRoutes(frigateApp.page);
await frigateApp.goto(SETTINGS_URL);
await openZoneAction(frigateApp.page, frigateApp.isMobile, "Edit");
const speedEstimation = frigateApp.page
.getByText("Speed Estimation", { exact: true })
.locator("..")
.getByRole("switch");
await speedEstimation.click();
for (const line of ["A", "B", "C", "D"]) {
await frigateApp.page
.getByRole("textbox", { name: new RegExp(`^Line ${line} distance`) })
.fill("5");
}
await speedEstimation.click();
await frigateApp.page.getByRole("button", { name: "Save" }).click();
await expect.poll(() => requests.length).toBe(1);
expect(requests[0].body).toMatchObject({
config_data: { cameras: { front_door: { zones: { driveway: {} } } } },
});
const body = requests[0].body as {
config_data: {
cameras: { front_door: { zones: { driveway: object } } };
};
};
expect(
body.config_data.cameras.front_door.zones.driveway,
).not.toHaveProperty("distances");
});
test("renaming a referenced zone moves every reference in one request", async ({
frigateApp,
}) => {
const requests = await installRoutes(frigateApp.page);
await frigateApp.goto(SETTINGS_URL);
await openZoneAction(frigateApp.page, frigateApp.isMobile, "Edit");
await frigateApp.page
.getByRole("textbox", { name: "ID", exact: true })
.fill("front_drive");
await frigateApp.page.getByRole("button", { name: "Save" }).click();
await expect.poll(() => requests.length).toBe(1);
expect(new URL(requests[0].url).search).toBe("");
expect(requests[0].body).toEqual({
requires_restart: 1,
update_topic: "config/cameras/front_door/zones",
config_data: {
cameras: {
front_door: {
zones: {
driveway: null,
front_drive: {
coordinates: COORDINATES,
enabled: true,
filters: FILTERS,
inertia: 3,
loitering_time: 0,
},
},
review: { alerts: { required_zones: ["front_drive"] } },
snapshots: { required_zones: ["front_drive"] },
mqtt: { required_zones: ["front_drive"] },
profiles: {
armed: {
review: { alerts: { required_zones: ["front_drive"] } },
zones: {
driveway: null,
front_drive: { coordinates: COORDINATES, inertia: 6 },
},
},
},
},
},
},
});
});
test("deleting a referenced zone drops every reference in one request", async ({
frigateApp,
}) => {
const requests = await installRoutes(frigateApp.page);
await frigateApp.goto(SETTINGS_URL);
await openZoneAction(frigateApp.page, frigateApp.isMobile, "Delete");
await frigateApp.page
.getByRole("alertdialog")
.getByRole("button", { name: "Delete" })
.click();
await expect.poll(() => requests.length).toBe(1);
expect(new URL(requests[0].url).search).toBe("");
expect(requests[0].body).toEqual({
requires_restart: 1,
update_topic: "config/cameras/front_door/zones",
config_data: {
cameras: {
front_door: {
zones: { driveway: null },
review: { alerts: { required_zones: [] } },
snapshots: { required_zones: [] },
mqtt: { required_zones: [] },
profiles: {
armed: {
review: { alerts: { required_zones: [] } },
zones: { driveway: null },
},
},
},
},
},
});
});
});
+102 -239
View File
@@ -1,8 +1,7 @@
/**
* Health tab tests -- MEDIUM tier.
*
* Default tab, notice list rendering, acknowledge and mute, empty state,
* update notice.
* Default tab, notice list rendering, dismiss, empty state, update notice.
*/
import { test, expect } from "../fixtures/frigate-test";
@@ -24,9 +23,7 @@ const ERROR_NOTICE = {
first_seen: NOW - 600,
last_seen: NOW,
count: 2,
acknowledgeable: true,
acknowledged_at: null,
muted_at: null,
dismissed_at: null,
};
const EVENT_NOTICE = {
@@ -40,9 +37,7 @@ const EVENT_NOTICE = {
first_seen: NOW - 7200,
last_seen: NOW - 60,
count: 3,
acknowledgeable: true,
acknowledged_at: null,
muted_at: null,
dismissed_at: null,
};
test.describe("System — Health tab @medium", () => {
@@ -68,61 +63,54 @@ test.describe("System — Health tab @medium", () => {
"Downloading model.onnx for yolo failed: timeout",
);
await expect(rows.nth(0)).toContainText("2 times");
await expect(
rows.nth(0).getByRole("button", { name: "Dismiss" }),
).toBeVisible();
await expect(rows.nth(1)).toContainText("Detector ov was restarted");
await expect(rows.nth(1)).toContainText("3 times");
for (const index of [0, 1]) {
await expect(
rows.nth(index).getByRole("button", { name: "Acknowledge" }),
).toBeVisible();
await expect(
rows.nth(index).getByRole("button", { name: "Mute", exact: true }),
).toBeVisible();
}
await expect(
rows.nth(1).getByRole("button", { name: "Dismiss" }),
).toBeVisible();
});
for (const action of ["acknowledge", "mute"] as const) {
test(`${action} posts and removes the row`, async ({ frigateApp }) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [EVENT_NOTICE],
});
// the list shrinks after the POST so the refetch shows the row gone
let hidden = false;
await frigateApp.page.route("**/api/notices", (route) =>
route.fulfill({ json: hidden ? [] : [EVENT_NOTICE] }),
);
await frigateApp.page.route(
`**/api/notices/detector_stuck/${action}`,
(route) => {
hidden = true;
return route.fulfill({ json: { success: true } });
},
);
await frigateApp.goto("/system#health");
const request = frigateApp.page.waitForRequest(
(req) =>
req.url().includes(`/api/notices/detector_stuck/${action}`) &&
req.method() === "POST",
);
await frigateApp.page
.getByTestId("health-problem-notice:detector_stuck")
.getByRole("button", {
name: action === "mute" ? "Mute" : "Acknowledge",
exact: true,
})
.click();
await request;
await expect(
frigateApp.page.locator("[data-testid^='health-problem-']"),
).toHaveCount(0, { timeout: 5_000 });
await expect(
frigateApp.page.getByText("Your Frigate installation is healthy"),
).toBeVisible();
test("dismiss posts and removes the row", async ({ frigateApp }) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [EVENT_NOTICE],
});
}
// the list shrinks after the dismiss so the refetch shows the row gone
let dismissed = false;
await frigateApp.page.route("**/api/notices", (route) =>
route.fulfill({ json: dismissed ? [] : [EVENT_NOTICE] }),
);
await frigateApp.page.route(
"**/api/notices/detector_stuck/dismiss",
(route) => {
dismissed = true;
return route.fulfill({ json: { success: true } });
},
);
await frigateApp.goto("/system#health");
const request = frigateApp.page.waitForRequest(
(req) =>
req.url().includes("/api/notices/detector_stuck/dismiss") &&
req.method() === "POST",
);
await frigateApp.page
.getByTestId("health-problem-notice:detector_stuck")
.getByRole("button", { name: "Dismiss" })
.click();
await request;
await expect(
frigateApp.page.locator("[data-testid^='health-problem-']"),
).toHaveCount(0, { timeout: 5_000 });
await expect(
frigateApp.page.getByText("Your Frigate installation is healthy"),
).toBeVisible();
});
test("empty state with no notices", async ({ frigateApp }) => {
await frigateApp.installDefaults({ stats: QUIET_STATS });
@@ -152,9 +140,7 @@ test.describe("System — Health tab @medium", () => {
first_seen: NOW - 3600,
last_seen: NOW,
count: 1,
acknowledgeable: false,
acknowledged_at: null,
muted_at: null,
dismissed_at: null,
},
],
});
@@ -170,23 +156,10 @@ test.describe("System — Health tab @medium", () => {
"href",
"https://github.com/blakeblackshear/frigate/releases/tag/v0.19.0",
);
await expect(
row.getByRole("button", { name: "Mute", exact: true }),
).toBeVisible();
await expect(row.getByRole("button", { name: "Acknowledge" })).toHaveCount(
0,
);
await expect(row.getByRole("button", { name: "Dismiss" })).toBeVisible();
});
const ACKNOWLEDGED_NOTICE = {
...EVENT_NOTICE,
id: "detector_stuck:coral",
params: { detector: "coral" },
acknowledged_at: NOW - 60,
};
const MUTED_NOTICE = { ...ERROR_NOTICE, muted_at: NOW - 120 };
test("the filter shows hidden notices marked by how they were hidden", async ({
test("the filter shows dismissed notices without a Dismiss button", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
@@ -196,85 +169,33 @@ test.describe("System — Health tab @medium", () => {
await frigateApp.page.route(
(url) =>
url.pathname.endsWith("/api/notices") &&
url.searchParams.get("include_hidden") === "true",
url.searchParams.get("include_dismissed") === "true",
(route) =>
route.fulfill({
json: [EVENT_NOTICE, ACKNOWLEDGED_NOTICE, MUTED_NOTICE],
json: [EVENT_NOTICE, { ...ERROR_NOTICE, dismissed_at: NOW - 120 }],
}),
);
await frigateApp.goto("/system#health");
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
const showHidden = frigateApp.page.getByRole("switch", {
name: "Show hidden",
const showDismissed = frigateApp.page.getByRole("switch", {
name: "Show dismissed",
});
await expect(showHidden).toHaveAttribute("aria-checked", "false");
await showHidden.click();
// mobile opens the filter as a modal drawer, which hides the rows' roles
await frigateApp.page.keyboard.press("Escape");
await expect(showDismissed).toHaveAttribute("aria-checked", "false");
await showDismissed.click();
const acknowledged = frigateApp.page.getByTestId(
"health-problem-notice:detector_stuck:coral",
);
await expect(acknowledged).toBeVisible({ timeout: 15_000 });
await expect(acknowledged).toContainText("Acknowledged");
await expect(
acknowledged.getByRole("button", { name: "Show again" }),
).toBeVisible();
await expect(
acknowledged.getByRole("button", { name: "Acknowledge" }),
).toHaveCount(0);
const muted = frigateApp.page.getByTestId(
const row = frigateApp.page.getByTestId(
"health-problem-notice:model_download_failed:yolo/model.onnx",
);
await expect(muted).toContainText("Muted");
await expect(muted.getByRole("button", { name: "Unmute" })).toBeVisible();
await expect(
muted.getByRole("button", { name: "Mute", exact: true }),
).toHaveCount(0);
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(row).toContainText("Dismissed");
await expect(row.getByRole("button", { name: "Dismiss" })).toHaveCount(0);
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
await showHidden.click();
await expect(muted).toHaveCount(0);
await showDismissed.click();
await expect(row).toHaveCount(0);
});
test("unmute deletes the row's hidden state", async ({ frigateApp }) => {
await frigateApp.installDefaults({ stats: QUIET_STATS, notices: [] });
await frigateApp.page.route(
(url) =>
url.pathname.endsWith("/api/notices") &&
url.searchParams.get("include_hidden") === "true",
(route) => route.fulfill({ json: [MUTED_NOTICE] }),
);
await frigateApp.page.route(
"**/api/notices/model_download_failed:yolo/model.onnx/hidden",
(route) => route.fulfill({ json: { success: true } }),
);
await frigateApp.goto("/system#health");
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
await frigateApp.page.getByRole("switch", { name: "Show hidden" }).click();
await frigateApp.page.keyboard.press("Escape");
const request = frigateApp.page.waitForRequest(
(req) =>
req
.url()
.endsWith(
"/api/notices/model_download_failed:yolo/model.onnx/hidden",
) && req.method() === "DELETE",
);
await frigateApp.page
.getByTestId(
"health-problem-notice:model_download_failed:yolo/model.onnx",
)
.getByRole("button", { name: "Unmute" })
.click({ timeout: 15_000 });
await request;
});
test("show all again unhides every row after confirming", async ({
test("clear dismissed deletes the dismissed rows after confirming", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
@@ -282,25 +203,29 @@ test.describe("System — Health tab @medium", () => {
notices: [EVENT_NOTICE],
});
// the hidden list loses its muted row once the DELETE lands
// the history loses its dismissed row once the DELETE lands
let cleared = false;
await frigateApp.page.route(
(url) =>
url.pathname.endsWith("/api/notices") &&
url.searchParams.get("include_hidden") === "true",
url.searchParams.get("include_dismissed") === "true",
(route) =>
route.fulfill({
json: cleared ? [EVENT_NOTICE] : [EVENT_NOTICE, MUTED_NOTICE],
json: cleared
? [EVENT_NOTICE]
: [EVENT_NOTICE, { ...ERROR_NOTICE, dismissed_at: NOW - 120 }],
}),
);
await frigateApp.page.route("**/api/notices/hidden", (route) => {
await frigateApp.page.route("**/api/notices/dismissed", (route) => {
cleared = true;
return route.fulfill({ json: { success: true } });
});
await frigateApp.goto("/system#health");
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
await frigateApp.page.getByRole("switch", { name: "Show hidden" }).click();
await frigateApp.page
.getByRole("switch", { name: "Show dismissed" })
.click();
const row = frigateApp.page.getByTestId(
"health-problem-notice:model_download_failed:yolo/model.onnx",
);
@@ -308,20 +233,23 @@ test.describe("System — Health tab @medium", () => {
await frigateApp.page.keyboard.press("Escape");
await frigateApp.page
.getByRole("button", { name: "Show all again" })
.getByRole("button", { name: "Clear dismissed" })
.click();
const request = frigateApp.page.waitForRequest(
(req) =>
req.url().endsWith("/api/notices/hidden") && req.method() === "DELETE",
req.url().endsWith("/api/notices/dismissed") &&
req.method() === "DELETE",
);
await frigateApp.page
.getByRole("alertdialog")
.getByRole("button", { name: "Show all again" })
.getByRole("button", { name: "Clear dismissed" })
.click();
await request;
await expect(row).toHaveCount(0);
await expect(frigateApp.page.getByText("No hidden notices")).toBeVisible();
await expect(
frigateApp.page.getByText("No dismissed notices"),
).toBeVisible();
});
test("severity switches hide notices of that severity", async ({
@@ -362,9 +290,7 @@ test.describe("System — Health tab @medium", () => {
first_seen: start,
last_seen: start + 60,
count,
acknowledgeable: true,
acknowledged_at: null,
muted_at: null,
dismissed_at: null,
});
await frigateApp.installDefaults({
stats: QUIET_STATS,
@@ -406,9 +332,7 @@ test.describe("System — Health tab @medium", () => {
first_seen: NOW - 600,
last_seen: NOW - 600,
count: 1,
acknowledgeable: true,
acknowledged_at: null,
muted_at: null,
dismissed_at: null,
},
],
});
@@ -439,9 +363,7 @@ test.describe("System — Health tab @medium", () => {
first_seen: NOW - 600,
last_seen: NOW - 600,
count: 1,
acknowledgeable: true,
acknowledged_at: null,
muted_at: null,
dismissed_at: null,
},
],
});
@@ -773,7 +695,7 @@ test.describe("System — Health notices sources @medium", () => {
// the status bar shows a problem, so stats have loaded
await expect(
frigateApp.page.getByText("Recordings are being deleted"),
frigateApp.page.getByText("Front Door is offline"),
).toBeVisible({ timeout: 15_000 });
await expect(
frigateApp.page.locator("[data-testid^='health-problem-']"),
@@ -1124,7 +1046,7 @@ test.describe("System — Health notices sources @medium", () => {
await expect(frigateApp.page).toHaveURL(/\/system#health/);
});
test("status bar counts shown notices next to the health text", async ({
test("status bar counts undismissed notices next to the health text", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Status bar is desktop-only");
@@ -1167,63 +1089,7 @@ test.describe("System — Health notices sources @medium", () => {
await expect(frigateApp.page.getByText("System is healthy")).toHaveCount(0);
});
// the slow detector warning is added before the offline error
const TWO_PROBLEM_STATS = {
detectors: { cpu: { inference_speed: 60 } },
cameras: { front_door: { camera_fps: 0 } },
};
test("status bar collapses several problems behind the most severe", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Status bar is desktop-only");
await frigateApp.installDefaults({ stats: TWO_PROBLEM_STATS });
await frigateApp.goto("/");
const summary = frigateApp.page.getByRole("button", {
name: "Front Door is offline +1",
});
await expect(summary).toBeVisible({ timeout: 15_000 });
await expect(frigateApp.page.getByText("Cpu is slow")).toHaveCount(0);
await summary.click();
const list = frigateApp.page.getByTestId("status-message-list");
await expect(list.getByRole("link")).toHaveText([
"Front Door is offline",
"Cpu is slow (60 ms)",
]);
await list.getByRole("link", { name: "Cpu is slow (60 ms)" }).click();
await expect(frigateApp.page).toHaveURL(/\/system#general/);
await expect(list).toHaveCount(0);
});
test("mobile status drawer stacks every problem", async ({ frigateApp }) => {
test.skip(!frigateApp.isMobile, "Mobile-only");
await frigateApp.installDefaults({ stats: TWO_PROBLEM_STATS });
await frigateApp.goto("/");
await frigateApp.page
.getByTestId("status-alert-trigger")
.click({ timeout: 15_000 });
const items = frigateApp.page
.getByTestId("status-message-list")
.getByRole("link");
await expect(items).toHaveText([
"Front Door is offline",
"Cpu is slow (60 ms)",
]);
const [first, second] = await Promise.all([
items.nth(0).boundingBox(),
items.nth(1).boundingBox(),
]);
expect(second!.y).toBeGreaterThan(first!.y);
});
test("a config row can be muted but not acknowledged", async ({
frigateApp,
}) => {
test("a config row can be dismissed", async ({ frigateApp }) => {
const id = "config:detect:fps-greater-than-five:camera.garage";
await frigateApp.installDefaults({
config: {
@@ -1234,33 +1100,32 @@ test.describe("System — Health notices sources @medium", () => {
stats: QUIET_STATS,
});
// the list gains the mute after the POST so the refetch hides the row
let muted = false;
await frigateApp.page.route(`**/api/notices/${id}/mute`, (route) => {
muted = true;
// the list gains the dismissal after the POST so the refetch hides the row
let dismissed = false;
await frigateApp.page.route(`**/api/notices/${id}/dismiss`, (route) => {
dismissed = true;
return route.fulfill({ json: { success: true } });
});
await frigateApp.page.route("**/api/notices/muted_checks", (route) =>
route.fulfill({ json: muted ? [{ id, muted_at: NOW }] : [] }),
await frigateApp.page.route("**/api/notices/dismissed_checks", (route) =>
route.fulfill({ json: dismissed ? [{ id, dismissed_at: NOW }] : [] }),
);
await frigateApp.goto("/system#health");
const row = frigateApp.page.getByTestId(`health-problem-${id}`);
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(row.getByRole("button", { name: "Acknowledge" })).toHaveCount(
0,
);
const request = frigateApp.page.waitForRequest(
(req) =>
req.url().includes(`/api/notices/${id}/mute`) &&
req.url().includes(`/api/notices/${id}/dismiss`) &&
req.method() === "POST",
);
await row.getByRole("button", { name: "Mute", exact: true }).click();
await row.getByRole("button", { name: "Dismiss" }).click();
await request;
await expect(row).toHaveCount(0);
});
test("muted config rows move to the hidden list", async ({ frigateApp }) => {
test("dismissed config rows move to the dismissed list", async ({
frigateApp,
}) => {
const id = "config:detect:fps-greater-than-five:camera.garage";
await frigateApp.installDefaults({
config: {
@@ -1269,12 +1134,12 @@ test.describe("System — Health notices sources @medium", () => {
},
},
stats: QUIET_STATS,
mutedChecks: [{ id, muted_at: NOW - 120 }],
dismissedChecks: [{ id, dismissed_at: NOW - 120 }],
});
await frigateApp.page.route(
(url) =>
url.pathname.endsWith("/api/notices") &&
url.searchParams.get("include_hidden") === "true",
url.searchParams.get("include_dismissed") === "true",
(route) => route.fulfill({ json: [] }),
);
await frigateApp.goto("/system#health");
@@ -1288,14 +1153,12 @@ test.describe("System — Health notices sources @medium", () => {
await expect(row).toHaveCount(0);
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
await frigateApp.page.getByRole("switch", { name: "Show hidden" }).click();
await frigateApp.page.keyboard.press("Escape");
await frigateApp.page
.getByRole("switch", { name: "Show dismissed" })
.click();
await expect(row).toBeVisible();
await expect(row).toContainText("Muted");
await expect(row.getByRole("button", { name: "Unmute" })).toBeVisible();
await expect(
row.getByRole("button", { name: "Mute", exact: true }),
).toHaveCount(0);
await expect(row).toContainText("Dismissed");
await expect(row.getByRole("button", { name: "Dismiss" })).toHaveCount(0);
});
});
+1 -50
View File
@@ -72,15 +72,7 @@ test.describe("System — tabs @medium", () => {
{ timeout: 15_000 },
);
await expect(frigateApp.page.getByText("0.15.0-test")).toBeVisible();
if (frigateApp.isMobile) {
// the "Last refreshed" label is dropped on mobile so the timestamp
// clears the centered logo
await expect(frigateApp.page.getByText(/Last refreshed/)).toHaveCount(0);
await expect(frigateApp.page.getByText(/Just now|ago/)).toBeVisible();
} else {
await expect(frigateApp.page.getByText(/Last refreshed/)).toBeVisible();
}
await expect(frigateApp.page.getByText(/Last refreshed/)).toBeVisible();
});
test("storage tab renders content after switching", async ({
@@ -242,45 +234,4 @@ test.describe("System — mobile @medium @mobile", () => {
{ timeout: 5_000 },
);
});
test("header controls leave the logo uncovered on a narrow phone", async ({
frigateApp,
}) => {
await frigateApp.goto("/system#general");
await expect(frigateApp.page.getByLabel("Select general")).toHaveAttribute(
"data-state",
"on",
{ timeout: 15_000 },
);
await frigateApp.page.setViewportSize({ width: 320, height: 740 });
const logo = frigateApp.page.locator("svg.fill-current").first();
const tabs = frigateApp.page
.locator("[data-radix-scroll-area-viewport]")
.filter({ has: frigateApp.page.getByLabel("Select general") });
const refreshed = frigateApp.page.getByText(/Just now|ago/);
const logoBox = await logo.boundingBox();
const tabsBox = await tabs.boundingBox();
const refreshedBox = await refreshed.boundingBox();
expect(tabsBox!.x + tabsBox!.width).toBeLessThanOrEqual(logoBox!.x + 1);
expect(refreshedBox!.x).toBeGreaterThanOrEqual(logoBox!.x + logoBox!.width);
// the clipped tabs stay reachable by scrolling
const overflow = await tabs.evaluate((el) => ({
scroll: el.scrollWidth,
client: el.clientWidth,
}));
expect(overflow.scroll).toBeGreaterThan(overflow.client);
await tabs.evaluate((el) => {
el.scrollLeft = el.scrollWidth;
});
await frigateApp.page.getByLabel("Select cameras").click();
await expect(frigateApp.page.getByLabel("Select cameras")).toHaveAttribute(
"data-state",
"on",
{ timeout: 5_000 },
);
});
});
@@ -15,6 +15,10 @@
"bandwidth": {
"title": "العرض الترددي:",
"short": "العرض الترددي"
},
"latency": {
"title": "التأخير:",
"value": "{{seconds}} ثانية"
}
},
"streamOffline": {
+6 -40
View File
@@ -21,8 +21,8 @@
"1hour": "1 гадзіна",
"12hours": "12 гадзін",
"24hours": "24 гадзіны",
"pm": "pm",
"am": "am",
"pm": "вечара",
"am": "раніцы",
"yr": "{{time}} г",
"year_one": "{{time}} год",
"year_few": "{{time}} гады",
@@ -168,16 +168,13 @@
"saveAll": "Захаваць усё",
"savingAll": "Захаванне ўсяго…",
"undoAll": "Адрабіць усё",
"retry": "Паўтарыць",
"expand": "Разгарнуць",
"collapse": "Згарнуць",
"reload": "Перазагрузіць"
"retry": "Паўтарыць"
},
"menu": {
"system": "Сістэма",
"systemMetrics": "Стан і метрыкі",
"systemMetrics": "Сістэмныя метрыкі",
"configuration": "Канфігурацыя",
"systemLogs": "Журналы",
"systemLogs": "Журналы сістэмы",
"profiles": "Профілі",
"settings": "Налады",
"configurationEditor": "Рэдактар канфігурацыі",
@@ -228,7 +225,7 @@
"withSystem": {
"label": "Выкарыстоўваць мову з налад сістэмы"
},
"be": "Беларуская (беларуская)"
"be": "Беларуская"
},
"appearance": "Выгляд",
"darkMode": {
@@ -331,36 +328,5 @@
"validation_errors": "Памылкі праверкі",
"credentialField": {
"savedPlaceholder": "Захавана - пакіньце пустым, каб не мяняць"
},
"error": {
"title": "Гэтая старонка перастала працаваць",
"desc": "На старонцы адбылася нечаканая памылка. Звычайна дапамагае перазагрузка. Калі гэта паўтараецца, скапіюйце падрабязнасці і дадайце іх да абмеркавання на GitHub.",
"staleTitle": "Патрэбна перазагрузка",
"staleDesc": "Частка інтэрфейсу не загрузілася - звычайна гэта значыць, што Frigate абнавіўся, пакуль гэтая ўкладка была адкрытая. Перазагрузіце, каб атрымаць новую версію.",
"partial": "Частка інтэрфейсу перастала працаваць.",
"copyDetails": "Скапіяваць падрабязнасці",
"copyFailed": "Не ўдалося скапіяваць падрабязнасці ў буфер абмену"
},
"commandMenu": {
"title": "Меню каманд",
"description": "Пошук старонак, камер, налад і дзеянняў",
"placeholder": "Пошук старонак, камер, налад і дзеянняў",
"empty": "Няма супадзенняў",
"hint": "{{key}} або / каб адкрыць, Enter каб выканаць",
"section": {
"recent": "Апошнія",
"pages": "Старонкі",
"cameras": "Камеры",
"cameraGroups": "Групы камер",
"settings": "Налады",
"actions": "Дзеянні"
},
"camera": {
"live": "Эфір",
"review": "Разгляд"
},
"action": {
"toggleTheme": "Пераключыць светлы і цёмны рэжым"
}
}
}

Some files were not shown because too many files have changed in this diff Show More