mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-09-25 02:36:52 +03:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]andGitHub
|
ff0ee9fd16 |
@@ -46,6 +46,9 @@ jobs:
|
||||
- name: Build web
|
||||
run: npm run build
|
||||
working-directory: ./web
|
||||
# - name: Test
|
||||
# run: npm run test
|
||||
# working-directory: ./web
|
||||
|
||||
web_e2e:
|
||||
name: Web - E2E Tests
|
||||
@@ -124,7 +127,5 @@ jobs:
|
||||
run: devcontainer exec --workspace-folder . bash -lc "python3 -u -m mypy --config-file frigate/mypy.ini frigate"
|
||||
- name: Check API spec is up to date
|
||||
run: devcontainer exec --workspace-folder . bash -lc "python3 generate_api_auth_spec.py --check"
|
||||
- name: Check analytics schema is up to date
|
||||
run: devcontainer exec --workspace-folder . bash -lc "python3 generate_analytics_schema.py --check"
|
||||
- name: Run unit tests in devcontainer
|
||||
run: devcontainer exec --workspace-folder . bash -lc "python3 -u -m unittest"
|
||||
|
||||
@@ -5,4 +5,3 @@
|
||||
/docker/rockchip/ @MarcA711
|
||||
/docker/rocm/ @harakas
|
||||
/docker/hailo8l/ @spanner3003
|
||||
/docker/deepx/ @sixfab
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Installs the DEEPX NPU kernel driver and the DX-RT runtime on the Docker host,
|
||||
# then enables the vendor's dxrt.service. A container cannot load kernel
|
||||
# modules, so this runs outside the image; the driver creates the /dev/dxrt*
|
||||
# nodes and the daemon multiplexes the NPU across host and container.
|
||||
#
|
||||
# Driver, runtime and firmware versions must agree or inference hangs instead
|
||||
# of failing at startup. The set this script installs is pinned in
|
||||
# driver_version, runtime_version and firmware_version below; move them
|
||||
# together, never one at a time.
|
||||
#
|
||||
# DEEPX NPU support in Frigate is maintained by Sixfab (https://sixfab.com).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
driver_version="v2.6.0"
|
||||
# the commit the tag resolves to, since DEEPX signs neither tags nor releases
|
||||
# and this is compiled and installed as root. Update both together
|
||||
driver_commit="7074748e7104f470b02f517583abba652b3f05fa"
|
||||
firmware_version="v2.7.4"
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git build-essential "linux-headers-$(uname -r)" pciutils wget
|
||||
|
||||
if ! lspci -d 1ff4: | grep -q .; then
|
||||
echo "No DEEPX device found on the PCIe bus (lspci -d 1ff4:)."
|
||||
echo "Check that the module is seated correctly before continuing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# fetch the pinned commit rather than cloning the tag, so a retag cannot swap
|
||||
# in different source. The build directory is reused so a second run after a
|
||||
# failure does not stop on the directory already being there
|
||||
mkdir -p dx_rt_npu_linux_driver
|
||||
cd dx_rt_npu_linux_driver
|
||||
git init -q
|
||||
git remote get-url origin > /dev/null 2>&1 ||
|
||||
git remote add origin https://github.com/DEEPX-AI/dx_rt_npu_linux_driver.git
|
||||
git fetch --depth 1 origin "${driver_commit}"
|
||||
git checkout -q FETCH_HEAD
|
||||
|
||||
fetched_commit=$(git rev-parse HEAD)
|
||||
if [[ "${fetched_commit}" != "${driver_commit}" ]]; then
|
||||
echo "Fetched commit ${fetched_commit} does not match pinned driver_commit ${driver_commit}."
|
||||
echo "Refusing to build unverified driver source."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd modules
|
||||
|
||||
sudo ./build.sh -c install --reload
|
||||
|
||||
sudo depmod -A
|
||||
|
||||
# dx_dma is the PCIe transport, dxrt_driver the NPU driver on top of it
|
||||
for module in dx_dma dxrt_driver; do
|
||||
if ! sudo modprobe "${module}"; then
|
||||
echo "Unable to load the ${module} kernel module, common reasons are:"
|
||||
echo "- Secure Boot is enabled and is rejecting the unsigned module."
|
||||
echo "- The running kernel does not match the installed linux-headers."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if ! compgen -G "/dev/dxrt*" > /dev/null; then
|
||||
echo "Modules loaded but no /dev/dxrt* device node appeared."
|
||||
echo "Run ./sanity_check.sh from the driver repo to diagnose."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
runtime_version="v3.4.0"
|
||||
declare -A runtime_sha256=(
|
||||
[amd64]="736cfef009ce9e974ab1ab610d867239d19d72a426a53e367ddcbd53297b6e20"
|
||||
[arm64]="eb6107f5f02f2ad76ae89f414e8b5f346f34fbc6f0888236136853a26be6f6a0"
|
||||
)
|
||||
|
||||
runtime_release="${runtime_version#v}"
|
||||
deb_arch=$(dpkg --print-architecture)
|
||||
deb_file="/tmp/libdxrt-bin_${runtime_release}_${deb_arch}.deb"
|
||||
|
||||
wget -qO "${deb_file}" \
|
||||
"https://raw.githubusercontent.com/DEEPX-AI/dx_rt/${runtime_version}/release/${runtime_release}/libdxrt-bin_${runtime_release}_${deb_arch}.deb"
|
||||
|
||||
expected_sha256="${runtime_sha256[${deb_arch}]:-}"
|
||||
if [[ -z "${expected_sha256}" ]]; then
|
||||
echo "No pinned SHA-256 for architecture ${deb_arch}; refusing to install."
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$(sha256sum "${deb_file}" | cut -d' ' -f1)" != "${expected_sha256}" ]]; then
|
||||
echo "SHA-256 mismatch for ${deb_file}; refusing to install."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo dpkg -i "${deb_file}"
|
||||
sudo ldconfig
|
||||
rm -f "${deb_file}"
|
||||
|
||||
sudo cp /usr/share/libdxrt-bin/service/dxrt.service /etc/systemd/system/
|
||||
|
||||
# With an endpoint set, dxrtd binds that path only, so the socket goes in a
|
||||
# directory Frigate can mount (kept across restarts so the mount stays valid)
|
||||
# and a symlink at the default /tmp path keeps host tools that do not set the
|
||||
# variable working through their own fallback.
|
||||
sudo mkdir -p /etc/systemd/system/dxrt.service.d
|
||||
sudo tee /etc/systemd/system/dxrt.service.d/frigate.conf > /dev/null <<'UNIT'
|
||||
[Service]
|
||||
RuntimeDirectory=dxrt
|
||||
RuntimeDirectoryMode=0755
|
||||
RuntimeDirectoryPreserve=yes
|
||||
Environment=DXRT_DYNAMIC_IPC_ENDPOINT=/run/dxrt/dxrt_dynamic_ipc.sock
|
||||
ExecStartPost=/bin/ln -sfn /run/dxrt/dxrt_dynamic_ipc.sock /tmp/dxrt_dynamic_ipc.sock
|
||||
UNIT
|
||||
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable dxrt.service
|
||||
sudo systemctl restart dxrt.service
|
||||
|
||||
if ! sudo systemctl is-active --quiet dxrt.service; then
|
||||
echo "dxrt.service did not start. Check: sudo journalctl -u dxrt.service"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "DEEPX driver and runtime installation complete."
|
||||
echo "Driver version: $(modinfo -F version dxrt_driver) (expected ${driver_version#v})"
|
||||
echo "Runtime version: ${runtime_release}"
|
||||
echo "Device node(s): $(echo /dev/dxrt*)"
|
||||
echo
|
||||
echo "This driver expects NPU firmware ${firmware_version}. Check it with:"
|
||||
echo " dxrt-cli --status"
|
||||
echo "Update the module if it does not match before starting Frigate."
|
||||
@@ -381,7 +381,6 @@ FROM deps AS frigate
|
||||
|
||||
WORKDIR /opt/frigate/
|
||||
COPY --from=rootfs / /
|
||||
ENV FRIGATE_IMAGE_VARIANT=standard
|
||||
|
||||
# Pre-compile bytecode so a read-only rootfs doesn't force re-parsing the
|
||||
# source tree on every boot (pip-installed packages are already compiled)
|
||||
|
||||
@@ -5,7 +5,7 @@ aiohttp == 3.12.*
|
||||
starlette == 0.47.*
|
||||
starlette-context == 0.5.*
|
||||
fastapi[standard-no-fastapi-cloud-cli] == 0.116.*
|
||||
uvicorn == 0.52.*
|
||||
uvicorn == 0.46.*
|
||||
slowapi == 0.1.*
|
||||
joserfc == 1.6.*
|
||||
cryptography == 46.0.*
|
||||
@@ -61,7 +61,7 @@ rapidfuzz==3.12.*
|
||||
argcomplete==2.0.*
|
||||
contextlib2==0.6.*
|
||||
future==0.18.*
|
||||
netaddr==1.3.*
|
||||
netaddr==0.8.*
|
||||
netifaces==0.10.*
|
||||
prometheus-client == 0.26.*
|
||||
# TFLite
|
||||
@@ -73,4 +73,4 @@ faster-whisper==1.2.*
|
||||
librosa==0.11.*
|
||||
soundfile==0.13.*
|
||||
# Memory profiling
|
||||
memray == 1.20.*
|
||||
memray == 1.15.*
|
||||
|
||||
@@ -37,7 +37,6 @@ device_globs=(
|
||||
"/dev/nvmap"
|
||||
"/dev/nvidia*"
|
||||
"/dev/memx*"
|
||||
"/dev/dxrt*"
|
||||
)
|
||||
|
||||
IFS=',' read -ra extra_globs <<< "${DEVICE_ACL_PATHS:-}"
|
||||
|
||||
@@ -343,6 +343,13 @@ http {
|
||||
add_header Cache-Control "public";
|
||||
}
|
||||
|
||||
location /fonts/ {
|
||||
access_log off;
|
||||
expires 1y;
|
||||
include security_headers.conf;
|
||||
add_header Cache-Control "public";
|
||||
}
|
||||
|
||||
location /locales/ {
|
||||
access_log off;
|
||||
include security_headers.conf;
|
||||
|
||||
@@ -25,7 +25,6 @@ RUN --mount=type=bind,from=rk-wheels,source=/rk-wheels,target=/deps/rk-wheels \
|
||||
|
||||
WORKDIR /opt/frigate/
|
||||
COPY --from=rootfs / /
|
||||
ENV FRIGATE_IMAGE_VARIANT=rk
|
||||
COPY docker/rockchip/COCO /COCO
|
||||
COPY docker/rockchip/conv2rknn.py /opt/conv2rknn.py
|
||||
|
||||
|
||||
@@ -44,7 +44,6 @@ RUN echo "deb http://deb.debian.org/debian trixie main" > /etc/apt/sources.list.
|
||||
|
||||
WORKDIR /opt/frigate
|
||||
COPY --from=rootfs / /
|
||||
ENV FRIGATE_IMAGE_VARIANT=rocm
|
||||
|
||||
RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
|
||||
&& sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \
|
||||
|
||||
@@ -15,4 +15,3 @@ ENV INCLUDED_FFMPEG_VERSIONS="${DEFAULT_FFMPEG_VERSION}:${INCLUDED_FFMPEG_VERSIO
|
||||
|
||||
WORKDIR /opt/frigate/
|
||||
COPY --from=rootfs / /
|
||||
ENV FRIGATE_IMAGE_VARIANT=rpi
|
||||
|
||||
@@ -22,7 +22,6 @@ pip3 install --no-deps -U /deps/synap-wheels/*.whl
|
||||
|
||||
WORKDIR /opt/frigate/
|
||||
COPY --from=rootfs / /
|
||||
ENV FRIGATE_IMAGE_VARIANT=synaptics
|
||||
|
||||
COPY --from=synap1680-wheels /rootfs/usr/local/lib/*.so /usr/lib
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels
|
||||
&& pip3 install -U /deps/trt-wheels/*.whl
|
||||
|
||||
COPY --from=rootfs / /
|
||||
ENV FRIGATE_IMAGE_VARIANT=tensorrt
|
||||
COPY docker/tensorrt/detector/rootfs/etc/ld.so.conf.d /etc/ld.so.conf.d
|
||||
RUN ldconfig
|
||||
|
||||
|
||||
@@ -151,7 +151,6 @@ RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels
|
||||
|
||||
WORKDIR /opt/frigate/
|
||||
COPY --from=rootfs / /
|
||||
ENV FRIGATE_IMAGE_VARIANT=tensorrt-jp6
|
||||
|
||||
# Fixes "Error importing detector runtime: /usr/lib/aarch64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block"
|
||||
ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libstdc++.so.6
|
||||
@@ -14,4 +14,4 @@ nvidia-nccl-cu12==2.26.2.post1; platform_machine == 'x86_64'
|
||||
nvidia-nvjitlink-cu12==12.8.93; platform_machine == 'x86_64'
|
||||
onnx==1.16.*; platform_machine == 'x86_64'
|
||||
onnxruntime-gpu==1.24.*; platform_machine == 'x86_64'
|
||||
protobuf==3.20.3; platform_machine == 'x86_64'
|
||||
protobuf==5.29.6; platform_machine == 'x86_64'
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
onnx == 1.14.0; platform_machine == 'aarch64'
|
||||
protobuf == 3.20.3; platform_machine == 'aarch64'
|
||||
protobuf == 5.29.6; platform_machine == 'aarch64'
|
||||
|
||||
@@ -967,65 +967,6 @@ memryx:
|
||||
# The .zip file must contain:
|
||||
# ├── ssdlite_mobilenet.dfp (a file ending with .dfp)
|
||||
# └── ssdlite_mobilenet_post.onnx (optional; only if the model includes a cropped post-processing network)
|
||||
deepx:
|
||||
title: DEEPX NPU
|
||||
models:
|
||||
- key: yolo
|
||||
label: YOLO
|
||||
recommended: true
|
||||
download: No model is bundled with Frigate. Download a pre-compiled YOLO `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) or compile your own with DX-COM, then bind-mount it into the container and point the model's `path` at it. The recommended model is `yolox-s_640x640_ppu.dxnn`. Its Post-Processing Unit (PPU) compile moves candidate selection onto the NPU, which makes it the fastest ModelZoo model measured through Frigate (about 13 ms on a DX-M1). The output layout is read from the compiled model, so anchor-based, anchor-free, NMS-in-head and PPU models (anchor-based or anchor-free) all need no extra configuration; prefer a `PPU` variant whenever the ModelZoo offers one. PPU models must be compiled with DX-COM 2.4.0 or later, which writes the head layout Frigate reads into the file.
|
||||
ui: |-
|
||||
Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure:
|
||||
|
||||
| Field | Value |
|
||||
| ---------------------------------------- | ---------------------------------------------------- |
|
||||
| **Custom object detector model path** | `/config/model_cache/deepx/yolox-s_640x640_ppu.dxnn` |
|
||||
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
|
||||
| **Object detection model input width** | `640` |
|
||||
| **Object detection model input height** | `640` |
|
||||
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
|
||||
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
|
||||
| **Model Input D Type** | `int` (Frigate's default value) |
|
||||
| **Object Detection Model Type** | `yolo-generic` |
|
||||
|
||||
Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden.
|
||||
yaml: |-
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
path: /config/model_cache/deepx/yolox-s_640x640_ppu.dxnn
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
model_type: yolo-generic
|
||||
width: 640
|
||||
height: 640
|
||||
- key: yolox
|
||||
label: YOLOX
|
||||
recommended: false
|
||||
download: No model is bundled with Frigate. Download a pre-compiled YOLOX `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point the model's `path` at it. The `_ppu` variant is faster and also works with the `yolo-generic` model type; the plain export needs `yolox` so its raw head is decoded.
|
||||
ui: |-
|
||||
Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure:
|
||||
|
||||
| Field | Value |
|
||||
| ---------------------------------------- | ---------------------------------------- |
|
||||
| **Custom object detector model path** | `/config/model_cache/deepx/yolox-s_640x640.dxnn`|
|
||||
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
|
||||
| **Object detection model input width** | `640` |
|
||||
| **Object detection model input height** | `640` |
|
||||
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
|
||||
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
|
||||
| **Model Input D Type** | `int` (Frigate's default value) |
|
||||
| **Object Detection Model Type** | `yolox` |
|
||||
|
||||
The width and height must match the resolution the `.dxnn` file was compiled for.
|
||||
yaml: |-
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
path: /config/model_cache/deepx/yolox-s_640x640.dxnn
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
model_type: yolox
|
||||
width: 640
|
||||
height: 640
|
||||
tensorrt:
|
||||
title: TensorRT
|
||||
models:
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
---
|
||||
id: analytics
|
||||
title: Anonymous Analytics
|
||||
---
|
||||
|
||||
import AnalyticsFields from "@site/src/components/AnalyticsFields";
|
||||
import NavPath from "@site/src/components/NavPath";
|
||||
|
||||
Frigate can send one anonymous usage report a day. The reports show the maintainers which hardware to support, which features people use, and how releases perform. Sharing is off until you turn it on.
|
||||
|
||||
## Turning it on
|
||||
|
||||
Enable **Share anonymous analytics** at <NavPath path="Settings > System > Telemetry" />, or set it in your config:
|
||||
|
||||
```yaml
|
||||
telemetry:
|
||||
analytics: true
|
||||
```
|
||||
|
||||
The same page has a **Preview the report** button that shows exactly what the next report contains.
|
||||
|
||||
## How it's sent
|
||||
|
||||
- Once a day, as a JSON POST to `https://analytics.frigate.video/report`
|
||||
- The server looks up your country and region from your IP address and never stores the address
|
||||
- Raw reports are kept for 60 days; only aggregate totals are published
|
||||
- A random install ID, stored in `/config/.analytics.json`, keeps your install from being counted twice. Turning sharing off deletes it
|
||||
|
||||
## What's never sent
|
||||
|
||||
- Camera, zone, group, profile, or user names
|
||||
- Object labels, face names, or license plate text
|
||||
- IP addresses, hostnames, URLs, or stream paths
|
||||
- Credentials or API keys
|
||||
- Events, recordings, or anything from them
|
||||
|
||||
## Every field
|
||||
|
||||
Fields marked public appear in the published totals. The machine-readable schema is [frigate-analytics-schema.json](pathname:///frigate-analytics-schema.json).
|
||||
|
||||
<AnalyticsFields />
|
||||
@@ -800,7 +800,7 @@ lpr:
|
||||
# to Google or OpenAI's LLMs to generate descriptions. GenAI features can be configured at
|
||||
# the camera level to enhance privacy for indoor cameras.
|
||||
# NOTE: genai is a map of named providers. Each key is a name you choose for the provider,
|
||||
# and each role (chat, descriptions, embeddings, transcribe) may be assigned to exactly one provider.
|
||||
# and each role (chat, descriptions, embeddings) may be assigned to exactly one provider.
|
||||
genai:
|
||||
# Required: name of the provider (chosen by you, used to reference it elsewhere)
|
||||
my_provider:
|
||||
@@ -813,13 +813,11 @@ genai:
|
||||
# Required: The model to use with the provider.
|
||||
model: gemini-1.5-flash
|
||||
# Optional: Roles this provider handles (default: shown below)
|
||||
# Each role (chat, descriptions, embeddings, transcribe) must be assigned to exactly
|
||||
# one provider.
|
||||
# Each role (chat, descriptions, embeddings) must be assigned to exactly one provider.
|
||||
roles:
|
||||
- chat
|
||||
- descriptions
|
||||
- embeddings
|
||||
- transcribe
|
||||
# Optional additional args to pass to the GenAI Provider (default: None)
|
||||
provider_options:
|
||||
keep_alive: -1
|
||||
@@ -832,19 +830,13 @@ genai:
|
||||
audio_transcription:
|
||||
# Optional: Enable live and speech event audio transcription (default: shown below)
|
||||
enabled: False
|
||||
# Optional: The transcription backend (default: shown below)
|
||||
# Either 'whisper' for Frigate's built-in local models, or the name of a genai
|
||||
# provider that has 'transcribe' in its roles. device and model_size are ignored
|
||||
# when a genai provider is named.
|
||||
model: whisper
|
||||
# Optional: The device to run the models on for live transcription. (default: shown below)
|
||||
device: CPU
|
||||
# Optional: Set the model size used for live transcription. (default: shown below)
|
||||
model_size: small
|
||||
# Optional: Set the language used for transcription translation. (default: shown below)
|
||||
# Use 'auto' to let the model detect the language, or a language code from
|
||||
# https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
|
||||
language: auto
|
||||
# List of language codes: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
|
||||
language: en
|
||||
|
||||
# Optional: Configuration for classification models
|
||||
classification:
|
||||
@@ -1194,9 +1186,6 @@ ui:
|
||||
|
||||
# Optional: Telemetry configuration
|
||||
telemetry:
|
||||
# Optional: Share one anonymous usage report a day (default: shown below)
|
||||
# NOTE: See https://docs.frigate.video/configuration/advanced/analytics for what is sent
|
||||
analytics: False
|
||||
# Optional: Enabled network interfaces for bandwidth stats monitoring (default: empty list, let nethogs search all)
|
||||
network_interfaces:
|
||||
- eth
|
||||
|
||||
@@ -204,7 +204,7 @@ Frequently-heard labels like `speech` can generate a lot of events, and each eve
|
||||
|
||||
### Audio Transcription
|
||||
|
||||
Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`, and can alternatively offload transcription to a [GenAI provider](#genai-provider). The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
|
||||
Frigate supports fully local audio transcription using either `sherpa-onnx` or OpenAI's open-source Whisper models via `faster-whisper`. The goal of this feature is to support Semantic Search for `speech` audio events. Frigate is not intended to act as a continuous, fully-automatic speech transcription service. Automatically transcribing all speech (or queuing many audio events for transcription) requires substantial CPU (or GPU) resources and is impractical on most systems. For this reason, transcriptions for events are initiated manually from the UI or the API rather than being run continuously in the background.
|
||||
|
||||
:::info
|
||||
|
||||
@@ -224,7 +224,6 @@ To enable transcription, configure it globally and optionally disable for specif
|
||||
**Global:** Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
|
||||
|
||||
- Set **Enable audio transcription** to on
|
||||
- Set **Audio transcription model or GenAI provider name** to `whisper` for Frigate's built-in local models, or to the name of a GenAI provider
|
||||
- Set **Transcription device** to the desired device
|
||||
- Set **Model size** to the desired size
|
||||
|
||||
@@ -236,7 +235,6 @@ To enable transcription, configure it globally and optionally disable for specif
|
||||
```yaml
|
||||
audio_transcription:
|
||||
enabled: True
|
||||
model: whisper
|
||||
device: ...
|
||||
model_size: ...
|
||||
```
|
||||
@@ -265,88 +263,20 @@ The optional config parameters that can be set at the global level include:
|
||||
- **`enabled`**: Enable or disable the audio transcription feature.
|
||||
- Default: `False`
|
||||
- It is recommended to only configure the features at the global level, and enable it at the individual camera level.
|
||||
- **`model`**: The transcription backend.
|
||||
- Default: `whisper`
|
||||
- `whisper` uses Frigate's built-in local models, described by `device` and `model_size` below.
|
||||
- Any other value must name a key in your `genai` config whose entry has `transcribe` in its `roles`. See [GenAI Provider](#genai-provider).
|
||||
- **`device`**: Device to use to run transcription and translation models.
|
||||
- Default: `CPU`
|
||||
- This can be `CPU` or `GPU`. The `sherpa-onnx` models are lightweight and run on the CPU only. The `whisper` models can run on GPU but are only supported on CUDA hardware.
|
||||
- Ignored when `model` names a GenAI provider.
|
||||
- **`model_size`**: The size of the model used for live transcription.
|
||||
- Default: `small`
|
||||
- This can be `small` or `large`. The `small` setting uses `sherpa-onnx` models that are fast, lightweight, and always run on the CPU but are not as accurate as the `whisper` model.
|
||||
- This config option applies to **live transcription only**. With `model: whisper`, recorded `speech` events always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
|
||||
- Ignored when `model` names a GenAI provider.
|
||||
- **`language`**: Defines the language used to transcribe and translate `speech` audio events (and live audio only if using the `large` model or a GenAI provider).
|
||||
- Default: `auto`
|
||||
- `auto` lets the model detect the language itself, which most models do well. Set an explicit language only if detection is picking the wrong one.
|
||||
- Otherwise you must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
|
||||
- This config option applies to **live transcription only**. Recorded `speech` events will always use a different `whisper` model (and can be accelerated for CUDA hardware if available with `device: GPU`).
|
||||
- **`language`**: Defines the language used by `whisper` to translate `speech` audio events (and live audio only if using the `large` model).
|
||||
- Default: `en`
|
||||
- You must use a valid [language code](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10).
|
||||
- Transcriptions for `speech` events are translated.
|
||||
- Live audio is translated only if you are using the `large` model. The `small` `sherpa-onnx` model is English-only.
|
||||
|
||||
The only field that is valid at the camera level is `enabled`. In particular `model` is global only: the transcription backend is a process-wide resource shared by every camera.
|
||||
|
||||
#### GenAI Provider
|
||||
|
||||
Frigate can send audio to a GenAI provider for transcription when that provider has the `transcribe` role. This is useful if you already run a GenAI provider, or if you do not have the CPU/GPU headroom for a local whisper model. Supported providers are **OpenAI**, **Azure OpenAI**, **Gemini**, and **llama.cpp** with an audio-capable model (a dedicated ASR model such as Qwen3-ASR, or a general multimodal model that accepts audio). Ollama is not supported as it has no audio input.
|
||||
|
||||
To use a GenAI provider for audio transcription:
|
||||
|
||||
1. Configure a GenAI provider with `transcribe` in its `roles`.
|
||||
2. Set the audio transcription model to that GenAI config key (e.g. `whisper_cloud`).
|
||||
|
||||
<ConfigTabs>
|
||||
<TabItem value="ui">
|
||||
|
||||
Navigate to <NavPath path="Settings > Enrichments > Audio transcription" />.
|
||||
|
||||
| Field | Description |
|
||||
| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
|
||||
| **Audio transcription model or GenAI provider name** | Set to the GenAI config key (e.g. `whisper_cloud`) to use a configured GenAI provider for transcription |
|
||||
|
||||
The GenAI provider must also be configured with the `transcribe` role under <NavPath path="Settings > Enrichments > Generative AI" />.
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="yaml">
|
||||
|
||||
```yaml
|
||||
genai:
|
||||
whisper_cloud:
|
||||
provider: openai
|
||||
api_key: your-api-key
|
||||
model: gpt-transcribe
|
||||
roles:
|
||||
- transcribe
|
||||
|
||||
audio_transcription:
|
||||
enabled: True
|
||||
model: whisper_cloud
|
||||
language: en
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</ConfigTabs>
|
||||
|
||||
:::warning
|
||||
|
||||
**Give `transcribe` its own `genai` entry.** A `genai` entry has a single `model` string that is shared by every role it holds, so `roles: [descriptions, transcribe]` would send the same model name to both the chat endpoint and the transcription endpoint. Transcription models and chat models are almost never the same model, so define a dedicated entry as shown above.
|
||||
|
||||
:::
|
||||
|
||||
:::warning
|
||||
|
||||
**Live transcription against a metered provider is billed continuously.** In live mode Frigate uploads an overlapping ~2 second window of audio roughly once per second, per camera, for as long as audio stays above that camera's `audio.min_volume`. Windows below that threshold are never uploaded, which is what keeps a quiet camera near zero requests, but a camera pointed at a busy street will keep sending.
|
||||
|
||||
Three things keep this opt-in: `transcribe` is not one of the default roles, live transcription is off by default, and the volume gate suppresses silence. Transcription of recorded `speech` events is unaffected - it remains a manual, one-request-per-event action.
|
||||
|
||||
:::
|
||||
|
||||
`device` and `model_size` have no effect on this path and no local model is ever downloaded.
|
||||
|
||||
`language` defaults to `auto`, which sends no language hint and lets the model detect it. Most audio models detect language well, so leave it on `auto` unless detection is picking the wrong one.
|
||||
|
||||
When set explicitly, it is sent as the transcription endpoint's native `language` parameter for OpenAI, Azure, and llama.cpp, and as part of the prompt for Gemini. This matters for dedicated ASR models such as Qwen3-ASR: they read the prompt as contextual biasing rather than as an instruction, so a language named in the prompt is ignored, while the endpoint parameter is honored.
|
||||
The only field that is valid at the camera level is `enabled`.
|
||||
|
||||
#### Live transcription
|
||||
|
||||
@@ -362,8 +292,6 @@ Results can be error-prone due to a number of factors, including:
|
||||
|
||||
For speech sources close to the camera with minimal background noise, use the `small` model.
|
||||
|
||||
A [GenAI provider](#genai-provider) is generally the most accurate option for live transcription, at the cost of a network round trip per window. That round trip has to stay under about a second to keep up with the audio; if it does not, Frigate drops the oldest buffered audio rather than letting the backlog grow.
|
||||
|
||||
If you have CUDA hardware, you can experiment with the `large` `whisper` model on GPU. Performance is not quite as fast as the `sherpa-onnx` `small` model, but live transcription is far more accurate. Using the `large` model with CPU will likely be too slow for real-time transcription.
|
||||
|
||||
#### Transcription and translation of `speech` audio events
|
||||
@@ -380,7 +308,7 @@ Only one `speech` event may be transcribed at a time. Frigate does not automatic
|
||||
|
||||
:::
|
||||
|
||||
With `model: whisper`, recorded `speech` events always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient. With a [GenAI provider](#genai-provider), the recorded clip is sent to the provider instead and no local model is used.
|
||||
Recorded `speech` events will always use a `whisper` model, regardless of the `model_size` config setting. Without a supported Nvidia GPU, generating transcriptions for longer `speech` events may take a fair amount of time, so be patient.
|
||||
|
||||
#### FAQ
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ genai:
|
||||
|
||||
The examples on this page all use `my_provider`, but the name is arbitrary and is only used to reference the provider elsewhere in the config (for example, `semantic_search.model`).
|
||||
|
||||
Each provider handles one or more **roles**: `chat`, `descriptions`, `embeddings`, and `transcribe`. A provider handles the first three by default; `transcribe` must always be listed explicitly, and is not available on Ollama, which has no audio input. Each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
|
||||
Each provider handles one or more **roles**: `chat`, `descriptions`, and `embeddings`. A provider handles all three by default, and each role may be assigned to exactly one provider. Define a single provider if you want it to do everything, or split the roles across several providers using the `roles` option.
|
||||
|
||||
If the provider you choose requires an API key, you may either directly paste it in your configuration, or store it in an environment variable prefixed with `FRIGATE_`.
|
||||
|
||||
@@ -63,11 +63,11 @@ Running Generative AI models on CPU is not recommended, as high inference times
|
||||
|
||||
You must use a vision-capable model with Frigate. The following models are recommended for local deployment of the `descriptions` and `chat` roles:
|
||||
|
||||
| Model | Review [frame mode](/configuration/genai/genai_review#frame-mode) | Notes |
|
||||
| ------------------- | --------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `qwen3-vl` | `frames` | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object. Follows a sequence of frames on its own. |
|
||||
| `qwen3.6`/`qwen3.8` | `frames` | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
|
||||
| `gemma4` | `annotated_frames` | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action. Loses track of activity that repeats or reverses, so it benefits from annotated frames. |
|
||||
| Model | Notes |
|
||||
| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `qwen3-vl` | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object. |
|
||||
| `qwen3.6`/`qwen3.8` | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
|
||||
| `gemma4` | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action. |
|
||||
|
||||
#### Embedding models
|
||||
|
||||
@@ -77,17 +77,6 @@ The `embeddings` role needs a different kind of model. Text queries are matched
|
||||
| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `qwen3-vl-embedding` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Must be served by llama.cpp started with `--embeddings` and `--mmproj`. |
|
||||
|
||||
#### Transcription models
|
||||
|
||||
The `transcribe` role needs a model that accepts audio input. A text-only or vision-only model cannot serve this role. The following are recommended for local deployment of the `transcribe` role:
|
||||
|
||||
| Model | Notes |
|
||||
| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `qwen3-asr` | Dedicated speech recognition model covering 30 languages, and the better choice for transcription quality. It only transcribes, so it cannot be shared with the `descriptions` or `chat` roles. |
|
||||
| `gemma4` | General multimodal model that accepts audio as well as images, so one served model can cover `transcribe` alongside the other roles. Transcript quality is below `qwen3-asr`, particularly on noisy audio. |
|
||||
|
||||
Both must be served by llama.cpp started with the matching audio `--mmproj`. llama.cpp only reports audio support when an audio projector is loaded. Without it Frigate sees the model as text-only and the `transcribe` role is unavailable in the UI. Frigate transcribes through the server's `/v1/audio/transcriptions` route, which llama.cpp serves for any audio-capable model.
|
||||
|
||||
:::info
|
||||
|
||||
Each model is available in multiple parameter sizes (3b, 4b, 8b, etc.). Larger sizes are more capable of complex tasks and understanding of situations, but requires more memory and computational resources. It is recommended to try multiple models and experiment to see which performs best.
|
||||
|
||||
@@ -192,43 +192,6 @@ review:
|
||||
</TabItem>
|
||||
</ConfigTabs>
|
||||
|
||||
### Frame Mode
|
||||
|
||||
Review items are sent to the model as a sequence of still frames. Some models follow that sequence well on their own; others lose track of activity that repeats or reverses, and describe a single trip when the subject actually made several. The `frame_mode` option controls how those frames are presented.
|
||||
|
||||
- `frames` (default): the prompt followed by the frames, exactly as earlier versions of Frigate sent them.
|
||||
- `annotated_frames`: each frame is preceded by its frame number and elapsed time, along with notes describing what the object tracker recorded at that moment, such as an object being first detected, starting to move, turning around, stopping, or no longer being detected.
|
||||
|
||||
The notes come from tracking data rather than from the images, so they describe activity the model may not have picked up on its own. In testing with a person carrying three waste bins to the curb one at a time, `gemma4` described a single trip on every attempt with `frames`, and consistently described multiple trips with `annotated_frames`. Models that already handle these sequences well, such as the `qwen3-vl` family, gain little and should stay on `frames`.
|
||||
|
||||
Annotated mode also caps the number of frames, since the notes already establish the order of events and extra near-duplicate frames tend to crowd out the middle of a clip. Longer review items are sampled more sparsely as a result, and typically use fewer tokens than `frames` mode for the same item.
|
||||
|
||||
:::note
|
||||
|
||||
Annotated mode needs tracking data for the review item. If none is available, Frigate falls back to sending plain frames for that item.
|
||||
|
||||
:::
|
||||
|
||||
<ConfigTabs>
|
||||
<TabItem value="ui">
|
||||
|
||||
Navigate to <NavPath path="Settings > Global configuration > Review" />.
|
||||
|
||||
- Set **GenAI config > Frame mode** to the desired mode (e.g., `annotated_frames`)
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="yaml">
|
||||
|
||||
```yaml {4}
|
||||
review:
|
||||
genai:
|
||||
enabled: true
|
||||
frame_mode: annotated_frames
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</ConfigTabs>
|
||||
|
||||
### Response Style
|
||||
|
||||
Different models respond to the built-in prompt with very different writing styles: some produce natural narration while others sound short and mechanical. The `response_style` option selects a writing style preset that rewords the prompt's instructions for the user-facing fields (the title, short summary, and scene description). Presets replace those instructions rather than adding extra ones, so the model never receives competing style directions.
|
||||
|
||||
@@ -28,24 +28,6 @@ WebRTC may use an external STUN server for NAT traversal. MSE and HLS streaming
|
||||
|
||||
:::
|
||||
|
||||
### Selecting a streaming technology
|
||||
|
||||
Frigate [defaults to MSE](#why-does-frigate-prefer-mse-over-webrtc-for-live-view) for restreamed cameras by design. To use WebRTC, select it explicitly from a camera's single-camera Live view settings (the settings menu in the camera's Live view header on desktop, or the settings drawer on mobile). Three related controls work together:
|
||||
|
||||
- **Stream**: _what_ to play. This lists the [streams you've configured](#setting-streams-for-live-ui) (for example `Main Stream` and `Sub Stream`).
|
||||
- **Force low-bandwidth mode**: a switch that always plays Frigate's built-in low-bandwidth feed (the stream assigned the `detect` role, using JSMpeg) instead of the selected stream. It works anywhere without go2rtc and is useful on slow or metered connections. While it is enabled, the stream and streaming technology selectors are disabled; your stream and technology choices are restored when you turn it off.
|
||||
- **Streaming Technology**: _how_ to play the selected stream, listing **MSE** and **WebRTC**. It is only shown for a restreamed stream.
|
||||
|
||||
- The choices are saved **per device, per camera** in your browser's local storage.
|
||||
- **WebRTC is only selectable when it can actually work for that stream.** When it can't, the option is shown disabled with the reason inline, and a more detailed reason (the failing codecs, or why the connectivity check failed) is logged to your browser's console. Common reasons:
|
||||
- **Not configured**: no `candidates` or `ice_servers` are set under `go2rtc.webrtc` (see [WebRTC extra configuration](#webrtc-extra-configuration)).
|
||||
- **Could not connect**: e.g. port `8555` isn't reachable, or a STUN/TURN server is misconfigured. Frigate runs a one-time WebRTC connectivity check when the Live view opens; the option may briefly show as "checking" while it runs.
|
||||
- **Unsupported video codec**: the stream's video codec can't be played over WebRTC in your browser, most commonly H.265/HEVC in Firefox or Edge.
|
||||
- **Unsupported audio codec**: WebRTC needs opus or G.711 audio, so a stream whose playback audio is only AAC (without an added opus/G.711 track) can't carry audio over WebRTC. See [Audio Support](#audio-support) for how to add one.
|
||||
- **Unsupported browser**: the browser doesn't support WebRTC.
|
||||
|
||||
When WebRTC isn't available, Frigate automatically uses MSE (or falls back to JSMpeg), so live view keeps working regardless of the selection.
|
||||
|
||||
### Camera Settings Recommendations
|
||||
|
||||
If you are using go2rtc, you should adjust the following settings in your camera's firmware for the best experience with Live view:
|
||||
@@ -175,17 +157,6 @@ WebRTC works by creating a TCP or UDP connection on port `8555`. However, it req
|
||||
- stun:8555
|
||||
```
|
||||
|
||||
- The web UI uses the STUN and TURN servers in `ice_servers` and falls back to Google's public STUN server when none are set:
|
||||
|
||||
```yaml title="config.yml"
|
||||
go2rtc:
|
||||
webrtc:
|
||||
ice_servers:
|
||||
- urls: [turn:turn.example.com:3478]
|
||||
username: frigate
|
||||
credential: password
|
||||
```
|
||||
|
||||
- For access through Tailscale, the Frigate system's Tailscale IP must be added as a WebRTC candidate. Tailscale IPs all start with `100.`, and are reserved within the `100.64.0.0/10` CIDR block.
|
||||
|
||||
- Note that some browsers may not support H.265 (HEVC). You can check your browser's current version for H.265 compatibility [here](https://github.com/AlexxIT/go2rtc?tab=readme-ov-file#codecs-madness).
|
||||
@@ -235,8 +206,6 @@ For devices that support two way talk, Frigate can be configured to use the feat
|
||||
- Ensure you access Frigate via https (may require [opening port 8971](/frigate/installation/#ports)).
|
||||
- For the Home Assistant Frigate card, [follow the docs](http://card.camera/#/usage/2-way-audio) for the correct source.
|
||||
|
||||
The two-way talk control in the single-camera Live view is only enabled when WebRTC is available; if WebRTC isn't configured or can't connect, the control is shown disabled.
|
||||
|
||||
To use the Reolink Doorbell with two way talk, you should use the [recommended Reolink configuration](/configuration/camera_specific#reolink-cameras)
|
||||
|
||||
As a starting point to check compatibility for your camera, view the list of cameras supported for two-way talk on the [go2rtc repository](https://github.com/AlexxIT/go2rtc?tab=readme-ov-file#two-way-audio). For cameras in the category `ONVIF Profile T`, you can use the [ONVIF Conformant Products Database](https://www.onvif.org/conformant-products/)'s FeatureList to check for the presence of `AudioOutput`. A camera that supports `ONVIF Profile T` _usually_ supports this, but due to inconsistent support, a camera that explicitly lists this feature may still not work. If no entry for your camera exists on the database, it is recommended not to buy it or to consult with the manufacturer's support on the feature availability.
|
||||
|
||||
@@ -24,7 +24,6 @@ Frigate supports multiple different detectors that work on different types of ha
|
||||
- [Coral EdgeTPU](#edge-tpu-detector): The Google Coral EdgeTPU is available in USB, Mini PCIe, and m.2 formats allowing for a wide range of compatibility with devices.
|
||||
- [Hailo](#hailo): The Hailo-8, Hailo-8L and Hailo-8R AI Acceleration modules are available in m.2 format with a HAT for RPi devices, offering a wide range of compatibility with devices.
|
||||
- <CommunityBadge /> [MemryX](#memryx-mx3): The MX3 Acceleration module is available in m.2 format, offering broad compatibility across various platforms.
|
||||
- <CommunityBadge /> [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, offering broad compatibility across various platforms.
|
||||
|
||||
**AMD**
|
||||
|
||||
@@ -613,63 +612,6 @@ For detailed instructions on compiling models, refer to the [MemryX Compiler](ht
|
||||
|
||||
---
|
||||
|
||||
## DEEPX NPU
|
||||
|
||||
This detector is available for use with the DEEPX NPU, both the DX-M1 M.2 module and the DX-M1M on the Sixfab AI HAT+ for the Raspberry Pi 5. The configuration below applies unchanged to either form factor. DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
|
||||
|
||||
See the [installation docs](../frigate/installation.md#deepx-npu) for information on installing the DEEPX kernel driver and runtime on the host and passing the NPU through to the container.
|
||||
|
||||
To run a model on a DEEPX NPU, list a `deepx` device on that model.
|
||||
|
||||
:::info
|
||||
|
||||
The DX-RT Python bindings are not part of the Frigate image. They are downloaded and installed into `/config/.local` the first time a DEEPX device is configured, verified against pinned checksums, and updated automatically when a Frigate release pins a new version. If the container has no internet access, see [Detector runtimes](/frigate/network_requirements#detector-runtimes) for how to provide the files yourself.
|
||||
|
||||
:::
|
||||
|
||||
### Configuration {#configuration-deepx}
|
||||
|
||||
<ModelConfigDropdown detectorTitle="DEEPX" models={objectDetectorsModels.deepx.models} />
|
||||
|
||||
Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format. Two model types are supported:
|
||||
|
||||
- `yolo-generic` for YOLO object detection models, the recommended default. The detector reads the model's output layout from the compiled file, so anchor-based, anchor-free and NMS-in-head models all work with the same configuration, as do models compiled with DEEPX's Post-Processing Unit (PPU) support.
|
||||
- `yolox` for YOLOX models compiled without PPU support, whose raw head needs Frigate's YOLOX decoder. A YOLOX model compiled with PPU support works under either `yolox` or `yolo-generic`.
|
||||
|
||||
The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of YOLO object detection models. Download the `.dxnn`, bind-mount it into the container, and point the model's `path` at it. Alternatively, compile your own model with the DX-COM compiler. The recommended starting point is `yolox-s_640x640_ppu.dxnn`, the fastest ModelZoo model measured through Frigate:
|
||||
|
||||
```yaml
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
path: /config/model_cache/deepx/yolox-s_640x640_ppu.dxnn
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
model_type: yolo-generic
|
||||
width: 640
|
||||
height: 640
|
||||
```
|
||||
|
||||
For PPU models, use a `.dxnn` compiled with DX-COM 2.4.0 or later. Frigate reads the PPU head layout the compiler writes into the file and refuses to load a PPU model without it.
|
||||
|
||||
`model_type` must be set to `yolo-generic` or `yolox` to match the model; `yolo-generic` is the recommended default unless the model is a raw YOLOX export. Frigate defaults it to `ssd`, which this detector does not support, so the detector refuses to start on a model that leaves it unset.
|
||||
|
||||
`width` and `height` must match the resolution the model was compiled for. Quantization parameters are baked into the `.dxnn` file at compile time, so no normalization is applied on the host and Frigate's default `input_tensor`, `input_pixel_format`, and `input_dtype` values do not need to be overridden.
|
||||
|
||||
A DEEPX device is `PCIe:<index>`, as reported on the detector settings page. The NPU daemon multiplexes across processes, so the same device may be listed more than once to run additional inference processes against it:
|
||||
|
||||
```yaml
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
- deepx:PCIe:0
|
||||
```
|
||||
|
||||
#### Label maps
|
||||
|
||||
The object detection models in the DEEPX ModelZoo are trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. Frigate's default label map uses an extended 91-class COCO scheme, and leaving it in place will cause detections to be reported as the wrong object type. For `yolo-generic` models the label map is also what the detector uses to tell the output layout, so a label map with the wrong number of classes is reported as an error at startup.
|
||||
|
||||
---
|
||||
|
||||
## NVidia TensorRT Detector
|
||||
|
||||
Nvidia Jetson devices may be used for object detection using the TensorRT libraries. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt-jp6` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt-jp6`. This detector is designed to work with Yolo models for object detection.
|
||||
|
||||
@@ -204,20 +204,11 @@ Light guidelines and advice:
|
||||
npm run lint
|
||||
```
|
||||
|
||||
- Ensure the backend [unit tests](#unit-tests) pass. Your PR cannot be merged unless tests pass.
|
||||
|
||||
```shell
|
||||
python3 -u -m unittest
|
||||
```
|
||||
|
||||
- Ensure the end-to-end tests pass. They run in Playwright against a production build with mocked API data, so they don't need a running Frigate instance. Add or update tests in `web/e2e/specs/` when you change UI behavior.
|
||||
- Add to unit tests and ensure they pass. As much as possible, you should strive to _increase_ test coverage whenever making changes. This will help ensure features do not accidentally become broken in the future.
|
||||
- If you run into error messages like "TypeError: Cannot read properties of undefined (reading 'context')" when running tests, this may be due to these issues (https://github.com/vitest-dev/vitest/issues/1910, https://github.com/vitest-dev/vitest/issues/1652) in vitest, but I haven't been able to resolve them.
|
||||
|
||||
```console
|
||||
# First-time setup
|
||||
npx playwright install chromium
|
||||
|
||||
# Build the app and run all tests
|
||||
npm run e2e:build && npm run e2e
|
||||
npm run test
|
||||
```
|
||||
|
||||
- Test in different browsers. Firefox, Chrome, and Safari all have different quirks that make them unique targets to interact with.
|
||||
|
||||
@@ -65,11 +65,6 @@ Frigate supports multiple different detectors that work on different types of ha
|
||||
- [Supports many model architectures](../../configuration/object_detectors#memryx-mx3)
|
||||
- Runs best with tiny, small, or medium-size models
|
||||
|
||||
- <CommunityBadge /> [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, allowing for a wide range of compatibility with devices.
|
||||
- [Supports YOLO model architectures](../../configuration/object_detectors#deepx-npu)
|
||||
- Runs best with tiny or small size models
|
||||
- Runs efficiently on low power hardware
|
||||
|
||||
**AMD**
|
||||
|
||||
- [ROCm](#rocm---amd-gpu): ROCm can run on AMD Discrete GPUs to provide efficient object detection
|
||||
@@ -262,32 +257,6 @@ The MX3 is a pipelined architecture, where the maximum frames per second support
|
||||
|
||||
Inference speeds may vary depending on the host platform. The above data was measured on an **Intel 13700 CPU**. Platforms like Raspberry Pi, Orange Pi, and other ARM-based SBCs have different levels of processing capability, which may limit total FPS.
|
||||
|
||||
### DEEPX NPU
|
||||
|
||||
Frigate supports the DEEPX NPU in both of its form factors: the **DX-M1** M.2 module, which works on x86 (Intel/AMD) and ARM-based SBCs such as the Raspberry Pi 5, and the **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart) for the Raspberry Pi 5. Both use the same driver and runtime, so the configuration is identical for either one. DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
|
||||
|
||||
The DEEPX driver and runtime run on the Docker host rather than inside the Frigate container and must be installed before the NPU can be used. See the [installation docs](installation.md#deepx-npu) for the setup steps and [the detector docs](/configuration/object_detectors#deepx-npu) for the configuration.
|
||||
|
||||
Frigate does not bundle a model for this detector. Models use DEEPX's `.dxnn` format, and pre-compiled YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo). Prefer a model with a `_ppu` suffix whenever one is available for the architecture you want: these run part of the post-processing on the NPU itself and are considerably faster, roughly 2.5x for the same architecture and input size. **YOLOX-S with PPU is the recommended starting point.**
|
||||
|
||||
Inference times for a few recommended models, measured through Frigate's own stats on a DX-M1:
|
||||
|
||||
| Model | Input Size | DX-M1 Inference Time |
|
||||
| ----------------- | ---------- | -------------------- |
|
||||
| YOLOX-S (PPU) | 640 | ~ 13 ms |
|
||||
| YOLOv9-t (PPU) | 640 | ~ 18 ms |
|
||||
| YOLOv4 (PPU) | 512 | ~ 20 ms |
|
||||
| YOLOX-S | 640 | ~ 34 ms |
|
||||
| YOLOv9-s | 640 | ~ 39 ms |
|
||||
|
||||
Other ModelZoo YOLO variants are also supported but have not been measured. Inference speeds vary with the host platform, so a slower host such as a Raspberry Pi 5 will report higher times than those above.
|
||||
|
||||
:::note
|
||||
|
||||
A few ModelZoo models can not be used with Frigate: SSD models (they are trained on Pascal VOC, so their labels do not match Frigate's), DAMO-YOLO models, face and pose models, and the PPU builds of YOLOv7.
|
||||
|
||||
:::
|
||||
|
||||
### Nvidia Jetson
|
||||
|
||||
Jetson devices are supported via the TensorRT or ONNX detectors when running Jetpack 6. It will [make use of the Jetson's hardware media engine](/configuration/hardware_acceleration_video#nvidia-jetson) when configured with the [appropriate presets](/configuration/ffmpeg_presets#hwaccel-presets), and will make use of the Jetson's GPU and DLA for object detection when configured with the [TensorRT detector](/configuration/object_detectors#nvidia-tensorrt-detector).
|
||||
|
||||
@@ -381,99 +381,6 @@ If you can't use Docker Compose, you can run the container with something simila
|
||||
|
||||
Finally, configure [hardware object detection](/configuration/object_detectors#memryx-mx3) to complete the setup.
|
||||
|
||||
### DEEPX NPU
|
||||
|
||||
The DEEPX NPU is available in two form factors, and Frigate supports both:
|
||||
|
||||
- **DX-M1** in the M.2 2280 form factor (like an NVMe SSD), for x86 (Intel/AMD) PCs, the Raspberry Pi 5, and other ARM SBCs with an exposed PCIe M.2 slot.
|
||||
- **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart), a HAT+ board that connects to the Raspberry Pi 5 over PCIe Gen 3 x1.
|
||||
|
||||
Both present the NPU through the same PCIe driver and DX-RT runtime, so the setup below and the detector configuration are identical for either one. Nothing needs to change when moving between them.
|
||||
|
||||
DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
|
||||
|
||||
#### Versions
|
||||
|
||||
A DEEPX install has several separately versioned pieces, and they all have to agree. The driver, the runtime, and the daemon live on the Docker host; Frigate itself carries only the Python bindings, which it downloads on first start:
|
||||
|
||||
| Component | Version | Installed on | Installed by |
|
||||
| -------------- | -------- | ------------ | ------------------------- |
|
||||
| Kernel driver | `v2.6.0` | Host | `user_installation.sh` |
|
||||
| DX-RT runtime | `v3.4.0` | Host | `user_installation.sh` |
|
||||
| NPU firmware | `v2.7.4` | The module | Flashed from the host |
|
||||
| DX-RT bindings | `v3.4.0` | Frigate | Downloaded at first start |
|
||||
|
||||
:::warning
|
||||
|
||||
A version mismatch does not produce a startup error. It typically shows up as inference requests that are accepted but never return a result, so detections simply stop appearing while Frigate looks healthy. If that happens after a Frigate upgrade, check every version in the table before anything else.
|
||||
|
||||
:::
|
||||
|
||||
The installation script installs the DX-RT runtime on the host and enables `dxrt.service`, so the daemon starts at boot and any other program on the host can share the NPU with Frigate. Check the firmware version with `dxrt-cli --status` and update the module if it does not match the table above.
|
||||
|
||||
#### Installation
|
||||
|
||||
The DEEPX kernel driver must be installed on the host rather than in the container, because containers share the host kernel and cannot load kernel modules. Installing it creates the `/dev/dxrt*` device nodes that are passed through to Frigate. The same script installs the DX-RT runtime and enables `dxrt.service`, the daemon that owns the NPU and hands work to it on behalf of Frigate and anything else on the host.
|
||||
|
||||
1. Copy or download [this script](https://github.com/blakeblackshear/frigate/blob/dev/docker/deepx/user_installation.sh).
|
||||
2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh`
|
||||
3. Run the script with `./user_installation.sh`
|
||||
4. **Restart your computer** to complete driver installation.
|
||||
|
||||
Confirm the NPU is visible before continuing:
|
||||
|
||||
```bash
|
||||
ls /dev/dxrt*
|
||||
```
|
||||
|
||||
Then confirm the daemon is running and listening in `/run/dxrt`:
|
||||
|
||||
```bash
|
||||
systemctl is-active dxrt.service
|
||||
ls /run/dxrt/
|
||||
```
|
||||
|
||||
#### Setup
|
||||
|
||||
To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable`
|
||||
|
||||
#### Docker configuration
|
||||
|
||||
Frigate needs the NPU device node and the directory holding the daemon's socket:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
frigate:
|
||||
devices:
|
||||
- /dev/dxrt0:/dev/dxrt0
|
||||
volumes:
|
||||
- /run/dxrt:/run/dxrt
|
||||
```
|
||||
|
||||
If you can't use Docker Compose, add `--device /dev/dxrt0:/dev/dxrt0 -v /run/dxrt:/run/dxrt` to your `docker run` command.
|
||||
|
||||
Add one `--device` per NPU, contiguously from `/dev/dxrt0`, since the client stops enumerating at the first gap.
|
||||
|
||||
The installation script configures `dxrt.service` to place its socket in `/run/dxrt` through a systemd drop-in. Mounting the directory rather than the socket file means the container sees the new socket after `dxrt.service` is restarted, rather than holding on to a deleted one.
|
||||
|
||||
`dxrtd` listens on an abstract socket as well, but that one does not cross into a container, so Frigate names the filesystem socket through `DXRT_DYNAMIC_IPC_ENDPOINT` on your behalf. Set that variable on the container yourself only if the daemon listens somewhere else, which means you also set it for `dxrtd` through its own systemd drop-in. The script writes `/etc/systemd/system/dxrt.service.d/frigate.conf` for exactly that, and has `dxrt.service` link the socket to `/tmp/dxrt_dynamic_ipc.sock` when it starts, so the host's own `dxrt-cli` and `dxtop` keep finding it at the default path they fall back to.
|
||||
|
||||
:::note
|
||||
|
||||
The DX-RT client exits when `dxrt.service` stops, so restart the Frigate container after restarting `dxrt.service`.
|
||||
|
||||
:::
|
||||
|
||||
The device node is needed as well as the socket, because the client opens the NPU directly even though the daemon arbitrates access. Without it, inference fails with `Device not found`.
|
||||
|
||||
`/dev/shm` does not need sharing.
|
||||
|
||||
The DX-RT python bindings are not shipped in the Frigate image. Frigate downloads them on first start when a DEEPX detector is configured, and caches them under `/config`.
|
||||
|
||||
#### Configuration
|
||||
|
||||
Finally, configure [hardware object detection](/configuration/object_detectors#deepx-npu) to complete the setup.
|
||||
|
||||
### Rockchip platform
|
||||
|
||||
Make sure that you use a linux distribution that comes with the rockchip BSP kernel 5.10 or 6.1 and necessary drivers (especially rkvdec2 and rknpu). To check, enter the following commands:
|
||||
|
||||
@@ -134,15 +134,6 @@ telemetry:
|
||||
version_check: false
|
||||
```
|
||||
|
||||
### Anonymous Analytics
|
||||
|
||||
If [anonymous analytics](/configuration/advanced/analytics) sharing is turned on, Frigate sends one report a day to `https://analytics.frigate.video`. It's off by default, so no outbound connection happens unless you enable it:
|
||||
|
||||
```yaml
|
||||
telemetry:
|
||||
analytics: true
|
||||
```
|
||||
|
||||
### Push Notifications
|
||||
|
||||
When [notifications](/configuration/notifications) are enabled and users have registered for push notifications in the web UI, Frigate sends push messages through the browser vendor's push service (e.g., Google FCM, Mozilla autopush). This requires internet access from the Frigate server to these push endpoints.
|
||||
@@ -156,7 +147,7 @@ If an [MQTT broker](/integrations/mqtt) is configured, Frigate maintains a conne
|
||||
For [WebRTC live streaming](/configuration/live), Frigate uses STUN for NAT traversal:
|
||||
|
||||
- **go2rtc** defaults to a local STUN listener (`stun:8555`), no internet required.
|
||||
- **The web UI** uses the servers in `go2rtc.webrtc.ice_servers` for its WebRTC player and for the WebRTC connectivity check it runs when the Live view loads. If none are set, it uses Google's public STUN server (`stun:stun.l.google.com:19302`), which requires internet access from the browser. Set `ice_servers` to a STUN or TURN server on your network to avoid this.
|
||||
- **The web UI's WebRTC player** includes a fallback to Google's public STUN server (`stun:stun.l.google.com:19302`), which requires internet.
|
||||
|
||||
## Home Assistant Supervisor
|
||||
|
||||
@@ -179,7 +170,7 @@ To run Frigate in an air-gapped or offline environment:
|
||||
2. **Pre-download the training base weights**: If you plan to train custom classification models, set `TF_KERAS_MOBILENET_V2_WEIGHTS_URL` before training, then run one training job while online. Without this variable the base weights are cached outside `/config/` and are lost whenever the container is recreated, so a later training run will fail offline. If the machine never has internet access, copy the weights in manually as described below.
|
||||
3. **Disable version check**: Set `telemetry.version_check: false` in your configuration.
|
||||
4. **Block outbound model requests**: Set the `HF_HUB_OFFLINE=1` and `TRANSFORMERS_OFFLINE=1` environment variables to prevent HuggingFace and Transformers from attempting any network requests.
|
||||
5. **Avoid cloud features**: Do not configure Frigate+, Generative AI providers that require internet, or cloud MQTT brokers, and leave anonymous analytics off (its default).
|
||||
5. **Avoid cloud features**: Do not configure Frigate+, Generative AI providers that require internet, or cloud MQTT brokers.
|
||||
6. **Use local model mirrors**: If limited internet is available, set the `HF_ENDPOINT`, `GITHUB_ENDPOINT`, `GITHUB_RAW_ENDPOINT`, and `TF_KERAS_MOBILENET_V2_WEIGHTS_URL` environment variables to point to local mirrors.
|
||||
|
||||
After these steps, Frigate will operate with no outbound internet connections.
|
||||
|
||||
@@ -130,7 +130,6 @@ const sidebars: SidebarsConfig = {
|
||||
label: "Advanced Configuration",
|
||||
items: [
|
||||
"configuration/advanced/system",
|
||||
"configuration/advanced/analytics",
|
||||
"configuration/advanced/reference",
|
||||
{
|
||||
type: "link",
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
import React from "react";
|
||||
import schema from "@site/static/frigate-analytics-schema.json";
|
||||
|
||||
function resolve(node) {
|
||||
if (!node) return node;
|
||||
if (node.$ref) return schema.$defs[node.$ref.split("/").pop()];
|
||||
if (node.anyOf) {
|
||||
const inner = node.anyOf.find((option) => option.type !== "null");
|
||||
return inner ? resolve(inner) : node;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
function rows(properties, prefix = "") {
|
||||
return Object.entries(properties).flatMap(([name, field]) => {
|
||||
const path = prefix ? `${prefix}.${name}` : name;
|
||||
const row = {
|
||||
path,
|
||||
description: field.description,
|
||||
isPublic: field["x-public"],
|
||||
};
|
||||
const target = resolve(field);
|
||||
|
||||
if (target?.properties) return [row, ...rows(target.properties, path)];
|
||||
|
||||
const values = resolve(target?.additionalProperties);
|
||||
if (values?.properties)
|
||||
return [row, ...rows(values.properties, `${path}.<key>`)];
|
||||
|
||||
const items = resolve(target?.items);
|
||||
if (items?.properties) return [row, ...rows(items.properties, `${path}[]`)];
|
||||
|
||||
return [row];
|
||||
});
|
||||
}
|
||||
|
||||
export default function AnalyticsFields() {
|
||||
return (
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Description</th>
|
||||
<th>Public</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows(schema.properties).map((row) => (
|
||||
<tr key={row.path}>
|
||||
<td>
|
||||
<code>{row.path}</code>
|
||||
</td>
|
||||
<td>{row.description}</td>
|
||||
<td>{row.isPublic ? "Yes" : "No"}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
);
|
||||
}
|
||||
-1482
File diff suppressed because it is too large
Load Diff
Vendored
-19
@@ -10,25 +10,6 @@ servers:
|
||||
- url: https://demo.frigate.video/api
|
||||
- url: http://localhost:5001/api
|
||||
paths:
|
||||
/analytics/preview:
|
||||
get:
|
||||
tags:
|
||||
- Analytics
|
||||
summary: Get Analytics Preview
|
||||
description: |-
|
||||
**Access:** Admin role required.
|
||||
|
||||
Get the analytics report Frigate would send next, without sending it.
|
||||
operationId: get_analytics_preview_analytics_preview_get
|
||||
responses:
|
||||
'200':
|
||||
description: Successful Response
|
||||
content:
|
||||
application/json:
|
||||
schema: {}
|
||||
security:
|
||||
- frigateAdminAuth: []
|
||||
x-required-role: admin
|
||||
/auth/first_time_login:
|
||||
get:
|
||||
tags:
|
||||
|
||||
@@ -99,10 +99,6 @@ def main() -> None:
|
||||
print("*** End Config Validation Errors ***")
|
||||
print("*************************************************************")
|
||||
|
||||
# force a non-zero exit code for config failures
|
||||
if args.validate_config:
|
||||
sys.exit(1)
|
||||
|
||||
# attempt to start Frigate in recovery mode
|
||||
try:
|
||||
config = FrigateConfig.load(install=True, safe_load=True)
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""Opt-in anonymous analytics reports."""
|
||||
@@ -1 +0,0 @@
|
||||
"""One collector per report section."""
|
||||
@@ -1,223 +0,0 @@
|
||||
"""Cameras section: counts and histograms across cameras, never per camera."""
|
||||
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from frigate.analytics.collectors.common import closed, histogram
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.schema import (
|
||||
CamerasSection,
|
||||
ConnectionQuality,
|
||||
FpsBucket,
|
||||
HeightBucket,
|
||||
HwaccelKey,
|
||||
InputPresetKey,
|
||||
RetainBucket,
|
||||
RetainDays,
|
||||
)
|
||||
from frigate.config import CameraConfig
|
||||
from frigate.config.camera.camera import CameraTypeEnum
|
||||
from frigate.config.camera.ffmpeg import CameraInput, CameraRoleEnum
|
||||
from frigate.const import REPLAY_CAMERA_PREFIX
|
||||
|
||||
# the upper edge of every height bucket but the last
|
||||
HEIGHT_BUCKETS = (
|
||||
(360, HeightBucket.le_360),
|
||||
(540, HeightBucket.h480),
|
||||
(900, HeightBucket.h720),
|
||||
(1260, HeightBucket.h1080),
|
||||
(1800, HeightBucket.h1440),
|
||||
)
|
||||
RESTREAM_HOSTS = frozenset({"127.0.0.1", "localhost"})
|
||||
RESTREAM_PORT = 8554
|
||||
QUALITIES = frozenset(ConnectionQuality)
|
||||
|
||||
|
||||
def height_bucket(height: int) -> HeightBucket:
|
||||
for limit, bucket in HEIGHT_BUCKETS:
|
||||
if height <= limit:
|
||||
return bucket
|
||||
|
||||
return HeightBucket.ge_2160
|
||||
|
||||
|
||||
def fps_bucket(fps: int) -> FpsBucket:
|
||||
if fps <= 5:
|
||||
return FpsBucket.le_5
|
||||
|
||||
if fps <= 10:
|
||||
return FpsBucket.f6_10
|
||||
|
||||
return FpsBucket.gt_10
|
||||
|
||||
|
||||
def retain_bucket(days: float) -> RetainBucket:
|
||||
if days <= 0:
|
||||
return RetainBucket.zero
|
||||
|
||||
if days <= 7:
|
||||
return RetainBucket.d1_7
|
||||
|
||||
if days <= 30:
|
||||
return RetainBucket.d8_30
|
||||
|
||||
return RetainBucket.gt_30
|
||||
|
||||
|
||||
def preset_key(args: str | list[str], enum: Any) -> Any:
|
||||
"""The preset's name, custom for hand written args, or none."""
|
||||
if not args:
|
||||
return enum("none")
|
||||
|
||||
if isinstance(args, str) and args.startswith("preset-"):
|
||||
return closed(enum, args.removeprefix("preset-"), enum("custom"))
|
||||
|
||||
return enum("custom")
|
||||
|
||||
|
||||
def is_restream(path: str) -> bool:
|
||||
try:
|
||||
url = urlsplit(path)
|
||||
return url.hostname in RESTREAM_HOSTS and url.port == RESTREAM_PORT
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def role_input(camera: CameraConfig, role: CameraRoleEnum) -> CameraInput | None:
|
||||
return next((i for i in camera.ffmpeg.inputs if role in i.roles), None)
|
||||
|
||||
|
||||
def object_masks(camera: CameraConfig) -> int:
|
||||
# parsing copies camera-wide masks into every filter as global_<id>
|
||||
own = sum(1 for mask in camera.objects.mask.values() if mask is not None)
|
||||
per_label = sum(
|
||||
1
|
||||
for label_filter in camera.objects.filters.values()
|
||||
for mask_id, mask in label_filter.mask.items()
|
||||
if mask is not None and not mask_id.startswith("global_")
|
||||
)
|
||||
return own + per_label
|
||||
|
||||
|
||||
def collect(ctx: ReportContext) -> CamerasSection:
|
||||
cameras = {
|
||||
name: camera
|
||||
for name, camera in ctx.config.cameras.items()
|
||||
if not name.startswith(REPLAY_CAMERA_PREFIX)
|
||||
}
|
||||
camera_stats = ctx.stats.get("cameras", {})
|
||||
|
||||
flags: Counter[str] = Counter()
|
||||
types: Counter[CameraTypeEnum] = Counter()
|
||||
heights: Counter[HeightBucket] = Counter()
|
||||
fps: Counter[FpsBucket] = Counter()
|
||||
hwaccel: Counter[Any] = Counter()
|
||||
input_presets: Counter[Any] = Counter()
|
||||
quality: Counter[ConnectionQuality] = Counter()
|
||||
retain: dict[str, Counter[RetainBucket]] = {
|
||||
period: Counter() for period in ("continuous", "motion", "alerts", "detections")
|
||||
}
|
||||
|
||||
for name, camera in cameras.items():
|
||||
detect_input = role_input(camera, CameraRoleEnum.detect)
|
||||
record_input = role_input(camera, CameraRoleEnum.record)
|
||||
|
||||
# config validation requires a detect input, so this never skips
|
||||
if detect_input is None:
|
||||
continue
|
||||
|
||||
types[camera.type] += 1
|
||||
fps[fps_bucket(camera.detect.fps)] += 1
|
||||
hwaccel[
|
||||
preset_key(
|
||||
detect_input.hwaccel_args or camera.ffmpeg.hwaccel_args, HwaccelKey
|
||||
)
|
||||
] += 1
|
||||
input_presets[
|
||||
preset_key(
|
||||
detect_input.input_args or camera.ffmpeg.input_args, InputPresetKey
|
||||
)
|
||||
] += 1
|
||||
|
||||
if camera.detect.height:
|
||||
heights[height_bucket(camera.detect.height)] += 1
|
||||
|
||||
state = camera_stats.get(name, {}).get("connection_quality")
|
||||
|
||||
if state in QUALITIES:
|
||||
quality[ConnectionQuality(state)] += 1
|
||||
|
||||
if camera.record.enabled:
|
||||
retain["continuous"][retain_bucket(camera.record.continuous.days)] += 1
|
||||
retain["motion"][retain_bucket(camera.record.motion.days)] += 1
|
||||
retain["alerts"][retain_bucket(camera.record.alerts.retain.days)] += 1
|
||||
retain["detections"][
|
||||
retain_bucket(camera.record.detections.retain.days)
|
||||
] += 1
|
||||
|
||||
zones = len(camera.zones)
|
||||
flags["enabled"] += camera.enabled
|
||||
flags["go2rtc_restream"] += any(
|
||||
is_restream(i.path) for i in camera.ffmpeg.inputs
|
||||
)
|
||||
flags["separate_detect_stream"] += (
|
||||
record_input is not None and record_input.path != detect_input.path
|
||||
)
|
||||
flags["detect"] += camera.detect.enabled
|
||||
flags["record"] += camera.record.enabled
|
||||
flags["sub_stream_record"] += camera.record.sub.enabled
|
||||
flags["snapshots"] += camera.snapshots.enabled
|
||||
flags["audio"] += camera.audio.enabled
|
||||
flags["audio_transcription"] += camera.audio_transcription.enabled
|
||||
flags["birdseye"] += camera.birdseye.enabled
|
||||
flags["onvif"] += bool(camera.onvif.host)
|
||||
flags["autotracking"] += camera.onvif.autotracking.enabled
|
||||
flags["face_recognition"] += camera.face_recognition.enabled
|
||||
flags["lpr"] += camera.lpr.enabled
|
||||
flags["review_genai"] += camera.review.genai.enabled
|
||||
flags["object_genai"] += camera.objects.genai.enabled
|
||||
flags["notifications"] += camera.notifications.enabled
|
||||
flags["zones"] += zones
|
||||
flags["cameras_with_zones"] += zones > 0
|
||||
flags["motion_masks"] += sum(
|
||||
1 for mask in camera.motion.mask.values() if mask is not None
|
||||
)
|
||||
flags["object_masks"] += object_masks(camera)
|
||||
|
||||
return CamerasSection(
|
||||
total=len(cameras),
|
||||
enabled=flags["enabled"],
|
||||
types=histogram(types),
|
||||
detect_height=histogram(heights),
|
||||
detect_fps=histogram(fps),
|
||||
hwaccel=histogram(hwaccel),
|
||||
input_preset=histogram(input_presets),
|
||||
go2rtc_restream=flags["go2rtc_restream"],
|
||||
separate_detect_stream=flags["separate_detect_stream"],
|
||||
detect=flags["detect"],
|
||||
record=flags["record"],
|
||||
sub_stream_record=flags["sub_stream_record"],
|
||||
snapshots=flags["snapshots"],
|
||||
audio=flags["audio"],
|
||||
audio_transcription=flags["audio_transcription"],
|
||||
birdseye=flags["birdseye"],
|
||||
onvif=flags["onvif"],
|
||||
autotracking=flags["autotracking"],
|
||||
face_recognition=flags["face_recognition"],
|
||||
lpr=flags["lpr"],
|
||||
review_genai=flags["review_genai"],
|
||||
object_genai=flags["object_genai"],
|
||||
notifications=flags["notifications"],
|
||||
zones=flags["zones"],
|
||||
cameras_with_zones=flags["cameras_with_zones"],
|
||||
motion_masks=flags["motion_masks"],
|
||||
object_masks=flags["object_masks"],
|
||||
connection_quality=histogram(quality),
|
||||
retain_days=RetainDays(
|
||||
continuous=histogram(retain["continuous"]),
|
||||
motion=histogram(retain["motion"]),
|
||||
alerts=histogram(retain["alerts"]),
|
||||
detections=histogram(retain["detections"]),
|
||||
),
|
||||
)
|
||||
@@ -1,37 +0,0 @@
|
||||
"""Helpers the section collectors share."""
|
||||
|
||||
import math
|
||||
from collections import Counter
|
||||
from enum import Enum
|
||||
from typing import Any, TypeVar
|
||||
|
||||
E = TypeVar("E", bound=Enum)
|
||||
|
||||
|
||||
def closed(enum: type[E], value: Any, fallback: E) -> E:
|
||||
"""The member for a value, or the fallback for one the enum doesn't know."""
|
||||
try:
|
||||
return enum(str(value))
|
||||
except ValueError:
|
||||
return fallback
|
||||
|
||||
|
||||
def rate(value: Any) -> float:
|
||||
"""A finite, non-negative number rounded to 2 decimals, else 0.
|
||||
|
||||
A NaN would serialize as null and fail the schema's number type.
|
||||
"""
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
if not math.isfinite(number):
|
||||
return 0.0
|
||||
|
||||
return round(max(number, 0.0), 2)
|
||||
|
||||
|
||||
def histogram(counter: "Counter[E]") -> dict[E, int]:
|
||||
"""Drop the empty buckets, since an absent key means zero."""
|
||||
return {key: total for key, total in counter.items() if total > 0}
|
||||
@@ -1,65 +0,0 @@
|
||||
"""Detection section: models, the detectors they run on, and inference speed."""
|
||||
|
||||
import os
|
||||
|
||||
from frigate.analytics.collectors.common import rate
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.schema import DetectionModel, DetectionSection, ModelSource
|
||||
from frigate.config.config import DEFAULT_MODEL
|
||||
from frigate.const import MODEL_CACHE_DIR
|
||||
from frigate.detectors.detector_config import SceneEnum
|
||||
from frigate.detectors.detector_types import DetectorTypeEnum
|
||||
from frigate.detectors.device import runner_names
|
||||
|
||||
# the paths FrigateConfig fills in for a model that sets none
|
||||
BUNDLED_MODEL_PATHS = frozenset(
|
||||
{"/cpu_model.tflite", "/edgetpu_model.tflite", str(DEFAULT_MODEL["path"])}
|
||||
)
|
||||
|
||||
|
||||
def model_source(path: str | None) -> ModelSource:
|
||||
"""Default, Frigate+ (a cached model next to its info file), or custom.
|
||||
|
||||
Parsing rewrites plus://<id> to the model cache, so the prefix is gone by now.
|
||||
"""
|
||||
if path is None or path in BUNDLED_MODEL_PATHS:
|
||||
return ModelSource.default
|
||||
|
||||
if path.startswith(f"{MODEL_CACHE_DIR}/") and os.path.isfile(f"{path}.json"):
|
||||
return ModelSource.plus
|
||||
|
||||
return ModelSource.custom
|
||||
|
||||
|
||||
def collect(ctx: ReportContext) -> DetectionSection:
|
||||
config = ctx.config
|
||||
detectors = ctx.stats.get("detectors", {})
|
||||
model_specs = [(model, config.devices_for_model(model)) for model in config.models]
|
||||
|
||||
# FrigateApp.start_detectors names the processes in this same order
|
||||
names = iter(runner_names([spec for _, specs in model_specs for spec in specs]))
|
||||
models: dict[SceneEnum, DetectionModel] = {}
|
||||
|
||||
for model, specs in model_specs:
|
||||
speeds: list[float] = []
|
||||
|
||||
for _ in specs:
|
||||
speed = detectors.get(next(names), {}).get("inference_speed")
|
||||
|
||||
if isinstance(speed, int | float) and speed > 0:
|
||||
speeds.append(float(speed))
|
||||
|
||||
models[model.scene] = DetectionModel(
|
||||
detector=DetectorTypeEnum(specs[0].detector),
|
||||
devices=len(specs),
|
||||
model_type=model.model_type,
|
||||
input=f"{model.width}x{model.height}",
|
||||
source=model_source(model.path),
|
||||
inference_ms=rate(sum(speeds) / len(speeds)) if speeds else None,
|
||||
)
|
||||
|
||||
return DetectionSection(
|
||||
models=models,
|
||||
detection_fps=rate(ctx.stats.get("detection_fps")),
|
||||
skipped_fps=rate(ctx.stats.get("skipped_fps")),
|
||||
)
|
||||
@@ -1,145 +0,0 @@
|
||||
"""Features section: enrichments, GenAI, integrations, and users."""
|
||||
|
||||
from collections import Counter
|
||||
|
||||
from frigate.analytics.collectors.common import closed, histogram
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.schema import (
|
||||
BirdseyeUsage,
|
||||
ClassificationUsage,
|
||||
EnrichmentDevice,
|
||||
EnrichmentUsage,
|
||||
FeaturesSection,
|
||||
GenAIUsage,
|
||||
SemanticSearchModel,
|
||||
SemanticSearchUsage,
|
||||
TranscriptionModel,
|
||||
TranscriptionUsage,
|
||||
UserRole,
|
||||
)
|
||||
from frigate.config.camera.genai import GenAIProviderEnum, GenAIRoleEnum
|
||||
from frigate.const import REPLAY_CAMERA_PREFIX
|
||||
from frigate.models import User
|
||||
|
||||
RUNTIME_DEVICES = {
|
||||
"cpu": EnrichmentDevice.cpu,
|
||||
"cuda": EnrichmentDevice.cuda,
|
||||
"tensorrt": EnrichmentDevice.tensorrt,
|
||||
"migraphx": EnrichmentDevice.migraphx,
|
||||
}
|
||||
OPENVINO_DEVICES = {
|
||||
"cpu": EnrichmentDevice.openvino_cpu,
|
||||
"gpu": EnrichmentDevice.openvino_gpu,
|
||||
"npu": EnrichmentDevice.openvino_npu,
|
||||
}
|
||||
|
||||
|
||||
def enrichment_device(label: object) -> EnrichmentDevice | None:
|
||||
"""Map a runner's device label, like "CUDA" or "OpenVINO GPU.0,CPU"."""
|
||||
if not isinstance(label, str) or not label:
|
||||
return None
|
||||
|
||||
runtime, _, target = label.partition(" ")
|
||||
|
||||
if runtime == "OpenVINO":
|
||||
first = target.split(",")[0].split(".")[0].strip().lower()
|
||||
return OPENVINO_DEVICES.get(first, EnrichmentDevice.other)
|
||||
|
||||
return RUNTIME_DEVICES.get(label.lower(), EnrichmentDevice.other)
|
||||
|
||||
|
||||
def model_name(model: object) -> str | None:
|
||||
if model is None:
|
||||
return None
|
||||
|
||||
return str(getattr(model, "value", model))
|
||||
|
||||
|
||||
def semantic_model(model: object) -> SemanticSearchModel | None:
|
||||
# any string that isn't a built-in model names a GenAI provider
|
||||
name = model_name(model)
|
||||
|
||||
if name is None:
|
||||
return None
|
||||
|
||||
if name in ("jinav1", "jinav2"):
|
||||
return SemanticSearchModel(name)
|
||||
|
||||
return SemanticSearchModel.genai
|
||||
|
||||
|
||||
def transcription_model(model: object) -> TranscriptionModel | None:
|
||||
name = model_name(model)
|
||||
|
||||
if name is None:
|
||||
return None
|
||||
|
||||
return TranscriptionModel.whisper if name == "whisper" else TranscriptionModel.genai
|
||||
|
||||
|
||||
def users() -> dict[UserRole, int]:
|
||||
roles: Counter[UserRole] = Counter(
|
||||
closed(UserRole, user.role, UserRole.custom) for user in User.select(User.role)
|
||||
)
|
||||
return histogram(roles)
|
||||
|
||||
|
||||
def collect(ctx: ReportContext) -> FeaturesSection:
|
||||
config = ctx.config
|
||||
devices = ctx.stats.get("embeddings", {}).get("devices", {})
|
||||
cameras = [
|
||||
camera
|
||||
for name, camera in config.cameras.items()
|
||||
if not name.startswith(REPLAY_CAMERA_PREFIX)
|
||||
]
|
||||
providers: Counter[GenAIProviderEnum] = Counter(
|
||||
genai.provider for genai in config.genai.values()
|
||||
)
|
||||
roles: Counter[GenAIRoleEnum] = Counter(
|
||||
role for genai in config.genai.values() for role in genai.roles
|
||||
)
|
||||
custom = list(config.classification.custom.values())
|
||||
|
||||
return FeaturesSection(
|
||||
face_recognition=EnrichmentUsage(
|
||||
enabled=config.face_recognition.enabled,
|
||||
model_size=config.face_recognition.model_size,
|
||||
device=enrichment_device(devices.get("face_recognition")),
|
||||
),
|
||||
lpr=EnrichmentUsage(
|
||||
enabled=config.lpr.enabled,
|
||||
model_size=config.lpr.model_size,
|
||||
device=enrichment_device(devices.get("lpr")),
|
||||
),
|
||||
semantic_search=SemanticSearchUsage(
|
||||
enabled=config.semantic_search.enabled,
|
||||
model=semantic_model(config.semantic_search.model),
|
||||
model_size=config.semantic_search.model_size,
|
||||
device=enrichment_device(devices.get("semantic_search")),
|
||||
triggers=sum(len(camera.semantic_search.triggers) for camera in cameras),
|
||||
),
|
||||
audio_transcription=TranscriptionUsage(
|
||||
enabled=config.audio_transcription.enabled,
|
||||
model=transcription_model(config.audio_transcription.model),
|
||||
model_size=config.audio_transcription.model_size,
|
||||
),
|
||||
genai=GenAIUsage(providers=histogram(providers), roles=histogram(roles)),
|
||||
classification_models=ClassificationUsage(
|
||||
state=sum(1 for model in custom if model.state_config is not None),
|
||||
object=sum(1 for model in custom if model.object_config is not None),
|
||||
),
|
||||
birdseye=BirdseyeUsage(
|
||||
enabled=config.birdseye.enabled,
|
||||
modes=list(config.birdseye.modes),
|
||||
restream=config.birdseye.restream,
|
||||
),
|
||||
mqtt=config.mqtt.enabled,
|
||||
notifications=config.notifications.enabled,
|
||||
auth=config.auth.enabled,
|
||||
proxy_auth=config.proxy.header_map.user is not None,
|
||||
tls=config.tls.enabled,
|
||||
users=users(),
|
||||
camera_groups=len(config.camera_groups),
|
||||
profiles=len(config.profiles),
|
||||
plus_api_key=config.plus_api.is_active(),
|
||||
)
|
||||
@@ -1,107 +0,0 @@
|
||||
"""Hardware section: CPU, memory, GPUs, decode and detection hardware, storage."""
|
||||
|
||||
import os
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
import psutil
|
||||
|
||||
from frigate.analytics.collectors.common import closed, histogram
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.schema import (
|
||||
DecodeFamily,
|
||||
GpuInfo,
|
||||
GpuVendor,
|
||||
HardwareKey,
|
||||
HardwareSection,
|
||||
StorageInfo,
|
||||
)
|
||||
from frigate.const import RECORD_DIR
|
||||
from frigate.detectors.hardware import hardware_prober
|
||||
from frigate.util.hwaccel import hwaccel_options
|
||||
|
||||
DEVICE_TREE_MODEL = "/proc/device-tree/model"
|
||||
CPUINFO = "/proc/cpuinfo"
|
||||
|
||||
|
||||
def cpu_model() -> str:
|
||||
"""The board model on ARM boards, else the CPU's model name."""
|
||||
try:
|
||||
with open(DEVICE_TREE_MODEL) as f:
|
||||
board = f.read().strip("\x00\n ")
|
||||
|
||||
if board:
|
||||
return board[:64]
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(CPUINFO) as f:
|
||||
for line in f:
|
||||
key, _, value = line.partition(":")
|
||||
|
||||
if key.strip() == "model name" and value.strip():
|
||||
return value.strip()[:64]
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return "unknown"
|
||||
|
||||
|
||||
def gpus(stats: dict[str, Any]) -> list[GpuInfo]:
|
||||
found: list[GpuInfo] = []
|
||||
|
||||
for name, entry in stats.get("gpu_usages", {}).items():
|
||||
vendor = entry.get("vendor") if isinstance(entry, dict) else None
|
||||
found.append(
|
||||
GpuInfo(
|
||||
vendor=closed(GpuVendor, vendor, GpuVendor.other),
|
||||
name=str(name)[:64],
|
||||
)
|
||||
)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def decode_families() -> list[DecodeFamily]:
|
||||
_, available = hwaccel_options()
|
||||
families = [
|
||||
closed(DecodeFamily, family.key, DecodeFamily.other) for family in available
|
||||
]
|
||||
return list(dict.fromkeys(families))
|
||||
|
||||
|
||||
def detection_hardware() -> dict[HardwareKey, int]:
|
||||
units: Counter[HardwareKey] = Counter()
|
||||
|
||||
for found in hardware_prober.probe():
|
||||
units[closed(HardwareKey, found.key, HardwareKey.other)] += found.count
|
||||
|
||||
return histogram(units)
|
||||
|
||||
|
||||
def storage(stats: dict[str, Any]) -> StorageInfo:
|
||||
# stats report sizes in MB
|
||||
entry = stats.get("service", {}).get("storage", {}).get(RECORD_DIR) or {}
|
||||
total_mb = float(entry.get("total") or 0)
|
||||
used_mb = float(entry.get("used") or 0)
|
||||
|
||||
return StorageInfo(
|
||||
record_fs=str(entry.get("mount_type") or "unknown")[:16],
|
||||
record_total_gb=round(total_mb / 1024),
|
||||
record_used_pct=min(round(used_mb / total_mb * 100), 100)
|
||||
if total_mb > 0
|
||||
else 0,
|
||||
)
|
||||
|
||||
|
||||
def collect(ctx: ReportContext) -> HardwareSection:
|
||||
return HardwareSection(
|
||||
cpu_model=cpu_model(),
|
||||
cpu_cores=os.cpu_count() or 0,
|
||||
memory_gb=round(psutil.virtual_memory().total / 2**30),
|
||||
gpus=gpus(ctx.stats),
|
||||
decode_families=decode_families(),
|
||||
detection_hardware=detection_hardware(),
|
||||
storage=storage(ctx.stats),
|
||||
)
|
||||
@@ -1,59 +0,0 @@
|
||||
"""Health section: uptime, CPU, enrichment speed, and notice counts."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from frigate.analytics.collectors.common import rate
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.schema import (
|
||||
EnrichmentTiming,
|
||||
HealthSection,
|
||||
NoticeCounts,
|
||||
NoticeKindKey,
|
||||
)
|
||||
|
||||
TIMING_STATS = {
|
||||
EnrichmentTiming.face: "face_recognition_speed",
|
||||
EnrichmentTiming.lpr: "plate_recognition_speed",
|
||||
EnrichmentTiming.plate_detection: "yolov9_plate_detection_speed",
|
||||
EnrichmentTiming.image_embedding: "image_embedding_speed",
|
||||
EnrichmentTiming.text_embedding: "text_embedding_speed",
|
||||
EnrichmentTiming.review_description: "review_description_speed",
|
||||
EnrichmentTiming.object_description: "object_description_speed",
|
||||
}
|
||||
REPORTABLE_KINDS = frozenset(key.value for key in NoticeKindKey)
|
||||
|
||||
|
||||
def notice_deltas(notice_stats: list[dict[str, Any]]) -> dict[Any, NoticeCounts]:
|
||||
"""What changed since the last accepted report, per reportable kind."""
|
||||
deltas: dict[Any, NoticeCounts] = {}
|
||||
|
||||
for row in notice_stats:
|
||||
if row["kind"] not in REPORTABLE_KINDS:
|
||||
continue
|
||||
|
||||
occurrences = max(row["occurrences"] - row["reported_occurrences"], 0)
|
||||
dismissals = max(row["dismissals"] - row["reported_dismissals"], 0)
|
||||
|
||||
if occurrences or dismissals:
|
||||
deltas[NoticeKindKey(row["kind"])] = NoticeCounts(
|
||||
occurrences=occurrences, dismissals=dismissals
|
||||
)
|
||||
|
||||
return deltas
|
||||
|
||||
|
||||
def collect(ctx: ReportContext) -> HealthSection:
|
||||
service = ctx.stats.get("service", {})
|
||||
embeddings = ctx.stats.get("embeddings", {})
|
||||
cpu = ctx.stats.get("cpu_usages", {}).get("frigate.full_system", {}).get("cpu")
|
||||
timings = {
|
||||
timing: rate(embeddings.get(key)) for timing, key in TIMING_STATS.items()
|
||||
}
|
||||
|
||||
return HealthSection(
|
||||
uptime_hours=int(rate(service.get("uptime")) // 3600),
|
||||
cpu_percent=min(round(rate(cpu)), 100),
|
||||
enrichment_ms={timing: value for timing, value in timings.items() if value > 0},
|
||||
retention_unmet=bool(service.get("retention_unmet", False)),
|
||||
notices=notice_deltas(ctx.notice_stats),
|
||||
)
|
||||
@@ -1,63 +0,0 @@
|
||||
"""Install section: version, image variant, install type, platform."""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
|
||||
from frigate.analytics.collectors.common import closed
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.schema import Arch, ImageVariant, InstallSection, InstallType
|
||||
from frigate.version import VERSION
|
||||
|
||||
KERNEL_PATTERN = re.compile(r"^(\d{1,3})\.(\d{1,3})")
|
||||
|
||||
|
||||
def image_variant(value: str | None) -> ImageVariant:
|
||||
"""The published image from the build-time FRIGATE_IMAGE_VARIANT, dev when unset."""
|
||||
if not value:
|
||||
return ImageVariant.dev
|
||||
|
||||
return closed(ImageVariant, value, ImageVariant.other)
|
||||
|
||||
|
||||
def install_type() -> InstallType:
|
||||
# the add-on is a container too, so it has to be checked first
|
||||
if os.path.isfile("/data/options.json"):
|
||||
return InstallType.ha_addon
|
||||
|
||||
if os.environ.get("KUBERNETES_SERVICE_HOST"):
|
||||
return InstallType.kubernetes
|
||||
|
||||
if os.path.exists("/run/.containerenv"):
|
||||
return InstallType.podman
|
||||
|
||||
if os.path.exists("/.dockerenv"):
|
||||
return InstallType.docker
|
||||
|
||||
return InstallType.unknown
|
||||
|
||||
|
||||
def arch(machine: str) -> Arch:
|
||||
match machine.lower():
|
||||
case "x86_64" | "amd64":
|
||||
return Arch.x86_64
|
||||
case "aarch64" | "arm64":
|
||||
return Arch.aarch64
|
||||
case _:
|
||||
return Arch.other
|
||||
|
||||
|
||||
def kernel(release: str) -> str:
|
||||
match = KERNEL_PATTERN.match(release)
|
||||
return f"{match.group(1)}.{match.group(2)}" if match else "unknown"
|
||||
|
||||
|
||||
def collect(ctx: ReportContext) -> InstallSection:
|
||||
return InstallSection(
|
||||
version=VERSION[:32],
|
||||
image_variant=image_variant(os.environ.get("FRIGATE_IMAGE_VARIANT")),
|
||||
install_type=install_type(),
|
||||
arch=arch(platform.machine()),
|
||||
kernel=kernel(platform.release()),
|
||||
run_as_root=os.geteuid() == 0,
|
||||
)
|
||||
@@ -1,13 +0,0 @@
|
||||
"""What the collectors read, gathered once per report."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from frigate.config import FrigateConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReportContext:
|
||||
config: FrigateConfig
|
||||
stats: dict[str, Any]
|
||||
notice_stats: list[dict[str, Any]] = field(default_factory=list)
|
||||
@@ -1,88 +0,0 @@
|
||||
"""Build a report from the section collectors."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from frigate.analytics.collectors import (
|
||||
cameras,
|
||||
detection,
|
||||
features,
|
||||
hardware,
|
||||
health,
|
||||
install,
|
||||
)
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.schema import SCHEMA_VERSION, AnalyticsReport
|
||||
from frigate.analytics.state import load_state
|
||||
from frigate.config import FrigateConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from frigate.notices.registry import NoticeRegistry
|
||||
from frigate.stats.emitter import StatsEmitter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COLLECTORS: dict[str, Callable[[ReportContext], BaseModel | None]] = {
|
||||
"install": install.collect,
|
||||
"hardware": hardware.collect,
|
||||
"detection": detection.collect,
|
||||
"cameras": cameras.collect,
|
||||
"features": features.collect,
|
||||
"health": health.collect,
|
||||
}
|
||||
|
||||
# shown in the preview until the first report creates a real ID
|
||||
PREVIEW_INSTALL_ID = "0" * 32
|
||||
|
||||
|
||||
def build_report(
|
||||
ctx: ReportContext, install_id: str, sent_at: int | None = None
|
||||
) -> AnalyticsReport:
|
||||
"""Run every collector; one that fails sends its section as null."""
|
||||
sections: dict[str, Any] = {}
|
||||
|
||||
for name, collect in COLLECTORS.items():
|
||||
try:
|
||||
sections[name] = collect(ctx)
|
||||
except Exception:
|
||||
# a collector bug must cost one section, never the whole report
|
||||
logger.warning("Analytics %s section failed", name, exc_info=True)
|
||||
sections[name] = None
|
||||
|
||||
return AnalyticsReport.model_validate(
|
||||
{
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"install_id": install_id,
|
||||
"report_id": str(uuid4()),
|
||||
"sent_at": int(time.time()) if sent_at is None else sent_at,
|
||||
**sections,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def gather_context(
|
||||
config: FrigateConfig,
|
||||
stats_emitter: "StatsEmitter | None",
|
||||
notice_registry: "NoticeRegistry | None",
|
||||
) -> ReportContext:
|
||||
return ReportContext(
|
||||
config=config,
|
||||
stats=stats_emitter.get_latest_stats() if stats_emitter is not None else {},
|
||||
notice_stats=notice_registry.stats() if notice_registry is not None else [],
|
||||
)
|
||||
|
||||
|
||||
def preview_report(
|
||||
config: FrigateConfig,
|
||||
stats_emitter: "StatsEmitter | None",
|
||||
notice_registry: "NoticeRegistry | None",
|
||||
) -> AnalyticsReport:
|
||||
"""The report the next send would carry, without sending it."""
|
||||
state = load_state()
|
||||
ctx = gather_context(config, stats_emitter, notice_registry)
|
||||
return build_report(ctx, state.install_id if state else PREVIEW_INSTALL_ID)
|
||||
@@ -1,177 +0,0 @@
|
||||
"""Send one analytics report a day while the admin has opted in."""
|
||||
|
||||
import logging
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from multiprocessing.synchronize import Event as MpEvent
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.analytics.report import build_report
|
||||
from frigate.analytics.state import (
|
||||
STATE_PATH,
|
||||
AnalyticsState,
|
||||
delete_state,
|
||||
load_state,
|
||||
new_state,
|
||||
save_state,
|
||||
)
|
||||
from frigate.analytics.transport import SendOutcome, send_report
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.holder import ConfigHolder
|
||||
from frigate.const import ANALYTICS_URL
|
||||
from frigate.notices import raise_notice, resolve_notice
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from frigate.notices.registry import NoticeRegistry
|
||||
from frigate.stats.emitter import StatsEmitter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WAKE_S = 10 * 60
|
||||
INTERVAL_S = 24 * 60 * 60
|
||||
JITTER_S = 60 * 60
|
||||
FIRST_DELAY_S = (15 * 60, 45 * 60)
|
||||
PROMPT_KIND = "analytics_prompt"
|
||||
|
||||
|
||||
class AnalyticsReporter(threading.Thread):
|
||||
"""Follows the live config, so a settings save needs no restart."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config_holder: ConfigHolder,
|
||||
stats_emitter: "StatsEmitter",
|
||||
notice_registry: "NoticeRegistry",
|
||||
stop_event: MpEvent | threading.Event,
|
||||
*,
|
||||
state_path: str = STATE_PATH,
|
||||
url: str = ANALYTICS_URL,
|
||||
send: Callable[[str, str], SendOutcome] = send_report,
|
||||
clock: Callable[[], float] = time.time,
|
||||
rng: random.Random | None = None,
|
||||
) -> None:
|
||||
super().__init__(name="analytics_reporter", daemon=True)
|
||||
self.config_holder = config_holder
|
||||
self.stats_emitter = stats_emitter
|
||||
self.notice_registry = notice_registry
|
||||
self.stop_event = stop_event
|
||||
self.state_path = state_path
|
||||
self.url = url
|
||||
self.send = send
|
||||
self.clock = clock
|
||||
self.rng = rng or random.Random()
|
||||
self.first_due = clock() + self.rng.uniform(*FIRST_DELAY_S)
|
||||
self.interval = self._next_interval()
|
||||
self.opted_in: bool | None = None
|
||||
self.warned_unwritable = False
|
||||
# a settings save runs on an API thread while a wake may be mid-attempt
|
||||
self._lock = threading.Lock()
|
||||
config_holder.subscribe(self._on_config)
|
||||
|
||||
def _next_interval(self) -> float:
|
||||
return INTERVAL_S + self.rng.uniform(-JITTER_S, JITTER_S)
|
||||
|
||||
def run(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
self.tick()
|
||||
except Exception:
|
||||
logger.exception("Analytics reporter failed")
|
||||
|
||||
if self.stop_event.wait(WAKE_S):
|
||||
break
|
||||
|
||||
def _on_config(self, config: FrigateConfig) -> None:
|
||||
# a save can turn sharing off and back on between two wakes, so an
|
||||
# opt-out is handled when it's saved rather than at the next wake
|
||||
with self._lock:
|
||||
if not config.safe_mode:
|
||||
self._apply_consent(config.telemetry.analytics)
|
||||
|
||||
def _apply_consent(self, opted_in: bool) -> None:
|
||||
# called with the lock held
|
||||
if opted_in == self.opted_in:
|
||||
return
|
||||
|
||||
self.opted_in = opted_in
|
||||
|
||||
if opted_in:
|
||||
resolve_notice(PROMPT_KIND)
|
||||
else:
|
||||
raise_notice(PROMPT_KIND)
|
||||
delete_state(self.state_path)
|
||||
|
||||
def tick(self) -> None:
|
||||
with self._lock:
|
||||
config = self.config_holder.config
|
||||
|
||||
# safe mode parses a default config where analytics reads as off,
|
||||
# and handling that as an opt-out would delete the install ID
|
||||
if config.safe_mode:
|
||||
return
|
||||
|
||||
self._apply_consent(config.telemetry.analytics)
|
||||
|
||||
if not config.telemetry.analytics:
|
||||
return
|
||||
|
||||
now = self.clock()
|
||||
|
||||
if now < self.first_due:
|
||||
return
|
||||
|
||||
state = load_state(self.state_path) or new_state()
|
||||
|
||||
if not self._due(state.last_attempt_at, now):
|
||||
return
|
||||
|
||||
# saved before sending, so a failing endpoint or a crash mid-send
|
||||
# still waits a full interval; without saved state every boot would
|
||||
# send under a new install ID
|
||||
if not save_state(AnalyticsState(state.install_id, now), self.state_path):
|
||||
if not self.warned_unwritable:
|
||||
logger.warning(
|
||||
"Analytics is on, but %s isn't writable, so no report is sent",
|
||||
self.state_path,
|
||||
)
|
||||
self.warned_unwritable = True
|
||||
|
||||
return
|
||||
|
||||
self.interval = self._next_interval()
|
||||
|
||||
# the lock stays free during the request, so a save never waits on it
|
||||
self._send(state.install_id, now)
|
||||
|
||||
def _due(self, last_attempt_at: float, now: float) -> bool:
|
||||
# a last attempt stamped in the future came from a wrong clock
|
||||
return (
|
||||
now >= last_attempt_at + self.interval or last_attempt_at > now + INTERVAL_S
|
||||
)
|
||||
|
||||
def _send(self, install_id: str, now: float) -> None:
|
||||
notice_stats = self.notice_registry.stats()
|
||||
ctx = ReportContext(
|
||||
config=self.config_holder.config,
|
||||
stats=self.stats_emitter.get_latest_stats(),
|
||||
notice_stats=notice_stats,
|
||||
)
|
||||
report = build_report(ctx, install_id, sent_at=int(now))
|
||||
body = report.model_dump_json()
|
||||
|
||||
# consent can be withdrawn while the report builds
|
||||
if not self.config_holder.config.telemetry.analytics:
|
||||
return
|
||||
|
||||
if self.send(self.url, body) is not SendOutcome.accepted:
|
||||
return
|
||||
|
||||
# a failed health section sent no notice counts, so they stay pending
|
||||
if report.health is not None:
|
||||
self.notice_registry.mark_reported(notice_stats)
|
||||
|
||||
logger.info("Sent the daily analytics report")
|
||||
logger.debug("Analytics report: %s", body)
|
||||
@@ -1,407 +0,0 @@
|
||||
"""Models for the analytics report, the contract with the ingest endpoint.
|
||||
|
||||
Every field carries a description and an x-public flag, and no field accepts
|
||||
user-entered text, so a report can't carry camera names or other free text.
|
||||
"""
|
||||
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from frigate.config.camera.birdseye import BirdseyeModeEnum
|
||||
from frigate.config.camera.camera import CameraTypeEnum
|
||||
from frigate.config.camera.genai import GenAIProviderEnum, GenAIRoleEnum
|
||||
from frigate.config.classification import ModelSizeEnum
|
||||
from frigate.detectors.detector_config import ModelTypeEnum, SceneEnum
|
||||
from frigate.detectors.detector_types import DetectorTypeEnum
|
||||
from frigate.ffmpeg_presets import PRESETS_HW_ACCEL_DECODE, PRESETS_INPUT
|
||||
from frigate.notices.types import NOTICE_KINDS
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
class ImageVariant(StrEnum):
|
||||
standard = "standard"
|
||||
rpi = "rpi"
|
||||
tensorrt = "tensorrt"
|
||||
tensorrt_jp6 = "tensorrt-jp6"
|
||||
rocm = "rocm"
|
||||
rk = "rk"
|
||||
synaptics = "synaptics"
|
||||
dev = "dev"
|
||||
other = "other"
|
||||
|
||||
|
||||
class InstallType(StrEnum):
|
||||
ha_addon = "ha_addon"
|
||||
docker = "docker"
|
||||
podman = "podman"
|
||||
kubernetes = "kubernetes"
|
||||
unknown = "unknown"
|
||||
|
||||
|
||||
class Arch(StrEnum):
|
||||
x86_64 = "x86_64"
|
||||
aarch64 = "aarch64"
|
||||
other = "other"
|
||||
|
||||
|
||||
class GpuVendor(StrEnum):
|
||||
intel = "intel"
|
||||
amd = "amd"
|
||||
nvidia = "nvidia"
|
||||
rockchip = "rockchip"
|
||||
rpi = "rpi"
|
||||
other = "other"
|
||||
|
||||
|
||||
class DecodeFamily(StrEnum):
|
||||
nvidia = "nvidia"
|
||||
vaapi = "vaapi"
|
||||
rkmpp = "rkmpp"
|
||||
intel_qsv = "intel-qsv"
|
||||
jetson = "jetson"
|
||||
rpi = "rpi"
|
||||
other = "other"
|
||||
|
||||
|
||||
class HardwareKey(StrEnum):
|
||||
edgetpu_pci = "edgetpu:pci"
|
||||
edgetpu_usb = "edgetpu:usb"
|
||||
openvino_gpu = "openvino:GPU"
|
||||
openvino_npu = "openvino:NPU"
|
||||
onnx_amd = "onnx:amd"
|
||||
onnx_nvidia = "onnx:nvidia"
|
||||
tensorrt = "tensorrt"
|
||||
hailo = "hailo"
|
||||
memryx = "memryx"
|
||||
deepx = "deepx"
|
||||
rknn = "rknn"
|
||||
axengine = "axengine"
|
||||
synaptics = "synaptics"
|
||||
cpu = "cpu"
|
||||
other = "other"
|
||||
|
||||
|
||||
class ModelSource(StrEnum):
|
||||
default = "default"
|
||||
plus = "plus"
|
||||
custom = "custom"
|
||||
|
||||
|
||||
class HeightBucket(StrEnum):
|
||||
le_360 = "le_360"
|
||||
h480 = "480"
|
||||
h720 = "720"
|
||||
h1080 = "1080"
|
||||
h1440 = "1440"
|
||||
ge_2160 = "ge_2160"
|
||||
|
||||
|
||||
class FpsBucket(StrEnum):
|
||||
le_5 = "le_5"
|
||||
f6_10 = "6_10"
|
||||
gt_10 = "gt_10"
|
||||
|
||||
|
||||
class RetainBucket(StrEnum):
|
||||
zero = "0"
|
||||
d1_7 = "1_7"
|
||||
d8_30 = "8_30"
|
||||
gt_30 = "gt_30"
|
||||
|
||||
|
||||
class ConnectionQuality(StrEnum):
|
||||
excellent = "excellent"
|
||||
fair = "fair"
|
||||
poor = "poor"
|
||||
unusable = "unusable"
|
||||
|
||||
|
||||
class EnrichmentDevice(StrEnum):
|
||||
cpu = "cpu"
|
||||
cuda = "cuda"
|
||||
tensorrt = "tensorrt"
|
||||
migraphx = "migraphx"
|
||||
openvino_cpu = "openvino_cpu"
|
||||
openvino_gpu = "openvino_gpu"
|
||||
openvino_npu = "openvino_npu"
|
||||
other = "other"
|
||||
|
||||
|
||||
class SemanticSearchModel(StrEnum):
|
||||
jinav1 = "jinav1"
|
||||
jinav2 = "jinav2"
|
||||
genai = "genai"
|
||||
|
||||
|
||||
class TranscriptionModel(StrEnum):
|
||||
whisper = "whisper"
|
||||
genai = "genai"
|
||||
|
||||
|
||||
class UserRole(StrEnum):
|
||||
admin = "admin"
|
||||
viewer = "viewer"
|
||||
custom = "custom"
|
||||
|
||||
|
||||
class EnrichmentTiming(StrEnum):
|
||||
face = "face"
|
||||
lpr = "lpr"
|
||||
plate_detection = "plate_detection"
|
||||
image_embedding = "image_embedding"
|
||||
text_embedding = "text_embedding"
|
||||
review_description = "review_description"
|
||||
object_description = "object_description"
|
||||
|
||||
|
||||
def _preset_keys(presets: dict[str, Any]) -> dict[str, str]:
|
||||
keys = [name.removeprefix("preset-") for name in presets]
|
||||
return {key: key for key in [*keys, "custom", "none"]}
|
||||
|
||||
|
||||
# built from the registries they mirror, so a new preset or notice kind reaches
|
||||
# the schema through generate_analytics_schema.py instead of a hand edit
|
||||
HwaccelKey = StrEnum("HwaccelKey", _preset_keys(PRESETS_HW_ACCEL_DECODE)) # type: ignore[misc]
|
||||
InputPresetKey = StrEnum("InputPresetKey", _preset_keys(PRESETS_INPUT)) # type: ignore[misc]
|
||||
NoticeKindKey = StrEnum( # type: ignore[misc]
|
||||
"NoticeKindKey",
|
||||
{key: key for key, kind in NOTICE_KINDS.items() if kind.reportable},
|
||||
)
|
||||
|
||||
|
||||
class AnalyticsModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
def metric(description: str, *, public: bool = True, **kwargs: Any) -> Any:
|
||||
"""Declare a report field; the description and flag land in the schema."""
|
||||
return Field(
|
||||
description=description, json_schema_extra={"x-public": public}, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def count(description: str) -> Any:
|
||||
return metric(description, ge=0)
|
||||
|
||||
|
||||
class InstallSection(AnalyticsModel):
|
||||
version: str = metric("Frigate version string", max_length=32)
|
||||
image_variant: ImageVariant = metric("Published image the install runs")
|
||||
install_type: InstallType = metric("How Frigate is installed")
|
||||
arch: Arch = metric("CPU architecture")
|
||||
kernel: str = metric(
|
||||
"Host kernel as major.minor", pattern=r"^(\d{1,3}\.\d{1,3}|unknown)$"
|
||||
)
|
||||
run_as_root: bool = metric("Whether the main process runs as root")
|
||||
|
||||
|
||||
class GpuInfo(AnalyticsModel):
|
||||
vendor: GpuVendor = metric("GPU vendor")
|
||||
name: str = metric("GPU name as the hardware reports it", max_length=64)
|
||||
|
||||
|
||||
class StorageInfo(AnalyticsModel):
|
||||
record_fs: str = metric("Filesystem of the recordings volume", max_length=16)
|
||||
record_total_gb: int = count("Size of the recordings volume in GB")
|
||||
record_used_pct: int = metric(
|
||||
"Percent of the recordings volume in use", ge=0, le=100
|
||||
)
|
||||
|
||||
|
||||
class HardwareSection(AnalyticsModel):
|
||||
cpu_model: str = metric(
|
||||
"CPU or board model as the hardware reports it", max_length=64
|
||||
)
|
||||
cpu_cores: int = count("Logical CPU count")
|
||||
memory_gb: int = count("Total memory in GB")
|
||||
gpus: list[GpuInfo] = metric("GPUs the stats collector found")
|
||||
decode_families: list[DecodeFamily] = metric(
|
||||
"Hardware decode families this system can use"
|
||||
)
|
||||
detection_hardware: dict[HardwareKey, int] = metric(
|
||||
"Detection hardware found, as unit counts by kind"
|
||||
)
|
||||
storage: StorageInfo = metric("Recordings storage")
|
||||
|
||||
|
||||
class DetectionModel(AnalyticsModel):
|
||||
detector: DetectorTypeEnum = metric("Detector type the model runs on")
|
||||
devices: int = count("Devices the model runs on")
|
||||
model_type: ModelTypeEnum = metric("Model architecture")
|
||||
input: str = metric("Model input size as WxH", pattern=r"^\d{1,5}x\d{1,5}$")
|
||||
source: ModelSource = metric("Where the model came from", public=False)
|
||||
inference_ms: float | None = metric(
|
||||
"Mean inference time across the model's detector processes, null before the first stats",
|
||||
ge=0,
|
||||
)
|
||||
|
||||
|
||||
class DetectionSection(AnalyticsModel):
|
||||
models: dict[SceneEnum, DetectionModel] = metric("Detection models keyed by scene")
|
||||
detection_fps: float = metric("Detections per second across cameras", ge=0)
|
||||
skipped_fps: float = metric("Frames per second skipped across cameras", ge=0)
|
||||
|
||||
|
||||
class RetainDays(AnalyticsModel):
|
||||
continuous: dict[RetainBucket, int] = metric(
|
||||
"Recording cameras by continuous retention in days"
|
||||
)
|
||||
motion: dict[RetainBucket, int] = metric(
|
||||
"Recording cameras by motion retention in days"
|
||||
)
|
||||
alerts: dict[RetainBucket, int] = metric(
|
||||
"Recording cameras by alert retention in days"
|
||||
)
|
||||
detections: dict[RetainBucket, int] = metric(
|
||||
"Recording cameras by detection retention in days"
|
||||
)
|
||||
|
||||
|
||||
class CamerasSection(AnalyticsModel):
|
||||
total: int = count("Configured cameras")
|
||||
enabled: int = count("Enabled cameras")
|
||||
types: dict[CameraTypeEnum, int] = metric("Cameras by type")
|
||||
detect_height: dict[HeightBucket, int] = metric(
|
||||
"Cameras by detect resolution height"
|
||||
)
|
||||
detect_fps: dict[FpsBucket, int] = metric("Cameras by detect fps")
|
||||
hwaccel: dict[HwaccelKey, int] = metric(
|
||||
"Cameras by the resolved hwaccel preset of the detect input"
|
||||
)
|
||||
input_preset: dict[InputPresetKey, int] = metric(
|
||||
"Cameras by the input preset of the detect input"
|
||||
)
|
||||
go2rtc_restream: int = count("Cameras with an input from the go2rtc restream")
|
||||
separate_detect_stream: int = count(
|
||||
"Cameras whose detect input differs from their record input"
|
||||
)
|
||||
detect: int = count("Cameras with detection on")
|
||||
record: int = count("Cameras with recording on")
|
||||
sub_stream_record: int = count("Cameras with sub stream recording on")
|
||||
snapshots: int = count("Cameras with snapshots on")
|
||||
audio: int = count("Cameras with audio detection on")
|
||||
audio_transcription: int = count("Cameras with audio transcription on")
|
||||
birdseye: int = count("Cameras in birdseye")
|
||||
onvif: int = count("Cameras with an ONVIF host")
|
||||
autotracking: int = count("Cameras with PTZ autotracking on")
|
||||
face_recognition: int = count("Cameras with face recognition on")
|
||||
lpr: int = count("Cameras with license plate recognition on")
|
||||
review_genai: int = count("Cameras with GenAI review summaries on")
|
||||
object_genai: int = count("Cameras with GenAI object descriptions on")
|
||||
notifications: int = count("Cameras with notifications on")
|
||||
zones: int = count("Zones across cameras")
|
||||
cameras_with_zones: int = count("Cameras with at least one zone")
|
||||
motion_masks: int = count("Motion masks across cameras")
|
||||
object_masks: int = count("Object masks across cameras")
|
||||
connection_quality: dict[ConnectionQuality, int] = metric(
|
||||
"Cameras by connection quality at send time"
|
||||
)
|
||||
retain_days: RetainDays = metric("Recording retention")
|
||||
|
||||
|
||||
class EnrichmentUsage(AnalyticsModel):
|
||||
enabled: bool = metric("Whether the enrichment is on")
|
||||
model_size: ModelSizeEnum = metric("Configured model size")
|
||||
device: EnrichmentDevice | None = metric(
|
||||
"Device the model loaded on, null when it isn't loaded"
|
||||
)
|
||||
|
||||
|
||||
class SemanticSearchUsage(AnalyticsModel):
|
||||
enabled: bool = metric("Whether semantic search is on")
|
||||
model: SemanticSearchModel | None = metric(
|
||||
"Embedding model, genai for a GenAI provider"
|
||||
)
|
||||
model_size: ModelSizeEnum = metric("Configured model size")
|
||||
device: EnrichmentDevice | None = metric(
|
||||
"Device the model loaded on, null when it isn't loaded"
|
||||
)
|
||||
triggers: int = count("Semantic search triggers across cameras")
|
||||
|
||||
|
||||
class TranscriptionUsage(AnalyticsModel):
|
||||
enabled: bool = metric("Whether audio transcription is on")
|
||||
model: TranscriptionModel | None = metric(
|
||||
"Transcription model, genai for a GenAI provider"
|
||||
)
|
||||
model_size: ModelSizeEnum = metric("Configured model size")
|
||||
|
||||
|
||||
class GenAIUsage(AnalyticsModel):
|
||||
providers: dict[GenAIProviderEnum, int] = metric(
|
||||
"Configured GenAI providers by type"
|
||||
)
|
||||
roles: dict[GenAIRoleEnum, int] = metric("Configured GenAI providers by role")
|
||||
|
||||
|
||||
class ClassificationUsage(AnalyticsModel):
|
||||
state: int = count("Custom state classification models")
|
||||
object: int = count("Custom object classification models")
|
||||
|
||||
|
||||
class BirdseyeUsage(AnalyticsModel):
|
||||
enabled: bool = metric("Whether birdseye is on")
|
||||
modes: list[BirdseyeModeEnum] = metric("Birdseye modes")
|
||||
restream: bool = metric("Whether birdseye is restreamed")
|
||||
|
||||
|
||||
class FeaturesSection(AnalyticsModel):
|
||||
face_recognition: EnrichmentUsage = metric("Face recognition")
|
||||
lpr: EnrichmentUsage = metric("License plate recognition")
|
||||
semantic_search: SemanticSearchUsage = metric("Semantic search")
|
||||
audio_transcription: TranscriptionUsage = metric("Audio transcription")
|
||||
genai: GenAIUsage = metric("Generative AI providers")
|
||||
classification_models: ClassificationUsage = metric("Custom classification models")
|
||||
birdseye: BirdseyeUsage = metric("Birdseye")
|
||||
mqtt: bool = metric("Whether MQTT is on")
|
||||
notifications: bool = metric("Whether web push notifications are on")
|
||||
auth: bool = metric("Whether authentication is on")
|
||||
proxy_auth: bool = metric("Whether a proxy supplies the user header")
|
||||
tls: bool = metric("Whether TLS is on")
|
||||
users: dict[UserRole, int] = metric("Users by role")
|
||||
camera_groups: int = count("Camera groups")
|
||||
profiles: int = count("Profiles")
|
||||
plus_api_key: bool = metric("Whether a Frigate+ API key is set", public=False)
|
||||
|
||||
|
||||
class NoticeCounts(AnalyticsModel):
|
||||
occurrences: int = count("Occurrences since the last accepted report")
|
||||
dismissals: int = count("Dismissals since the last accepted report")
|
||||
|
||||
|
||||
class HealthSection(AnalyticsModel):
|
||||
uptime_hours: int = count("Hours since Frigate started")
|
||||
cpu_percent: int = metric("System CPU use at send time", ge=0, le=100)
|
||||
enrichment_ms: dict[EnrichmentTiming, float] = metric(
|
||||
"Mean enrichment inference times in milliseconds"
|
||||
)
|
||||
retention_unmet: bool = metric(
|
||||
"Whether storage can't keep the configured retention"
|
||||
)
|
||||
notices: dict[NoticeKindKey, NoticeCounts] = metric(
|
||||
"Notice counts by kind since the last accepted report", public=False
|
||||
)
|
||||
|
||||
|
||||
class AnalyticsReport(AnalyticsModel):
|
||||
schema_version: int = metric("Report format version", ge=1)
|
||||
install_id: str = metric(
|
||||
"Random install identifier", public=False, pattern=r"^[0-9a-f]{32}$"
|
||||
)
|
||||
report_id: str = metric(
|
||||
"Random identifier of this report",
|
||||
public=False,
|
||||
pattern=r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
|
||||
)
|
||||
sent_at: int = count("Unix time the report was built")
|
||||
install: InstallSection | None = metric("Install, null if its collector failed")
|
||||
hardware: HardwareSection | None = metric("Hardware, null if its collector failed")
|
||||
detection: DetectionSection | None = metric(
|
||||
"Object detection, null if its collector failed"
|
||||
)
|
||||
cameras: CamerasSection | None = metric("Cameras, null if its collector failed")
|
||||
features: FeaturesSection | None = metric("Features, null if its collector failed")
|
||||
health: HealthSection | None = metric("Health, null if its collector failed")
|
||||
@@ -1,85 +0,0 @@
|
||||
"""The install's analytics identity and last attempt time, kept under /config."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from uuid import uuid4
|
||||
|
||||
from frigate.const import CONFIG_DIR
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STATE_PATH = os.path.join(CONFIG_DIR, ".analytics.json")
|
||||
INSTALL_ID_PATTERN = re.compile(r"[0-9a-f]{32}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AnalyticsState:
|
||||
install_id: str
|
||||
last_attempt_at: float
|
||||
|
||||
|
||||
def new_state() -> AnalyticsState:
|
||||
return AnalyticsState(install_id=uuid4().hex, last_attempt_at=0.0)
|
||||
|
||||
|
||||
def load_state(path: str = STATE_PATH) -> AnalyticsState | None:
|
||||
"""The saved state, or None when the file is missing or unusable."""
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (OSError, ValueError):
|
||||
logger.warning("Ignoring unreadable analytics state at %s", path)
|
||||
return None
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
|
||||
install_id = data.get("install_id")
|
||||
last_attempt_at = data.get("last_attempt_at")
|
||||
|
||||
if (
|
||||
not isinstance(install_id, str)
|
||||
or not INSTALL_ID_PATTERN.fullmatch(install_id)
|
||||
or isinstance(last_attempt_at, bool)
|
||||
or not isinstance(last_attempt_at, int | float)
|
||||
):
|
||||
logger.warning("Ignoring invalid analytics state at %s", path)
|
||||
return None
|
||||
|
||||
return AnalyticsState(install_id=install_id, last_attempt_at=float(last_attempt_at))
|
||||
|
||||
|
||||
def save_state(state: AnalyticsState, path: str = STATE_PATH) -> bool:
|
||||
"""Write the state atomically, returning False when it can't be written."""
|
||||
temp_path = f"{path}.tmp"
|
||||
|
||||
try:
|
||||
with open(temp_path, "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"install_id": state.install_id,
|
||||
"last_attempt_at": state.last_attempt_at,
|
||||
},
|
||||
f,
|
||||
)
|
||||
|
||||
os.replace(temp_path, path)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def delete_state(path: str = STATE_PATH) -> None:
|
||||
"""Forget the install ID, so a later opt-in starts a fresh identity."""
|
||||
try:
|
||||
os.remove(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError:
|
||||
logger.warning("Unable to delete analytics state at %s", path)
|
||||
@@ -1,54 +0,0 @@
|
||||
"""POST a report to the ingest endpoint."""
|
||||
|
||||
import logging
|
||||
from enum import Enum
|
||||
|
||||
import requests
|
||||
|
||||
from frigate.version import VERSION
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TIMEOUT_S = 30
|
||||
|
||||
|
||||
class SendOutcome(Enum):
|
||||
accepted = "accepted"
|
||||
rejected = "rejected"
|
||||
rate_limited = "rate_limited"
|
||||
failed = "failed"
|
||||
|
||||
|
||||
def send_report(url: str, body: str) -> SendOutcome:
|
||||
"""Send one report. Never raises; the outcome says what happened."""
|
||||
try:
|
||||
# a redirect would turn the POST into a GET, so it counts as a failure
|
||||
response = requests.post(
|
||||
url,
|
||||
data=body.encode(),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": f"Frigate/{VERSION}",
|
||||
},
|
||||
timeout=TIMEOUT_S,
|
||||
allow_redirects=False,
|
||||
)
|
||||
except requests.RequestException as err:
|
||||
logger.warning("Unable to send the analytics report: %s", err)
|
||||
return SendOutcome.failed
|
||||
|
||||
status = response.status_code
|
||||
|
||||
if 200 <= status < 300:
|
||||
return SendOutcome.accepted
|
||||
|
||||
if status == 400:
|
||||
logger.warning("The analytics report was rejected: %s", response.text[:200])
|
||||
return SendOutcome.rejected
|
||||
|
||||
if status == 429:
|
||||
logger.debug("The analytics endpoint is rate limiting this install")
|
||||
return SendOutcome.rate_limited
|
||||
|
||||
logger.warning("The analytics endpoint returned %s", status)
|
||||
return SendOutcome.failed
|
||||
@@ -1,25 +0,0 @@
|
||||
"""Analytics APIs."""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from frigate.analytics.report import preview_report
|
||||
from frigate.api.auth import require_role
|
||||
from frigate.api.defs.tags import Tags
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(tags=[Tags.analytics])
|
||||
|
||||
|
||||
@router.get("/analytics/preview", dependencies=[Depends(require_role(["admin"]))])
|
||||
def get_analytics_preview(request: Request) -> JSONResponse:
|
||||
"""Get the analytics report Frigate would send next, without sending it."""
|
||||
report = preview_report(
|
||||
request.app.frigate_config,
|
||||
request.app.stats_emitter,
|
||||
request.app.notice_registry,
|
||||
)
|
||||
return JSONResponse(content=report.model_dump(mode="json"))
|
||||
@@ -786,13 +786,6 @@ def auth(request: Request):
|
||||
|
||||
user = token.claims.get("sub")
|
||||
role = token.claims.get("role")
|
||||
|
||||
# the token keeps the role it was issued with, so a role removed from
|
||||
# the config since then must send the user back through login
|
||||
if role not in auth_config.roles:
|
||||
logger.debug("jwt role %s is not in the config", role)
|
||||
return fail_response
|
||||
|
||||
current_time = int(time.time())
|
||||
|
||||
# if the jwt is expired
|
||||
|
||||
@@ -2,7 +2,6 @@ from enum import Enum
|
||||
|
||||
|
||||
class Tags(Enum):
|
||||
analytics = "Analytics"
|
||||
app = "App"
|
||||
auth = "Auth"
|
||||
camera = "Camera"
|
||||
|
||||
+4
-14
@@ -1038,10 +1038,6 @@ def export_recording_custom(
|
||||
if camera_validation_error is not None:
|
||||
return camera_validation_error
|
||||
|
||||
# Validate user-provided ffmpeg args to prevent injection and add to cases.
|
||||
# Admin users are trusted and skip validation.
|
||||
is_admin = request.headers.get("remote-role", "") == "admin"
|
||||
|
||||
playback_source = body.source
|
||||
friendly_name = body.name
|
||||
existing_image, image_validation_error = _sanitize_existing_image(body.image_path)
|
||||
@@ -1052,16 +1048,6 @@ def export_recording_custom(
|
||||
cpu_fallback = body.cpu_fallback
|
||||
|
||||
export_case_id = body.export_case_id
|
||||
|
||||
if export_case_id is not None and not is_admin:
|
||||
return JSONResponse(
|
||||
content={
|
||||
"success": False,
|
||||
"message": "Only admins can attach exports to an existing case.",
|
||||
},
|
||||
status_code=403,
|
||||
)
|
||||
|
||||
case_validation_error = _validate_export_case(export_case_id)
|
||||
if case_validation_error is not None:
|
||||
return case_validation_error
|
||||
@@ -1078,6 +1064,10 @@ def export_recording_custom(
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
# Validate user-provided ffmpeg args to prevent injection.
|
||||
# Admin users are trusted and skip validation.
|
||||
is_admin = request.headers.get("remote-role", "") == "admin"
|
||||
|
||||
if not is_admin:
|
||||
for args_label, args_value in [
|
||||
("input", ffmpeg_input_args),
|
||||
|
||||
@@ -12,8 +12,8 @@ from slowapi.middleware import SlowAPIMiddleware
|
||||
from starlette_context import middleware, plugins
|
||||
from starlette_context.plugins import Plugin
|
||||
|
||||
from frigate.api import app as main_app
|
||||
from frigate.api import (
|
||||
analytics,
|
||||
auth,
|
||||
camera,
|
||||
chat,
|
||||
@@ -30,7 +30,6 @@ from frigate.api import (
|
||||
record,
|
||||
review,
|
||||
)
|
||||
from frigate.api import app as main_app
|
||||
from frigate.api.auth import get_jwt_secret, limiter, require_admin_by_default
|
||||
from frigate.comms.dispatcher import Dispatcher
|
||||
from frigate.comms.event_metadata_updater import (
|
||||
@@ -141,7 +140,6 @@ def create_fastapi_app(
|
||||
|
||||
# Routes
|
||||
# Order of include_router matters: https://fastapi.tiangolo.com/tutorial/path-params/#order-matters
|
||||
app.include_router(analytics.router)
|
||||
app.include_router(auth.router)
|
||||
app.include_router(camera.router)
|
||||
app.include_router(chat.router)
|
||||
|
||||
+62
-83
@@ -189,7 +189,7 @@ async def camera_ptz_info(request: Request, camera_name: str):
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
request.app.onvif.get_camera_info(camera_name), request.app.onvif.loop
|
||||
)
|
||||
result = await asyncio.wrap_future(future)
|
||||
result = future.result()
|
||||
return JSONResponse(content=result)
|
||||
else:
|
||||
return JSONResponse(
|
||||
@@ -258,15 +258,15 @@ async def latest_frame(
|
||||
|
||||
frame = request.app.camera_error_image
|
||||
|
||||
height = int(params.height or str(frame.shape[0]))
|
||||
width = int(height * frame.shape[1] / frame.shape[0])
|
||||
|
||||
if frame is None:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Unable to get valid frame"},
|
||||
status_code=500,
|
||||
)
|
||||
|
||||
height = int(params.height or str(frame.shape[0]))
|
||||
width = int(height * frame.shape[1] / frame.shape[0])
|
||||
|
||||
if height < 1 or width < 1:
|
||||
return JSONResponse(
|
||||
content="Invalid height / width requested :: {} / {}".format(
|
||||
@@ -886,8 +886,9 @@ async def vod_event(
|
||||
# If the recordings are not found and the event started more than 5 minutes ago, set has_clip to false
|
||||
if (
|
||||
event.start_time < datetime.now().timestamp() - 300
|
||||
and isinstance(vod_response, JSONResponse)
|
||||
and vod_response.status_code == 404
|
||||
and type(vod_response) is tuple
|
||||
and len(vod_response) == 2
|
||||
and vod_response[1] == 404
|
||||
):
|
||||
Event.update(has_clip=False).where(Event.id == event_id).execute()
|
||||
|
||||
@@ -955,80 +956,64 @@ async def event_snapshot(
|
||||
event_complete = False
|
||||
jpg_bytes = None
|
||||
frame_time = 0
|
||||
|
||||
try:
|
||||
event = Event.get(Event.id == event_id, Event.end_time != None)
|
||||
await require_camera_access(event.camera, request=request)
|
||||
except DoesNotExist:
|
||||
event = None
|
||||
|
||||
if event is not None:
|
||||
event_complete = True
|
||||
|
||||
await require_camera_access(event.camera, request=request)
|
||||
if not event.has_snapshot:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Snapshot not available"},
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
snapshot_settings = _resolve_snapshot_settings(
|
||||
request.app.frigate_config.cameras[event.camera].snapshots, params
|
||||
)
|
||||
jpg_bytes, frame_time = get_event_snapshot_bytes(
|
||||
event,
|
||||
ext="jpg",
|
||||
timestamp=snapshot_settings["timestamp"],
|
||||
bounding_box=snapshot_settings["bounding_box"],
|
||||
crop=snapshot_settings["crop"],
|
||||
height=snapshot_settings["height"],
|
||||
quality=snapshot_settings["quality"],
|
||||
timestamp_style=request.app.frigate_config.cameras[
|
||||
event.camera
|
||||
].timestamp_style,
|
||||
colormap=request.app.frigate_config.model_for_camera(event.camera).colormap,
|
||||
)
|
||||
except DoesNotExist:
|
||||
# see if the object is currently being tracked
|
||||
try:
|
||||
snapshot_settings = _resolve_snapshot_settings(
|
||||
request.app.frigate_config.cameras[event.camera].snapshots, params
|
||||
)
|
||||
jpg_bytes, frame_time = get_event_snapshot_bytes(
|
||||
event,
|
||||
ext="jpg",
|
||||
timestamp=snapshot_settings["timestamp"],
|
||||
bounding_box=snapshot_settings["bounding_box"],
|
||||
crop=snapshot_settings["crop"],
|
||||
height=snapshot_settings["height"],
|
||||
quality=snapshot_settings["quality"],
|
||||
timestamp_style=request.app.frigate_config.cameras[
|
||||
event.camera
|
||||
].timestamp_style,
|
||||
colormap=request.app.frigate_config.model_for_camera(
|
||||
event.camera
|
||||
).colormap,
|
||||
camera_states: list[CameraState] = (
|
||||
request.app.detected_frames_processor.get_camera_states()
|
||||
)
|
||||
for camera_state in camera_states:
|
||||
if event_id in camera_state.tracked_objects:
|
||||
tracked_obj = camera_state.tracked_objects.get(event_id)
|
||||
if tracked_obj is not None:
|
||||
snapshot_settings = _resolve_snapshot_settings(
|
||||
camera_state.camera_config.snapshots, params
|
||||
)
|
||||
jpg_bytes, frame_time = tracked_obj.get_img_bytes(
|
||||
ext="jpg",
|
||||
timestamp=snapshot_settings["timestamp"],
|
||||
bounding_box=snapshot_settings["bounding_box"],
|
||||
crop=snapshot_settings["crop"],
|
||||
height=snapshot_settings["height"],
|
||||
quality=snapshot_settings["quality"],
|
||||
)
|
||||
await require_camera_access(camera_state.name, request=request)
|
||||
except Exception:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Unknown error occurred"},
|
||||
content={"success": False, "message": "Ongoing event not found"},
|
||||
status_code=404,
|
||||
)
|
||||
else:
|
||||
# see if the object is currently being tracked
|
||||
camera_states: list[CameraState] = (
|
||||
request.app.detected_frames_processor.get_camera_states()
|
||||
except Exception:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Unknown error occurred"},
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
for camera_state in camera_states:
|
||||
tracked_obj = camera_state.tracked_objects.get(event_id)
|
||||
|
||||
if tracked_obj is None:
|
||||
continue
|
||||
|
||||
await require_camera_access(camera_state.name, request=request)
|
||||
|
||||
try:
|
||||
snapshot_settings = _resolve_snapshot_settings(
|
||||
camera_state.camera_config.snapshots, params
|
||||
)
|
||||
jpg_bytes, frame_time = tracked_obj.get_img_bytes(
|
||||
ext="jpg",
|
||||
timestamp=snapshot_settings["timestamp"],
|
||||
bounding_box=snapshot_settings["bounding_box"],
|
||||
crop=snapshot_settings["crop"],
|
||||
height=snapshot_settings["height"],
|
||||
quality=snapshot_settings["quality"],
|
||||
)
|
||||
except Exception:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Ongoing event not found"},
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
break
|
||||
|
||||
if jpg_bytes is None:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Live frame not available"},
|
||||
@@ -1077,25 +1062,19 @@ async def event_thumbnail(
|
||||
|
||||
if not thumbnail_bytes:
|
||||
# see if the object is currently being tracked
|
||||
camera_states = request.app.detected_frames_processor.get_camera_states()
|
||||
|
||||
for camera_state in camera_states:
|
||||
tracked_obj = camera_state.tracked_objects.get(event_id)
|
||||
|
||||
if tracked_obj is None:
|
||||
continue
|
||||
|
||||
await require_camera_access(camera_state.name, request=request)
|
||||
|
||||
try:
|
||||
thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
|
||||
except Exception:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Event not found"},
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
break
|
||||
try:
|
||||
camera_states = request.app.detected_frames_processor.get_camera_states()
|
||||
for camera_state in camera_states:
|
||||
if event_id in camera_state.tracked_objects:
|
||||
tracked_obj = camera_state.tracked_objects.get(event_id)
|
||||
if tracked_obj is not None:
|
||||
await require_camera_access(camera_state.name, request=request)
|
||||
thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
|
||||
except Exception:
|
||||
return JSONResponse(
|
||||
content={"success": False, "message": "Event not found"},
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
if not thumbnail_bytes:
|
||||
return JSONResponse(
|
||||
|
||||
@@ -412,8 +412,11 @@ async def no_recordings(
|
||||
if not camera_list:
|
||||
return JSONResponse(content=[])
|
||||
|
||||
before = params.before or datetime.now().timestamp()
|
||||
after = params.after or (datetime.now() - timedelta(hours=1)).timestamp()
|
||||
before = params.before or datetime.datetime.now().timestamp()
|
||||
after = (
|
||||
params.after
|
||||
or (datetime.datetime.now() - datetime.timedelta(hours=1)).timestamp()
|
||||
)
|
||||
scale = params.scale
|
||||
|
||||
recordings: list[tuple[float, float]] = []
|
||||
|
||||
+2
-25
@@ -15,7 +15,6 @@ import uvicorn
|
||||
from peewee_migrate import Router
|
||||
from playhouse.sqlite_ext import SqliteExtDatabase
|
||||
|
||||
from frigate.analytics.reporter import AnalyticsReporter
|
||||
from frigate.api.auth import hash_password
|
||||
from frigate.api.fastapi_app import create_fastapi_app
|
||||
from frigate.camera import CameraMetrics, PTZMetrics
|
||||
@@ -484,7 +483,7 @@ class FrigateApp:
|
||||
|
||||
def start_audio_processor(self) -> None:
|
||||
self.audio_process = AudioProcessor(
|
||||
self.config, self.camera_metrics, self.embeddings_metrics, self.stop_event
|
||||
self.config, self.camera_metrics, self.stop_event
|
||||
)
|
||||
self.audio_process.start()
|
||||
self.processes["audio_detector"] = self.audio_process.pid or 0
|
||||
@@ -530,15 +529,6 @@ class FrigateApp:
|
||||
)
|
||||
self.stats_emitter.start()
|
||||
|
||||
def start_analytics_reporter(self) -> None:
|
||||
self.analytics_reporter = AnalyticsReporter(
|
||||
self.config_holder,
|
||||
self.stats_emitter,
|
||||
self.notice_registry,
|
||||
self.stop_event,
|
||||
)
|
||||
self.analytics_reporter.start()
|
||||
|
||||
def start_watchdog(self) -> None:
|
||||
self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event)
|
||||
|
||||
@@ -566,20 +556,10 @@ class FrigateApp:
|
||||
"output",
|
||||
lambda: OutputProcess(self.config, self.stop_event),
|
||||
),
|
||||
(
|
||||
"audio_process",
|
||||
"audio_detector",
|
||||
lambda: AudioProcessor(
|
||||
self.config,
|
||||
self.camera_metrics,
|
||||
self.embeddings_metrics,
|
||||
self.stop_event,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
for attr, key, factory in specs:
|
||||
if getattr(self, attr, None) is None:
|
||||
if not hasattr(self, attr):
|
||||
continue
|
||||
|
||||
def on_restart(
|
||||
@@ -690,7 +670,6 @@ class FrigateApp:
|
||||
self.start_audio_processor()
|
||||
self.start_storage_maintainer()
|
||||
self.start_stats_emitter()
|
||||
self.start_analytics_reporter()
|
||||
self.start_timeline_processor()
|
||||
self.start_event_processor()
|
||||
self.start_event_cleanup()
|
||||
@@ -790,8 +769,6 @@ class FrigateApp:
|
||||
self.event_cleanup.join()
|
||||
self.record_cleanup.join()
|
||||
self.stats_emitter.join()
|
||||
# a send in flight can hold the thread for the whole request timeout
|
||||
self.analytics_reporter.join(timeout=5)
|
||||
self.frigate_watchdog.join()
|
||||
self.camera_maintainer.join()
|
||||
self.db.stop()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from enum import Enum
|
||||
from typing import Any, Self
|
||||
from typing import Any
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic import Field
|
||||
|
||||
from ..base import FrigateBaseModel
|
||||
from ..env import EnvString
|
||||
@@ -21,17 +21,6 @@ class GenAIRoleEnum(str, Enum):
|
||||
chat = "chat"
|
||||
descriptions = "descriptions"
|
||||
embeddings = "embeddings"
|
||||
transcribe = "transcribe"
|
||||
|
||||
|
||||
# Providers that can accept audio input for the transcribe role. Ollama has no
|
||||
# audio input support, so claiming the role there would fail at request time.
|
||||
TRANSCRIBE_CAPABLE_PROVIDERS = {
|
||||
GenAIProviderEnum.openai,
|
||||
GenAIProviderEnum.azure_openai,
|
||||
GenAIProviderEnum.gemini,
|
||||
GenAIProviderEnum.llamacpp,
|
||||
}
|
||||
|
||||
|
||||
class GenAIConfig(FrigateBaseModel):
|
||||
@@ -63,7 +52,7 @@ class GenAIConfig(FrigateBaseModel):
|
||||
GenAIRoleEnum.chat,
|
||||
],
|
||||
title="Roles",
|
||||
description="GenAI roles (chat, descriptions, embeddings, transcribe); one provider per role. Only chat, descriptions, and embeddings are granted by default; transcribe must be listed explicitly.",
|
||||
description="GenAI roles (chat, descriptions, embeddings); one provider per role.",
|
||||
)
|
||||
provider_options: dict[str, Any] = Field(
|
||||
default={},
|
||||
@@ -77,17 +66,3 @@ class GenAIConfig(FrigateBaseModel):
|
||||
description="Runtime options passed to the provider for each inference call.",
|
||||
json_schema_extra={"additionalProperties": {}},
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_transcribe_provider(self) -> Self:
|
||||
"""Reject the transcribe role on providers that cannot accept audio input."""
|
||||
if (
|
||||
GenAIRoleEnum.transcribe in self.roles
|
||||
and self.provider not in TRANSCRIBE_CAPABLE_PROVIDERS
|
||||
):
|
||||
raise ValueError(
|
||||
f"GenAI provider '{self.provider.value}' does not support audio input "
|
||||
"and cannot be given the 'transcribe' role."
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
@@ -9,7 +9,6 @@ __all__ = [
|
||||
"DetectionsConfig",
|
||||
"AlertsConfig",
|
||||
"ImageSourceEnum",
|
||||
"ReviewFrameModeEnum",
|
||||
"ReviewResponseStyleEnum",
|
||||
]
|
||||
|
||||
@@ -21,13 +20,6 @@ class ImageSourceEnum(str, Enum):
|
||||
recordings = "recordings"
|
||||
|
||||
|
||||
class ReviewFrameModeEnum(str, Enum):
|
||||
"""How review frames are presented to the GenAI provider."""
|
||||
|
||||
frames = "frames"
|
||||
annotated_frames = "annotated_frames"
|
||||
|
||||
|
||||
class ReviewResponseStyleEnum(str, Enum):
|
||||
"""Writing style presets for GenAI review descriptions."""
|
||||
|
||||
@@ -161,11 +153,6 @@ class GenAIReviewConfig(FrigateBaseModel):
|
||||
description="Preferred language to request from the GenAI provider for generated responses.",
|
||||
default=None,
|
||||
)
|
||||
frame_mode: ReviewFrameModeEnum = Field(
|
||||
default=ReviewFrameModeEnum.frames,
|
||||
title="Frame mode",
|
||||
description="How frames are presented to the model. 'frames' sends the prompt followed by the frames, which suits models that track a sequence well on their own. 'annotated_frames' labels each frame and interleaves notes derived from object tracking, which helps models that lose track of activity that repeats or reverses.",
|
||||
)
|
||||
response_style: ReviewResponseStyleEnum = Field(
|
||||
default=ReviewResponseStyleEnum.default,
|
||||
title="Response style",
|
||||
|
||||
@@ -5,7 +5,6 @@ from pydantic import ConfigDict, Field, field_validator
|
||||
from .base import FrigateBaseModel
|
||||
|
||||
__all__ = [
|
||||
"AudioTranscriptionModelEnum",
|
||||
"CameraFaceRecognitionConfig",
|
||||
"CameraLicensePlateRecognitionConfig",
|
||||
"CameraAudioTranscriptionConfig",
|
||||
@@ -21,10 +20,6 @@ class SemanticSearchModelEnum(str, Enum):
|
||||
jinav2 = "jinav2"
|
||||
|
||||
|
||||
class AudioTranscriptionModelEnum(str, Enum):
|
||||
whisper = "whisper"
|
||||
|
||||
|
||||
class EnrichmentsDeviceEnum(str, Enum):
|
||||
GPU = "GPU"
|
||||
CPU = "CPU"
|
||||
@@ -58,35 +53,10 @@ class AudioTranscriptionConfig(FrigateBaseModel):
|
||||
description="Enable or disable automatic audio transcription for all cameras; can be overridden per-camera.",
|
||||
)
|
||||
language: str = Field(
|
||||
default="auto",
|
||||
default="en",
|
||||
title="Transcription language",
|
||||
description="Language code used for transcription/translation (for example 'en' for English), or 'auto' to let the model detect it. See https://whisper-api.com/docs/languages/ for supported language codes.",
|
||||
description="Language code used for transcription/translation (for example 'en' for English). See https://whisper-api.com/docs/languages/ for supported language codes.",
|
||||
)
|
||||
model: AudioTranscriptionModelEnum | str | None = Field(
|
||||
default=AudioTranscriptionModelEnum.whisper,
|
||||
title="Audio transcription model or GenAI provider name",
|
||||
description="The transcription backend: 'whisper' for Frigate's built-in local models, or the name of a GenAI provider with the transcribe role.",
|
||||
)
|
||||
|
||||
@field_validator("model", mode="before")
|
||||
@classmethod
|
||||
def coerce_model_enum(cls, v):
|
||||
# An absent value ("model:" with nothing after it, or an explicit null)
|
||||
# means unspecified, so fall back to the built-in backend. Left as None
|
||||
# it would pass the GenAI-provider validation, which only inspects
|
||||
# strings, and then be treated as a provider name that resolves to no
|
||||
# client, turning transcription into a silent no-op.
|
||||
if v is None or (isinstance(v, str) and not v.strip()):
|
||||
return AudioTranscriptionModelEnum.whisper
|
||||
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
return AudioTranscriptionModelEnum(v)
|
||||
except ValueError:
|
||||
return v
|
||||
|
||||
return v
|
||||
|
||||
device: EnrichmentsDeviceEnum = Field(
|
||||
default=EnrichmentsDeviceEnum.CPU,
|
||||
title="Transcription device",
|
||||
|
||||
@@ -56,7 +56,6 @@ from .camera.timestamp import TimestampStyleConfig
|
||||
from .camera_group import CameraGroupConfig
|
||||
from .classification import (
|
||||
AudioTranscriptionConfig,
|
||||
AudioTranscriptionModelEnum,
|
||||
ClassificationConfig,
|
||||
FaceRecognitionConfig,
|
||||
LicensePlateRecognitionConfig,
|
||||
@@ -885,7 +884,7 @@ class FrigateConfig(FrigateBaseModel):
|
||||
# set notifications state
|
||||
self.notifications.enabled_in_config = self.notifications.enabled
|
||||
|
||||
# validate genai: each role (chat, descriptions, embeddings, transcribe) at most once
|
||||
# validate genai: each role (chat, descriptions, embeddings) at most once
|
||||
role_to_name: dict[GenAIRoleEnum, str] = {}
|
||||
for name, genai_cfg in self.genai.items():
|
||||
for role in genai_cfg.roles:
|
||||
@@ -1246,35 +1245,6 @@ class FrigateConfig(FrigateBaseModel):
|
||||
for model in self.models:
|
||||
model.create_colormap(colored_labels)
|
||||
|
||||
# validate audio_transcription.model when it is a GenAI provider name.
|
||||
# this runs here rather than beside the semantic_search check because the
|
||||
# global->camera merge above is what resolves camera-level enablement.
|
||||
transcription_active = self.audio_transcription.enabled or any(
|
||||
camera.audio_transcription.enabled for camera in self.cameras.values()
|
||||
)
|
||||
|
||||
if (
|
||||
transcription_active
|
||||
and isinstance(self.audio_transcription.model, str)
|
||||
and not isinstance(
|
||||
self.audio_transcription.model, AudioTranscriptionModelEnum
|
||||
)
|
||||
):
|
||||
if self.audio_transcription.model not in self.genai:
|
||||
raise ValueError(
|
||||
f"audio_transcription.model '{self.audio_transcription.model}' is not a "
|
||||
"valid GenAI config key. Must match a key in genai config."
|
||||
)
|
||||
|
||||
if (
|
||||
GenAIRoleEnum.transcribe
|
||||
not in self.genai[self.audio_transcription.model].roles
|
||||
):
|
||||
raise ValueError(
|
||||
f"GenAI provider '{self.audio_transcription.model}' must have "
|
||||
"'transcribe' in its roles for audio transcription."
|
||||
)
|
||||
|
||||
# Check audio transcription and audio detection requirements
|
||||
if self.audio_transcription.enabled:
|
||||
# If audio transcription is enabled globally, at least one camera must have audio detection enabled
|
||||
|
||||
@@ -1,14 +1,9 @@
|
||||
"""Shared handle on the config object that is current for this instance."""
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
|
||||
from .config import FrigateConfig
|
||||
|
||||
__all__ = ["ConfigHolder"]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConfigHolder:
|
||||
"""Indirection for the most recently parsed config.
|
||||
@@ -28,24 +23,12 @@ class ConfigHolder:
|
||||
|
||||
def __init__(self, config: FrigateConfig) -> None:
|
||||
self._config = config
|
||||
self._listeners: list[Callable[[FrigateConfig], None]] = []
|
||||
|
||||
@property
|
||||
def config(self) -> FrigateConfig:
|
||||
"""The config as of the most recent successful save."""
|
||||
return self._config
|
||||
|
||||
def subscribe(self, listener: Callable[[FrigateConfig], None]) -> None:
|
||||
"""Call listener on the saving thread with each config installed later."""
|
||||
self._listeners.append(listener)
|
||||
|
||||
def set(self, config: FrigateConfig) -> None:
|
||||
"""Install a freshly parsed config as the current one."""
|
||||
self._config = config
|
||||
|
||||
for listener in self._listeners:
|
||||
try:
|
||||
listener(config)
|
||||
except Exception:
|
||||
# a listener bug must not fail the save that has already applied
|
||||
logger.exception("Config listener failed")
|
||||
|
||||
@@ -29,11 +29,6 @@ class StatsConfig(FrigateBaseModel):
|
||||
|
||||
|
||||
class TelemetryConfig(FrigateBaseModel):
|
||||
analytics: bool = Field(
|
||||
default=False,
|
||||
title="Share anonymous analytics",
|
||||
description="Send one anonymous usage report a day to help the Frigate maintainers decide what to support. Nothing is sent until this is on.",
|
||||
)
|
||||
network_interfaces: list[str] = Field(
|
||||
default=[],
|
||||
title="Network interfaces",
|
||||
|
||||
@@ -101,9 +101,6 @@ MAX_WAL_SIZE = 10 # MB
|
||||
|
||||
DEFAULT_FFMPEG_VERSION = os.environ.get("DEFAULT_FFMPEG_VERSION", "")
|
||||
INCLUDED_FFMPEG_VERSIONS = os.environ.get("INCLUDED_FFMPEG_VERSIONS", "").split(":")
|
||||
ANALYTICS_URL = os.environ.get(
|
||||
"FRIGATE_ANALYTICS_URL", "https://analytics.frigate.video/report"
|
||||
)
|
||||
LIBAVFORMAT_VERSION_MAJOR = int(os.environ.get("LIBAVFORMAT_VERSION_MAJOR", "59"))
|
||||
FFMPEG_HWACCEL_NVIDIA = "preset-nvidia"
|
||||
FFMPEG_HWACCEL_VAAPI = "preset-vaapi"
|
||||
|
||||
@@ -10,7 +10,6 @@ from peewee import DoesNotExist
|
||||
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.classification import AudioTranscriptionModelEnum
|
||||
from frigate.const import (
|
||||
CACHE_DIR,
|
||||
MODEL_CACHE_DIR,
|
||||
@@ -19,13 +18,8 @@ from frigate.const import (
|
||||
)
|
||||
from frigate.data_processing.types import PostProcessDataEnum
|
||||
from frigate.embeddings.embeddings import Embeddings
|
||||
from frigate.genai.manager import GenAIClientManager
|
||||
from frigate.types import TrackedObjectUpdateTypesEnum
|
||||
from frigate.util.audio import (
|
||||
clean_transcript,
|
||||
get_audio_from_recording,
|
||||
resolve_language,
|
||||
)
|
||||
from frigate.util.audio import get_audio_from_recording
|
||||
|
||||
from ..types import DataProcessorMetrics
|
||||
from .api import PostProcessorApi
|
||||
@@ -40,25 +34,15 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
|
||||
requestor: InterProcessRequestor,
|
||||
embeddings: Embeddings,
|
||||
metrics: DataProcessorMetrics,
|
||||
genai_manager: GenAIClientManager | None = None,
|
||||
):
|
||||
super().__init__(config, metrics, None)
|
||||
self.config = config
|
||||
self.requestor = requestor
|
||||
self.embeddings = embeddings
|
||||
self.genai_manager = genai_manager
|
||||
self.recognizer = None
|
||||
self.transcription_lock = threading.Lock()
|
||||
self.transcription_thread: threading.Thread | None = None
|
||||
self.transcription_running = False
|
||||
self._use_genai = not isinstance(
|
||||
config.audio_transcription.model, AudioTranscriptionModelEnum
|
||||
)
|
||||
|
||||
if self._use_genai:
|
||||
# never build the local recognizer on the GenAI path; WhisperModel
|
||||
# downloads several hundred MB on first use
|
||||
return
|
||||
|
||||
# faster-whisper handles model downloading automatically
|
||||
self.model_path = os.path.join(MODEL_CACHE_DIR, "whisper")
|
||||
@@ -163,31 +147,6 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
|
||||
logger.error(f"Error in audio transcription post-processing: {e}")
|
||||
|
||||
def __transcribe_audio(self, audio_data: bytes) -> str | None:
|
||||
"""Transcribe WAV audio data with the configured backend."""
|
||||
if self._use_genai:
|
||||
return self.__transcribe_audio_genai(audio_data)
|
||||
|
||||
return self.__transcribe_audio_whisper(audio_data)
|
||||
|
||||
def __transcribe_audio_genai(self, audio_data: bytes) -> str | None:
|
||||
"""Hand the WAV bytes to the GenAI provider holding the transcribe role."""
|
||||
client = self.genai_manager.transcribe_client if self.genai_manager else None
|
||||
|
||||
if not client:
|
||||
logger.error(
|
||||
"audio_transcription.model is '%s' (GenAI provider) but no transcribe "
|
||||
"client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
|
||||
self.config.audio_transcription.model,
|
||||
)
|
||||
return None
|
||||
|
||||
text = client.transcribe(
|
||||
audio_data,
|
||||
language=resolve_language(self.config.audio_transcription.language),
|
||||
)
|
||||
return clean_transcript(text) or None
|
||||
|
||||
def __transcribe_audio_whisper(self, audio_data: bytes) -> str | None:
|
||||
"""Transcribe WAV audio data using faster-whisper."""
|
||||
if not self.recognizer:
|
||||
logger.debug("Recognizer not initialized")
|
||||
@@ -201,7 +160,7 @@ class AudioTranscriptionPostProcessor(PostProcessorApi):
|
||||
|
||||
segments, info = self.recognizer.transcribe(
|
||||
temp_wav,
|
||||
language=resolve_language(self.config.audio_transcription.language),
|
||||
language=self.config.audio_transcription.language,
|
||||
beam_size=5,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,394 +0,0 @@
|
||||
"""Frame annotations derived from object tracking data.
|
||||
|
||||
Builds short notes describing what changed during a review item, keyed to the
|
||||
frames sampled from it. Everything here comes from tracked object data already
|
||||
in the database (each event's `path_data` trajectory and the timeline's
|
||||
stationary/active changes), so the notes can be stated to the model as fact
|
||||
rather than as something it must perceive.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from frigate.models import Event, Timeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Movement smaller than this (normalized frame units) between two path points
|
||||
# is treated as the object holding still rather than travelling.
|
||||
STILL_THRESHOLD = 0.02
|
||||
|
||||
# A heading change beyond this (dot product against the leg's own heading)
|
||||
# counts as the object turning back rather than curving.
|
||||
REVERSAL_DOT = -0.3
|
||||
|
||||
# A run of travel shorter than this (normalized frame units) is treated as
|
||||
# milling about rather than going somewhere. Without it, a subject pacing in
|
||||
# one spot produces a burst of contradictory "turns around" notes on a single
|
||||
# frame.
|
||||
MIN_LEG_DISTANCE = 0.08
|
||||
|
||||
# Movement that begins within this many seconds of detection is folded into
|
||||
# the detection note, so each arrival reads as one event instead of several.
|
||||
DETECT_MOVE_MERGE_SECONDS = 2.0
|
||||
|
||||
STATE_CHANGE_PHRASES = {
|
||||
"stationary": "has stopped moving",
|
||||
"active": "starts moving again",
|
||||
}
|
||||
|
||||
Point = tuple[float, float, float]
|
||||
Leg = tuple[int, int]
|
||||
|
||||
|
||||
def describe_position(x: float, y: float) -> str:
|
||||
"""Name a normalized frame position in plain terms."""
|
||||
horizontal = "left" if x < 0.34 else ("right" if x > 0.66 else "center")
|
||||
vertical = "top" if y < 0.34 else ("bottom" if y > 0.66 else "middle")
|
||||
|
||||
if horizontal == "center" and vertical == "middle":
|
||||
return "the middle of the frame"
|
||||
|
||||
if horizontal == "center":
|
||||
return f"the {vertical} of the frame"
|
||||
|
||||
if vertical == "middle":
|
||||
return f"the {horizontal} of the frame"
|
||||
|
||||
return f"the {vertical} {horizontal} of the frame"
|
||||
|
||||
|
||||
def describe_heading(dx: float, dy: float) -> str:
|
||||
"""Name a direction of travel in frame terms.
|
||||
|
||||
y grows downward in normalized coordinates, so a falling y reads as moving
|
||||
toward the top of the frame.
|
||||
"""
|
||||
parts = []
|
||||
|
||||
if abs(dy) > abs(dx) * 0.4:
|
||||
parts.append("down" if dy > 0 else "up")
|
||||
|
||||
if abs(dx) > abs(dy) * 0.4:
|
||||
parts.append("right" if dx > 0 else "left")
|
||||
|
||||
return " and ".join(parts) if parts else "in place"
|
||||
|
||||
|
||||
def event_name(event: dict[str, Any]) -> str:
|
||||
"""Name an object for the notes, e.g. 'a person' or 'waste bin "Compost"'.
|
||||
|
||||
Objects are never numbered or given track identifiers. Frigate opens a new
|
||||
tracked object whenever a subject is re-detected, so the tracking data
|
||||
cannot say whether two entries are the same subject, and the notes stay
|
||||
ambiguous rather than implying either answer.
|
||||
"""
|
||||
label = str(event["label"]).replace("_", " ").replace("-verified", "")
|
||||
sub_label = event.get("sub_label")
|
||||
|
||||
if sub_label:
|
||||
return f'{label} "{sub_label}"'
|
||||
|
||||
article = "an" if label[:1].lower() in "aeiou" else "a"
|
||||
return f"{article} {label}"
|
||||
|
||||
|
||||
def path_legs(points: list[Point]) -> list[Leg]:
|
||||
"""Split a trajectory into runs of travel in a consistent direction.
|
||||
|
||||
A leg ends when the subject starts moving back against the direction that
|
||||
leg established, and only once the leg has covered MIN_LEG_DISTANCE, so
|
||||
jitter around a standing subject does not register as a turn.
|
||||
|
||||
Returns (start, end) index pairs into `points`.
|
||||
"""
|
||||
legs: list[Leg] = []
|
||||
start = 0
|
||||
|
||||
for i in range(1, len(points)):
|
||||
lx = points[i][0] - points[start][0]
|
||||
ly = points[i][1] - points[start][1]
|
||||
leg_distance = (lx * lx + ly * ly) ** 0.5
|
||||
|
||||
if leg_distance < MIN_LEG_DISTANCE:
|
||||
continue
|
||||
|
||||
sx = points[i][0] - points[i - 1][0]
|
||||
sy = points[i][1] - points[i - 1][1]
|
||||
step = (sx * sx + sy * sy) ** 0.5
|
||||
|
||||
if step < STILL_THRESHOLD:
|
||||
continue
|
||||
|
||||
dot = (lx / leg_distance) * (sx / step) + (ly / leg_distance) * (sy / step)
|
||||
|
||||
if dot < REVERSAL_DOT:
|
||||
legs.append((start, i - 1))
|
||||
start = i - 1
|
||||
|
||||
if start < len(points) - 1:
|
||||
legs.append((start, len(points) - 1))
|
||||
|
||||
return [
|
||||
(a, b)
|
||||
for a, b in legs
|
||||
if ((points[b][0] - points[a][0]) ** 2 + (points[b][1] - points[a][1]) ** 2)
|
||||
** 0.5
|
||||
>= MIN_LEG_DISTANCE
|
||||
]
|
||||
|
||||
|
||||
def path_points(path_data: list[Any]) -> list[Point]:
|
||||
"""Flatten path_data into (x, y, timestamp) tuples, or [] if malformed."""
|
||||
try:
|
||||
return [(p[0][0], p[0][1], p[1]) for p in path_data or []]
|
||||
except (IndexError, TypeError):
|
||||
logger.debug("Malformed path_data, skipping trajectory notes")
|
||||
return []
|
||||
|
||||
|
||||
def leg_start_time(points: list[Point], leg: Leg) -> float:
|
||||
"""When a leg's movement actually began.
|
||||
|
||||
path_data always keeps an object's first two samples, so a leg can open
|
||||
with points recorded long before the object moved. The first sample that
|
||||
has left the leg's origin is the earliest evidence of movement.
|
||||
"""
|
||||
a, b = leg
|
||||
x0, y0, t0 = points[a]
|
||||
|
||||
for x, y, t in points[a + 1 : b + 1]:
|
||||
if math.hypot(x - x0, y - y0) >= STILL_THRESHOLD:
|
||||
return t
|
||||
|
||||
return t0
|
||||
|
||||
|
||||
def leg_heading(points: list[Point], leg: Leg) -> str:
|
||||
a, b = leg
|
||||
return describe_heading(points[b][0] - points[a][0], points[b][1] - points[a][1])
|
||||
|
||||
|
||||
def leg_phrase(points: list[Point], leg: Leg, first: bool) -> str:
|
||||
"""Describe the start of a leg, e.g. 'turns around at ... and heads left'."""
|
||||
heading = leg_heading(points, leg)
|
||||
place = describe_position(points[leg[0]][0], points[leg[0]][1])
|
||||
|
||||
if first:
|
||||
return f"starts moving {heading} from {place}"
|
||||
|
||||
return f"turns around at {place} and heads {heading}"
|
||||
|
||||
|
||||
def path_moments(path_data: list[Any]) -> list[tuple[float, str]]:
|
||||
"""Key moments in one trajectory as (timestamp, phrase).
|
||||
|
||||
Emits one note per leg of travel. Where the last leg ends is left out:
|
||||
path_data only records significant movement, so its final point cannot
|
||||
distinguish an object coming to rest from one leaving the frame.
|
||||
"""
|
||||
points = path_points(path_data)
|
||||
|
||||
if len(points) < 2:
|
||||
return []
|
||||
|
||||
return [
|
||||
(leg_start_time(points, leg), leg_phrase(points, leg, index == 0))
|
||||
for index, leg in enumerate(path_legs(points))
|
||||
]
|
||||
|
||||
|
||||
def build_timeline(
|
||||
events: list[dict[str, Any]],
|
||||
span_end: float,
|
||||
state_changes: Sequence[dict[str, Any]] = (),
|
||||
) -> list[tuple[float, str]]:
|
||||
"""All annotated moments across every event, in time order.
|
||||
|
||||
Only changes are noted, since those are what sparse frames miss; an
|
||||
object's state at the end of the clip is visible in the last frame.
|
||||
`span_end` is the timestamp of the last sampled frame, and moments past it
|
||||
describe nothing the model can see. A track ending means the object
|
||||
stopped being detected, which may or may not mean it left the frame.
|
||||
|
||||
`state_changes` are timeline rows (timestamp, source_id, class_type); the
|
||||
stationary and active ones become "has stopped moving" / "starts moving
|
||||
again". Frigate only marks an object stationary after it has been still
|
||||
for a while, which the past-tense wording reflects.
|
||||
|
||||
Each object keeps its own notes. Folding an object into the note of the
|
||||
person moving it ("alongside ...") was tried and made models lose track of
|
||||
where the object went.
|
||||
"""
|
||||
changes_by_event: dict[str, list[tuple[float, str]]] = {}
|
||||
|
||||
for change in state_changes:
|
||||
phrase = STATE_CHANGE_PHRASES.get(change["class_type"])
|
||||
|
||||
if phrase:
|
||||
changes_by_event.setdefault(change["source_id"], []).append(
|
||||
(change["timestamp"], phrase)
|
||||
)
|
||||
|
||||
timeline: list[tuple[float, str]] = []
|
||||
|
||||
for event in sorted(events, key=lambda e: e["start_time"]):
|
||||
# Frame extraction can come up short at the end of a clip, leaving
|
||||
# objects that only appear after the last frame we actually have.
|
||||
if event["start_time"] > span_end:
|
||||
continue
|
||||
|
||||
points = path_points(event.get("path_data") or [])
|
||||
legs = path_legs(points) if len(points) >= 2 else []
|
||||
name = event_name(event)
|
||||
detected_at = event["start_time"]
|
||||
where = describe_position(points[0][0], points[0][1]) if points else "the frame"
|
||||
merges = bool(legs) and leg_start_time(points, legs[0]) - detected_at <= (
|
||||
DETECT_MOVE_MERGE_SECONDS
|
||||
)
|
||||
remaining = list(enumerate(legs))
|
||||
moments: list[tuple[float, str]] = []
|
||||
|
||||
if merges:
|
||||
heading = leg_heading(points, legs[0])
|
||||
timeline.append(
|
||||
(detected_at, f"{name} first detected at {where}, moving {heading}")
|
||||
)
|
||||
remaining = remaining[1:]
|
||||
else:
|
||||
timeline.append((detected_at, f"{name} first detected at {where}"))
|
||||
|
||||
for index, leg in remaining:
|
||||
moments.append(
|
||||
(
|
||||
leg_start_time(points, leg),
|
||||
leg_phrase(points, leg, index == 0),
|
||||
)
|
||||
)
|
||||
|
||||
moments.extend(
|
||||
(timestamp, phrase)
|
||||
for timestamp, phrase in changes_by_event.get(event["id"], [])
|
||||
if timestamp >= detected_at
|
||||
)
|
||||
|
||||
for timestamp, phrase in moments:
|
||||
if timestamp <= span_end:
|
||||
timeline.append((timestamp, f"{name} {phrase}"))
|
||||
|
||||
if event["end_time"] and event["end_time"] <= span_end:
|
||||
timeline.append((event["end_time"], f"{name} is no longer detected"))
|
||||
|
||||
return sorted(timeline, key=lambda m: m[0])
|
||||
|
||||
|
||||
def annotations_by_frame(
|
||||
timeline: list[tuple[float, str]], frame_times: list[float]
|
||||
) -> dict[int, list[str]]:
|
||||
"""Bucket timeline moments onto the frame that follows each one.
|
||||
|
||||
A moment is attached to the first frame at or after it happened, so the
|
||||
note always precedes the image in which the change becomes visible.
|
||||
"""
|
||||
buckets: dict[int, list[str]] = {}
|
||||
|
||||
if not frame_times:
|
||||
return buckets
|
||||
|
||||
for timestamp, phrase in timeline:
|
||||
index = next(
|
||||
(i for i, ft in enumerate(frame_times) if ft >= timestamp),
|
||||
len(frame_times) - 1,
|
||||
)
|
||||
buckets.setdefault(index, []).append(phrase)
|
||||
|
||||
return buckets
|
||||
|
||||
|
||||
def get_tracked_events(detection_ids: list[str]) -> list[dict[str, Any]]:
|
||||
"""Load the tracked objects behind a review item's detections."""
|
||||
if not detection_ids:
|
||||
return []
|
||||
|
||||
rows = list(
|
||||
Event.select(
|
||||
Event.id,
|
||||
Event.label,
|
||||
Event.sub_label,
|
||||
Event.start_time,
|
||||
Event.end_time,
|
||||
Event.data,
|
||||
)
|
||||
.where(Event.id << detection_ids)
|
||||
.dicts()
|
||||
.iterator()
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
"id": row["id"],
|
||||
"label": row["label"],
|
||||
"sub_label": row["sub_label"],
|
||||
"start_time": row["start_time"],
|
||||
"end_time": row["end_time"],
|
||||
"path_data": (row["data"] or {}).get("path_data") or [],
|
||||
}
|
||||
for row in rows
|
||||
if row["start_time"] is not None
|
||||
]
|
||||
|
||||
|
||||
def get_state_changes(detection_ids: list[str]) -> list[dict[str, Any]]:
|
||||
"""Stationary/active changes the timeline recorded for these objects."""
|
||||
if not detection_ids:
|
||||
return []
|
||||
|
||||
return list(
|
||||
Timeline.select(Timeline.timestamp, Timeline.source_id, Timeline.class_type)
|
||||
.where(
|
||||
(Timeline.source_id << detection_ids)
|
||||
& (Timeline.class_type << list(STATE_CHANGE_PHRASES))
|
||||
)
|
||||
.dicts()
|
||||
.iterator()
|
||||
)
|
||||
|
||||
|
||||
def build_frame_captions(
|
||||
detection_ids: list[str],
|
||||
frame_times: list[float],
|
||||
) -> list[str]:
|
||||
"""A caption for each sampled frame, in frame order.
|
||||
|
||||
Every frame gets its index and elapsed time so the model can tell them
|
||||
apart; frames where something changed also carry the tracker notes for
|
||||
that moment. Returns an empty list when there is nothing to say, which
|
||||
callers treat as a reason to fall back to sending plain frames.
|
||||
"""
|
||||
if not frame_times:
|
||||
return []
|
||||
|
||||
events = get_tracked_events(detection_ids)
|
||||
|
||||
if not events:
|
||||
logger.debug("No tracked events found for review item, skipping annotations")
|
||||
return []
|
||||
|
||||
timeline = build_timeline(events, frame_times[-1], get_state_changes(detection_ids))
|
||||
buckets = annotations_by_frame(timeline, frame_times)
|
||||
|
||||
if not buckets:
|
||||
return []
|
||||
|
||||
total = len(frame_times)
|
||||
origin = frame_times[0]
|
||||
captions: list[str] = []
|
||||
|
||||
for index, timestamp in enumerate(frame_times):
|
||||
lines = [f"Frame {index + 1} of {total} (+{timestamp - origin:.1f}s):"]
|
||||
lines.extend(f"[tracker] {note}" for note in buckets.get(index, []))
|
||||
captions.append("\n".join(lines))
|
||||
|
||||
return captions
|
||||
@@ -19,11 +19,7 @@ from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.camera import CameraConfig
|
||||
from frigate.config.camera.review import (
|
||||
GenAIReviewConfig,
|
||||
ImageSourceEnum,
|
||||
ReviewFrameModeEnum,
|
||||
)
|
||||
from frigate.config.camera.review import GenAIReviewConfig, ImageSourceEnum
|
||||
from frigate.const import (
|
||||
ATTRIBUTE_LABEL_DISPLAY_MAP,
|
||||
CACHE_DIR,
|
||||
@@ -40,7 +36,6 @@ from frigate.util.image import get_image_from_recording
|
||||
|
||||
from ..post.api import PostProcessorApi
|
||||
from ..types import DataProcessorMetrics
|
||||
from .review_annotations import build_frame_captions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -48,7 +43,6 @@ RECORDING_BUFFER_EXTENSION_PERCENT = 0.10
|
||||
MIN_RECORDING_DURATION = 10
|
||||
MAX_IMAGE_TOKENS = 24000
|
||||
MAX_FRAMES_PER_SECOND = 1
|
||||
MAX_ANNOTATED_FRAMES = 28
|
||||
|
||||
|
||||
class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
@@ -73,7 +67,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
duration: float,
|
||||
image_source: ImageSourceEnum = ImageSourceEnum.preview,
|
||||
height: int = 480,
|
||||
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
|
||||
) -> int:
|
||||
"""Calculate optimal number of frames based on event duration, context size,
|
||||
image source, and resolution.
|
||||
@@ -87,8 +80,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
- MAX_FRAMES_PER_SECOND x duration, to avoid drowning short events in
|
||||
near-duplicate frames where the model latches onto the redundant middle
|
||||
and skips the start/end action
|
||||
- MAX_ANNOTATED_FRAMES in annotated mode, where the tracking notes
|
||||
already carry the sequence
|
||||
"""
|
||||
client = self.genai_manager.description_client
|
||||
|
||||
@@ -134,10 +125,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
max_frames_by_tokens = int(image_token_budget / tokens_per_image)
|
||||
max_frames_by_duration = int(duration * MAX_FRAMES_PER_SECOND)
|
||||
max_frames = min(max_frames_by_tokens, max_frames_by_duration)
|
||||
|
||||
if frame_mode == ReviewFrameModeEnum.annotated_frames:
|
||||
max_frames = min(max_frames, MAX_ANNOTATED_FRAMES)
|
||||
|
||||
return max(max_frames, 3)
|
||||
|
||||
def process_data(
|
||||
@@ -179,7 +166,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
return
|
||||
|
||||
image_source = camera_config.review.genai.image_source
|
||||
frame_mode = camera_config.review.genai.frame_mode
|
||||
|
||||
if image_source == ImageSourceEnum.recordings:
|
||||
buffer_extension = get_recording_buffer_extension(
|
||||
@@ -188,43 +174,40 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
final_data["start_time"] -= buffer_extension
|
||||
final_data["end_time"] += buffer_extension
|
||||
|
||||
frames = self.get_recording_frames(
|
||||
thumbs = self.get_recording_frames(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
height=480, # Use 480p for good balance between quality and token usage
|
||||
frame_mode=frame_mode,
|
||||
)
|
||||
|
||||
if not frames:
|
||||
if not thumbs:
|
||||
# Fallback to preview frames if no recordings available
|
||||
logger.warning(
|
||||
f"No recording frames found for {camera}, falling back to preview frames"
|
||||
)
|
||||
frames = self.get_preview_frames_as_bytes(
|
||||
thumbs = self.get_preview_frames_as_bytes(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["thumb_path"],
|
||||
id,
|
||||
camera_config.review.genai.debug_save_thumbnails,
|
||||
frame_mode,
|
||||
)
|
||||
elif camera_config.review.genai.debug_save_thumbnails:
|
||||
self.save_debug_recording_frames(id, frames)
|
||||
self.save_debug_recording_frames(id, thumbs)
|
||||
else:
|
||||
# Use preview frames
|
||||
frames = self.get_preview_frames_as_bytes(
|
||||
thumbs = self.get_preview_frames_as_bytes(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["thumb_path"],
|
||||
id,
|
||||
camera_config.review.genai.debug_save_thumbnails,
|
||||
frame_mode,
|
||||
)
|
||||
|
||||
self.start_analysis(camera_config, final_data, frames)
|
||||
self.start_analysis(camera_config, final_data, thumbs)
|
||||
|
||||
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
|
||||
if topic == EmbeddingsRequestEnum.regenerate_review_description.value:
|
||||
@@ -403,50 +386,31 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
buffer_extension = get_recording_buffer_extension(
|
||||
final_data["end_time"] - final_data["start_time"]
|
||||
)
|
||||
frames = self.get_recording_frames(
|
||||
thumbs = self.get_recording_frames(
|
||||
str(review.camera),
|
||||
final_data["start_time"] - buffer_extension,
|
||||
final_data["end_time"] + buffer_extension,
|
||||
height=480,
|
||||
frame_mode=camera_config.review.genai.frame_mode,
|
||||
)
|
||||
|
||||
if not frames:
|
||||
if not thumbs:
|
||||
logger.error(
|
||||
"No recording frames are available for review item %s", review_id
|
||||
)
|
||||
return
|
||||
|
||||
if camera_config.review.genai.debug_save_thumbnails:
|
||||
self.save_debug_recording_frames(review_id, frames)
|
||||
self.save_debug_recording_frames(review_id, thumbs)
|
||||
|
||||
self.start_analysis(camera_config, final_data, frames)
|
||||
self.start_analysis(camera_config, final_data, thumbs)
|
||||
|
||||
def start_analysis(
|
||||
self,
|
||||
camera_config: CameraConfig,
|
||||
final_data: dict[str, Any],
|
||||
frames: list[tuple[bytes, float]],
|
||||
thumbs: list[bytes],
|
||||
) -> None:
|
||||
"""Kick off description generation for a review item in the background."""
|
||||
thumbs = [frame for frame, _ in frames]
|
||||
captions: list[str] = []
|
||||
|
||||
if (
|
||||
camera_config.review.genai.frame_mode
|
||||
== ReviewFrameModeEnum.annotated_frames
|
||||
):
|
||||
captions = build_frame_captions(
|
||||
final_data["data"].get("detections") or [],
|
||||
[timestamp for _, timestamp in frames],
|
||||
)
|
||||
|
||||
if not captions:
|
||||
logger.debug(
|
||||
"No tracking annotations for review item %s, sending plain frames",
|
||||
final_data["id"],
|
||||
)
|
||||
|
||||
self.review_desc_dps.update()
|
||||
threading.Thread(
|
||||
target=run_analysis,
|
||||
@@ -457,22 +421,19 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
camera_config,
|
||||
final_data,
|
||||
thumbs,
|
||||
captions,
|
||||
camera_config.review.genai,
|
||||
sorted(self.config.all_labels),
|
||||
self.config.all_attributes,
|
||||
),
|
||||
).start()
|
||||
|
||||
def save_debug_recording_frames(
|
||||
self, review_id: str, frames: list[tuple[bytes, float]]
|
||||
) -> None:
|
||||
def save_debug_recording_frames(self, review_id: str, thumbs: list[bytes]) -> None:
|
||||
"""Write the recording frames sent to the provider out for debugging."""
|
||||
Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
|
||||
for idx, (frame_bytes, _) in enumerate(frames):
|
||||
for idx, frame_bytes in enumerate(thumbs):
|
||||
with open(
|
||||
os.path.join(CLIPS_DIR, f"genai-requests/{review_id}/{idx}.jpg"),
|
||||
"wb",
|
||||
@@ -484,9 +445,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
camera: str,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
|
||||
) -> list[tuple[str, float]]:
|
||||
"""Preview frame paths paired with the time each one was captured."""
|
||||
) -> list[str]:
|
||||
preview_dir = os.path.join(CACHE_DIR, "preview_frames")
|
||||
file_start = f"preview_{camera}-"
|
||||
start_file = f"{file_start}{start_time}.webp"
|
||||
@@ -518,29 +477,18 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
|
||||
frame_count = len(all_frames)
|
||||
desired_frame_count = self.calculate_frame_count(
|
||||
camera,
|
||||
duration=end_time - start_time,
|
||||
frame_mode=frame_mode,
|
||||
camera, duration=end_time - start_time
|
||||
)
|
||||
|
||||
def with_timestamp(path: str) -> tuple[str, float]:
|
||||
# Preview frames are named preview_<camera>-<timestamp>.webp
|
||||
stem = os.path.basename(path).removesuffix(".webp")
|
||||
|
||||
try:
|
||||
return (path, float(stem.removeprefix(file_start)))
|
||||
except ValueError:
|
||||
return (path, start_time)
|
||||
|
||||
if frame_count <= desired_frame_count:
|
||||
return [with_timestamp(f) for f in all_frames]
|
||||
return all_frames
|
||||
|
||||
selected_frames = []
|
||||
step_size = (frame_count - 1) / (desired_frame_count - 1)
|
||||
|
||||
for i in range(desired_frame_count):
|
||||
index = round(i * step_size)
|
||||
selected_frames.append(with_timestamp(all_frames[index]))
|
||||
selected_frames.append(all_frames[index])
|
||||
|
||||
return selected_frames
|
||||
|
||||
@@ -550,12 +498,11 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
height: int = 480,
|
||||
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
|
||||
) -> list[tuple[bytes, float]]:
|
||||
"""Get frames from recordings paired with the time each was captured."""
|
||||
) -> list[bytes]:
|
||||
"""Get frames from recordings at specified timestamps."""
|
||||
duration = end_time - start_time
|
||||
desired_frame_count = self.calculate_frame_count(
|
||||
camera, duration, ImageSourceEnum.recordings, height, frame_mode
|
||||
camera, duration, ImageSourceEnum.recordings, height
|
||||
)
|
||||
|
||||
# Calculate evenly spaced timestamps throughout the duration
|
||||
@@ -593,7 +540,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
except DoesNotExist:
|
||||
return None
|
||||
|
||||
frames: list[tuple[bytes, float]] = []
|
||||
frames = []
|
||||
|
||||
for timestamp in timestamps:
|
||||
try:
|
||||
@@ -606,7 +553,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
image_data = extract_frame_from_recording(rounded_timestamp)
|
||||
|
||||
if image_data:
|
||||
frames.append((image_data, timestamp))
|
||||
frames.append(image_data)
|
||||
else:
|
||||
logger.warning(
|
||||
f"No recording found for {camera} at timestamp {timestamp}"
|
||||
@@ -627,8 +574,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
thumb_path_fallback: str,
|
||||
review_id: str,
|
||||
save_debug: bool,
|
||||
frame_mode: ReviewFrameModeEnum = ReviewFrameModeEnum.frames,
|
||||
) -> list[tuple[bytes, float]]:
|
||||
) -> list[bytes]:
|
||||
"""Get preview frames and convert them to JPEG bytes.
|
||||
|
||||
Args:
|
||||
@@ -640,14 +586,14 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
save_debug: Whether to save debug thumbnails
|
||||
|
||||
Returns:
|
||||
List of (JPEG image bytes, capture timestamp) pairs
|
||||
List of JPEG image bytes
|
||||
"""
|
||||
frame_paths = self.get_cache_frames(camera, start_time, end_time, frame_mode)
|
||||
frame_paths = self.get_cache_frames(camera, start_time, end_time)
|
||||
if not frame_paths:
|
||||
frame_paths = [(thumb_path_fallback, start_time)]
|
||||
frame_paths = [thumb_path_fallback]
|
||||
|
||||
thumbs: list[tuple[bytes, float]] = []
|
||||
for idx, (thumb_path, timestamp) in enumerate(frame_paths):
|
||||
thumbs = []
|
||||
for idx, thumb_path in enumerate(frame_paths):
|
||||
thumb_data = cv2.imread(thumb_path)
|
||||
|
||||
if thumb_data is None:
|
||||
@@ -660,7 +606,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100]
|
||||
)
|
||||
if ret:
|
||||
thumbs.append((jpg.tobytes(), timestamp))
|
||||
thumbs.append(jpg.tobytes())
|
||||
|
||||
if save_debug:
|
||||
Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
|
||||
@@ -695,7 +641,6 @@ def run_analysis(
|
||||
camera_config: CameraConfig,
|
||||
final_data: dict[str, Any],
|
||||
thumbs: list[bytes],
|
||||
frame_captions: list[str],
|
||||
genai_config: GenAIReviewConfig,
|
||||
labelmap_objects: list[str],
|
||||
attribute_labels: list[str],
|
||||
@@ -752,7 +697,6 @@ def run_analysis(
|
||||
genai_config.debug_save_thumbnails,
|
||||
genai_config.activity_context_prompt,
|
||||
genai_config.response_style,
|
||||
frame_captions,
|
||||
)
|
||||
review_inference_speed.update(datetime.datetime.now().timestamp() - start)
|
||||
|
||||
|
||||
@@ -1,19 +1,16 @@
|
||||
"""Handle processing audio for speech transcription using sherpa-onnx with FFmpeg pipe."""
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import CameraConfig, FrigateConfig
|
||||
from frigate.config.classification import AudioTranscriptionModelEnum
|
||||
from frigate.const import AUDIO_DURATION, MODEL_CACHE_DIR
|
||||
from frigate.const import MODEL_CACHE_DIR
|
||||
from frigate.data_processing.common.audio_transcription.model import (
|
||||
AudioTranscriptionModelRunner,
|
||||
)
|
||||
@@ -21,38 +18,12 @@ from frigate.data_processing.real_time.whisper_online import (
|
||||
FasterWhisperASR,
|
||||
OnlineASRProcessor,
|
||||
)
|
||||
from frigate.util.audio import (
|
||||
clean_transcript,
|
||||
pcm16_to_wav,
|
||||
resolve_language,
|
||||
stitch_transcripts,
|
||||
)
|
||||
|
||||
from ..types import DataProcessorMetrics
|
||||
from .api import RealTimeProcessorApi
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# importing frigate.genai eagerly would pull the provider SDKs into the
|
||||
# audio process even when transcription runs on a local model
|
||||
from frigate.genai.manager import GenAIClientManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Number of ~0.975s audio detector chunks per GenAI request. The window advances
|
||||
# one chunk at a time, so two chunks means a 50% overlap: every word lands whole
|
||||
# in at least one window, which whisper-family models need to avoid hallucinating
|
||||
# on a clipped clip. The cadence is fixed by the audio detector's frame size, so
|
||||
# this is a constant rather than a config knob.
|
||||
GENAI_WINDOW_CHUNKS = 2
|
||||
|
||||
# Bound the queue at ~30s of audio so a slow or hung provider cannot grow it
|
||||
# without limit. The producer is the ffmpeg read thread and must never block.
|
||||
AUDIO_QUEUE_MAXSIZE = int(30 / AUDIO_DURATION)
|
||||
|
||||
# A backed-up queue drops a chunk per cycle, so warning on each one would spam
|
||||
# the log once a second per camera for as long as the provider stays slow.
|
||||
AUDIO_DROP_WARN_INTERVAL = 10.0
|
||||
|
||||
|
||||
class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
def __init__(
|
||||
@@ -60,10 +31,9 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
config: FrigateConfig,
|
||||
camera_config: CameraConfig,
|
||||
requestor: InterProcessRequestor,
|
||||
model_runner: AudioTranscriptionModelRunner | None,
|
||||
model_runner: AudioTranscriptionModelRunner,
|
||||
metrics: DataProcessorMetrics,
|
||||
stop_event: threading.Event,
|
||||
genai_manager: "GenAIClientManager | None" = None,
|
||||
):
|
||||
super().__init__(config, metrics)
|
||||
self.config = config
|
||||
@@ -72,31 +42,11 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
self.stream: Any = None
|
||||
self.whisper_model: FasterWhisperASR | None = None
|
||||
self.model_runner = model_runner
|
||||
self.genai_manager = genai_manager
|
||||
self.transcription_segments: list[str] = []
|
||||
self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue(
|
||||
maxsize=AUDIO_QUEUE_MAXSIZE
|
||||
)
|
||||
self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue()
|
||||
self.stop_event = stop_event
|
||||
self._use_genai = not isinstance(
|
||||
config.audio_transcription.model, AudioTranscriptionModelEnum
|
||||
)
|
||||
# sliding window of raw int16 chunks; the deque's maxlen is what evicts
|
||||
# the oldest chunk and so produces the overlap
|
||||
self._genai_window: collections.deque[np.ndarray] = collections.deque(
|
||||
maxlen=GENAI_WINDOW_CHUNKS
|
||||
)
|
||||
self._genai_committed = ""
|
||||
# set by the producer when it discards a chunk, so the consumer knows the
|
||||
# audio it is about to receive is not contiguous with what it buffered
|
||||
self._audio_dropped = threading.Event()
|
||||
self._last_drop_warning = 0.0
|
||||
|
||||
def __build_recognizer(self) -> None:
|
||||
if self._use_genai:
|
||||
# nothing local to load; never import sherpa or FasterWhisperASR
|
||||
return
|
||||
|
||||
try:
|
||||
if self.config.audio_transcription.model_size == "large":
|
||||
# Whisper models need to be per-process and can only run one stream at a time
|
||||
@@ -114,7 +64,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
self.stream = OnlineASRProcessor(
|
||||
asr=self.whisper_model,
|
||||
)
|
||||
elif self.model_runner is not None:
|
||||
else:
|
||||
logger.debug(f"Loading sherpa stream for {self.camera_config.name}")
|
||||
self.stream = self.model_runner.model.create_stream()
|
||||
logger.debug(
|
||||
@@ -126,15 +76,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
)
|
||||
|
||||
def __process_audio_stream(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
|
||||
# must precede both the model_runner guard (model_runner is None on this
|
||||
# path) and the float32 normalization below (GenAI wants untouched int16)
|
||||
if self._use_genai:
|
||||
return self.__process_audio_genai(audio_data)
|
||||
|
||||
if self.model_runner is None:
|
||||
logger.debug("Audio transcription (live) model runner not initialized")
|
||||
return None
|
||||
|
||||
if (
|
||||
self.model_runner.model is None
|
||||
and self.config.audio_transcription.model_size == "small"
|
||||
@@ -199,82 +140,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
logger.error(f"Error processing audio stream: {e}")
|
||||
return None
|
||||
|
||||
def __process_audio_genai(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
|
||||
"""Transcribe a sliding overlapped window through the GenAI provider."""
|
||||
client = self.genai_manager.transcribe_client if self.genai_manager else None
|
||||
|
||||
if not client:
|
||||
logger.error(
|
||||
"audio_transcription.model is '%s' (GenAI provider) but no transcribe "
|
||||
"client is configured. Ensure the GenAI provider has 'transcribe' in its roles",
|
||||
self.config.audio_transcription.model,
|
||||
)
|
||||
return None
|
||||
|
||||
if self._audio_dropped.is_set():
|
||||
self._audio_dropped.clear()
|
||||
|
||||
# Chunks were discarded between what is buffered and this one, so
|
||||
# concatenating them would splice non-adjacent audio into one window
|
||||
# and destroy the overlap the stitcher depends on.
|
||||
self._genai_window.clear()
|
||||
|
||||
if self._genai_committed:
|
||||
# the transcript has a gap in it; close the utterance out rather
|
||||
# than stitching across missing speech
|
||||
return self.__end_genai_utterance()
|
||||
|
||||
self._genai_window.append(audio_data)
|
||||
|
||||
if len(self._genai_window) < GENAI_WINDOW_CHUNKS:
|
||||
# wait for a full window so the first request is never a clipped clip
|
||||
return None
|
||||
|
||||
window = np.concatenate(list(self._genai_window))
|
||||
|
||||
# Silence gate, using the same threshold audio detection uses. Gate the
|
||||
# whole window rather than individual chunks; this is the primary cost
|
||||
# and privacy brake and is what keeps a quiet camera near zero requests.
|
||||
window_as_float = window.astype(np.float32)
|
||||
rms = float(np.sqrt(np.mean(np.absolute(np.square(window_as_float)))))
|
||||
|
||||
if rms < self.camera_config.audio.min_volume:
|
||||
logger.debug(
|
||||
f"Window RMS {rms:.1f} below min_volume, skipping transcription"
|
||||
)
|
||||
return self.__end_genai_utterance()
|
||||
|
||||
text = client.transcribe(
|
||||
pcm16_to_wav(window),
|
||||
language=resolve_language(self.config.audio_transcription.language),
|
||||
)
|
||||
|
||||
# cleaning has to come first: a silent window often comes back as the
|
||||
# model's preamble alone, which is silence, not a word to commit
|
||||
cleaned = clean_transcript(text)
|
||||
|
||||
if not cleaned:
|
||||
return self.__end_genai_utterance()
|
||||
|
||||
self._genai_committed = stitch_transcripts(self._genai_committed, cleaned)
|
||||
|
||||
# no VAD on this path, so mirror the whisper branch's heuristic endpoint
|
||||
is_endpoint = (
|
||||
self._genai_committed.endswith((".", "!", "?"))
|
||||
and len(self._genai_committed) > 300
|
||||
)
|
||||
|
||||
logger.debug(f"GenAI transcription: '{self._genai_committed}'")
|
||||
|
||||
return self._genai_committed, is_endpoint
|
||||
|
||||
def __end_genai_utterance(self) -> tuple[str, bool] | None:
|
||||
"""Close out the current utterance when a window carries no speech."""
|
||||
if not self._genai_committed:
|
||||
return None
|
||||
|
||||
return self._genai_committed, True
|
||||
|
||||
def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
|
||||
pass
|
||||
|
||||
@@ -283,38 +148,8 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
logger.debug("No audio data provided for transcription")
|
||||
return None
|
||||
|
||||
# enqueue audio data for processing in the thread. never block: the
|
||||
# producer is the ffmpeg read thread that audio detection depends on,
|
||||
# so on a backlog drop the oldest chunk instead.
|
||||
try:
|
||||
self.audio_queue.put_nowait((obj_data, audio))
|
||||
except queue.Full:
|
||||
try:
|
||||
self.audio_queue.get_nowait()
|
||||
self.audio_queue.task_done()
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
# the stream now has a hole in it, which the consumer has to know
|
||||
# about before it splices the next chunk onto what it already holds
|
||||
self._audio_dropped.set()
|
||||
|
||||
now = time.monotonic()
|
||||
|
||||
if now - self._last_drop_warning >= AUDIO_DROP_WARN_INTERVAL:
|
||||
self._last_drop_warning = now
|
||||
logger.warning(
|
||||
"Audio transcription queue for %s is full, dropping audio. The "
|
||||
"provider is not keeping up with the %.2fs chunk rate",
|
||||
self.camera_config.name,
|
||||
AUDIO_DURATION,
|
||||
)
|
||||
|
||||
try:
|
||||
self.audio_queue.put_nowait((obj_data, audio))
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
# enqueue audio data for processing in the thread
|
||||
self.audio_queue.put((obj_data, audio))
|
||||
return None
|
||||
|
||||
def run(self) -> None:
|
||||
@@ -370,14 +205,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
break
|
||||
|
||||
def reset(self) -> None:
|
||||
if self._use_genai:
|
||||
self._genai_committed = ""
|
||||
# stale audio carried across an utterance boundary would be
|
||||
# re-transcribed into the next one
|
||||
self._genai_window.clear()
|
||||
logger.debug("Stream reset")
|
||||
return
|
||||
|
||||
if self.config.audio_transcription.model_size == "large":
|
||||
# get final output from whisper
|
||||
output = self.stream.finish()
|
||||
@@ -391,7 +218,7 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
# reset whisper
|
||||
self.stream.init()
|
||||
self.transcription_segments = []
|
||||
elif self.model_runner is not None:
|
||||
else:
|
||||
# reset sherpa
|
||||
self.model_runner.model.reset(self.stream)
|
||||
|
||||
@@ -399,24 +226,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
|
||||
def check_unload_model(self) -> None:
|
||||
# regularly called in the loop in audio maintainer
|
||||
if self._use_genai:
|
||||
# no model to unload, but this is the hook that fires when
|
||||
# live_enabled flips off. guard on emptiness: called ~1x/s per camera.
|
||||
if self._genai_committed or self._genai_window:
|
||||
logger.debug(
|
||||
f"Clearing GenAI transcription state for {self.camera_config.name}"
|
||||
)
|
||||
self.clear_audio_queue()
|
||||
self._genai_committed = ""
|
||||
self._genai_window.clear()
|
||||
|
||||
self.requestor.send_data(
|
||||
f"{self.camera_config.name}/audio/transcription",
|
||||
"",
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
if (
|
||||
self.config.audio_transcription.model_size == "large"
|
||||
and self.whisper_model is not None
|
||||
@@ -461,10 +270,6 @@ class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
|
||||
self, topic: str, request_data: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
if topic == "clear_audio_recognizer":
|
||||
if self._use_genai:
|
||||
self.reset()
|
||||
return {"message": "Audio transcription state cleared", "success": True}
|
||||
|
||||
self.stream = None
|
||||
self.__build_recognizer()
|
||||
return {"message": "Audio recognizer cleared and rebuilt", "success": True}
|
||||
|
||||
@@ -273,16 +273,6 @@ def detect_memryx() -> DetectionHardware | None:
|
||||
return _hardware("memryx", "memryx", "MemryX MX3", units)
|
||||
|
||||
|
||||
def detect_deepx() -> DetectionHardware | None:
|
||||
"""Find DEEPX NPUs by their device nodes."""
|
||||
units = _dev_units("dxrt*", "deepx:PCIe:{index}", "PCIe")
|
||||
|
||||
if not units:
|
||||
return None
|
||||
|
||||
return _hardware("deepx", "deepx", "DEEPX NPU", units)
|
||||
|
||||
|
||||
def detect_rockchip() -> DetectionHardware | None:
|
||||
"""Find a Rockchip NPU by reading the SoC from the device tree."""
|
||||
compatible = _read(f"{PROC_ROOT}/device-tree/compatible")
|
||||
@@ -329,7 +319,6 @@ PROBES = (
|
||||
detect_coral_usb,
|
||||
detect_hailo,
|
||||
detect_memryx,
|
||||
detect_deepx,
|
||||
detect_intel_npu,
|
||||
detect_intel_gpu,
|
||||
detect_nvidia_gpu,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -68,7 +68,7 @@ from frigate.events.types import (
|
||||
RegenerateDescriptionEnum,
|
||||
)
|
||||
from frigate.genai import GenAIClientManager
|
||||
from frigate.models import Event, Recordings, ReviewSegment, Timeline, Trigger
|
||||
from frigate.models import Event, Recordings, ReviewSegment, Trigger
|
||||
from frigate.types import TrackedObjectUpdateTypesEnum
|
||||
from frigate.util.builtin import serialize
|
||||
from frigate.util.file import get_event_thumbnail_bytes
|
||||
@@ -139,7 +139,7 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
),
|
||||
load_vec_extension=True,
|
||||
)
|
||||
models = [Event, Recordings, ReviewSegment, Timeline, Trigger]
|
||||
models = [Event, Recordings, ReviewSegment, Trigger]
|
||||
db.bind(models)
|
||||
|
||||
self.genai_manager = GenAIClientManager(config)
|
||||
@@ -251,11 +251,7 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
):
|
||||
self.post_processors.append(
|
||||
AudioTranscriptionPostProcessor(
|
||||
self.config,
|
||||
self.requestor,
|
||||
self.embeddings,
|
||||
metrics,
|
||||
self.genai_manager,
|
||||
self.config, self.requestor, self.embeddings, metrics
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
+38
-71
@@ -11,7 +11,6 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from frigate.camera import CameraMetrics
|
||||
from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import CameraConfig, CameraInput, FrigateConfig
|
||||
@@ -20,7 +19,6 @@ from frigate.config.camera.updater import (
|
||||
CameraConfigUpdateEnum,
|
||||
CameraConfigUpdateSubscriber,
|
||||
)
|
||||
from frigate.config.classification import AudioTranscriptionModelEnum
|
||||
from frigate.const import (
|
||||
AUDIO_DURATION,
|
||||
AUDIO_FORMAT,
|
||||
@@ -37,7 +35,6 @@ from frigate.data_processing.common.audio_transcription.model import (
|
||||
from frigate.data_processing.real_time.audio_transcription import (
|
||||
AudioTranscriptionRealTimeProcessor,
|
||||
)
|
||||
from frigate.data_processing.types import DataProcessorMetrics
|
||||
from frigate.ffmpeg_presets import parse_preset_input
|
||||
from frigate.log import LogPipe, suppress_stderr_during
|
||||
from frigate.util.builtin import get_ffmpeg_arg_list, load_labels
|
||||
@@ -88,7 +85,6 @@ class AudioProcessor(FrigateProcess):
|
||||
self,
|
||||
config: FrigateConfig,
|
||||
camera_metrics: DictProxy,
|
||||
embeddings_metrics: DataProcessorMetrics,
|
||||
stop_event: MpEvent,
|
||||
):
|
||||
super().__init__(
|
||||
@@ -96,38 +92,8 @@ class AudioProcessor(FrigateProcess):
|
||||
)
|
||||
|
||||
self.camera_metrics = camera_metrics
|
||||
self.embeddings_metrics = embeddings_metrics
|
||||
self.config = config
|
||||
|
||||
def spawn_if_needed(self, camera: CameraConfig) -> None:
|
||||
"""Start an audio maintainer for the camera once everything it needs
|
||||
has arrived. Returning early leaves the camera for the next poll."""
|
||||
name = camera.name
|
||||
if name is None or name in self.audio_threads:
|
||||
return
|
||||
if not camera.enabled or not camera.audio.enabled:
|
||||
return
|
||||
# ffmpeg update may not have arrived yet
|
||||
if not any("audio" in i.roles for i in camera.ffmpeg.inputs):
|
||||
return
|
||||
# the camera maintainer creates metrics on its own poll of the same
|
||||
# add update and may not have gotten there yet
|
||||
metrics = self.camera_metrics.get(name)
|
||||
if metrics is None:
|
||||
return
|
||||
thread = AudioEventMaintainer(
|
||||
camera,
|
||||
self.config,
|
||||
metrics,
|
||||
self.embeddings_metrics,
|
||||
self.transcription_model_runner,
|
||||
self.stop_event, # type: ignore[arg-type]
|
||||
self.genai_manager,
|
||||
)
|
||||
self.audio_threads[name] = thread
|
||||
thread.start()
|
||||
self.logger.info(f"Audio maintainer started for {name}")
|
||||
|
||||
def __stop_audio_thread(self, camera: str) -> None:
|
||||
thread = self.audio_threads.pop(camera, None)
|
||||
if thread is None:
|
||||
@@ -146,31 +112,18 @@ class AudioProcessor(FrigateProcess):
|
||||
|
||||
threading.current_thread().name = "process:audio_manager"
|
||||
|
||||
self.transcription_model_runner: AudioTranscriptionModelRunner | None = None
|
||||
self.genai_manager: Any = None
|
||||
|
||||
if any(
|
||||
c.enabled_in_config and c.audio_transcription.enabled
|
||||
for c in self.config.cameras.values()
|
||||
):
|
||||
if isinstance(
|
||||
self.config.audio_transcription.model, AudioTranscriptionModelEnum
|
||||
):
|
||||
# AudioTranscriptionModelRunner.__init__ unconditionally fetches
|
||||
# sherpa-onnx or whisper weights, so only build it on the local path
|
||||
self.transcription_model_runner = AudioTranscriptionModelRunner(
|
||||
self.transcription_model_runner: AudioTranscriptionModelRunner | None = (
|
||||
AudioTranscriptionModelRunner(
|
||||
self.config.audio_transcription.device or "AUTO",
|
||||
self.config.audio_transcription.model_size,
|
||||
)
|
||||
else:
|
||||
# imported here rather than at module scope: frigate.genai pulls in
|
||||
# numpy, the provider SDKs, frigate.models, and the prompt builders,
|
||||
# and this process runs at PROCESS_PRIORITY_HIGH. built after the
|
||||
# fork because SDK clients hold sockets and TLS state that must not
|
||||
# cross it; clients themselves stay lazy behind the role property.
|
||||
from frigate.genai.manager import GenAIClientManager
|
||||
|
||||
self.genai_manager = GenAIClientManager(self.config)
|
||||
)
|
||||
else:
|
||||
self.transcription_model_runner = None
|
||||
|
||||
config_subscriber = CameraConfigUpdateSubscriber(
|
||||
self.config,
|
||||
@@ -183,8 +136,28 @@ class AudioProcessor(FrigateProcess):
|
||||
],
|
||||
)
|
||||
|
||||
def spawn_if_needed(camera: CameraConfig) -> None:
|
||||
name = camera.name
|
||||
if name is None or name in self.audio_threads:
|
||||
return
|
||||
if not camera.enabled or not camera.audio.enabled:
|
||||
return
|
||||
# ffmpeg update may not have arrived yet; wait for next poll
|
||||
if not any("audio" in i.roles for i in camera.ffmpeg.inputs):
|
||||
return
|
||||
thread = AudioEventMaintainer(
|
||||
camera,
|
||||
self.config,
|
||||
self.camera_metrics,
|
||||
self.transcription_model_runner,
|
||||
self.stop_event, # type: ignore[arg-type]
|
||||
)
|
||||
self.audio_threads[name] = thread
|
||||
thread.start()
|
||||
self.logger.info(f"Audio maintainer started for {name}")
|
||||
|
||||
for camera in self.config.cameras.values():
|
||||
self.spawn_if_needed(camera)
|
||||
spawn_if_needed(camera)
|
||||
|
||||
self.logger.info(f"Audio processor started (pid: {self.pid})")
|
||||
|
||||
@@ -194,14 +167,15 @@ class AudioProcessor(FrigateProcess):
|
||||
updated_topics = config_subscriber.check_for_updates()
|
||||
|
||||
# stop maintainers for removed cameras so their ffmpeg process is
|
||||
# torn down
|
||||
# torn down and they stop touching camera_metrics (which the camera
|
||||
# maintainer has already popped for the removed camera)
|
||||
for removed_camera in updated_topics.get(
|
||||
CameraConfigUpdateEnum.remove.name, []
|
||||
):
|
||||
self.__stop_audio_thread(removed_camera)
|
||||
|
||||
for camera in self.config.cameras.values():
|
||||
self.spawn_if_needed(camera)
|
||||
spawn_if_needed(camera)
|
||||
|
||||
config_subscriber.stop()
|
||||
|
||||
@@ -223,21 +197,15 @@ class AudioEventMaintainer(threading.Thread):
|
||||
self,
|
||||
camera: CameraConfig,
|
||||
config: FrigateConfig,
|
||||
metrics: CameraMetrics,
|
||||
embeddings_metrics: DataProcessorMetrics,
|
||||
camera_metrics: DictProxy,
|
||||
audio_transcription_model_runner: AudioTranscriptionModelRunner | None,
|
||||
stop_event: threading.Event,
|
||||
genai_manager: Any = None,
|
||||
) -> None:
|
||||
super().__init__(name=f"{camera.name}_audio_event_processor")
|
||||
|
||||
self.config = config
|
||||
self.camera_config = camera
|
||||
# hold the metrics object rather than indexing the manager dict per
|
||||
# chunk, which costs an IPC round trip and breaks once the camera
|
||||
# maintainer pops the entry on removal
|
||||
self.metrics = metrics
|
||||
self.embeddings_metrics = embeddings_metrics
|
||||
self.camera_metrics = camera_metrics
|
||||
self.stop_event = stop_event
|
||||
# per-camera stop signal so a single maintainer can be torn down at
|
||||
# runtime (e.g. on camera removal) without stopping the whole process
|
||||
@@ -254,7 +222,6 @@ class AudioEventMaintainer(threading.Thread):
|
||||
self.logpipe = LogPipe(f"ffmpeg.{self.camera_config.name}.audio")
|
||||
self.audio_listener: subprocess.Popen[Any] | None = None
|
||||
self.audio_transcription_model_runner = audio_transcription_model_runner
|
||||
self.genai_manager = genai_manager
|
||||
self.transcription_processor = None
|
||||
self.transcription_thread = None
|
||||
|
||||
@@ -271,9 +238,9 @@ class AudioEventMaintainer(threading.Thread):
|
||||
)
|
||||
self.detection_publisher = DetectionPublisher(DetectionTypeEnum.audio.value)
|
||||
|
||||
if self.camera_config.audio_transcription.enabled and (
|
||||
self.audio_transcription_model_runner is not None
|
||||
or self.genai_manager is not None
|
||||
if (
|
||||
self.camera_config.audio_transcription.enabled
|
||||
and self.audio_transcription_model_runner is not None
|
||||
):
|
||||
# init the transcription processor for this camera
|
||||
self.transcription_processor = AudioTranscriptionRealTimeProcessor(
|
||||
@@ -281,9 +248,8 @@ class AudioEventMaintainer(threading.Thread):
|
||||
camera_config=self.camera_config,
|
||||
requestor=self.requestor,
|
||||
model_runner=self.audio_transcription_model_runner,
|
||||
metrics=self.embeddings_metrics,
|
||||
metrics=self.camera_metrics[self.camera_config.name],
|
||||
stop_event=self.stop_event,
|
||||
genai_manager=self.genai_manager,
|
||||
)
|
||||
|
||||
self.transcription_thread = threading.Thread(
|
||||
@@ -307,8 +273,8 @@ class AudioEventMaintainer(threading.Thread):
|
||||
audio_as_float: np.ndarray = audio.astype(np.float32)
|
||||
rms, dBFS = self.calculate_audio_levels(audio_as_float)
|
||||
|
||||
self.metrics.audio_rms.value = rms
|
||||
self.metrics.audio_dBFS.value = dBFS
|
||||
self.camera_metrics[self.camera_config.name].audio_rms.value = rms
|
||||
self.camera_metrics[self.camera_config.name].audio_dBFS.value = dBFS
|
||||
|
||||
audio_detections: list[tuple[str, float]] = []
|
||||
|
||||
@@ -393,6 +359,7 @@ class AudioEventMaintainer(threading.Thread):
|
||||
return
|
||||
|
||||
time.sleep(self.camera_config.ffmpeg.retry_interval)
|
||||
self.logpipe.dump()
|
||||
self.start_or_restart_ffmpeg()
|
||||
|
||||
if self.audio_listener is None or self.audio_listener.stdout is None:
|
||||
|
||||
@@ -105,21 +105,8 @@ class GenAIClient:
|
||||
debug_save: bool,
|
||||
activity_context_prompt: str,
|
||||
response_style: str = "default",
|
||||
frame_captions: list[str] | None = None,
|
||||
) -> ReviewMetadata | None:
|
||||
"""Generate a description for the review item activity.
|
||||
|
||||
`frame_captions` holds one caption per thumbnail for the annotated
|
||||
frame mode; each is sent directly before its frame.
|
||||
"""
|
||||
if frame_captions and len(frame_captions) != len(thumbnails):
|
||||
logger.warning(
|
||||
"Got %d frame captions for %d thumbnails, sending plain frames",
|
||||
len(frame_captions),
|
||||
len(thumbnails),
|
||||
)
|
||||
frame_captions = None
|
||||
|
||||
"""Generate a description for the review item activity."""
|
||||
context_prompt = build_review_description_prompt(
|
||||
review_data,
|
||||
thumbnails,
|
||||
@@ -127,7 +114,6 @@ class GenAIClient:
|
||||
preferred_language,
|
||||
activity_context_prompt,
|
||||
response_style,
|
||||
frame_captions,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
@@ -143,30 +129,9 @@ class GenAIClient:
|
||||
) as f:
|
||||
f.write(context_prompt)
|
||||
|
||||
if frame_captions:
|
||||
# One file per frame, numbered to match the image it precedes
|
||||
# (0.txt goes with 0.jpg), so the debug folder replays without
|
||||
# having to re-derive the mapping.
|
||||
for index, caption in enumerate(frame_captions):
|
||||
with open(
|
||||
os.path.join(
|
||||
CLIPS_DIR,
|
||||
"genai-requests",
|
||||
review_data["id"],
|
||||
f"{index}.txt",
|
||||
),
|
||||
"w",
|
||||
) as f:
|
||||
f.write(caption)
|
||||
|
||||
response_format = build_review_description_response_format(concerns)
|
||||
|
||||
response = self._send(
|
||||
context_prompt,
|
||||
thumbnails,
|
||||
response_format,
|
||||
image_captions=frame_captions,
|
||||
)
|
||||
response = self._send(context_prompt, thumbnails, response_format)
|
||||
|
||||
if debug_save and response:
|
||||
with open(
|
||||
@@ -304,7 +269,6 @@ class GenAIClient:
|
||||
images: list[bytes],
|
||||
response_format: dict | None = None,
|
||||
enable_thinking: bool = False,
|
||||
image_captions: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Submit a request to the provider.
|
||||
|
||||
@@ -312,10 +276,6 @@ class GenAIClient:
|
||||
``supports_toggleable_thinking``. Description-style callers leave it
|
||||
at the default (off) since synthesis tasks don't benefit from
|
||||
reasoning traces.
|
||||
|
||||
``image_captions`` carries one caption per image, to be placed
|
||||
immediately before its image so the model can tell the frames apart.
|
||||
Providers build their request order with ``interleave_images``.
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -338,11 +298,6 @@ class GenAIClient:
|
||||
"""Whether the configured model can generate embeddings via embed()."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def supports_transcription(self) -> bool:
|
||||
"""Whether the configured model can transcribe audio via transcribe()."""
|
||||
return False
|
||||
|
||||
def list_models(self) -> list[str]:
|
||||
"""Return the list of model names available from this provider.
|
||||
|
||||
@@ -350,21 +305,6 @@ class GenAIClient:
|
||||
"""
|
||||
return []
|
||||
|
||||
def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
|
||||
"""Return capability flags for each model the provider serves.
|
||||
|
||||
Only providers whose backend advertises capabilities per model can
|
||||
populate this; llama.cpp reports input modalities for every model it
|
||||
serves, so one request describes them all. An empty mapping means "no
|
||||
per-model information available", and callers fall back to this
|
||||
client's own capability properties, which describe only the configured
|
||||
model. A model absent from a non-empty mapping means the same thing.
|
||||
|
||||
Returns:
|
||||
Model name (including aliases) to its capability flags
|
||||
"""
|
||||
return {}
|
||||
|
||||
def get_context_size(self) -> int:
|
||||
"""Get the context window size for this provider in tokens."""
|
||||
return 4096
|
||||
@@ -396,33 +336,6 @@ class GenAIClient:
|
||||
)
|
||||
return []
|
||||
|
||||
def transcribe(
|
||||
self,
|
||||
audio: bytes,
|
||||
language: str | None = None,
|
||||
mime_type: str = "audio/wav",
|
||||
) -> str | None:
|
||||
"""Transcribe speech audio to text.
|
||||
|
||||
Audio is passed as a self-describing blob rather than raw samples so
|
||||
every provider receives a container it can declare, and WAV framing
|
||||
lives in one place instead of in each plugin.
|
||||
|
||||
Args:
|
||||
audio: The encoded audio payload (WAV bytes by default)
|
||||
language: Optional ISO language hint for the provider
|
||||
mime_type: Media type of ``audio``
|
||||
|
||||
Returns:
|
||||
The transcript, or None when the provider cannot produce one
|
||||
"""
|
||||
logger.warning(
|
||||
"%s does not support transcription. "
|
||||
"This method should be overridden by the provider implementation.",
|
||||
self.__class__.__name__,
|
||||
)
|
||||
return None
|
||||
|
||||
def chat_with_tools(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
|
||||
@@ -110,12 +110,6 @@ class GenAIClientManager:
|
||||
name = self._role_map.get(GenAIRoleEnum.embeddings)
|
||||
return self._get_client(name) if name else None
|
||||
|
||||
@property
|
||||
def transcribe_client(self) -> "GenAIClient | None":
|
||||
"""Client configured for the transcribe role."""
|
||||
name = self._role_map.get(GenAIRoleEnum.transcribe)
|
||||
return self._get_client(name) if name else None
|
||||
|
||||
def role_info(self) -> dict[str, dict[str, Any]]:
|
||||
"""Return the model selected for each configured role and its context size.
|
||||
|
||||
@@ -150,11 +144,5 @@ class GenAIClientManager:
|
||||
"roles": [r.value for r in genai_cfg.roles],
|
||||
"supports_toggleable_thinking": client.supports_toggleable_thinking,
|
||||
"supports_embeddings": client.supports_embeddings,
|
||||
"supports_transcription": client.supports_transcription,
|
||||
# Capabilities of the configured model are above; this maps every
|
||||
# model the provider serves to its own, so the UI can react to a
|
||||
# model selected but not yet saved. Empty when the provider
|
||||
# cannot report capabilities without loading a model.
|
||||
"model_capabilities": client.list_model_capabilities(),
|
||||
}
|
||||
return result
|
||||
|
||||
@@ -13,19 +13,6 @@ overrides what is genuinely Azure-specific:
|
||||
- Context size: Azure does not expose a per-model ``max_model_len`` field
|
||||
reliably, so we keep the historical 128K default rather than the
|
||||
model-name heuristic used by OpenAI.
|
||||
|
||||
Transcription is inherited too: :class:`openai.AzureOpenAI` exposes the same
|
||||
``audio.transcriptions.create``. Two Azure-specific caveats apply when using
|
||||
the ``transcribe`` role:
|
||||
|
||||
- ``model`` must be the Azure *deployment* name, not the underlying model name.
|
||||
- The ``api-version`` parsed from ``base_url`` must be 2024-06-01 or later;
|
||||
earlier versions have no transcriptions route and the 404 surfaces only as a
|
||||
generic provider error.
|
||||
- Because ``model`` is a deployment name, the inherited check that picks
|
||||
``languages`` over ``language`` for gpt-transcribe cannot fire unless the
|
||||
deployment happens to be named after the model. Name the deployment
|
||||
``gpt-transcribe`` to get the right field, or leave the language on ``auto``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
@@ -13,14 +13,9 @@ from google.genai.types import FunctionCallingConfigMode
|
||||
|
||||
from frigate.config import GenAIProviderEnum
|
||||
from frigate.genai import GenAIClient, register_genai_provider
|
||||
from frigate.genai.utils import interleave_images
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Gemini requests carrying inline data are capped at ~20 MB total; stay well
|
||||
# under it so the request fails as a log line rather than a 400.
|
||||
GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024
|
||||
|
||||
|
||||
def _decode_thought_signature(value: Any) -> bytes | None:
|
||||
"""Decode a base64-encoded thought_signature carried across conversation turns."""
|
||||
@@ -123,16 +118,11 @@ class GeminiClient(GenAIClient):
|
||||
images: list[bytes],
|
||||
response_format: dict | None = None,
|
||||
enable_thinking: bool = False,
|
||||
image_captions: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Submit a request to Gemini."""
|
||||
contents: list[Any] = [
|
||||
part
|
||||
if isinstance(part, str)
|
||||
else types.Part.from_bytes(data=part, mime_type="image/jpeg")
|
||||
for part in interleave_images(prompt, images, image_captions)
|
||||
contents = [prompt] + [
|
||||
types.Part.from_bytes(data=img, mime_type="image/jpeg") for img in images
|
||||
]
|
||||
|
||||
try:
|
||||
# Merge runtime_options into generation_config if provided
|
||||
generation_config_dict: dict[str, Any] = {"candidate_count": 1}
|
||||
@@ -146,7 +136,7 @@ class GeminiClient(GenAIClient):
|
||||
|
||||
response = self.provider.models.generate_content(
|
||||
model=self.genai_config.model,
|
||||
contents=contents,
|
||||
contents=contents, # type: ignore[arg-type]
|
||||
config=types.GenerateContentConfig(
|
||||
**generation_config_dict,
|
||||
),
|
||||
@@ -167,58 +157,6 @@ class GeminiClient(GenAIClient):
|
||||
return None
|
||||
return description
|
||||
|
||||
@property
|
||||
def supports_transcription(self) -> bool:
|
||||
"""Gemini models accept inline audio parts."""
|
||||
return True
|
||||
|
||||
def transcribe(
|
||||
self,
|
||||
audio: bytes,
|
||||
language: str | None = None,
|
||||
mime_type: str = "audio/wav",
|
||||
) -> str | None:
|
||||
"""Transcribe audio by sending it as an inline part alongside a prompt."""
|
||||
if len(audio) > GEMINI_MAX_INLINE_BYTES:
|
||||
logger.warning(
|
||||
"Audio payload of %d bytes exceeds the Gemini inline limit; skipping transcription",
|
||||
len(audio),
|
||||
)
|
||||
return None
|
||||
|
||||
prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
|
||||
|
||||
if language:
|
||||
prompt += f" The speech is in language '{language}'."
|
||||
|
||||
try:
|
||||
contents: list[Any] = [
|
||||
prompt,
|
||||
types.Part.from_bytes(data=audio, mime_type=mime_type),
|
||||
]
|
||||
response = self.provider.models.generate_content(
|
||||
model=self.genai_config.model,
|
||||
contents=contents,
|
||||
config=types.GenerateContentConfig(candidate_count=1),
|
||||
)
|
||||
except errors.APIError as e:
|
||||
logger.warning("Gemini returned an error: %s", str(e))
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("An unexpected error occurred with Gemini: %s", str(e))
|
||||
return None
|
||||
|
||||
try:
|
||||
if response.text is None:
|
||||
return None
|
||||
|
||||
transcript = response.text.strip()
|
||||
except (ValueError, AttributeError):
|
||||
# No transcript was generated
|
||||
return None
|
||||
|
||||
return transcript or None
|
||||
|
||||
def list_models(self) -> list[str]:
|
||||
"""Return available model names from Gemini."""
|
||||
try:
|
||||
|
||||
@@ -14,7 +14,7 @@ from PIL import Image
|
||||
|
||||
from frigate.config import GenAIProviderEnum
|
||||
from frigate.genai import GenAIClient, register_genai_provider
|
||||
from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
|
||||
from frigate.genai.utils import parse_tool_calls_from_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -333,7 +333,6 @@ class LlamaCppClient(GenAIClient):
|
||||
images: list[bytes],
|
||||
response_format: dict | None = None,
|
||||
enable_thinking: bool = False,
|
||||
image_captions: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Submit a request to llama.cpp server."""
|
||||
if self.provider is None:
|
||||
@@ -343,17 +342,18 @@ class LlamaCppClient(GenAIClient):
|
||||
return None
|
||||
|
||||
try:
|
||||
content: list[dict[str, Any]] = []
|
||||
for part in interleave_images(prompt, images, image_captions):
|
||||
if isinstance(part, str):
|
||||
content.append({"type": "text", "text": part})
|
||||
continue
|
||||
|
||||
encoded_image = base64.b64encode(part).decode("utf-8")
|
||||
content = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": prompt,
|
||||
}
|
||||
]
|
||||
for image in images:
|
||||
encoded_image = base64.b64encode(image).decode("utf-8")
|
||||
content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"image_url": { # type: ignore[dict-item]
|
||||
"url": f"data:image/jpeg;base64,{encoded_image}",
|
||||
},
|
||||
}
|
||||
@@ -408,126 +408,6 @@ class LlamaCppClient(GenAIClient):
|
||||
"""Whether the loaded model supports audio input."""
|
||||
return self._supports_audio
|
||||
|
||||
@property
|
||||
def supports_transcription(self) -> bool:
|
||||
"""Audio-capable models can transcribe through chat completions."""
|
||||
return self._supports_audio
|
||||
|
||||
def transcribe(
|
||||
self,
|
||||
audio: bytes,
|
||||
language: str | None = None,
|
||||
mime_type: str = "audio/wav",
|
||||
) -> str | None:
|
||||
"""Transcribe audio through the OpenAI-compatible transcriptions route.
|
||||
|
||||
llama.cpp serves /v1/audio/transcriptions for any audio-capable model,
|
||||
not only a separately loaded whisper (ggml-org/llama.cpp#21863), so it
|
||||
covers exactly the models supports_transcription detects. It takes the
|
||||
language as a native multipart field, which is the only thing dedicated
|
||||
ASR models honor: they read the chat prompt as contextual biasing, so
|
||||
asking one there to use a language does nothing.
|
||||
|
||||
Falls back to chat completions when the server predates that route.
|
||||
"""
|
||||
if self.provider is None:
|
||||
logger.warning(
|
||||
"llama.cpp provider has not been initialized, audio will not be transcribed. Check your llama.cpp configuration."
|
||||
)
|
||||
return None
|
||||
|
||||
if not self._supports_audio:
|
||||
logger.warning(
|
||||
"llama.cpp model '%s' does not accept audio input",
|
||||
self.genai_config.model,
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
data = {"model": self.genai_config.model, "response_format": "json"}
|
||||
|
||||
if language:
|
||||
data["language"] = language
|
||||
|
||||
response = self._post(
|
||||
f"{self.provider}/v1/audio/transcriptions",
|
||||
files={"file": ("audio.wav", audio, mime_type)},
|
||||
data=data,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
if response.status_code == 404:
|
||||
logger.debug(
|
||||
"llama.cpp server has no /v1/audio/transcriptions route, using chat completions"
|
||||
)
|
||||
return self._transcribe_via_chat(audio, language)
|
||||
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
text = result.get("text") if isinstance(result, dict) else None
|
||||
|
||||
return str(text).strip() or None if text else None
|
||||
except Exception as e:
|
||||
logger.warning("llama.cpp returned an error: %s", str(e))
|
||||
return None
|
||||
|
||||
def _transcribe_via_chat(self, audio: bytes, language: str | None) -> str | None:
|
||||
"""Transcribe through /v1/chat/completions, for servers without the
|
||||
transcriptions route.
|
||||
|
||||
The _media_marker / multimodal_data convention is an /embeddings-only
|
||||
protocol, so no marker-refresh retry is needed here.
|
||||
"""
|
||||
prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
|
||||
|
||||
if language:
|
||||
prompt += f" The speech is in language '{language}'."
|
||||
|
||||
try:
|
||||
encoded_audio = base64.b64encode(audio).decode("utf-8")
|
||||
payload: dict[str, Any] = {
|
||||
"model": self.genai_config.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": encoded_audio,
|
||||
"format": "wav",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
**self.provider_options,
|
||||
}
|
||||
|
||||
response = self._post(
|
||||
f"{self.provider}/v1/chat/completions",
|
||||
json=payload,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
|
||||
if (
|
||||
result is not None
|
||||
and "choices" in result
|
||||
and len(result["choices"]) > 0
|
||||
):
|
||||
choice = result["choices"][0]
|
||||
|
||||
if "message" in choice and choice["message"].get("content"):
|
||||
return str(choice["message"]["content"].strip()) or None
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("llama.cpp returned an error: %s", str(e))
|
||||
return None
|
||||
|
||||
@property
|
||||
def supports_tools(self) -> bool:
|
||||
"""Whether the loaded model supports tool/function calling."""
|
||||
@@ -537,73 +417,28 @@ class LlamaCppClient(GenAIClient):
|
||||
def supports_toggleable_thinking(self) -> bool:
|
||||
return self._supports_reasoning
|
||||
|
||||
def _fetch_models_data(self) -> list[dict[str, Any]]:
|
||||
"""Return the raw /v1/models entries, or an empty list if unreachable."""
|
||||
def list_models(self) -> list[str]:
|
||||
"""Return available model IDs from the llama.cpp server."""
|
||||
base_url = self.provider or (
|
||||
self.genai_config.base_url.rstrip("/")
|
||||
if self.genai_config.base_url
|
||||
else None
|
||||
)
|
||||
|
||||
if base_url is None:
|
||||
return []
|
||||
|
||||
try:
|
||||
response = self._get(f"{base_url}/v1/models", timeout=10)
|
||||
response.raise_for_status()
|
||||
data = response.json().get("data", [])
|
||||
models = []
|
||||
for m in response.json().get("data", []):
|
||||
models.append(m.get("id", "unknown"))
|
||||
for alias in m.get("aliases", []):
|
||||
models.append(alias)
|
||||
return sorted(models)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to list llama.cpp models: %s", e)
|
||||
return []
|
||||
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
def list_models(self) -> list[str]:
|
||||
"""Return available model IDs from the llama.cpp server."""
|
||||
models: set[str] = set()
|
||||
|
||||
# llama-server lists the id among the aliases when --alias is set
|
||||
for m in self._fetch_models_data():
|
||||
models.add(m.get("id", "unknown"))
|
||||
models.update(m.get("aliases", []))
|
||||
|
||||
return sorted(models)
|
||||
|
||||
def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
|
||||
"""Report input modalities for every model the server serves.
|
||||
|
||||
Since ggml-org/llama.cpp#22952 each /v1/models entry carries
|
||||
architecture.input_modalities, so a single request describes every
|
||||
model rather than just the configured one. That is what lets the UI
|
||||
answer "can the model I just picked transcribe" before the config is
|
||||
saved and a client for it exists.
|
||||
|
||||
Models whose entry predates that field are omitted rather than reported
|
||||
as incapable, so an older server falls back to the /props probe instead
|
||||
of silently losing capabilities it actually has.
|
||||
"""
|
||||
capabilities: dict[str, dict[str, bool]] = {}
|
||||
|
||||
for model in self._fetch_models_data():
|
||||
architecture = model.get("architecture") or {}
|
||||
modalities = architecture.get("input_modalities")
|
||||
|
||||
if not isinstance(modalities, list) or not modalities:
|
||||
continue
|
||||
|
||||
flags = {
|
||||
"supports_vision": "image" in modalities,
|
||||
"supports_transcription": "audio" in modalities,
|
||||
}
|
||||
|
||||
names = [model.get("id"), *(model.get("aliases") or [])]
|
||||
|
||||
for name in names:
|
||||
if isinstance(name, str) and name:
|
||||
capabilities[name] = flags
|
||||
|
||||
return capabilities
|
||||
|
||||
def get_context_size(self) -> int:
|
||||
"""Get the context window size for llama.cpp.
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from ollama import ResponseError
|
||||
|
||||
from frigate.config import GenAIProviderEnum
|
||||
from frigate.genai import GenAIClient, register_genai_provider
|
||||
from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
|
||||
from frigate.genai.utils import parse_tool_calls_from_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -50,28 +50,6 @@ def _extract_ollama_stats(response: Any) -> dict[str, Any] | None:
|
||||
return stats or None
|
||||
|
||||
|
||||
# Ollama replaces each occurrence of this marker in a message, in order, with
|
||||
# the next image from the message's images list. Without markers it puts every
|
||||
# image before the text.
|
||||
IMAGE_PLACEHOLDER = "[img]"
|
||||
|
||||
|
||||
def _flatten_parts(parts: list[str | bytes]) -> tuple[str, list[bytes] | None]:
|
||||
"""Collapse ordered text and image parts into Ollama's (content, images)
|
||||
shape, marking where each image goes so the order survives."""
|
||||
text: list[str] = []
|
||||
images: list[bytes] = []
|
||||
|
||||
for part in parts:
|
||||
if isinstance(part, bytes):
|
||||
text.append(IMAGE_PLACEHOLDER)
|
||||
images.append(part)
|
||||
elif part:
|
||||
text.append(part)
|
||||
|
||||
return "\n".join(text), (images or None)
|
||||
|
||||
|
||||
def _normalize_multimodal_content(
|
||||
content: Any,
|
||||
) -> tuple[str | None, list[bytes] | None]:
|
||||
@@ -80,13 +58,13 @@ def _normalize_multimodal_content(
|
||||
The chat API constructs user messages with content as a list of
|
||||
``{"type": "text"}`` and ``{"type": "image_url"}`` parts when a tool
|
||||
returns a live frame. Ollama's SDK requires content to be a string and
|
||||
images to be passed in a separate field, so images are pulled out and
|
||||
their positions marked with placeholders.
|
||||
images to be passed in a separate field, so we extract each.
|
||||
"""
|
||||
if not isinstance(content, list):
|
||||
return content, None
|
||||
|
||||
parts: list[str | bytes] = []
|
||||
text_parts: list[str] = []
|
||||
images: list[bytes] = []
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
@@ -94,20 +72,17 @@ def _normalize_multimodal_content(
|
||||
if part_type == "text":
|
||||
text = part.get("text")
|
||||
if text:
|
||||
parts.append(str(text))
|
||||
text_parts.append(str(text))
|
||||
elif part_type == "image_url":
|
||||
url = (part.get("image_url") or {}).get("url", "")
|
||||
if isinstance(url, str) and url.startswith("data:"):
|
||||
try:
|
||||
encoded = url.split(",", 1)[1]
|
||||
parts.append(base64.b64decode(encoded, validate=True))
|
||||
images.append(base64.b64decode(encoded, validate=True))
|
||||
except (ValueError, IndexError, binascii.Error) as e:
|
||||
logger.debug("Failed to decode multimodal image url: %s", e)
|
||||
|
||||
if not parts:
|
||||
return None, None
|
||||
|
||||
return _flatten_parts(parts)
|
||||
return ("\n".join(text_parts) if text_parts else None), (images or None)
|
||||
|
||||
|
||||
@register_genai_provider(GenAIProviderEnum.ollama)
|
||||
@@ -221,46 +196,58 @@ class OllamaClient(GenAIClient):
|
||||
images: list[bytes],
|
||||
response_format: dict | None = None,
|
||||
enable_thinking: bool = False,
|
||||
image_captions: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Submit a request to Ollama through the chat API, the same path the
|
||||
tool-calling chat uses, with image placeholders keeping any captions
|
||||
next to their frames."""
|
||||
"""Submit a request to Ollama"""
|
||||
if self.provider is None:
|
||||
logger.warning(
|
||||
"Ollama provider has not been initialized, a description will not be generated. Check your Ollama configuration."
|
||||
)
|
||||
return None
|
||||
|
||||
content, message_images = _flatten_parts(
|
||||
interleave_images(prompt, images, image_captions)
|
||||
)
|
||||
message: dict[str, Any] = {"role": "user", "content": content}
|
||||
|
||||
if message_images:
|
||||
message["images"] = message_images
|
||||
|
||||
request_params = self._build_request_params(
|
||||
[message], None, None, enable_thinking=enable_thinking
|
||||
)
|
||||
|
||||
if response_format and response_format.get("type") == "json_schema":
|
||||
schema = response_format.get("json_schema", {}).get("schema")
|
||||
if schema:
|
||||
request_params["format"] = self._clean_schema_for_ollama(schema)
|
||||
|
||||
logger.debug(
|
||||
"Ollama chat request: model=%s, prompt_len=%s, image_count=%s, "
|
||||
"has_format=%s, think=%s",
|
||||
self.genai_config.model,
|
||||
len(prompt),
|
||||
len(images),
|
||||
"format" in request_params,
|
||||
request_params.get("think"),
|
||||
)
|
||||
|
||||
try:
|
||||
response = self.provider.chat(**request_params)
|
||||
ollama_options = {
|
||||
**self.provider_options,
|
||||
**self.genai_config.runtime_options,
|
||||
}
|
||||
if response_format and response_format.get("type") == "json_schema":
|
||||
schema = response_format.get("json_schema", {}).get("schema")
|
||||
if schema:
|
||||
ollama_options["format"] = self._clean_schema_for_ollama(schema)
|
||||
if self.supports_toggleable_thinking:
|
||||
ollama_options["think"] = enable_thinking
|
||||
logger.debug(
|
||||
"Ollama generate request: model=%s, prompt_len=%s, image_count=%s, "
|
||||
"has_format=%s, options=%s",
|
||||
self.genai_config.model,
|
||||
len(prompt),
|
||||
len(images) if images else 0,
|
||||
"format" in ollama_options,
|
||||
{k: v for k, v in ollama_options.items() if k != "format"},
|
||||
)
|
||||
result = self.provider.generate(
|
||||
self.genai_config.model,
|
||||
prompt,
|
||||
images=images if images else None,
|
||||
**ollama_options,
|
||||
)
|
||||
logger.debug(
|
||||
"Ollama generate response: done=%s, done_reason=%s, eval_count=%s, "
|
||||
"prompt_eval_count=%s, response_len=%s",
|
||||
result.get("done"),
|
||||
result.get("done_reason"),
|
||||
result.get("eval_count"),
|
||||
result.get("prompt_eval_count"),
|
||||
len(result.get("response", "") or ""),
|
||||
)
|
||||
response_text = str(result["response"]).strip()
|
||||
if not response_text:
|
||||
logger.warning(
|
||||
"Ollama returned a blank response for model %s (done_reason=%s, "
|
||||
"eval_count=%s). Check model output, ensure thinking is disabled.",
|
||||
self.genai_config.model,
|
||||
result.get("done_reason"),
|
||||
result.get("eval_count"),
|
||||
)
|
||||
return response_text
|
||||
except (
|
||||
TimeoutException,
|
||||
ResponseError,
|
||||
@@ -270,27 +257,6 @@ class OllamaClient(GenAIClient):
|
||||
logger.warning("Ollama returned an error: %s", str(e))
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"Ollama chat response: done=%s, done_reason=%s, eval_count=%s, "
|
||||
"prompt_eval_count=%s",
|
||||
response.get("done"),
|
||||
response.get("done_reason"),
|
||||
response.get("eval_count"),
|
||||
response.get("prompt_eval_count"),
|
||||
)
|
||||
response_text = self._message_from_response(response)["content"] or ""
|
||||
|
||||
if not response_text:
|
||||
logger.warning(
|
||||
"Ollama returned a blank response for model %s (done_reason=%s, "
|
||||
"eval_count=%s). Check model output, ensure thinking is disabled.",
|
||||
self.genai_config.model,
|
||||
response.get("done_reason"),
|
||||
response.get("eval_count"),
|
||||
)
|
||||
|
||||
return response_text
|
||||
|
||||
def list_models(self) -> list[str]:
|
||||
"""Return available model names from the Ollama server."""
|
||||
client = self.provider
|
||||
@@ -340,8 +306,6 @@ class OllamaClient(GenAIClient):
|
||||
}
|
||||
if images:
|
||||
msg_dict["images"] = images
|
||||
elif msg.get("images"):
|
||||
msg_dict["images"] = msg["images"]
|
||||
if msg.get("tool_call_id"):
|
||||
msg_dict["tool_call_id"] = msg["tool_call_id"]
|
||||
if msg.get("name"):
|
||||
|
||||
@@ -11,16 +11,9 @@ from openai import OpenAI
|
||||
|
||||
from frigate.config import GenAIProviderEnum
|
||||
from frigate.genai import GenAIClient, register_genai_provider
|
||||
from frigate.genai.utils import interleave_images
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# gpt-transcribe replaced the singular `language` field with a `languages` array
|
||||
# and rejects a request that sends both. Older transcription models
|
||||
# (gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1) still take the singular
|
||||
# form. https://developers.openai.com/api/docs/guides/speech-to-text
|
||||
_LANGUAGES_ARRAY_MODEL_PREFIX = "gpt-transcribe"
|
||||
|
||||
|
||||
def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None:
|
||||
"""Build a stats dict from an OpenAI-compatible usage object."""
|
||||
@@ -70,21 +63,21 @@ class OpenAIClient(GenAIClient):
|
||||
images: list[bytes],
|
||||
response_format: dict | None = None,
|
||||
enable_thinking: bool = False,
|
||||
image_captions: list[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Submit a request to OpenAI."""
|
||||
messages_content: list[dict] = []
|
||||
for part in interleave_images(prompt, images, image_captions):
|
||||
if isinstance(part, str):
|
||||
messages_content.append({"type": "text", "text": part})
|
||||
continue
|
||||
|
||||
encoded = base64.b64encode(part).decode("utf-8")
|
||||
encoded_images = [base64.b64encode(image).decode("utf-8") for image in images]
|
||||
messages_content: list[dict] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": prompt,
|
||||
}
|
||||
]
|
||||
for image in encoded_images:
|
||||
messages_content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/jpeg;base64,{encoded}",
|
||||
"url": f"data:image/jpeg;base64,{image}",
|
||||
"detail": "low",
|
||||
},
|
||||
}
|
||||
@@ -140,51 +133,6 @@ class OpenAIClient(GenAIClient):
|
||||
logger.warning("OpenAI returned an error: %s", str(e))
|
||||
return None
|
||||
|
||||
@property
|
||||
def supports_transcription(self) -> bool:
|
||||
"""OpenAI exposes /v1/audio/transcriptions for its speech models."""
|
||||
return True
|
||||
|
||||
def transcribe(
|
||||
self,
|
||||
audio: bytes,
|
||||
language: str | None = None,
|
||||
mime_type: str = "audio/wav",
|
||||
) -> str | None:
|
||||
"""Transcribe audio via the OpenAI audio transcriptions endpoint."""
|
||||
try:
|
||||
# runtime_options are chat-completion parameters; the transcriptions
|
||||
# endpoint rejects unknown fields, so they are deliberately not splatted
|
||||
# in here the way _send() does.
|
||||
request_params: dict[str, Any] = {
|
||||
"model": self.genai_config.model,
|
||||
"file": ("audio.wav", audio, mime_type),
|
||||
"response_format": "text",
|
||||
"timeout": self.timeout,
|
||||
}
|
||||
|
||||
if language:
|
||||
if (
|
||||
self.genai_config.model.strip()
|
||||
.lower()
|
||||
.startswith(_LANGUAGES_ARRAY_MODEL_PREFIX)
|
||||
):
|
||||
# not a typed parameter on the SDK method, so it has to ride
|
||||
# along in extra_body
|
||||
request_params["extra_body"] = {"languages": [language]}
|
||||
else:
|
||||
request_params["language"] = language
|
||||
|
||||
result = self.provider.audio.transcriptions.create(**request_params)
|
||||
except (TimeoutException, Exception) as e:
|
||||
logger.warning("OpenAI returned an error: %s", str(e))
|
||||
return None
|
||||
|
||||
# response_format="text" yields a bare string, but some compatible
|
||||
# servers still return the object form
|
||||
text = result if isinstance(result, str) else getattr(result, "text", None)
|
||||
return text.strip() if text else None
|
||||
|
||||
def list_models(self) -> list[str]:
|
||||
"""Return available model IDs from the OpenAI-compatible API."""
|
||||
try:
|
||||
|
||||
@@ -59,13 +59,6 @@ def get_review_field_guidelines(response_style: str = "default") -> dict[str, st
|
||||
}
|
||||
|
||||
|
||||
# Explains the per-frame labels and tracker notes used by the annotated frame
|
||||
# mode. Neither the notes nor this guidance say whether repeated detections are
|
||||
# the same subject, since the tracking data cannot tell.
|
||||
FRAME_ANNOTATION_GUIDANCE = """- Each image below is immediately preceded by a text label giving its frame number and how many seconds into the sequence it was captured. Use these labels to track the order of events and the time between them.
|
||||
- Some images below are preceded by notes from the camera's object tracker recording what changed at that point: an object being first detected, starting to move, reversing direction, stopping, or no longer being detected. These notes come from tracking data rather than from the images, and they are reliable. Use them to establish how many distinct activities occur and in what order, and describe every one of them."""
|
||||
|
||||
|
||||
def build_review_description_prompt(
|
||||
review_data: dict[str, Any],
|
||||
thumbnails: list[bytes],
|
||||
@@ -73,13 +66,8 @@ def build_review_description_prompt(
|
||||
preferred_language: str | None,
|
||||
activity_context_prompt: str,
|
||||
response_style: str = "default",
|
||||
frame_captions: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Build the prompt for review activity description generation.
|
||||
|
||||
When `frame_captions` is set, each caption is sent directly before its
|
||||
image, so the prompt explains that layout.
|
||||
"""
|
||||
"""Build the prompt for review activity description generation."""
|
||||
|
||||
def get_concern_prompt() -> str:
|
||||
if concerns:
|
||||
@@ -105,7 +93,6 @@ def build_review_description_prompt(
|
||||
return "\n- (No objects detected)"
|
||||
|
||||
fields = get_review_field_guidelines(response_style)
|
||||
frame_guidance = f"\n{FRAME_ANNOTATION_GUIDANCE}" if frame_captions else ""
|
||||
|
||||
return f"""
|
||||
Your task is to analyze a sequence of images taken in chronological order from a security camera.
|
||||
@@ -143,7 +130,7 @@ Respond with a JSON object matching the provided schema. Field-specific guidance
|
||||
## Sequence Details
|
||||
|
||||
- Camera: {review_data["camera"]}
|
||||
- Total frames: {len(thumbnails)} (Frame 1 = earliest, Frame {len(thumbnails)} = latest){frame_guidance}
|
||||
- Total frames: {len(thumbnails)} (Frame 1 = earliest, Frame {len(thumbnails)} = latest)
|
||||
- Activity started at {review_data["start"]} and lasted {review_data["duration"]} seconds
|
||||
- Zones involved: {", ".join(review_data["zones"]) if review_data["zones"] else "None"}
|
||||
|
||||
|
||||
@@ -7,25 +7,6 @@ from typing import Any
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def interleave_images(
|
||||
prompt: str, images: list[bytes], captions: list[str] | None = None
|
||||
) -> list[str | bytes]:
|
||||
"""The prompt, then each image preceded by its caption when one is given.
|
||||
|
||||
Providers map the text and image parts onto their own request format, so
|
||||
every provider sends the same order.
|
||||
"""
|
||||
parts: list[str | bytes] = [prompt]
|
||||
|
||||
for index, image in enumerate(images):
|
||||
if captions and index < len(captions):
|
||||
parts.append(captions[index])
|
||||
|
||||
parts.append(image)
|
||||
|
||||
return parts
|
||||
|
||||
|
||||
def parse_tool_calls_from_message(
|
||||
message: dict[str, Any],
|
||||
) -> list[dict[str, Any]] | None:
|
||||
|
||||
@@ -144,14 +144,6 @@ class LogPipe(threading.Thread):
|
||||
self.pipeReader.close()
|
||||
|
||||
def dump(self) -> None:
|
||||
if not self.deque:
|
||||
return
|
||||
|
||||
self.logger.log(
|
||||
self.level,
|
||||
"The following ffmpeg logs include the last 100 lines prior to exit.",
|
||||
)
|
||||
|
||||
while len(self.deque) > 0:
|
||||
self.logger.log(self.level, self.deque.popleft())
|
||||
|
||||
|
||||
@@ -188,12 +188,8 @@ class ImprovedMotionDetector(MotionDetector):
|
||||
self.config.skip_motion_threshold is not None
|
||||
and pct_motion > self.config.skip_motion_threshold
|
||||
):
|
||||
# recalibrate so we transition to the new background. the frame
|
||||
# still has to be blended in here, otherwise the background stays
|
||||
# frozen and every subsequent frame skips as well
|
||||
# force a recalibration so we transition to the new background
|
||||
self.calibrating = True
|
||||
cv2.accumulateWeighted(resized_frame, self.avg_frame, 0.2)
|
||||
self.motion_frame_count = 0
|
||||
return []
|
||||
|
||||
# once the motion is less than 5% and the number of contours is < 4, assume its calibrated
|
||||
|
||||
@@ -319,19 +319,6 @@ class NoticeRegistry:
|
||||
if row.kind in NOTICE_KINDS
|
||||
]
|
||||
|
||||
def mark_reported(self, snapshot: list[dict[str, Any]]) -> None:
|
||||
"""Move the analytics watermarks to the counts a sent report was built from.
|
||||
|
||||
Using the snapshot rather than the current counts sends anything raised
|
||||
while the report was in flight with the next one.
|
||||
"""
|
||||
with self._lock:
|
||||
for row in snapshot:
|
||||
NoticeStats.update(
|
||||
reported_occurrences=row["occurrences"],
|
||||
reported_dismissals=row["dismissals"],
|
||||
).where(NoticeStats.kind == row["kind"]).execute()
|
||||
|
||||
def _write_repeats(
|
||||
self,
|
||||
row: Notice,
|
||||
|
||||
@@ -86,16 +86,6 @@ _KINDS = (
|
||||
link="https://github.com/blakeblackshear/frigate/releases/tag/v{version}",
|
||||
counts_repeats=False,
|
||||
keep_latest=1,
|
||||
reportable=False,
|
||||
),
|
||||
# raised while analytics is off; the row keeps a dismissal across restarts
|
||||
NoticeKind(
|
||||
"analytics_prompt",
|
||||
NoticeSeverity.info,
|
||||
"system",
|
||||
link="/settings?page=systemTelemetry",
|
||||
counts_repeats=False,
|
||||
reportable=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -66,12 +66,6 @@ def get_cache_image_name(camera: str, frame_time: float) -> str:
|
||||
)
|
||||
|
||||
|
||||
def is_camera_preview_frame(file_name: str, camera: str) -> bool:
|
||||
"""Check whether a cached preview frame file belongs to the camera."""
|
||||
# camera names may contain "-", so a prefix match would include "front-door"
|
||||
return file_name.rsplit("-", 1)[0] == f"preview_{camera}"
|
||||
|
||||
|
||||
def get_most_recent_preview_frame(
|
||||
camera: str, before: float | None = None
|
||||
) -> str | None:
|
||||
@@ -85,7 +79,7 @@ def get_most_recent_preview_frame(
|
||||
preview_files = [
|
||||
f
|
||||
for f in os.listdir(PREVIEW_CACHE_DIR)
|
||||
if is_camera_preview_frame(f, camera)
|
||||
if f.startswith(f"preview_{camera}-")
|
||||
and f.endswith(f".{PREVIEW_FRAME_TYPE}")
|
||||
]
|
||||
|
||||
@@ -282,7 +276,7 @@ class PreviewRecorder:
|
||||
start_file = f"{file_start}{start_ts}.webp"
|
||||
|
||||
for file in sorted(os.listdir(os.path.join(CACHE_DIR, FOLDER_PREVIEW_FRAMES))):
|
||||
if not is_camera_preview_frame(file, self.camera_name):
|
||||
if not file.startswith(file_start):
|
||||
continue
|
||||
|
||||
if file < start_file:
|
||||
|
||||
@@ -1120,18 +1120,6 @@ class OnvifController:
|
||||
f"Camera {camera_name} is still in ONVIF 'MOVING' status."
|
||||
)
|
||||
|
||||
async def _shutdown(self) -> None:
|
||||
"""Close the camera sessions and cancel the tasks running on the loop."""
|
||||
for cam_name in list(self.cams):
|
||||
await self._close_camera(cam_name)
|
||||
|
||||
tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
|
||||
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Gracefully shut down the ONVIF controller."""
|
||||
if not hasattr(self, "loop") or self.loop.is_closed():
|
||||
@@ -1139,16 +1127,6 @@ class OnvifController:
|
||||
return
|
||||
|
||||
logger.info("Exiting ONVIF controller...")
|
||||
|
||||
# anything left open here is garbage collected during interpreter
|
||||
# shutdown, where its warnings can no longer be logged cleanly
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(self._shutdown(), self.loop).result(
|
||||
timeout=5
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.debug("Timed out closing ONVIF sessions")
|
||||
|
||||
self.config_subscriber.stop()
|
||||
|
||||
def stop_and_cleanup():
|
||||
|
||||
@@ -36,7 +36,6 @@ from frigate.ffmpeg_presets import (
|
||||
parse_preset_hardware_acceleration_encode,
|
||||
)
|
||||
from frigate.models import Export, Previews, Recordings, ReviewSegment
|
||||
from frigate.output.preview import is_camera_preview_frame
|
||||
from frigate.util.ffmpeg import run_ffmpeg_with_progress
|
||||
from frigate.util.ownership import chown_to_runtime
|
||||
from frigate.util.recording_coverage import (
|
||||
@@ -1099,7 +1098,7 @@ class RecordingExporter(threading.Thread):
|
||||
fallback_preview = None
|
||||
|
||||
for file in sorted(os.listdir(preview_dir)):
|
||||
if not is_camera_preview_frame(file, self.camera):
|
||||
if not file.startswith(file_start):
|
||||
continue
|
||||
|
||||
if file < start_file:
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Shared fixtures for the analytics tests."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.config import FrigateConfig
|
||||
|
||||
FRONT_CAMERA: dict[str, Any] = {
|
||||
"ffmpeg": {"inputs": [{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}]},
|
||||
"detect": {"height": 720, "width": 1280, "fps": 5},
|
||||
}
|
||||
|
||||
|
||||
def make_config(config: dict[str, Any] | None = None) -> FrigateConfig:
|
||||
"""A parsed config; top-level keys in config replace the defaults here.
|
||||
|
||||
hwaccel_args is set so parsing never probes the test host's GPU.
|
||||
"""
|
||||
base: dict[str, Any] = {
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"ffmpeg": {"hwaccel_args": []},
|
||||
"cameras": {"front": FRONT_CAMERA},
|
||||
}
|
||||
return FrigateConfig(**{**base, **(config or {})})
|
||||
|
||||
|
||||
def make_context(
|
||||
config: FrigateConfig | None = None,
|
||||
stats: dict[str, Any] | None = None,
|
||||
notice_stats: list[dict[str, Any]] | None = None,
|
||||
) -> ReportContext:
|
||||
return ReportContext(
|
||||
config=config or make_config(),
|
||||
stats=stats or {},
|
||||
notice_stats=notice_stats or [],
|
||||
)
|
||||
@@ -1,49 +0,0 @@
|
||||
"""Tests for the analytics preview API."""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from frigate.analytics import report
|
||||
from frigate.analytics.collectors import hardware
|
||||
from frigate.models import Notice, NoticeStats, User
|
||||
from frigate.notices.registry import NoticeRegistry
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
|
||||
|
||||
class TestHttpAnalytics(BaseTestHttp):
|
||||
def setUp(self):
|
||||
super().setUp([Notice, NoticeStats, User])
|
||||
stats = Mock()
|
||||
stats.get_latest_stats.return_value = self.test_stats
|
||||
self.app = self.create_app(stats=stats, notice_registry=NoticeRegistry())
|
||||
|
||||
for target, name, value in (
|
||||
(report, "load_state", None),
|
||||
(hardware.hardware_prober, "probe", []),
|
||||
(hardware, "hwaccel_options", ("", [])),
|
||||
):
|
||||
patcher = patch.object(target, name, return_value=value)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def test_preview_returns_the_report_without_sending_it(self):
|
||||
with (
|
||||
patch("frigate.analytics.transport.requests.post") as post,
|
||||
AuthTestClient(self.app) as client,
|
||||
):
|
||||
response = client.get("/analytics/preview")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
body = response.json()
|
||||
self.assertEqual(body["install_id"], report.PREVIEW_INSTALL_ID)
|
||||
self.assertEqual(body["schema_version"], 1)
|
||||
self.assertEqual(body["cameras"]["total"], 1)
|
||||
post.assert_not_called()
|
||||
|
||||
def test_preview_is_admin_only(self):
|
||||
with AuthTestClient(self.app) as client:
|
||||
response = client.get(
|
||||
"/analytics/preview",
|
||||
headers={"remote-user": "viewer", "remote-role": "viewer"},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 403)
|
||||
@@ -1,81 +0,0 @@
|
||||
"""Tests that /auth only accepts a JWT whose role is still in the config."""
|
||||
|
||||
import os
|
||||
import time
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from frigate.api.auth import create_encoded_jwt
|
||||
from frigate.api.fastapi_app import create_fastapi_app
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.camera.updater import CameraConfigUpdatePublisher
|
||||
from frigate.const import JWT_SECRET_ENV_VAR
|
||||
from frigate.models import Event, Recordings, ReviewSegment
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
|
||||
|
||||
@patch.dict(os.environ, {JWT_SECRET_ENV_VAR: "test-secret"})
|
||||
class TestAuthJwtRole(BaseTestHttp):
|
||||
def setUp(self):
|
||||
super().setUp(models=[Event, Recordings, ReviewSegment])
|
||||
self.minimal_config = {
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"auth": {"enabled": True, "roles": {"garage": ["front_door"]}},
|
||||
"networking": {"listen": {"internal": 5000, "external": 8971}},
|
||||
"cameras": {
|
||||
"front_door": {
|
||||
"ffmpeg": {
|
||||
"inputs": [
|
||||
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
|
||||
]
|
||||
},
|
||||
"detect": {
|
||||
"height": 1080,
|
||||
"width": 1920,
|
||||
"fps": 5,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def _create_app(self):
|
||||
mock_publisher = Mock(spec=CameraConfigUpdatePublisher)
|
||||
mock_publisher.publisher = MagicMock()
|
||||
|
||||
return create_fastapi_app(
|
||||
FrigateConfig(**self.minimal_config),
|
||||
self.db,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
mock_publisher,
|
||||
None,
|
||||
enforce_default_admin=False,
|
||||
)
|
||||
|
||||
def _auth(self, app, role: str):
|
||||
token = create_encoded_jwt("bob", role, int(time.time()) + 3600, app.jwt_token)
|
||||
|
||||
with AuthTestClient(app) as client:
|
||||
return client.get(
|
||||
"/auth",
|
||||
headers={
|
||||
"x-server-port": "8971",
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
def test_configured_role_is_accepted(self):
|
||||
resp = self._auth(self._create_app(), "garage")
|
||||
|
||||
self.assertEqual(resp.status_code, 202)
|
||||
self.assertEqual(resp.headers["remote-user"], "bob")
|
||||
self.assertEqual(resp.headers["remote-role"], "garage")
|
||||
|
||||
def test_role_removed_from_config_is_rejected(self):
|
||||
resp = self._auth(self._create_app(), "removed_role")
|
||||
|
||||
self.assertEqual(resp.status_code, 401)
|
||||
self.assertNotIn("remote-role", resp.headers)
|
||||
@@ -1,193 +0,0 @@
|
||||
"""Tests for renaming and deleting a zone through config_set's JSON body."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import ruamel.yaml
|
||||
from fastapi import Request
|
||||
|
||||
from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user
|
||||
from frigate.api.fastapi_app import create_fastapi_app
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.camera.updater import CameraConfigUpdatePublisher
|
||||
from frigate.models import Event, Recordings, ReviewSegment
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
|
||||
|
||||
class TestConfigSetZones(BaseTestHttp):
|
||||
def setUp(self):
|
||||
super().setUp(models=[Event, Recordings, ReviewSegment])
|
||||
self.minimal_config = {
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"profiles": {"armed": {"friendly_name": "Armed"}},
|
||||
"snapshots": {"required_zones": ["driveway"]},
|
||||
"cameras": {
|
||||
"front": {
|
||||
"ffmpeg": {
|
||||
"inputs": [
|
||||
{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
|
||||
]
|
||||
},
|
||||
"detect": {"height": 1080, "width": 1920, "fps": 5},
|
||||
"zones": {
|
||||
"driveway": {
|
||||
"coordinates": "0,0,1,0,1,1",
|
||||
"inertia": 5,
|
||||
"filters": {"person": {"min_area": 5000}},
|
||||
},
|
||||
"porch": {"coordinates": "0,0,0.5,0,0.5,0.5"},
|
||||
},
|
||||
"review": {
|
||||
"alerts": {
|
||||
"labels": ["person"],
|
||||
"required_zones": ["driveway"],
|
||||
},
|
||||
},
|
||||
"mqtt": {"required_zones": ["driveway", "porch"]},
|
||||
"profiles": {
|
||||
"armed": {
|
||||
"zones": {
|
||||
"driveway": {
|
||||
"coordinates": "0,0,1,0,1,1",
|
||||
"objects": ["car"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
yaml = ruamel.yaml.YAML()
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".yml", delete=False
|
||||
) as config_file:
|
||||
yaml.dump(self.minimal_config, config_file)
|
||||
self.config_path = config_file.name
|
||||
self.addCleanup(os.unlink, self.config_path)
|
||||
|
||||
def _create_app(self):
|
||||
publisher = Mock(spec=CameraConfigUpdatePublisher)
|
||||
publisher.publisher = MagicMock()
|
||||
app = create_fastapi_app(
|
||||
FrigateConfig(**self.minimal_config),
|
||||
self.db,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
publisher,
|
||||
None,
|
||||
enforce_default_admin=False,
|
||||
)
|
||||
|
||||
async def mock_get_current_user(request: Request):
|
||||
return {
|
||||
"username": request.headers.get("remote-user"),
|
||||
"role": request.headers.get("remote-role"),
|
||||
}
|
||||
|
||||
async def mock_get_allowed_cameras_for_filter(request: Request):
|
||||
return ["front"]
|
||||
|
||||
app.dependency_overrides[get_current_user] = mock_get_current_user
|
||||
app.dependency_overrides[get_allowed_cameras_for_filter] = (
|
||||
mock_get_allowed_cameras_for_filter
|
||||
)
|
||||
return app
|
||||
|
||||
def _put(self, camera_data: dict):
|
||||
with patch("frigate.api.app.find_config_file", return_value=self.config_path):
|
||||
with AuthTestClient(self._create_app()) as client:
|
||||
return client.put(
|
||||
"/config/set",
|
||||
json={
|
||||
"config_data": {"cameras": {"front": camera_data}},
|
||||
"requires_restart": 0,
|
||||
"update_topic": "config/cameras/front/zones",
|
||||
},
|
||||
)
|
||||
|
||||
def _front(self) -> dict:
|
||||
with open(self.config_path) as f:
|
||||
return ruamel.yaml.YAML().load(f)["cameras"]["front"]
|
||||
|
||||
def test_rename_moves_zone_references_in_one_request(self):
|
||||
"""The new base zone exists when validation checks the moved override."""
|
||||
resp = self._put(
|
||||
{
|
||||
"zones": {
|
||||
"driveway": None,
|
||||
"front_drive": {
|
||||
"coordinates": "0,0,1,0,1,1",
|
||||
"enabled": True,
|
||||
# as /api/config returns them, with defaults filled in
|
||||
"filters": {
|
||||
"person": {
|
||||
"min_area": 5000,
|
||||
"max_area": 24000000,
|
||||
"min_ratio": 0.0,
|
||||
"max_ratio": 24000000.0,
|
||||
"threshold": 0.7,
|
||||
"min_score": 0.5,
|
||||
"mask": {},
|
||||
}
|
||||
},
|
||||
"inertia": 5,
|
||||
},
|
||||
},
|
||||
"review": {"alerts": {"required_zones": ["front_drive"]}},
|
||||
# inherited from the global snapshots config, so no camera key
|
||||
"snapshots": {"required_zones": ["front_drive"]},
|
||||
"mqtt": {"required_zones": ["front_drive", "porch"]},
|
||||
"profiles": {
|
||||
"armed": {
|
||||
"zones": {
|
||||
"driveway": None,
|
||||
"front_drive": {
|
||||
"coordinates": "0,0,1,0,1,1",
|
||||
"objects": ["car"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(resp.status_code, 200, resp.json())
|
||||
front = self._front()
|
||||
self.assertEqual(list(front["zones"]), ["porch", "front_drive"])
|
||||
self.assertEqual(front["zones"]["front_drive"]["inertia"], 5)
|
||||
self.assertEqual(
|
||||
front["zones"]["front_drive"]["filters"]["person"]["min_area"], 5000
|
||||
)
|
||||
self.assertEqual(front["review"]["alerts"]["labels"], ["person"])
|
||||
self.assertEqual(front["review"]["alerts"]["required_zones"], ["front_drive"])
|
||||
self.assertEqual(front["snapshots"]["required_zones"], ["front_drive"])
|
||||
self.assertEqual(front["mqtt"]["required_zones"], ["front_drive", "porch"])
|
||||
self.assertEqual(
|
||||
dict(front["profiles"]["armed"]["zones"]),
|
||||
{"front_drive": {"coordinates": "0,0,1,0,1,1", "objects": ["car"]}},
|
||||
)
|
||||
|
||||
def test_delete_empties_lists_without_dropping_their_section(self):
|
||||
resp = self._put(
|
||||
{
|
||||
"zones": {"driveway": None},
|
||||
"review": {"alerts": {"required_zones": []}},
|
||||
"snapshots": {"required_zones": []},
|
||||
"mqtt": {"required_zones": ["porch"]},
|
||||
"profiles": {"armed": {"zones": {"driveway": None}}},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(resp.status_code, 200, resp.json())
|
||||
front = self._front()
|
||||
self.assertEqual(list(front["zones"]), ["porch"])
|
||||
self.assertEqual(front["review"]["alerts"]["labels"], ["person"])
|
||||
self.assertEqual(front["review"]["alerts"]["required_zones"], [])
|
||||
self.assertEqual(front["mqtt"]["required_zones"], ["porch"])
|
||||
self.assertNotIn("driveway", front["profiles"]["armed"]["zones"] or {})
|
||||
@@ -1,112 +0,0 @@
|
||||
"""Tests for the cameras collector."""
|
||||
|
||||
import unittest
|
||||
|
||||
from frigate.analytics.collectors import cameras
|
||||
from frigate.analytics.schema import HeightBucket, HwaccelKey, RetainBucket
|
||||
from frigate.test.analytics_helpers import make_config, make_context
|
||||
|
||||
MASK = {"coordinates": "0,0,1,0,1,1"}
|
||||
|
||||
CONFIG = {
|
||||
"record": {
|
||||
"enabled": True,
|
||||
"continuous": {"days": 3},
|
||||
"alerts": {"retain": {"days": 14}},
|
||||
"detections": {"retain": {"days": 45}},
|
||||
},
|
||||
"cameras": {
|
||||
"front": {
|
||||
"ffmpeg": {
|
||||
"hwaccel_args": "preset-vaapi",
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://127.0.0.1:8554/front_sub",
|
||||
"roles": ["detect"],
|
||||
"input_args": "preset-rtsp-restream",
|
||||
},
|
||||
{"path": "rtsp://127.0.0.1:8554/front", "roles": ["record"]},
|
||||
],
|
||||
},
|
||||
"detect": {"enabled": True, "height": 720, "width": 1280, "fps": 5},
|
||||
"zones": {"porch": MASK, "yard": MASK},
|
||||
"motion": {"mask": {"tree": MASK}},
|
||||
"objects": {
|
||||
"mask": {"sign": MASK},
|
||||
"filters": {"person": {"mask": {"bush": MASK}}},
|
||||
},
|
||||
"onvif": {"host": "10.0.0.9"},
|
||||
},
|
||||
"garage": {
|
||||
"enabled": False,
|
||||
"type": "lpr",
|
||||
"ffmpeg": {
|
||||
"hwaccel_args": ["-hwaccel", "vaapi"],
|
||||
"inputs": [
|
||||
{"path": "rtsp://10.0.0.5/stream", "roles": ["detect", "record"]}
|
||||
],
|
||||
},
|
||||
"detect": {"height": 2160, "width": 3840, "fps": 15},
|
||||
"record": {"enabled": False},
|
||||
},
|
||||
"_replay_front": {
|
||||
"ffmpeg": {
|
||||
"inputs": [{"path": "rtsp://10.0.0.1/replay", "roles": ["detect"]}]
|
||||
},
|
||||
"detect": {"height": 1080, "width": 1920, "fps": 5},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
STATS = {
|
||||
"cameras": {
|
||||
"front": {"connection_quality": "excellent"},
|
||||
"garage": {"connection_quality": "offline"},
|
||||
"_replay_front": {"connection_quality": "fair"},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestCamerasCollector(unittest.TestCase):
|
||||
def test_aggregates_cameras_and_skips_replays(self):
|
||||
section = cameras.collect(make_context(make_config(CONFIG), STATS))
|
||||
|
||||
self.assertEqual((section.total, section.enabled), (2, 1))
|
||||
self.assertEqual(section.types, {"generic": 1, "lpr": 1})
|
||||
self.assertEqual(section.detect_height, {"720": 1, "ge_2160": 1})
|
||||
self.assertEqual(section.detect_fps, {"le_5": 1, "gt_10": 1})
|
||||
self.assertEqual(section.hwaccel, {"vaapi": 1, "custom": 1})
|
||||
self.assertEqual(section.input_preset, {"rtsp-restream": 1, "rtsp-generic": 1})
|
||||
self.assertEqual(section.go2rtc_restream, 1)
|
||||
self.assertEqual(section.separate_detect_stream, 1)
|
||||
self.assertEqual((section.detect, section.record, section.onvif), (1, 1, 1))
|
||||
self.assertEqual((section.zones, section.cameras_with_zones), (2, 1))
|
||||
self.assertEqual((section.motion_masks, section.object_masks), (1, 2))
|
||||
self.assertEqual(section.connection_quality, {"excellent": 1})
|
||||
self.assertEqual(section.retain_days.continuous, {"1_7": 1})
|
||||
self.assertEqual(section.retain_days.motion, {"0": 1})
|
||||
self.assertEqual(section.retain_days.alerts, {"8_30": 1})
|
||||
self.assertEqual(section.retain_days.detections, {"gt_30": 1})
|
||||
|
||||
def test_buckets(self):
|
||||
self.assertEqual(cameras.height_bucket(360), HeightBucket.le_360)
|
||||
self.assertEqual(cameras.height_bucket(361), HeightBucket.h480)
|
||||
self.assertEqual(cameras.height_bucket(1080), HeightBucket.h1080)
|
||||
self.assertEqual(cameras.height_bucket(1440), HeightBucket.h1440)
|
||||
self.assertEqual(cameras.height_bucket(2160), HeightBucket.ge_2160)
|
||||
self.assertEqual(cameras.retain_bucket(0.5), RetainBucket.d1_7)
|
||||
self.assertEqual(cameras.retain_bucket(0), RetainBucket.zero)
|
||||
|
||||
def test_preset_key(self):
|
||||
self.assertEqual(cameras.preset_key([], HwaccelKey), "none")
|
||||
self.assertEqual(cameras.preset_key("", HwaccelKey), "none")
|
||||
self.assertEqual(cameras.preset_key("preset-bogus", HwaccelKey), "custom")
|
||||
self.assertEqual(cameras.preset_key("-hwaccel vaapi", HwaccelKey), "custom")
|
||||
self.assertEqual(cameras.preset_key("preset-nvidia", HwaccelKey), "nvidia")
|
||||
|
||||
def test_is_restream(self):
|
||||
self.assertTrue(cameras.is_restream("rtsp://127.0.0.1:8554/front"))
|
||||
self.assertTrue(cameras.is_restream("rtsp://localhost:8554/front"))
|
||||
self.assertFalse(cameras.is_restream("rtsp://10.0.0.5:8554/front"))
|
||||
self.assertFalse(cameras.is_restream("rtsp://[bad"))
|
||||
self.assertFalse(cameras.is_restream("rtsp://127.0.0.1:notaport/x"))
|
||||
@@ -1,61 +0,0 @@
|
||||
"""Tests for the detection collector."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from frigate.analytics.collectors import detection
|
||||
from frigate.analytics.schema import ModelSource
|
||||
from frigate.detectors.detector_config import SceneEnum
|
||||
from frigate.test.analytics_helpers import make_config, make_context
|
||||
|
||||
|
||||
class TestDetectionCollector(unittest.TestCase):
|
||||
def test_reports_each_model_with_its_mean_inference_speed(self):
|
||||
config = make_config({"models": [{"devices": ["cpu", "cpu"]}]})
|
||||
stats = {
|
||||
"detectors": {
|
||||
"cpu": {"inference_speed": 10.0},
|
||||
"cpu#2": {"inference_speed": 20.0},
|
||||
},
|
||||
"detection_fps": 12.346,
|
||||
"skipped_fps": 0,
|
||||
}
|
||||
|
||||
section = detection.collect(make_context(config, stats))
|
||||
|
||||
model = section.models[SceneEnum.all]
|
||||
self.assertEqual(model.detector, "cpu")
|
||||
self.assertEqual(model.devices, 2)
|
||||
self.assertEqual(model.model_type, "ssd")
|
||||
self.assertEqual(model.input, "320x320")
|
||||
self.assertEqual(model.source, ModelSource.default)
|
||||
self.assertEqual(model.inference_ms, 15.0)
|
||||
self.assertEqual(section.detection_fps, 12.35)
|
||||
self.assertEqual(section.skipped_fps, 0.0)
|
||||
|
||||
def test_inference_is_null_before_the_first_stats(self):
|
||||
section = detection.collect(make_context())
|
||||
|
||||
self.assertIsNone(section.models[SceneEnum.all].inference_ms)
|
||||
|
||||
def test_model_source(self):
|
||||
with tempfile.TemporaryDirectory() as cache:
|
||||
plus_model = os.path.join(cache, "abc123")
|
||||
open(f"{plus_model}.json", "w").close()
|
||||
|
||||
with patch.object(detection, "MODEL_CACHE_DIR", cache):
|
||||
self.assertEqual(detection.model_source(plus_model), ModelSource.plus)
|
||||
self.assertEqual(
|
||||
detection.model_source(os.path.join(cache, "no_info")),
|
||||
ModelSource.custom,
|
||||
)
|
||||
|
||||
self.assertEqual(detection.model_source(None), ModelSource.default)
|
||||
self.assertEqual(
|
||||
detection.model_source("/cpu_model.tflite"), ModelSource.default
|
||||
)
|
||||
self.assertEqual(
|
||||
detection.model_source("/config/yolo.onnx"), ModelSource.custom
|
||||
)
|
||||
@@ -1,128 +0,0 @@
|
||||
"""Tests for the features collector."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from peewee_migrate import Router
|
||||
from playhouse.sqlite_ext import SqliteExtDatabase
|
||||
from playhouse.sqliteq import SqliteQueueDatabase
|
||||
|
||||
from frigate.analytics.collectors import features
|
||||
from frigate.analytics.schema import EnrichmentDevice, SemanticSearchModel
|
||||
from frigate.models import User
|
||||
from frigate.test.analytics_helpers import FRONT_CAMERA, make_config, make_context
|
||||
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
|
||||
|
||||
CONFIG = {
|
||||
"mqtt": {"host": "mqtt", "enabled": True},
|
||||
"genai": {
|
||||
"local": {
|
||||
"provider": "ollama",
|
||||
"model": "llava",
|
||||
"base_url": "http://ollama:11434",
|
||||
"roles": ["descriptions", "embeddings"],
|
||||
}
|
||||
},
|
||||
"semantic_search": {"enabled": True, "model": "local"},
|
||||
"face_recognition": {"enabled": True, "model_size": "large"},
|
||||
"birdseye": {"enabled": True, "modes": ["motion", "alerts"], "restream": True},
|
||||
"proxy": {"header_map": {"user": "x-forwarded-user"}},
|
||||
"camera_groups": {"outside": {"cameras": ["front"], "icon": "LuCar", "order": 0}},
|
||||
"cameras": {
|
||||
"front": {
|
||||
**FRONT_CAMERA,
|
||||
"semantic_search": {
|
||||
"triggers": {
|
||||
"cat": {
|
||||
"type": "description",
|
||||
"data": "a cat",
|
||||
"threshold": 0.8,
|
||||
"actions": ["notification"],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestFeaturesCollector(unittest.TestCase):
|
||||
def setUp(self):
|
||||
migrate_db = SqliteExtDatabase(TEST_DB)
|
||||
del logging.getLogger("peewee_migrate").handlers[:]
|
||||
Router(migrate_db).run()
|
||||
migrate_db.close()
|
||||
self.db = SqliteQueueDatabase(TEST_DB)
|
||||
self.db.bind([User])
|
||||
|
||||
def tearDown(self):
|
||||
# close() leaves the queue's writer thread running against the deleted file
|
||||
self.db.stop()
|
||||
|
||||
if not self.db.is_closed():
|
||||
self.db.close()
|
||||
|
||||
for file in TEST_DB_CLEANUPS:
|
||||
try:
|
||||
os.remove(file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_collects_enrichments_genai_integrations_and_users(self):
|
||||
for username, role in (("a", "admin"), ("v", "viewer"), ("o", "operator")):
|
||||
User.create(
|
||||
username=username, role=role, password_hash="x", notification_tokens=[]
|
||||
)
|
||||
|
||||
stats = {
|
||||
"embeddings": {
|
||||
"devices": {
|
||||
"face_recognition": "OpenVINO GPU.0",
|
||||
"semantic_search": "CUDA",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
section = features.collect(make_context(make_config(CONFIG), stats))
|
||||
|
||||
self.assertEqual(section.face_recognition.device, EnrichmentDevice.openvino_gpu)
|
||||
self.assertEqual(section.face_recognition.model_size, "large")
|
||||
self.assertIsNone(section.lpr.device)
|
||||
self.assertEqual(section.semantic_search.model, SemanticSearchModel.genai)
|
||||
self.assertEqual(section.semantic_search.device, EnrichmentDevice.cuda)
|
||||
self.assertEqual(section.semantic_search.triggers, 1)
|
||||
self.assertEqual(section.genai.providers, {"ollama": 1})
|
||||
self.assertEqual(section.genai.roles, {"descriptions": 1, "embeddings": 1})
|
||||
self.assertEqual(section.birdseye.modes, ["motion", "alerts"])
|
||||
self.assertTrue(section.mqtt)
|
||||
self.assertTrue(section.proxy_auth)
|
||||
self.assertEqual(section.users, {"admin": 1, "viewer": 1, "custom": 1})
|
||||
self.assertEqual((section.camera_groups, section.profiles), (1, 0))
|
||||
self.assertFalse(section.plus_api_key)
|
||||
|
||||
|
||||
class TestFeatureMappings(unittest.TestCase):
|
||||
def test_enrichment_device_labels(self):
|
||||
cases = {
|
||||
"CPU": EnrichmentDevice.cpu,
|
||||
"MIGraphX": EnrichmentDevice.migraphx,
|
||||
"OpenVINO NPU": EnrichmentDevice.openvino_npu,
|
||||
"OpenVINO GPU.0,CPU": EnrichmentDevice.openvino_gpu,
|
||||
"OpenVINO": EnrichmentDevice.other,
|
||||
"CoreML": EnrichmentDevice.other,
|
||||
}
|
||||
|
||||
for label, device in cases.items():
|
||||
with self.subTest(label=label):
|
||||
self.assertEqual(features.enrichment_device(label), device)
|
||||
|
||||
self.assertIsNone(features.enrichment_device(None))
|
||||
|
||||
def test_models_named_after_a_provider_are_genai(self):
|
||||
self.assertEqual(features.semantic_model("jinav2"), SemanticSearchModel.jinav2)
|
||||
self.assertEqual(
|
||||
features.semantic_model("my-ollama"), SemanticSearchModel.genai
|
||||
)
|
||||
self.assertIsNone(features.semantic_model(None))
|
||||
self.assertEqual(features.transcription_model("openai"), "genai")
|
||||
@@ -1,65 +0,0 @@
|
||||
"""Tests for the health collector."""
|
||||
|
||||
import unittest
|
||||
|
||||
from frigate.analytics.collectors import health
|
||||
from frigate.test.analytics_helpers import make_context
|
||||
|
||||
|
||||
class TestHealthCollector(unittest.TestCase):
|
||||
def test_collects_uptime_cpu_timings_and_notice_deltas(self):
|
||||
stats = {
|
||||
"service": {"uptime": 7300, "retention_unmet": True},
|
||||
"cpu_usages": {"frigate.full_system": {"cpu": "23.6", "mem": "40"}},
|
||||
"embeddings": {
|
||||
"face_recognition_speed": 41.234,
|
||||
"text_embedding_speed": 0,
|
||||
"devices": {},
|
||||
},
|
||||
}
|
||||
notice_stats = [
|
||||
{
|
||||
"kind": "detector_stuck",
|
||||
"occurrences": 5,
|
||||
"dismissals": 1,
|
||||
"reported_occurrences": 3,
|
||||
"reported_dismissals": 1,
|
||||
},
|
||||
{
|
||||
"kind": "shm_too_low",
|
||||
"occurrences": 2,
|
||||
"dismissals": 0,
|
||||
"reported_occurrences": 2,
|
||||
"reported_dismissals": 0,
|
||||
},
|
||||
{
|
||||
"kind": "update_available",
|
||||
"occurrences": 4,
|
||||
"dismissals": 0,
|
||||
"reported_occurrences": 0,
|
||||
"reported_dismissals": 0,
|
||||
},
|
||||
]
|
||||
|
||||
section = health.collect(make_context(stats=stats, notice_stats=notice_stats))
|
||||
|
||||
self.assertEqual(section.uptime_hours, 2)
|
||||
self.assertEqual(section.cpu_percent, 24)
|
||||
self.assertEqual(section.enrichment_ms, {"face": 41.23})
|
||||
self.assertTrue(section.retention_unmet)
|
||||
self.assertEqual(
|
||||
{
|
||||
kind: (c.occurrences, c.dismissals)
|
||||
for kind, c in section.notices.items()
|
||||
},
|
||||
{"detector_stuck": (2, 0)},
|
||||
)
|
||||
|
||||
def test_missing_stats_give_zeros(self):
|
||||
section = health.collect(make_context())
|
||||
|
||||
self.assertEqual(
|
||||
(section.uptime_hours, section.cpu_percent, section.enrichment_ms),
|
||||
(0, 0, {}),
|
||||
)
|
||||
self.assertEqual(section.notices, {})
|
||||
@@ -1,188 +0,0 @@
|
||||
"""Tests for the install and hardware collectors."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections import Counter
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from frigate.analytics.collectors import hardware, install
|
||||
from frigate.analytics.collectors.common import closed, histogram, rate
|
||||
from frigate.analytics.schema import (
|
||||
Arch,
|
||||
DecodeFamily,
|
||||
GpuVendor,
|
||||
HardwareKey,
|
||||
ImageVariant,
|
||||
InstallType,
|
||||
)
|
||||
from frigate.const import RECORD_DIR
|
||||
from frigate.test.analytics_helpers import make_context
|
||||
|
||||
|
||||
class TestCommon(unittest.TestCase):
|
||||
def test_closed_falls_back_for_unknown_values(self):
|
||||
self.assertEqual(closed(Arch, "x86_64", Arch.other), Arch.x86_64)
|
||||
self.assertEqual(closed(Arch, "sparc", Arch.other), Arch.other)
|
||||
self.assertEqual(closed(Arch, None, Arch.other), Arch.other)
|
||||
|
||||
def test_rate_rounds_and_rejects_bad_numbers(self):
|
||||
self.assertEqual(rate("12.346"), 12.35)
|
||||
self.assertEqual(rate(-3), 0.0)
|
||||
self.assertEqual(rate(float("nan")), 0.0)
|
||||
self.assertEqual(rate(float("inf")), 0.0)
|
||||
self.assertEqual(rate(None), 0.0)
|
||||
|
||||
def test_histogram_drops_empty_buckets(self):
|
||||
self.assertEqual(
|
||||
histogram(Counter({Arch.x86_64: 2, Arch.other: 0})), {"x86_64": 2}
|
||||
)
|
||||
|
||||
|
||||
class TestInstallCollector(unittest.TestCase):
|
||||
def test_collects_version_variant_and_platform(self):
|
||||
with (
|
||||
patch.dict(os.environ, {"FRIGATE_IMAGE_VARIANT": "tensorrt-jp6"}),
|
||||
patch.object(install, "install_type", return_value=InstallType.docker),
|
||||
patch("platform.machine", return_value="aarch64"),
|
||||
patch("platform.release", return_value="6.8.0-45-generic"),
|
||||
patch("os.geteuid", return_value=1000),
|
||||
):
|
||||
section = install.collect(make_context())
|
||||
|
||||
self.assertEqual(section.image_variant, ImageVariant.tensorrt_jp6)
|
||||
self.assertEqual(section.install_type, InstallType.docker)
|
||||
self.assertEqual(section.arch, Arch.aarch64)
|
||||
self.assertEqual(section.kernel, "6.8")
|
||||
self.assertFalse(section.run_as_root)
|
||||
|
||||
def test_image_variant_is_dev_when_unset_and_other_when_unknown(self):
|
||||
self.assertEqual(install.image_variant(None), ImageVariant.dev)
|
||||
self.assertEqual(install.image_variant(""), ImageVariant.dev)
|
||||
self.assertEqual(install.image_variant("h8l"), ImageVariant.other)
|
||||
|
||||
def test_install_type_checks_the_add_on_first(self):
|
||||
with patch("os.path.isfile", return_value=True):
|
||||
self.assertEqual(install.install_type(), InstallType.ha_addon)
|
||||
|
||||
with (
|
||||
patch("os.path.isfile", return_value=False),
|
||||
patch.dict(os.environ, {"KUBERNETES_SERVICE_HOST": "10.0.0.1"}),
|
||||
):
|
||||
self.assertEqual(install.install_type(), InstallType.kubernetes)
|
||||
|
||||
env = {k: v for k, v in os.environ.items() if k != "KUBERNETES_SERVICE_HOST"}
|
||||
|
||||
with (
|
||||
patch("os.path.isfile", return_value=False),
|
||||
patch.dict(os.environ, env, clear=True),
|
||||
patch("os.path.exists", side_effect=lambda path: path == "/.dockerenv"),
|
||||
):
|
||||
self.assertEqual(install.install_type(), InstallType.docker)
|
||||
|
||||
def test_kernel_and_arch_fall_back(self):
|
||||
self.assertEqual(install.kernel("weird"), "unknown")
|
||||
self.assertEqual(install.arch("armv7l"), Arch.other)
|
||||
self.assertEqual(install.arch("amd64"), Arch.x86_64)
|
||||
|
||||
|
||||
class TestHardwareCollector(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.dir.cleanup)
|
||||
|
||||
def write(self, name: str, content: str) -> str:
|
||||
path = os.path.join(self.dir.name, name)
|
||||
|
||||
with open(path, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
return path
|
||||
|
||||
def test_cpu_model_prefers_the_board_model(self):
|
||||
board = self.write("model", "Raspberry Pi 5 Model B Rev 1.0\x00")
|
||||
cpuinfo = self.write("cpuinfo", "model name\t: Cortex-A76\n")
|
||||
|
||||
with (
|
||||
patch.object(hardware, "DEVICE_TREE_MODEL", board),
|
||||
patch.object(hardware, "CPUINFO", cpuinfo),
|
||||
):
|
||||
self.assertEqual(hardware.cpu_model(), "Raspberry Pi 5 Model B Rev 1.0")
|
||||
|
||||
def test_cpu_model_reads_cpuinfo_and_truncates(self):
|
||||
cpuinfo = self.write("cpuinfo", "processor\t: 0\nmodel name\t: " + "x" * 80)
|
||||
|
||||
with (
|
||||
patch.object(hardware, "DEVICE_TREE_MODEL", self.dir.name + "/none"),
|
||||
patch.object(hardware, "CPUINFO", cpuinfo),
|
||||
):
|
||||
self.assertEqual(hardware.cpu_model(), "x" * 64)
|
||||
|
||||
def test_collects_gpus_decode_detection_and_storage(self):
|
||||
stats = {
|
||||
"gpu_usages": {
|
||||
"NVIDIA GeForce RTX 3060": {"vendor": "nvidia"},
|
||||
"mystery": {},
|
||||
},
|
||||
"service": {
|
||||
"storage": {
|
||||
RECORD_DIR: {
|
||||
"total": 2048000.0,
|
||||
"used": 512000.0,
|
||||
"mount_type": "nfs4",
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
probed = [
|
||||
SimpleNamespace(key="edgetpu:usb", count=2),
|
||||
SimpleNamespace(key="new:thing", count=1),
|
||||
SimpleNamespace(key="cpu", count=1),
|
||||
]
|
||||
families = [
|
||||
SimpleNamespace(key="vaapi"),
|
||||
SimpleNamespace(key="intel-qsv"),
|
||||
SimpleNamespace(key="future"),
|
||||
]
|
||||
|
||||
with (
|
||||
patch.object(hardware.hardware_prober, "probe", return_value=probed),
|
||||
patch.object(hardware, "hwaccel_options", return_value=("vaapi", families)),
|
||||
patch.object(hardware, "cpu_model", return_value="Intel(R) N100"),
|
||||
patch("os.cpu_count", return_value=4),
|
||||
patch(
|
||||
"psutil.virtual_memory", return_value=SimpleNamespace(total=16 * 2**30)
|
||||
),
|
||||
):
|
||||
section = hardware.collect(make_context(stats=stats))
|
||||
|
||||
self.assertEqual(section.cpu_model, "Intel(R) N100")
|
||||
self.assertEqual(section.cpu_cores, 4)
|
||||
self.assertEqual(section.memory_gb, 16)
|
||||
self.assertEqual(
|
||||
[(gpu.vendor, gpu.name) for gpu in section.gpus],
|
||||
[
|
||||
(GpuVendor.nvidia, "NVIDIA GeForce RTX 3060"),
|
||||
(GpuVendor.other, "mystery"),
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
section.decode_families,
|
||||
[DecodeFamily.vaapi, DecodeFamily.intel_qsv, DecodeFamily.other],
|
||||
)
|
||||
self.assertEqual(
|
||||
section.detection_hardware,
|
||||
{HardwareKey.edgetpu_usb: 2, HardwareKey.other: 1, HardwareKey.cpu: 1},
|
||||
)
|
||||
self.assertEqual(section.storage.record_fs, "nfs4")
|
||||
self.assertEqual(section.storage.record_total_gb, 2000)
|
||||
self.assertEqual(section.storage.record_used_pct, 25)
|
||||
|
||||
def test_storage_without_stats_is_unknown_and_empty(self):
|
||||
section = hardware.storage({})
|
||||
|
||||
self.assertEqual(
|
||||
(section.record_fs, section.record_total_gb, section.record_used_pct),
|
||||
("unknown", 0, 0),
|
||||
)
|
||||
@@ -1,107 +0,0 @@
|
||||
"""Tests for building a report from the collectors."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from peewee_migrate import Router
|
||||
from playhouse.sqlite_ext import SqliteExtDatabase
|
||||
from playhouse.sqliteq import SqliteQueueDatabase
|
||||
|
||||
from frigate.analytics import report
|
||||
from frigate.analytics.collectors import hardware
|
||||
from frigate.analytics.schema import AnalyticsReport, InstallSection
|
||||
from frigate.models import User
|
||||
from frigate.test.analytics_helpers import make_context
|
||||
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
|
||||
|
||||
INSTALL = InstallSection(
|
||||
version="0.19.0",
|
||||
image_variant="dev",
|
||||
install_type="docker",
|
||||
arch="x86_64",
|
||||
kernel="6.8",
|
||||
run_as_root=False,
|
||||
)
|
||||
|
||||
|
||||
class TestBuildReport(unittest.TestCase):
|
||||
def test_a_failing_collector_nulls_only_its_section(self):
|
||||
collectors = {
|
||||
"install": lambda ctx: INSTALL,
|
||||
"hardware": Mock(side_effect=RuntimeError("boom")),
|
||||
"detection": lambda ctx: None,
|
||||
"cameras": lambda ctx: None,
|
||||
"features": lambda ctx: None,
|
||||
"health": lambda ctx: None,
|
||||
}
|
||||
|
||||
with patch.dict(report.COLLECTORS, collectors):
|
||||
built = report.build_report(make_context(), "a" * 32, sent_at=1790000000)
|
||||
|
||||
self.assertEqual(built.install, INSTALL)
|
||||
self.assertIsNone(built.hardware)
|
||||
self.assertEqual(built.schema_version, 1)
|
||||
self.assertEqual(built.sent_at, 1790000000)
|
||||
self.assertRegex(built.report_id, r"^[0-9a-f-]{36}$")
|
||||
|
||||
|
||||
class TestRealCollectors(unittest.TestCase):
|
||||
def setUp(self):
|
||||
migrate_db = SqliteExtDatabase(TEST_DB)
|
||||
del logging.getLogger("peewee_migrate").handlers[:]
|
||||
Router(migrate_db).run()
|
||||
migrate_db.close()
|
||||
self.db = SqliteQueueDatabase(TEST_DB)
|
||||
self.db.bind([User])
|
||||
|
||||
def tearDown(self):
|
||||
# close() leaves the queue's writer thread running against the deleted file
|
||||
self.db.stop()
|
||||
|
||||
if not self.db.is_closed():
|
||||
self.db.close()
|
||||
|
||||
for file in TEST_DB_CLEANUPS:
|
||||
try:
|
||||
os.remove(file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_every_section_builds_and_round_trips_through_json(self):
|
||||
with (
|
||||
patch.object(hardware.hardware_prober, "probe", return_value=[]),
|
||||
patch.object(hardware, "hwaccel_options", return_value=("", [])),
|
||||
):
|
||||
built = report.build_report(make_context(), "a" * 32)
|
||||
|
||||
body = built.model_dump_json()
|
||||
parsed = json.loads(body)
|
||||
|
||||
for section in (
|
||||
"install",
|
||||
"hardware",
|
||||
"detection",
|
||||
"cameras",
|
||||
"features",
|
||||
"health",
|
||||
):
|
||||
with self.subTest(section=section):
|
||||
self.assertIsNotNone(parsed[section])
|
||||
|
||||
self.assertEqual(AnalyticsReport.model_validate_json(body), built)
|
||||
|
||||
def test_preview_uses_a_placeholder_id_before_the_first_report(self):
|
||||
stats_emitter = Mock()
|
||||
stats_emitter.get_latest_stats.return_value = {}
|
||||
|
||||
with (
|
||||
patch.object(report, "load_state", return_value=None),
|
||||
patch.object(hardware.hardware_prober, "probe", return_value=[]),
|
||||
patch.object(hardware, "hwaccel_options", return_value=("", [])),
|
||||
):
|
||||
built = report.preview_report(make_context().config, stats_emitter, None)
|
||||
|
||||
self.assertEqual(built.install_id, report.PREVIEW_INSTALL_ID)
|
||||
@@ -1,227 +0,0 @@
|
||||
"""Tests for the analytics reporter's schedule and consent handling."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from frigate.analytics import reporter as reporter_module
|
||||
from frigate.analytics.reporter import (
|
||||
FIRST_DELAY_S,
|
||||
INTERVAL_S,
|
||||
JITTER_S,
|
||||
PROMPT_KIND,
|
||||
WAKE_S,
|
||||
AnalyticsReporter,
|
||||
)
|
||||
from frigate.analytics.state import AnalyticsState, load_state, save_state
|
||||
from frigate.analytics.transport import SendOutcome
|
||||
from frigate.config.holder import ConfigHolder
|
||||
from frigate.test.analytics_helpers import make_config
|
||||
|
||||
SNAPSHOT = [
|
||||
{
|
||||
"kind": "detector_stuck",
|
||||
"occurrences": 3,
|
||||
"dismissals": 0,
|
||||
"reported_occurrences": 1,
|
||||
"reported_dismissals": 0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class TestAnalyticsReporter(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.dir.cleanup)
|
||||
self.path = os.path.join(self.dir.name, ".analytics.json")
|
||||
self.now = 1_790_000_000.0
|
||||
self.send = Mock(return_value=SendOutcome.accepted)
|
||||
self.registry = Mock()
|
||||
self.registry.stats.return_value = SNAPSHOT
|
||||
self.stats = Mock()
|
||||
self.stats.get_latest_stats.return_value = {}
|
||||
self.holder = ConfigHolder(make_config({"telemetry": {"analytics": True}}))
|
||||
|
||||
for name in ("raise_notice", "resolve_notice"):
|
||||
patcher = patch.object(reporter_module, name)
|
||||
setattr(self, name, patcher.start())
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
self.built = Mock()
|
||||
self.built.model_dump_json.return_value = '{"report": 1}'
|
||||
patcher = patch.object(reporter_module, "build_report", return_value=self.built)
|
||||
self.build_report = patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def reporter(self, path: str | None = None) -> AnalyticsReporter:
|
||||
# uniform(a, b) returns a: the shortest startup delay and interval
|
||||
rng = Mock()
|
||||
rng.uniform.side_effect = lambda low, high: low
|
||||
|
||||
return AnalyticsReporter(
|
||||
self.holder,
|
||||
self.stats,
|
||||
self.registry,
|
||||
threading.Event(),
|
||||
state_path=path or self.path,
|
||||
url="https://example.test/report",
|
||||
send=self.send,
|
||||
clock=lambda: self.now,
|
||||
rng=rng,
|
||||
)
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.now += seconds
|
||||
|
||||
def test_waits_for_the_startup_delay_then_sends(self):
|
||||
reporter = self.reporter()
|
||||
|
||||
reporter.tick()
|
||||
self.send.assert_not_called()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once_with(
|
||||
"https://example.test/report", '{"report": 1}'
|
||||
)
|
||||
self.registry.mark_reported.assert_called_once_with(SNAPSHOT)
|
||||
self.assertEqual(load_state(self.path).last_attempt_at, self.now)
|
||||
|
||||
def test_sends_once_per_interval(self):
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
self.assertEqual(self.send.call_count, 1)
|
||||
|
||||
self.advance(INTERVAL_S - JITTER_S)
|
||||
reporter.tick()
|
||||
self.assertEqual(self.send.call_count, 2)
|
||||
|
||||
def test_a_restart_inside_the_interval_does_not_send(self):
|
||||
save_state(AnalyticsState("a" * 32, self.now), self.path)
|
||||
reporter = self.reporter()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
|
||||
def test_a_failed_send_keeps_watermarks_and_waits_a_full_interval(self):
|
||||
self.send.return_value = SendOutcome.failed
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
reporter.tick()
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
|
||||
self.assertEqual(self.send.call_count, 1)
|
||||
self.registry.mark_reported.assert_not_called()
|
||||
|
||||
def test_an_accepted_report_without_health_keeps_the_watermarks(self):
|
||||
self.built.health = None
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
self.registry.mark_reported.assert_not_called()
|
||||
|
||||
def test_consent_withdrawn_while_the_report_builds_stops_the_send(self):
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
def withdraw(*args, **kwargs):
|
||||
self.holder.set(make_config())
|
||||
return self.built
|
||||
|
||||
self.build_report.side_effect = withdraw
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
|
||||
def test_turning_sharing_off_and_on_between_wakes_starts_a_fresh_identity(self):
|
||||
save_state(AnalyticsState("a" * 32, 0.0), self.path)
|
||||
reporter = self.reporter()
|
||||
reporter.tick()
|
||||
|
||||
self.holder.set(make_config())
|
||||
self.holder.set(make_config({"telemetry": {"analytics": True}}))
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.raise_notice.assert_called_once_with(PROMPT_KIND)
|
||||
self.assertNotEqual(load_state(self.path).install_id, "a" * 32)
|
||||
|
||||
def test_opting_out_raises_the_prompt_once_and_deletes_the_state(self):
|
||||
self.holder.set(make_config())
|
||||
save_state(AnalyticsState("a" * 32, 0.0), self.path)
|
||||
reporter = self.reporter()
|
||||
|
||||
reporter.tick()
|
||||
reporter.tick()
|
||||
|
||||
self.raise_notice.assert_called_once_with(PROMPT_KIND)
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
self.send.assert_not_called()
|
||||
|
||||
def test_opting_in_resolves_the_prompt(self):
|
||||
self.holder.set(make_config())
|
||||
reporter = self.reporter()
|
||||
reporter.tick()
|
||||
|
||||
self.holder.set(make_config({"telemetry": {"analytics": True}}))
|
||||
reporter.tick()
|
||||
|
||||
self.resolve_notice.assert_called_once_with(PROMPT_KIND)
|
||||
|
||||
def test_safe_mode_touches_nothing(self):
|
||||
self.holder.set(make_config({"safe_mode": True}))
|
||||
save_state(AnalyticsState("a" * 32, 0.0), self.path)
|
||||
reporter = self.reporter()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.raise_notice.assert_not_called()
|
||||
self.assertTrue(os.path.exists(self.path))
|
||||
self.send.assert_not_called()
|
||||
|
||||
def test_an_unwritable_config_dir_skips_sending_and_warns_once(self):
|
||||
reporter = self.reporter(os.path.join(self.dir.name, "missing", "x.json"))
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
with self.assertLogs("frigate.analytics.reporter", "WARNING") as logs:
|
||||
reporter.tick()
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
self.assertEqual(len(logs.output), 1)
|
||||
|
||||
def test_a_last_attempt_in_the_future_is_ignored(self):
|
||||
save_state(AnalyticsState("a" * 32, self.now + 10 * INTERVAL_S), self.path)
|
||||
reporter = self.reporter()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
|
||||
def test_the_loop_survives_a_failing_tick(self):
|
||||
reporter = self.reporter()
|
||||
reporter.stop_event.set()
|
||||
|
||||
with (
|
||||
patch.object(reporter, "tick", side_effect=RuntimeError("boom")),
|
||||
self.assertLogs("frigate.analytics.reporter", "ERROR"),
|
||||
):
|
||||
reporter.run()
|
||||
@@ -1,137 +0,0 @@
|
||||
"""Tests for the analytics report schema."""
|
||||
|
||||
import inspect
|
||||
import re
|
||||
import unittest
|
||||
from enum import Enum
|
||||
from typing import Any, get_args, get_origin
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
import frigate.detectors.hardware as detection_hardware
|
||||
import frigate.util.hwaccel as hwaccel
|
||||
from frigate.analytics.schema import (
|
||||
AnalyticsReport,
|
||||
DecodeFamily,
|
||||
HardwareKey,
|
||||
HwaccelKey,
|
||||
InputPresetKey,
|
||||
NoticeKindKey,
|
||||
)
|
||||
from frigate.ffmpeg_presets import PRESETS_HW_ACCEL_DECODE, PRESETS_INPUT
|
||||
from frigate.notices.types import NOTICE_KINDS
|
||||
from frigate.util.hwaccel import HwaccelFamily
|
||||
|
||||
# the only str fields: generated ids and strings the hardware reports
|
||||
ALLOWED_STRINGS = {
|
||||
("AnalyticsReport", "install_id"),
|
||||
("AnalyticsReport", "report_id"),
|
||||
("InstallSection", "version"),
|
||||
("InstallSection", "kernel"),
|
||||
("HardwareSection", "cpu_model"),
|
||||
("GpuInfo", "name"),
|
||||
("StorageInfo", "record_fs"),
|
||||
("DetectionModel", "input"),
|
||||
}
|
||||
|
||||
|
||||
def nested_models(annotation: Any) -> list[type[BaseModel]]:
|
||||
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
|
||||
return [annotation]
|
||||
|
||||
return [model for arg in get_args(annotation) for model in nested_models(arg)]
|
||||
|
||||
|
||||
def all_models(
|
||||
model: type[BaseModel], seen: set[type[BaseModel]] | None = None
|
||||
) -> list[type[BaseModel]]:
|
||||
seen = set() if seen is None else seen
|
||||
|
||||
if model in seen:
|
||||
return []
|
||||
|
||||
seen.add(model)
|
||||
found = [model]
|
||||
|
||||
for field in model.model_fields.values():
|
||||
for nested in nested_models(field.annotation):
|
||||
found += all_models(nested, seen)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def dict_key_types(annotation: Any) -> list[Any]:
|
||||
keys = [get_args(annotation)[0]] if get_origin(annotation) is dict else []
|
||||
return keys + [key for arg in get_args(annotation) for key in dict_key_types(arg)]
|
||||
|
||||
|
||||
class TestSchemaDiscipline(unittest.TestCase):
|
||||
def test_every_field_has_a_description_and_public_flag(self):
|
||||
for model in all_models(AnalyticsReport):
|
||||
for name, field in model.model_fields.items():
|
||||
with self.subTest(field=f"{model.__name__}.{name}"):
|
||||
self.assertTrue(field.description)
|
||||
self.assertIsInstance(field.json_schema_extra, dict)
|
||||
self.assertIsInstance(field.json_schema_extra.get("x-public"), bool)
|
||||
|
||||
def test_strings_are_limited_to_ids_and_hardware_names(self):
|
||||
for model in all_models(AnalyticsReport):
|
||||
properties = model.model_json_schema()["properties"]
|
||||
|
||||
for name, field in model.model_fields.items():
|
||||
if field.annotation is not str:
|
||||
continue
|
||||
|
||||
with self.subTest(field=f"{model.__name__}.{name}"):
|
||||
self.assertIn((model.__name__, name), ALLOWED_STRINGS)
|
||||
self.assertTrue(
|
||||
"maxLength" in properties[name] or "pattern" in properties[name]
|
||||
)
|
||||
|
||||
def test_map_keys_are_enums(self):
|
||||
for model in all_models(AnalyticsReport):
|
||||
for name, field in model.model_fields.items():
|
||||
for key_type in dict_key_types(field.annotation):
|
||||
with self.subTest(field=f"{model.__name__}.{name}"):
|
||||
self.assertTrue(issubclass(key_type, Enum))
|
||||
|
||||
|
||||
class TestEnumsTrackTheirSources(unittest.TestCase):
|
||||
def test_hardware_keys_cover_every_probe(self):
|
||||
probed = set(
|
||||
re.findall(
|
||||
r'_hardware\(\s*"([^"]+)"', inspect.getsource(detection_hardware)
|
||||
)
|
||||
)
|
||||
|
||||
self.assertTrue(probed)
|
||||
self.assertLessEqual(probed, {key.value for key in HardwareKey})
|
||||
|
||||
def test_decode_families_cover_every_hwaccel_family(self):
|
||||
families = {
|
||||
value.key
|
||||
for value in vars(hwaccel).values()
|
||||
if isinstance(value, HwaccelFamily)
|
||||
}
|
||||
|
||||
self.assertTrue(families)
|
||||
self.assertLessEqual(families, {family.value for family in DecodeFamily})
|
||||
|
||||
def test_preset_keys_mirror_the_presets(self):
|
||||
for enum, presets in (
|
||||
(HwaccelKey, PRESETS_HW_ACCEL_DECODE),
|
||||
(InputPresetKey, PRESETS_INPUT),
|
||||
):
|
||||
with self.subTest(enum=enum.__name__):
|
||||
self.assertEqual(
|
||||
{key.value for key in enum},
|
||||
{name.removeprefix("preset-") for name in presets}
|
||||
| {"custom", "none"},
|
||||
)
|
||||
|
||||
def test_notice_keys_are_the_reportable_kinds(self):
|
||||
self.assertEqual(
|
||||
{key.value for key in NoticeKindKey},
|
||||
{key for key, kind in NOTICE_KINDS.items() if kind.reportable},
|
||||
)
|
||||
self.assertNotIn("analytics_prompt", {key.value for key in NoticeKindKey})
|
||||
@@ -1,64 +0,0 @@
|
||||
"""Tests for the analytics state file."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from frigate.analytics.state import (
|
||||
AnalyticsState,
|
||||
delete_state,
|
||||
load_state,
|
||||
new_state,
|
||||
save_state,
|
||||
)
|
||||
|
||||
|
||||
class TestAnalyticsState(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.dir.cleanup)
|
||||
self.path = os.path.join(self.dir.name, ".analytics.json")
|
||||
|
||||
def write(self, content: str) -> None:
|
||||
with open(self.path, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
def test_missing_file_loads_as_none(self):
|
||||
self.assertIsNone(load_state(self.path))
|
||||
|
||||
def test_save_then_load_round_trips(self):
|
||||
state = AnalyticsState(install_id="a" * 32, last_attempt_at=1790000000.5)
|
||||
|
||||
self.assertTrue(save_state(state, self.path))
|
||||
self.assertEqual(load_state(self.path), state)
|
||||
|
||||
def test_unusable_files_load_as_none(self):
|
||||
for content in (
|
||||
"{not json",
|
||||
'["a"]',
|
||||
'{"install_id": "short", "last_attempt_at": 1}',
|
||||
'{"install_id": "' + "a" * 32 + '"}',
|
||||
'{"install_id": "' + "a" * 32 + '", "last_attempt_at": true}',
|
||||
):
|
||||
with self.subTest(content=content):
|
||||
self.write(content)
|
||||
self.assertIsNone(load_state(self.path))
|
||||
|
||||
def test_save_reports_failure_instead_of_raising(self):
|
||||
missing = os.path.join(self.dir.name, "missing", ".analytics.json")
|
||||
|
||||
self.assertFalse(save_state(new_state(), missing))
|
||||
|
||||
def test_new_state_has_a_hex_id_and_no_attempt(self):
|
||||
state = new_state()
|
||||
|
||||
self.assertRegex(state.install_id, r"^[0-9a-f]{32}$")
|
||||
self.assertEqual(state.last_attempt_at, 0.0)
|
||||
|
||||
def test_delete_removes_the_file_and_tolerates_a_missing_one(self):
|
||||
save_state(new_state(), self.path)
|
||||
|
||||
delete_state(self.path)
|
||||
delete_state(self.path)
|
||||
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
@@ -1,60 +0,0 @@
|
||||
"""Tests for sending a report."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import requests
|
||||
|
||||
from frigate.analytics.transport import TIMEOUT_S, SendOutcome, send_report
|
||||
from frigate.version import VERSION
|
||||
|
||||
URL = "https://example.test/report"
|
||||
|
||||
|
||||
def post(status: int = 204, text: str = "", side_effect: Exception | None = None):
|
||||
return patch(
|
||||
"frigate.analytics.transport.requests.post",
|
||||
return_value=Mock(status_code=status, text=text),
|
||||
side_effect=side_effect,
|
||||
)
|
||||
|
||||
|
||||
class TestSendReport(unittest.TestCase):
|
||||
def test_posts_the_body_as_json_with_a_user_agent(self):
|
||||
with post() as mock_post:
|
||||
outcome = send_report(URL, '{"a":1}')
|
||||
|
||||
self.assertIs(outcome, SendOutcome.accepted)
|
||||
args, kwargs = mock_post.call_args
|
||||
self.assertEqual(args[0], URL)
|
||||
self.assertEqual(kwargs["data"], b'{"a":1}')
|
||||
self.assertEqual(kwargs["headers"]["Content-Type"], "application/json")
|
||||
self.assertEqual(kwargs["headers"]["User-Agent"], f"Frigate/{VERSION}")
|
||||
self.assertFalse(kwargs["allow_redirects"])
|
||||
self.assertEqual(kwargs["timeout"], TIMEOUT_S)
|
||||
|
||||
def test_maps_statuses_to_outcomes(self):
|
||||
cases = (
|
||||
(200, SendOutcome.accepted),
|
||||
(400, SendOutcome.rejected),
|
||||
(429, SendOutcome.rate_limited),
|
||||
(302, SendOutcome.failed),
|
||||
(500, SendOutcome.failed),
|
||||
)
|
||||
|
||||
for status, outcome in cases:
|
||||
with self.subTest(status=status), post(status):
|
||||
self.assertIs(send_report(URL, "{}"), outcome)
|
||||
|
||||
def test_logs_the_rejection_reason(self):
|
||||
with (
|
||||
post(400, "install_id: bad"),
|
||||
self.assertLogs("frigate.analytics.transport", "WARNING") as logs,
|
||||
):
|
||||
send_report(URL, "{}")
|
||||
|
||||
self.assertIn("install_id: bad", logs.output[0])
|
||||
|
||||
def test_a_network_error_fails_without_raising(self):
|
||||
with post(side_effect=requests.ConnectionError("down")):
|
||||
self.assertIs(send_report(URL, "{}"), SendOutcome.failed)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user