From 01392e03ac6b81086ae418328b4a39aea787eb33 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 23 Mar 2026 16:16:54 -0600 Subject: [PATCH 001/291] Update docs for DEIMv2 support (#22598) --- docs/docs/configuration/object_detectors.md | 106 ++++++++++++++++++-- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 9bdacfb28d..4eb7933cdc 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -330,7 +330,7 @@ detectors: | [YOLO-NAS](#yolo-nas) | ✅ | ✅ | | | [MobileNet v2](#ssdlite-mobilenet-v2) | ✅ | ✅ | Fast and lightweight model, less accurate than larger models | | [YOLOX](#yolox) | ✅ | ? | | -| [D-FINE](#d-fine) | ❌ | ❌ | | +| [D-FINE / DEIMv2](#d-fine--deimv2) | ❌ | ❌ | | #### SSDLite MobileNet v2 @@ -464,13 +464,13 @@ model: -#### D-FINE +#### D-FINE / DEIMv2 -[D-FINE](https://github.com/Peterande/D-FINE) is a DETR based model. The ONNX exported models are supported, but not included by default. See [the models section](#downloading-d-fine-model) for more information on downloading the D-FINE model for use in Frigate. +[D-FINE](https://github.com/Peterande/D-FINE) and [DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) are DETR based models that share the same ONNX input/output format. The ONNX exported models are supported, but not included by default. See the models section for downloading [D-FINE](#downloading-d-fine-model) or [DEIMv2](#downloading-deimv2-model) for use in Frigate. :::warning -Currently D-FINE models only run on OpenVINO in CPU mode, GPUs currently fail to compile the model +Currently D-FINE / DEIMv2 models only run on OpenVINO in CPU mode, GPUs currently fail to compile the model ::: @@ -499,6 +499,31 @@ Note that the labelmap uses a subset of the complete COCO label set that has onl +
+ DEIMv2 Setup & Config + +After placing the downloaded onnx model in your `config/model_cache` folder, you can use the following configuration: + +```yaml +detectors: + ov: + type: openvino + device: CPU + +model: + model_type: dfine + width: 640 + height: 640 + input_tensor: nchw + input_dtype: float + path: /config/model_cache/deimv2_hgnetv2_n.onnx + labelmap_path: /labelmap/coco-80.txt +``` + +Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. + +
+ ## Apple Silicon detector The NPU in Apple Silicon can't be accessed from within a container, so the [Apple Silicon detector client](https://github.com/frigate-nvr/apple-silicon-detector) must first be setup. It is recommended to use the Frigate docker image with `-standard-arm64` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-standard-arm64`. @@ -648,7 +673,7 @@ The AMD GPU kernel is known problematic especially when converting models to mxr See [ONNX supported models](#supported-models) for supported models, there are some caveats: -- D-FINE models are not supported +- D-FINE / DEIMv2 models are not supported - YOLO-NAS models are known to not run well on integrated GPUs ## ONNX @@ -693,7 +718,7 @@ detectors: | [RF-DETR](#rf-detr) | ✅ | ❌ | Supports CUDA Graphs for optimal Nvidia performance | | [YOLO-NAS](#yolo-nas-1) | ⚠️ | ⚠️ | Not supported by CUDA Graphs | | [YOLOX](#yolox-1) | ✅ | ✅ | Supports CUDA Graphs for optimal Nvidia performance | -| [D-FINE](#d-fine) | ⚠️ | ❌ | Not supported by CUDA Graphs | +| [D-FINE / DEIMv2](#d-fine--deimv2-1) | ⚠️ | ❌ | Not supported by CUDA Graphs | There is no default model provided, the following formats are supported: @@ -822,9 +847,9 @@ model: -#### D-FINE +#### D-FINE / DEIMv2 -[D-FINE](https://github.com/Peterande/D-FINE) is a DETR based model. The ONNX exported models are supported, but not included by default. See [the models section](#downloading-d-fine-model) for more information on downloading the D-FINE model for use in Frigate. +[D-FINE](https://github.com/Peterande/D-FINE) and [DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) are DETR based models that share the same ONNX input/output format. The ONNX exported models are supported, but not included by default. See the models section for downloading [D-FINE](#downloading-d-fine-model) or [DEIMv2](#downloading-deimv2-model) for use in Frigate.
D-FINE Setup & Config @@ -848,6 +873,28 @@ model:
+
+ DEIMv2 Setup & Config + +After placing the downloaded onnx model in your `config/model_cache` folder, you can use the following configuration: + +```yaml +detectors: + onnx: + type: onnx + +model: + model_type: dfine + width: 640 + height: 640 + input_tensor: nchw + input_dtype: float + path: /config/model_cache/deimv2_hgnetv2_n.onnx + labelmap_path: /labelmap/coco-80.txt +``` + +
+ Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. ## CPU Detector (not recommended) @@ -1512,6 +1559,49 @@ COPY --from=build /dfine/output/dfine_${MODEL_SIZE}_obj2coco.onnx /dfine-${MODEL EOF ``` +### Downloading DEIMv2 Model + +[DEIMv2](https://github.com/Intellindust-AI-Lab/DEIMv2) can be exported as ONNX by running the command below. Pretrained weights are available on Hugging Face for two backbone families: + +- **HGNetv2** (smaller/faster): `atto`, `femto`, `pico`, `n` +- **DINOv3** (larger/more accurate): `s`, `m`, `l`, `x` + +Set `BACKBONE` and `MODEL_SIZE` in the first line to match your desired variant. Hugging Face model names use uppercase (e.g. `HGNetv2_N`, `DINOv3_S`), while config files use lowercase (e.g. `hgnetv2_n`, `dinov3_s`). + +```sh +docker build . --rm --build-arg BACKBONE=hgnetv2 --build-arg MODEL_SIZE=n --output . -f- <<'EOF' +FROM python:3.11-slim AS build +RUN apt-get update && apt-get install --no-install-recommends -y git libgl1 libglib2.0-0 && rm -rf /var/lib/apt/lists/* +COPY --from=ghcr.io/astral-sh/uv:0.8.0 /uv /bin/ +WORKDIR /deimv2 +RUN git clone https://github.com/Intellindust-AI-Lab/DEIMv2.git . +# Install CPU-only PyTorch first to avoid pulling CUDA variant +RUN uv pip install --no-cache --system torch torchvision --index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache --system -r requirements.txt +RUN uv pip install --no-cache --system onnx safetensors huggingface_hub +RUN mkdir -p output +ARG BACKBONE +ARG MODEL_SIZE +# Download from Hugging Face and convert safetensors to pth +RUN python3 -c "\ +from huggingface_hub import hf_hub_download; \ +from safetensors.torch import load_file; \ +import torch; \ +backbone = '${BACKBONE}'.replace('hgnetv2','HGNetv2').replace('dinov3','DINOv3'); \ +size = '${MODEL_SIZE}'.upper(); \ +st = load_file(hf_hub_download('Intellindust/DEIMv2_' + backbone + '_' + size + '_COCO', 'model.safetensors')); \ +torch.save({'model': st}, 'output/deimv2.pth')" +RUN sed -i "s/data = torch.rand(2/data = torch.rand(1/" tools/deployment/export_onnx.py +# HuggingFace safetensors omits frozen constants that the model constructor initializes +RUN sed -i "s/cfg.model.load_state_dict(state)/cfg.model.load_state_dict(state, strict=False)/" tools/deployment/export_onnx.py +RUN python3 tools/deployment/export_onnx.py -c configs/deimv2/deimv2_${BACKBONE}_${MODEL_SIZE}_coco.yml -r output/deimv2.pth +FROM scratch +ARG BACKBONE +ARG MODEL_SIZE +COPY --from=build /deimv2/output/deimv2.onnx /deimv2_${BACKBONE}_${MODEL_SIZE}.onnx +EOF +``` + ### Downloading RF-DETR Model RF-DETR can be exported as ONNX by running the command below. You can copy and paste the whole thing to your terminal and execute, altering `MODEL_SIZE=Nano` in the first line to `Nano`, `Small`, or `Medium` size. From 0371b60c715b8e53deed419f742f69d08ccef491 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 31 Mar 2026 08:51:55 -0500 Subject: [PATCH 002/291] limit access to admin-only websocket topics for viewer users (#22710) --- frigate/comms/ws.py | 102 ++++++++++++++++++++- frigate/test/test_ws_auth.py | 166 +++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 frigate/test/test_ws_auth.py diff --git a/frigate/comms/ws.py b/frigate/comms/ws.py index 6cfe4ecc0b..2f16ab7141 100644 --- a/frigate/comms/ws.py +++ b/frigate/comms/ws.py @@ -17,9 +17,90 @@ from ws4py.websocket import WebSocket as WebSocket_ from frigate.comms.base_communicator import Communicator from frigate.config import FrigateConfig +from frigate.const import ( + CLEAR_ONGOING_REVIEW_SEGMENTS, + EXPIRE_AUDIO_ACTIVITY, + INSERT_MANY_RECORDINGS, + INSERT_PREVIEW, + NOTIFICATION_TEST, + REQUEST_REGION_GRID, + UPDATE_AUDIO_ACTIVITY, + UPDATE_AUDIO_TRANSCRIPTION_STATE, + UPDATE_BIRDSEYE_LAYOUT, + UPDATE_CAMERA_ACTIVITY, + UPDATE_EMBEDDINGS_REINDEX_PROGRESS, + UPDATE_EVENT_DESCRIPTION, + UPDATE_MODEL_STATE, + UPDATE_REVIEW_DESCRIPTION, + UPSERT_REVIEW_SEGMENT, +) logger = logging.getLogger(__name__) +# Internal IPC topics — NEVER allowed from WebSocket, regardless of role +_WS_BLOCKED_TOPICS = frozenset( + { + INSERT_MANY_RECORDINGS, + INSERT_PREVIEW, + REQUEST_REGION_GRID, + UPSERT_REVIEW_SEGMENT, + CLEAR_ONGOING_REVIEW_SEGMENTS, + UPDATE_CAMERA_ACTIVITY, + UPDATE_AUDIO_ACTIVITY, + EXPIRE_AUDIO_ACTIVITY, + UPDATE_EVENT_DESCRIPTION, + UPDATE_REVIEW_DESCRIPTION, + UPDATE_MODEL_STATE, + UPDATE_EMBEDDINGS_REINDEX_PROGRESS, + UPDATE_BIRDSEYE_LAYOUT, + UPDATE_AUDIO_TRANSCRIPTION_STATE, + NOTIFICATION_TEST, + } +) + +# Read-only topics any authenticated user (including viewer) can send +_WS_VIEWER_TOPICS = frozenset( + { + "onConnect", + "modelState", + "audioTranscriptionState", + "birdseyeLayout", + "embeddingsReindexProgress", + } +) + + +def _check_ws_authorization( + topic: str, + role_header: str | None, + separator: str, +) -> bool: + """Check if a WebSocket message is authorized. + + Args: + topic: The message topic. + role_header: The HTTP_REMOTE_ROLE header value, or None. + separator: The role separator character from proxy config. + + Returns: + True if authorized, False if blocked. + """ + # Block IPC-only topics unconditionally + if topic in _WS_BLOCKED_TOPICS: + return False + + # No role header: default to viewer (fail-closed) + if role_header is None: + return topic in _WS_VIEWER_TOPICS + + # Check if any role is admin + roles = [r.strip() for r in role_header.split(separator)] + if "admin" in roles: + return True + + # Non-admin: only viewer topics allowed + return topic in _WS_VIEWER_TOPICS + class WebSocket(WebSocket_): # type: ignore[misc] def unhandled_error(self, error: Any) -> None: @@ -49,6 +130,7 @@ class WebSocketClient(Communicator): class _WebSocketHandler(WebSocket): receiver = self._dispatcher + role_separator = self.config.proxy.separator or "," def received_message(self, message: WebSocket.received_message) -> None: # type: ignore[name-defined] try: @@ -63,11 +145,25 @@ class WebSocketClient(Communicator): ) return - logger.debug( - f"Publishing mqtt message from websockets at {json_message['topic']}." + topic = json_message["topic"] + + # Authorization check (skip when environ is None — direct internal connection) + role_header = ( + self.environ.get("HTTP_REMOTE_ROLE") if self.environ else None ) + if self.environ is not None and not _check_ws_authorization( + topic, role_header, self.role_separator + ): + logger.warning( + "Blocked unauthorized WebSocket message: topic=%s, role=%s", + topic, + role_header, + ) + return + + logger.debug(f"Publishing mqtt message from websockets at {topic}.") self.receiver( - json_message["topic"], + topic, json_message["payload"], ) diff --git a/frigate/test/test_ws_auth.py b/frigate/test/test_ws_auth.py new file mode 100644 index 0000000000..b762f4384c --- /dev/null +++ b/frigate/test/test_ws_auth.py @@ -0,0 +1,166 @@ +"""Tests for WebSocket authorization checks.""" + +import unittest + +from frigate.comms.ws import _check_ws_authorization +from frigate.const import INSERT_MANY_RECORDINGS, UPDATE_CAMERA_ACTIVITY + + +class TestCheckWsAuthorization(unittest.TestCase): + """Tests for the _check_ws_authorization pure function.""" + + DEFAULT_SEPARATOR = "," + + # --- IPC topic blocking (unconditional, regardless of role) --- + + def test_ipc_topic_blocked_for_admin(self): + self.assertFalse( + _check_ws_authorization( + INSERT_MANY_RECORDINGS, "admin", self.DEFAULT_SEPARATOR + ) + ) + + def test_ipc_topic_blocked_for_viewer(self): + self.assertFalse( + _check_ws_authorization( + UPDATE_CAMERA_ACTIVITY, "viewer", self.DEFAULT_SEPARATOR + ) + ) + + def test_ipc_topic_blocked_when_no_role(self): + self.assertFalse( + _check_ws_authorization( + INSERT_MANY_RECORDINGS, None, self.DEFAULT_SEPARATOR + ) + ) + + # --- Viewer allowed topics --- + + def test_viewer_can_send_on_connect(self): + self.assertTrue( + _check_ws_authorization("onConnect", "viewer", self.DEFAULT_SEPARATOR) + ) + + def test_viewer_can_send_model_state(self): + self.assertTrue( + _check_ws_authorization("modelState", "viewer", self.DEFAULT_SEPARATOR) + ) + + def test_viewer_can_send_audio_transcription_state(self): + self.assertTrue( + _check_ws_authorization( + "audioTranscriptionState", "viewer", self.DEFAULT_SEPARATOR + ) + ) + + def test_viewer_can_send_birdseye_layout(self): + self.assertTrue( + _check_ws_authorization("birdseyeLayout", "viewer", self.DEFAULT_SEPARATOR) + ) + + def test_viewer_can_send_embeddings_reindex_progress(self): + self.assertTrue( + _check_ws_authorization( + "embeddingsReindexProgress", "viewer", self.DEFAULT_SEPARATOR + ) + ) + + # --- Viewer blocked from admin topics --- + + def test_viewer_blocked_from_restart(self): + self.assertFalse( + _check_ws_authorization("restart", "viewer", self.DEFAULT_SEPARATOR) + ) + + def test_viewer_blocked_from_camera_detect_set(self): + self.assertFalse( + _check_ws_authorization( + "front_door/detect/set", "viewer", self.DEFAULT_SEPARATOR + ) + ) + + def test_viewer_blocked_from_camera_ptz(self): + self.assertFalse( + _check_ws_authorization("front_door/ptz", "viewer", self.DEFAULT_SEPARATOR) + ) + + def test_viewer_blocked_from_global_notifications_set(self): + self.assertFalse( + _check_ws_authorization( + "notifications/set", "viewer", self.DEFAULT_SEPARATOR + ) + ) + + def test_viewer_blocked_from_camera_notifications_suspend(self): + self.assertFalse( + _check_ws_authorization( + "front_door/notifications/suspend", "viewer", self.DEFAULT_SEPARATOR + ) + ) + + def test_viewer_blocked_from_arbitrary_unknown_topic(self): + self.assertFalse( + _check_ws_authorization( + "some_random_topic", "viewer", self.DEFAULT_SEPARATOR + ) + ) + + # --- Admin access --- + + def test_admin_can_send_restart(self): + self.assertTrue( + _check_ws_authorization("restart", "admin", self.DEFAULT_SEPARATOR) + ) + + def test_admin_can_send_camera_detect_set(self): + self.assertTrue( + _check_ws_authorization( + "front_door/detect/set", "admin", self.DEFAULT_SEPARATOR + ) + ) + + def test_admin_can_send_camera_ptz(self): + self.assertTrue( + _check_ws_authorization("front_door/ptz", "admin", self.DEFAULT_SEPARATOR) + ) + + # --- Comma-separated roles --- + + def test_comma_separated_admin_viewer_grants_admin(self): + self.assertTrue( + _check_ws_authorization("restart", "admin,viewer", self.DEFAULT_SEPARATOR) + ) + + def test_comma_separated_viewer_admin_grants_admin(self): + self.assertTrue( + _check_ws_authorization("restart", "viewer,admin", self.DEFAULT_SEPARATOR) + ) + + def test_comma_separated_with_spaces(self): + self.assertTrue( + _check_ws_authorization("restart", "viewer, admin", self.DEFAULT_SEPARATOR) + ) + + # --- Custom separator --- + + def test_pipe_separator(self): + self.assertTrue(_check_ws_authorization("restart", "viewer|admin", "|")) + + def test_pipe_separator_no_admin(self): + self.assertFalse(_check_ws_authorization("restart", "viewer|editor", "|")) + + # --- No role header (fail-closed) --- + + def test_no_role_header_blocks_admin_topics(self): + self.assertFalse( + _check_ws_authorization("restart", None, self.DEFAULT_SEPARATOR) + ) + + def test_no_role_header_allows_viewer_topics(self): + self.assertTrue( + _check_ws_authorization("onConnect", None, self.DEFAULT_SEPARATOR) + ) + + +if __name__ == "__main__": + unittest.main() From 1c26bc289e585bc7844aecfd93b609b42132e8d5 Mon Sep 17 00:00:00 2001 From: Abinila Siva <163017635+abinila4@users.noreply.github.com> Date: Tue, 31 Mar 2026 13:22:23 -0400 Subject: [PATCH 003/291] docs: update MemryX docs (#22712) --- docs/docs/configuration/object_detectors.md | 23 ++++++++++++++++++--- docs/docs/frigate/installation.md | 8 ++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 4eb7933cdc..5c068e11c3 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -994,7 +994,7 @@ MemryX `.dfp` models are automatically downloaded at runtime, if enabled, to the #### YOLO-NAS -The [YOLO-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) model included in this detector is downloaded from the [Models Section](#downloading-yolo-nas-model) and compiled to DFP with [mx_nc](https://developer.memryx.com/tools/neural_compiler.html#usage). +The [YOLO-NAS](https://github.com/Deci-AI/super-gradients/blob/master/YOLONAS.md) model included in this detector is downloaded from the [Models Section](#downloading-yolo-nas-model) and compiled to DFP with [mx_nc](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage). **Note:** The default model for the MemryX detector is YOLO-NAS 320x320. @@ -1028,7 +1028,7 @@ model: #### YOLOv9 -The YOLOv9s model included in this detector is downloaded from [the original GitHub](https://github.com/WongKinYiu/yolov9) like in the [Models Section](#yolov9-1) and compiled to DFP with [mx_nc](https://developer.memryx.com/tools/neural_compiler.html#usage). +The YOLOv9s model included in this detector is downloaded from [the original GitHub](https://github.com/WongKinYiu/yolov9) like in the [Models Section](#yolov9-1) and compiled to DFP with [mx_nc](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage). ##### Configuration @@ -1122,7 +1122,24 @@ To use your own model: 5. Update the `labelmap_path` to match your custom model's labels. -For detailed instructions on compiling models, refer to the [MemryX Compiler](https://developer.memryx.com/tools/neural_compiler.html#usage) docs and [Tutorials](https://developer.memryx.com/tutorials/tutorials.html). +#### Compile the Model + +Custom models must be compiled using **MemryX SDK 2.1**. + +Before compiling your model, install the MemryX Neural Compiler tools from the +[Install Tools](https://developer.memryx.com/2p1/get_started/install_tools.html) page on the **host**. + +Once the SDK 2.1 environment is set up, follow the +[MemryX Compiler](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage) documentation to compile your model. + +Example: + +```bash +mx_nc -m ./yolov9.onnx --dfp_fname ./yolov9.dfp -is "1,3,640,640" -c 4 --autocrop -v +``` +> **Note:** `-is` specifies the input shape. Use your model's input dimensions. + +For detailed instructions on compiling models, refer to the [MemryX Compiler](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage) docs and [Tutorials](https://developer.memryx.com/2p1/tutorials/tutorials.html). ```yaml # The detector automatically selects the default model if nothing is provided in the config. diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index 8bc5cb6521..53e978c450 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -297,7 +297,7 @@ The MemryX MX3 Accelerator is available in the M.2 2280 form factor (like an NVM #### Installation -To get started with MX3 hardware setup for your system, refer to the [Hardware Setup Guide](https://developer.memryx.com/get_started/hardware_setup.html). +To get started with MX3 hardware setup for your system, refer to the [Hardware Setup Guide](https://developer.memryx.com/2p1/get_started/hardware_setup.html). Then follow these steps for installing the correct driver/runtime configuration: @@ -306,6 +306,12 @@ Then follow these steps for installing the correct driver/runtime configuration: 3. Run the script with `./user_installation.sh` 4. **Restart your computer** to complete driver installation. +:::warning + +For manual setup, use **MemryX SDK 2.1** only. Other SDK versions are not supported for this setup. See the [SDK 2.1 documentation](https://developer.memryx.com/2p1/index.html) + +::: + #### Setup To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable` From 6b71feffab8d7b4963c1b6b1405d2c70a0f8c693 Mon Sep 17 00:00:00 2001 From: Abinila Siva <163017635+abinila4@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:32:32 -0400 Subject: [PATCH 004/291] Memryx docs update (#22746) * docs: update MemryX documentation section * docs: update MemryX documentation section --- docs/docs/configuration/object_detectors.md | 29 ++++++++++++--------- docs/docs/frigate/installation.md | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 5c068e11c3..4a0f014d49 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -1110,17 +1110,7 @@ model: #### Using a Custom Model -To use your own model: - -1. Package your compiled model into a `.zip` file. - -2. The `.zip` must contain the compiled `.dfp` file. - -3. Depending on the model, the compiler may also generate a cropped post-processing network. If present, it will be named with the suffix `_post.onnx`. - -4. Bind-mount the `.zip` file into the container and specify its path using `model.path` in your config. - -5. Update the `labelmap_path` to match your custom model's labels. +To use your own custom model, first compile it into a [.dfp](https://developer.memryx.com/2p1/specs/files.html#dataflow-program) file, which is the format used by MemryX. #### Compile the Model @@ -1129,18 +1119,31 @@ Custom models must be compiled using **MemryX SDK 2.1**. Before compiling your model, install the MemryX Neural Compiler tools from the [Install Tools](https://developer.memryx.com/2p1/get_started/install_tools.html) page on the **host**. +> **Note:** It is recommended to compile the model on the host machine, or on another separate machine, rather than inside the Frigate Docker container. Installing the compiler inside Docker may conflict with container packages. It is recommended to create a Python virtual environment and install the compiler there. + Once the SDK 2.1 environment is set up, follow the [MemryX Compiler](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage) documentation to compile your model. Example: ```bash -mx_nc -m ./yolov9.onnx --dfp_fname ./yolov9.dfp -is "1,3,640,640" -c 4 --autocrop -v +mx_nc -m yolonas.onnx -c 4 --autocrop -v --dfp_fname yolonas.dfp ``` -> **Note:** `-is` specifies the input shape. Use your model's input dimensions. For detailed instructions on compiling models, refer to the [MemryX Compiler](https://developer.memryx.com/2p1/tools/neural_compiler.html#usage) docs and [Tutorials](https://developer.memryx.com/2p1/tutorials/tutorials.html). +#### Package the Compiled Model + +1. Package your compiled model into a `.zip` file. + +2. The `.zip` file must contain the compiled `.dfp` file. + +3. Depending on the model, the compiler may also generate a cropped post-processing network. If present, it will be named with the suffix `_post.onnx`. + +4. Bind-mount the `.zip` file into the container and specify its path using `model.path` in your config. + +5. Update `labelmap_path` to match your custom model's labels. + ```yaml # The detector automatically selects the default model if nothing is provided in the config. # diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index 53e978c450..3722a23ba2 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -297,7 +297,7 @@ The MemryX MX3 Accelerator is available in the M.2 2280 form factor (like an NVM #### Installation -To get started with MX3 hardware setup for your system, refer to the [Hardware Setup Guide](https://developer.memryx.com/2p1/get_started/hardware_setup.html). +To get started with MX3 hardware setup for your system, refer to the [Hardware Setup Guide](https://developer.memryx.com/2p1/get_started/install_hardware.html). Then follow these steps for installing the correct driver/runtime configuration: From 58c93c2e9e3a2f6c80ea6c5108a2d6949a63bd4a Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 13 Apr 2026 08:07:09 -0500 Subject: [PATCH 005/291] clarify emergency cleanup (#22864) --- docs/docs/configuration/record.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configuration/record.md b/docs/docs/configuration/record.md index 4d696dad0e..e805cc2d7d 100644 --- a/docs/docs/configuration/record.md +++ b/docs/docs/configuration/record.md @@ -68,7 +68,7 @@ record: ## Will Frigate delete old recordings if my storage runs out? -As of Frigate 0.12 if there is less than an hour left of storage, the oldest 2 hours of recordings will be deleted. +If there is less than an hour left of storage, the oldest hour of recordings will be deleted and a message will be printed in the Frigate logs. This emergency cleanup deletes the oldest recordings first regardless of retention settings to reclaim space as quickly as possible. ## Configuring Recording Retention From 67837f61d06254bb5220d69b0dbdc049aed8d753 Mon Sep 17 00:00:00 2001 From: Mark Date: Tue, 14 Apr 2026 21:00:18 +0200 Subject: [PATCH 006/291] Update restream.md docs and clarify output config (#22860) * Update restream.md Clarified that exec output must be put in curly braces ONLY in case of RTSP, not pipe, as per go2rtc docs. Added additional example use case for exec function (rpi5b cam set-up). * Cleanup Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --------- Co-authored-by: Nicolas Mowen Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --- docs/docs/configuration/restream.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/docs/configuration/restream.md b/docs/docs/configuration/restream.md index 875d9a2921..5955770a22 100644 --- a/docs/docs/configuration/restream.md +++ b/docs/docs/configuration/restream.md @@ -208,7 +208,7 @@ Enabling arbitrary exec sources allows execution of arbitrary commands through g ## Advanced Restream Configurations -The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below: +The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#source-exec) source in go2rtc can be used for custom ffmpeg commands and other applications. An example is below: :::warning @@ -216,16 +216,11 @@ The `exec:`, `echo:`, and `expr:` sources are disabled by default for security. ::: -:::warning - -The `exec:`, `echo:`, and `expr:` sources are disabled by default for security. You must set `GO2RTC_ALLOW_ARBITRARY_EXEC=true` to use them. See [Security: Restricted Stream Sources](#security-restricted-stream-sources) for more information. - -::: - -NOTE: The output will need to be passed with two curly braces `{{output}}` +NOTE: RTSP output will need to be passed with two curly braces `{{output}}`, whereas pipe output must be passed without curly braces. ```yaml go2rtc: streams: stream1: exec:ffmpeg -hide_banner -re -stream_loop -1 -i /media/BigBuckBunny.mp4 -c copy -rtsp_transport tcp -f rtsp {{output}} + stream2: exec:rpicam-vid -t 0 --libav-format h264 -o - ``` From cfb87f97444dac4887a7fb8039e91ac424f861b9 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 18 Apr 2026 08:10:50 -0500 Subject: [PATCH 007/291] Miscellaneous fixes (#22913) * add log when probing detect stream on startup when users don't explicitly set detect.width and detect.height, we probe for them. sometimes the probe hangs (camera doesn't support UDP, like some Reolinks), so this log message will make that clearer * add faq about probing detect stream * fix stuck activity ring when tracked object transitions to stationary * drop cache segments past retain cutoff regardless of retention mode * add maintainer test --- docs/docs/troubleshooting/faqs.md | 24 ++++++++++++++++ frigate/config/config.py | 3 ++ frigate/record/maintainer.py | 10 ++++--- frigate/test/test_maintainer.py | 41 ++++++++++++++++++++++++++++ web/src/hooks/use-camera-activity.ts | 8 ++++-- 5 files changed, 79 insertions(+), 7 deletions(-) diff --git a/docs/docs/troubleshooting/faqs.md b/docs/docs/troubleshooting/faqs.md index ff2379ea72..6cd67ba889 100644 --- a/docs/docs/troubleshooting/faqs.md +++ b/docs/docs/troubleshooting/faqs.md @@ -110,3 +110,27 @@ No. Frigate uses the TCP protocol to connect to your camera's RTSP URL. VLC auto TCP ensures that all data packets arrive in the correct order. This is crucial for video recording, decoding, and stream processing, which is why Frigate enforces a TCP connection. UDP is faster but less reliable, as it does not guarantee packet delivery or order, and VLC does not have the same requirements as Frigate. You can still configure Frigate to use UDP by using ffmpeg input args or the preset `preset-rtsp-udp`. See the [ffmpeg presets](/configuration/ffmpeg_presets) documentation. + +### Frigate hangs on startup with a "probing detect stream" message in the logs + +On startup, Frigate probes each camera's detect stream with OpenCV to auto-detect its resolution. OpenCV's FFmpeg backend may attempt RTSP over UDP during this probe regardless of the `-rtsp_transport tcp` in your `input_args` or preset. For cameras that do not respond to UDP (common on some Reolink models and others behind firewalls that block UDP), the probe can hang indefinitely and block Frigate from finishing startup, or it can return zeroed-out dimensions that show up as width `0` and height `0` in Camera Probe Info under System Metrics. + +There are two ways to avoid this: + +1. Set `detect.width` and `detect.height` explicitly in your camera config. When both are set, Frigate skips the auto-detect probe entirely: + + ```yaml + cameras: + my_camera: + detect: + width: 1280 + height: 720 + ``` + +2. Force OpenCV's FFmpeg backend to use TCP for RTSP by setting the environment variable on your Frigate container: + + ``` + OPENCV_FFMPEG_CAPTURE_OPTIONS=rtsp_transport;tcp + ``` + + This is a process-wide setting and applies to all cameras. If you have any cameras that require `preset-rtsp-udp`, use option 1 instead. diff --git a/frigate/config/config.py b/frigate/config/config.py index 1d09016f64..de3438cd01 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -730,6 +730,9 @@ class FrigateConfig(FrigateBaseModel): ) if need_detect_dimensions: + logger.info( + f"detect.width and detect.height not set for {camera_config.name}, probing detect stream to determine resolution." + ) stream_info = {"width": 0, "height": 0, "fourcc": None} try: stream_info = stream_info_retriever.get_stream_info( diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index 73868ea245..6d25622f49 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -464,10 +464,12 @@ class RecordingMaintainer(threading.Thread): self.drop_segment(cache_path) return None - # if it doesn't overlap with an review item, go ahead and drop the segment - # if it ends more than the configured pre_capture for the camera - # BUT only if continuous/motion is NOT enabled (otherwise wait for processing) - elif highest is None: + # if it doesn't overlap with a review item, drop the segment once it + # ends more than event_pre_capture before the most recently processed + # frame. at this point we've already decided not to keep it for + # continuous/motion retention (either disabled or segment_stats said + # discard), so waiting longer just fills the cache. + else: camera_info = self.object_recordings_info[camera] most_recently_processed_frame_time = ( camera_info[-1][0] if len(camera_info) > 0 else 0 diff --git a/frigate/test/test_maintainer.py b/frigate/test/test_maintainer.py index 49712749ef..3ac4d8a071 100644 --- a/frigate/test/test_maintainer.py +++ b/frigate/test/test_maintainer.py @@ -1,3 +1,4 @@ +import datetime import sys import unittest from unittest.mock import MagicMock, patch @@ -74,6 +75,46 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase): f"Expected a single warning for unexpected files, got {len(matching)}", ) + async def test_drops_quiet_segment_when_only_motion_retention(self): + # Regression: when motion retention is enabled but a segment has no + # motion and no review overlaps it, the segment must still be dropped. + # Otherwise it sits in cache forever, accumulates, and triggers the + # "Unable to keep up with recording segments in cache" warning every + # ~10s as the overflow trim in move_files discards the oldest one. + config = MagicMock(spec=FrigateConfig) + + camera_config = MagicMock() + camera_config.record.enabled = True + camera_config.record.continuous.days = 0 + camera_config.record.motion.days = 1 + camera_config.record.event_pre_capture = 5 + config.cameras = {"test_cam": camera_config} + + stop_event = MagicMock() + maintainer = RecordingMaintainer(config, stop_event) + + now = datetime.datetime.now(datetime.timezone.utc) + start_time = now - datetime.timedelta(seconds=20) + end_time = now - datetime.timedelta(seconds=10) + cache_path = "/tmp/cache/test_cam@20260417150000+0000.mp4" + + maintainer.end_time_cache = {cache_path: (end_time, 10.0)} + # Single processed frame well past end_time with no motion/objects. + maintainer.object_recordings_info["test_cam"] = [(now.timestamp(), [], [], [])] + maintainer.audio_recordings_info["test_cam"] = [] + + maintainer.drop_segment = MagicMock() + maintainer.recordings_publisher = MagicMock() + + result = await maintainer.validate_and_move_segment( + "test_cam", + reviews=[], + recording={"start_time": start_time, "cache_path": cache_path}, + ) + + self.assertIsNone(result) + maintainer.drop_segment.assert_called_once_with(cache_path) + if __name__ == "__main__": unittest.main() diff --git a/web/src/hooks/use-camera-activity.ts b/web/src/hooks/use-camera-activity.ts index 76a9827254..9900f277f2 100644 --- a/web/src/hooks/use-camera-activity.ts +++ b/web/src/hooks/use-camera-activity.ts @@ -137,9 +137,11 @@ export function useCameraActivity( } } - newObjects[updatedEventIndex].label = label; - newObjects[updatedEventIndex].stationary = - updatedEvent.after.stationary; + newObjects[updatedEventIndex] = { + ...newObjects[updatedEventIndex], + label, + stationary: updatedEvent.after.stationary, + }; } } From 81b0d947931f1dd436fdbc00d606cafe41cb6787 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 18 Apr 2026 17:50:28 -0500 Subject: [PATCH 008/291] fix broken docs links with hash fragments that resolve wrong on reload (#22925) --- docs/docs/configuration/face_recognition.md | 2 +- docs/docs/frigate/installation.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md index c44f76dea2..74fd810710 100644 --- a/docs/docs/configuration/face_recognition.md +++ b/docs/docs/configuration/face_recognition.md @@ -9,7 +9,7 @@ Face recognition identifies known individuals by matching detected faces with pr ### Face Detection -When running a Frigate+ model (or any custom model that natively detects faces) should ensure that `face` is added to the [list of objects to track](../plus/#available-label-types) either globally or for a specific camera. This will allow face detection to run at the same time as object detection and be more efficient. +When running a Frigate+ model (or any custom model that natively detects faces) should ensure that `face` is added to the [list of objects to track](../plus/index.md#available-label-types) either globally or for a specific camera. This will allow face detection to run at the same time as object detection and be more efficient. When running a default COCO model or another model that does not include `face` as a detectable label, face detection will run via CV2 using a lightweight DNN model that runs on the CPU. In this case, you should _not_ define `face` in your list of objects to track. diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index 3722a23ba2..8839c6bf0f 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -7,7 +7,7 @@ Frigate is a Docker container that can be run on any Docker host including as a :::tip -If you already have Frigate installed as a Home Assistant App, check out the [getting started guide](../guides/getting_started#configuring-frigate) to configure Frigate. +If you already have Frigate installed as a Home Assistant App, check out the [getting started guide](../guides/getting_started.md#configuring-frigate) to configure Frigate. ::: From d7f42735fcd890d5e1fa88ad420bd2789f069c4b Mon Sep 17 00:00:00 2001 From: Arun Rajiah Date: Sun, 19 Apr 2026 18:35:43 +0530 Subject: [PATCH 009/291] docs: add cctvQL to third-party extensions (#22930) --- docs/docs/integrations/third_party_extensions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/docs/integrations/third_party_extensions.md b/docs/docs/integrations/third_party_extensions.md index e0fe835633..c30c7d966b 100644 --- a/docs/docs/integrations/third_party_extensions.md +++ b/docs/docs/integrations/third_party_extensions.md @@ -17,6 +17,10 @@ Please use your own knowledge to assess and vet them before you install anything The [Advanced Camera Card](https://card.camera/#/README) is a Home Assistant dashboard card with deep Frigate integration. +## [cctvQL](https://github.com/arunrajiah/cctvql) + +[cctvQL](https://github.com/arunrajiah/cctvql) is a natural language query layer for Frigate and other CCTV systems. It connects to Frigate's REST API and MQTT broker to let you ask conversational questions about cameras and events (e.g. "Was there motion at the front door last night?"), with support for real-time event streaming, anomaly detection, PTZ control, alert rules, and a Home Assistant custom component. + ## [Double Take](https://github.com/skrashevich/double-take) [Double Take](https://github.com/skrashevich/double-take) provides an unified UI and API for processing and training images for facial recognition. From 423ee2fe7283fb0eaa8621b3ab6b1e47ff244086 Mon Sep 17 00:00:00 2001 From: Otto <36742501+0x464e@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:35:25 +0300 Subject: [PATCH 010/291] Feature: Share Timestamped URL for Camera Footage History (#22537) * Initial copy timestamp url implementation * revise url format * Implement share timestamp dialog * Use translations * Add comments * Add validations to shared link * Switch to searchEffect implementation * Add missing accessibility related dialog description * Change URL format to unix timestamps * Remove unnecessary useEffect * Remove duplicated dialog title * Fixes/improvements based off PR review comments * Add missing cancel button & separators to dialog * Make share description clearer * Bugfix: guard against showing toasts twice Because this effect ends up running multiple times * Clamp future timestamps to now * Revert "Bugfix: guard against showing toasts twice" This reverts commit 99fa5e1deebdc3a91a4b9015f5a46a36545bf349. * Use normal separator Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> * Fixes based off PR review comments * Bugfix: Share dialog was not receiving the player timestamp after removing key that triggered remounts * Defer `setRecording` and return true from hook for cleanup * Remove timeout defer hack in favor of refactored hook * Attempt to replay video muted on NotAllowedError * Use separate persistent mute and temporary forced mute states * Align cancel button with other dialogs * Prevent wrapping on dialog title * Remove extra "back" button on mobile drawer * Fix back navigation when coming from direct shared timestamp links * Use new timeformat hook * Simplify dialog radio buttons * Apply suggestions from code review Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --------- Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --- web/public/locales/en/components/dialog.json | 8 + web/public/locales/en/views/events.json | 4 +- .../components/overlay/ActionsDropdown.tsx | 7 +- .../overlay/MobileReviewSettingsDrawer.tsx | 63 ++- .../overlay/ShareTimestampDialog.tsx | 369 ++++++++++++++++++ web/src/components/player/HlsVideoPlayer.tsx | 35 +- .../player/dynamic/DynamicVideoController.ts | 3 +- web/src/pages/Events.tsx | 63 ++- web/src/types/record.ts | 1 + web/src/utils/recordingReviewUrl.ts | 56 +++ web/src/utils/videoUtil.ts | 17 + web/src/views/recording/RecordingView.tsx | 69 +++- 12 files changed, 678 insertions(+), 17 deletions(-) create mode 100644 web/src/components/overlay/ShareTimestampDialog.tsx create mode 100644 web/src/utils/recordingReviewUrl.ts diff --git a/web/public/locales/en/components/dialog.json b/web/public/locales/en/components/dialog.json index f4e54aa885..3630d68e09 100644 --- a/web/public/locales/en/components/dialog.json +++ b/web/public/locales/en/components/dialog.json @@ -152,6 +152,14 @@ } }, "recording": { + "shareTimestamp": { + "label": "Share Timestamp", + "title": "Share Timestamp", + "description": "Share a timestamped URL of current player position or choose a custom timestamp. Note that this is not a public share URL and is only accessible to users with access to Frigate and this camera.", + "custom": "Custom Timestamp", + "button": "Share Timestamp URL", + "shareTitle": "Frigate Review Timestamp: {{camera}}" + }, "confirmDelete": { "title": "Confirm Delete", "desc": { diff --git a/web/public/locales/en/views/events.json b/web/public/locales/en/views/events.json index a829d3687a..103d93b7eb 100644 --- a/web/public/locales/en/views/events.json +++ b/web/public/locales/en/views/events.json @@ -45,7 +45,9 @@ }, "documentTitle": "Review - Frigate", "recordings": { - "documentTitle": "Recordings - Frigate" + "documentTitle": "Recordings - Frigate", + "invalidSharedLink": "Unable to open timestamped recording link due to parsing error.", + "invalidSharedCamera": "Unable to open timestamped recording link due to an unknown or unauthorized camera." }, "calendarFilter": { "last24Hours": "Last 24 Hours" diff --git a/web/src/components/overlay/ActionsDropdown.tsx b/web/src/components/overlay/ActionsDropdown.tsx index 9ddb0bd353..7f841be4fb 100644 --- a/web/src/components/overlay/ActionsDropdown.tsx +++ b/web/src/components/overlay/ActionsDropdown.tsx @@ -11,12 +11,14 @@ import { FaFilm } from "react-icons/fa6"; type ActionsDropdownProps = { onDebugReplayClick: () => void; onExportClick: () => void; + onShareTimestampClick: () => void; }; export default function ActionsDropdown({ onDebugReplayClick, onExportClick, -}: ActionsDropdownProps) { + onShareTimestampClick, +}: Readonly) { const { t } = useTranslation(["components/dialog", "views/replay", "common"]); return ( @@ -37,6 +39,9 @@ export default function ActionsDropdown({ {t("menu.export", { ns: "common" })} + + {t("recording.shareTimestamp.label", { ns: "components/dialog" })} + {t("title", { ns: "views/replay" })} diff --git a/web/src/components/overlay/MobileReviewSettingsDrawer.tsx b/web/src/components/overlay/MobileReviewSettingsDrawer.tsx index 4adca54b22..63d391162d 100644 --- a/web/src/components/overlay/MobileReviewSettingsDrawer.tsx +++ b/web/src/components/overlay/MobileReviewSettingsDrawer.tsx @@ -2,7 +2,7 @@ import { useCallback, useState } from "react"; import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer"; import { Button } from "../ui/button"; import { FaArrowDown, FaCalendarAlt, FaCog, FaFilter } from "react-icons/fa"; -import { LuBug } from "react-icons/lu"; +import { LuBug, LuShare2 } from "react-icons/lu"; import { TimeRange } from "@/types/timeline"; import { ExportContent, ExportPreviewDialog, ExportTab } from "./ExportDialog"; import { @@ -27,6 +27,7 @@ import { isMobile } from "react-device-detect"; import { useTranslation } from "react-i18next"; import { useNavigate } from "react-router-dom"; import { StartExportResponse } from "@/types/export"; +import { ShareTimestampContent } from "./ShareTimestampDialog"; type DrawerMode = | "none" @@ -34,13 +35,15 @@ type DrawerMode = | "export" | "calendar" | "filter" - | "debug-replay"; + | "debug-replay" + | "share-timestamp"; const DRAWER_FEATURES = [ "export", "calendar", "filter", "debug-replay", + "share-timestamp", ] as const; export type DrawerFeatures = (typeof DRAWER_FEATURES)[number]; const DEFAULT_DRAWER_FEATURES: DrawerFeatures[] = [ @@ -48,6 +51,7 @@ const DEFAULT_DRAWER_FEATURES: DrawerFeatures[] = [ "calendar", "filter", "debug-replay", + "share-timestamp", ]; type MobileReviewSettingsDrawerProps = { @@ -68,6 +72,7 @@ type MobileReviewSettingsDrawerProps = { debugReplayRange?: TimeRange; setDebugReplayMode?: (mode: ExportMode) => void; setDebugReplayRange?: (range: TimeRange | undefined) => void; + onShareTimestamp?: (timestamp: number) => void; onUpdateFilter: (filter: ReviewFilter) => void; setRange: (range: TimeRange | undefined) => void; setMode: (mode: ExportMode) => void; @@ -91,6 +96,7 @@ export default function MobileReviewSettingsDrawer({ debugReplayRange, setDebugReplayMode = () => {}, setDebugReplayRange = () => {}, + onShareTimestamp = () => {}, onUpdateFilter, setRange, setMode, @@ -100,6 +106,7 @@ export default function MobileReviewSettingsDrawer({ "views/recording", "components/dialog", "views/replay", + "common", ]); const navigate = useNavigate(); const [drawerMode, setDrawerMode] = useState("none"); @@ -108,6 +115,15 @@ export default function MobileReviewSettingsDrawer({ "1" | "5" | "custom" | "timeline" >("1"); const [isDebugReplayStarting, setIsDebugReplayStarting] = useState(false); + const [selectedShareOption, setSelectedShareOption] = useState< + "current" | "custom" + >("current"); + const [shareTimestampAtOpen, setShareTimestampAtOpen] = useState( + Math.floor(currentTime), + ); + const [customShareTimestamp, setCustomShareTimestamp] = useState( + Math.floor(currentTime), + ); // exports @@ -323,6 +339,27 @@ export default function MobileReviewSettingsDrawer({ {t("export")} )} + {features.includes("share-timestamp") && ( + + )} {features.includes("calendar") && ( + )} + + + + ); +} + +type CustomTimestampSelectorProps = { + timestamp: number; + setTimestamp: (timestamp: number) => void; + label: string; +}; + +function CustomTimestampSelector({ + timestamp, + setTimestamp, + label, +}: Readonly) { + const { t } = useTranslation(["common"]); + const { data: config } = useSWR("config"); + const timeFormat = useTimeFormat(config); + + const timezoneOffset = useMemo( + () => + config?.ui.timezone + ? Math.round(getUTCOffset(new Date(), config.ui.timezone)) + : undefined, + [config?.ui.timezone], + ); + const localTimeOffset = useMemo( + () => + Math.round( + getUTCOffset( + new Date(), + Intl.DateTimeFormat().resolvedOptions().timeZone, + ), + ), + [], + ); + const offsetDeltaSeconds = useMemo(() => { + if (timezoneOffset === undefined) { + return 0; + } + + // the picker edits a timestamp in the configured UI timezone, + // but the stored value remains a unix timestamp + return (timezoneOffset - localTimeOffset) * 60; + }, [timezoneOffset, localTimeOffset]); + + const displayTimestamp = useMemo( + () => timestamp + offsetDeltaSeconds, + [timestamp, offsetDeltaSeconds], + ); + + const formattedTimestamp = useFormattedTimestamp( + displayTimestamp, + timeFormat == "24hour" + ? t("time.formattedTimestamp.24hour") + : t("time.formattedTimestamp.12hour"), + ); + + const clock = useMemo(() => { + const date = new Date(displayTimestamp * 1000); + return `${date.getHours().toString().padStart(2, "0")}:${date.getMinutes().toString().padStart(2, "0")}:${date.getSeconds().toString().padStart(2, "0")}`; + }, [displayTimestamp]); + + const [selectorOpen, setSelectorOpen] = useState(false); + + const setFromDisplayDate = useCallback( + (date: Date) => { + // convert the edited display time back into the underlying Unix timestamp + setTimestamp(date.getTime() / 1000 - offsetDeltaSeconds); + }, + [offsetDeltaSeconds, setTimestamp], + ); + + return ( +
+ +
+ { + if (!open) { + setSelectorOpen(false); + } + }} + > + + + + + { + if (!day) { + return; + } + + const nextTimestamp = new Date(displayTimestamp * 1000); + nextTimestamp.setFullYear( + day.getFullYear(), + day.getMonth(), + day.getDate(), + ); + setFromDisplayDate(nextTimestamp); + }} + /> +
+ { + const nextClock = e.target.value; + const [hour, minute, second] = isIOS + ? [...nextClock.split(":"), "00"] + : nextClock.split(":"); + const nextTimestamp = new Date(displayTimestamp * 1000); + nextTimestamp.setHours( + Number.parseInt(hour), + Number.parseInt(minute), + Number.parseInt(second ?? "0"), + 0, + ); + setFromDisplayDate(nextTimestamp); + }} + /> + + +
+
+ ); +} diff --git a/web/src/components/player/HlsVideoPlayer.tsx b/web/src/components/player/HlsVideoPlayer.tsx index 7d762912ca..7dfbf3bf3a 100644 --- a/web/src/components/player/HlsVideoPlayer.tsx +++ b/web/src/components/player/HlsVideoPlayer.tsx @@ -216,7 +216,11 @@ export default function HlsVideoPlayer({ const [tallCamera, setTallCamera] = useState(false); const [isPlaying, setIsPlaying] = useState(true); - const [muted, setMuted] = useUserPersistence("hlsPlayerMuted", true); + const [persistedMuted, setPersistedMuted] = useUserPersistence( + "hlsPlayerMuted", + true, + ); + const [temporaryMuted, setTemporaryMuted] = useState(false); const [volume, setVolume] = useOverlayState("playerVolume", 1.0); const [defaultPlaybackRate] = useUserPersistence("playbackRate", 1); const [playbackRate, setPlaybackRate] = useOverlayState( @@ -232,6 +236,16 @@ export default function HlsVideoPlayer({ height: number; }>({ width: 0, height: 0 }); + const muted = persistedMuted || temporaryMuted; + + const onSetMuted = useCallback( + (muted: boolean) => { + setTemporaryMuted(false); + setPersistedMuted(muted); + }, + [setPersistedMuted], + ); + useEffect(() => { if (!isDesktop) { return; @@ -297,7 +311,7 @@ export default function HlsVideoPlayer({ fullscreen: supportsFullscreen, }} setControlsOpen={setControlsOpen} - setMuted={(muted) => setMuted(muted)} + setMuted={onSetMuted} playbackRate={playbackRate ?? 1} hotKeys={hotKeys} onPlayPause={onPlayPause} @@ -404,9 +418,20 @@ export default function HlsVideoPlayer({ : undefined } onVolumeChange={() => { - setVolume(videoRef.current?.volume ?? 1.0, true); - if (!frigateControls) { - setMuted(videoRef.current?.muted); + if (!videoRef.current) { + return; + } + + setVolume(videoRef.current.volume ?? 1.0, true); + + if (frigateControls) { + if (videoRef.current.muted && !persistedMuted) { + setTemporaryMuted(true); + } else if (!videoRef.current.muted && temporaryMuted) { + setTemporaryMuted(false); + } + } else { + setPersistedMuted(videoRef.current.muted); } }} onPlay={() => { diff --git a/web/src/components/player/dynamic/DynamicVideoController.ts b/web/src/components/player/dynamic/DynamicVideoController.ts index e9da0064d6..151ea4022f 100644 --- a/web/src/components/player/dynamic/DynamicVideoController.ts +++ b/web/src/components/player/dynamic/DynamicVideoController.ts @@ -6,6 +6,7 @@ import { calculateInpointOffset, calculateSeekPosition, } from "@/utils/videoUtil"; +import { playWithTemporaryMuteFallback } from "@/utils/videoUtil.ts"; type PlayerMode = "playback" | "scrubbing"; @@ -107,7 +108,7 @@ export class DynamicVideoController { return new Promise((resolve) => { const onSeekedHandler = () => { this.playerController.removeEventListener("seeked", onSeekedHandler); - this.playerController.play(); + playWithTemporaryMuteFallback(this.playerController); resolve(undefined); }; diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index 9f8dd4e44e..3e01e7451f 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -21,6 +21,10 @@ import { getBeginningOfDayTimestamp, getEndOfDayTimestamp, } from "@/utils/dateUtil"; +import { + parseRecordingReviewLink, + RECORDING_REVIEW_LINK_PARAM, +} from "@/utils/recordingReviewUrl"; import EventView from "@/views/events/EventView"; import MotionSearchView from "@/views/motion-search/MotionSearchView"; import { RecordingView } from "@/views/recording/RecordingView"; @@ -28,6 +32,7 @@ import { useFrigateReviews } from "@/api/ws"; import axios from "axios"; import { useCallback, useEffect, useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; +import { toast } from "sonner"; import useSWR from "swr"; export default function Events() { @@ -128,6 +133,14 @@ export default function Events() { const [notificationTab, setNotificationTab] = useState("timeline"); + const getReviewDayBounds = useCallback((date: Date) => { + const now = Date.now() / 1000; + return { + after: getBeginningOfDayTimestamp(date), + before: Math.min(getEndOfDayTimestamp(date), now), + }; + }, []); + useSearchEffect("tab", (tab: string) => { if (tab === "timeline" || tab === "events" || tab === "detail") { setNotificationTab(tab as TimelineType); @@ -143,10 +156,7 @@ export default function Events() { const startTime = resp.data.start_time - REVIEW_PADDING; const date = new Date(startTime * 1000); - setReviewFilter({ - after: getBeginningOfDayTimestamp(date), - before: getEndOfDayTimestamp(date), - }); + setReviewFilter(getReviewDayBounds(date)); setRecording( { camera: resp.data.camera, @@ -234,6 +244,51 @@ export default function Events() { [recording, setRecording, setReviewFilter], ); + useSearchEffect(RECORDING_REVIEW_LINK_PARAM, (reviewLinkValue: string) => { + if (!config) { + return false; + } + + const reviewLink = parseRecordingReviewLink(reviewLinkValue); + + if (!reviewLink) { + toast.error(t("recordings.invalidSharedLink"), { + position: "top-center", + }); + return true; + } + + const validCamera = + config.cameras[reviewLink.camera] && + allowedCameras.includes(reviewLink.camera); + + if (!validCamera) { + toast.error(t("recordings.invalidSharedCamera"), { + position: "top-center", + }); + return true; + } + + setReviewFilter({ + ...reviewFilter, + ...getReviewDayBounds(new Date(reviewLink.timestamp * 1000)), + }); + setRecording( + { + camera: reviewLink.camera, + startTime: reviewLink.timestamp, + // severity not actually applicable here, but the type requires it + // this pattern is also used LiveCameraView to enter recording view + severity: "alert", + timelineType: notificationTab, + navigationSource: "shared-link", + }, + true, + ); + + return true; + }); + // review paging const [beforeTs, setBeforeTs] = useState(Math.ceil(Date.now() / 1000)); diff --git a/web/src/types/record.ts b/web/src/types/record.ts index 107a8d86eb..d8fd163bf9 100644 --- a/web/src/types/record.ts +++ b/web/src/types/record.ts @@ -40,6 +40,7 @@ export type RecordingStartingPoint = { startTime: number; severity: ReviewSeverity; timelineType?: TimelineType; + navigationSource?: "shared-link"; }; export type RecordingPlayerError = "stalled" | "startup"; diff --git a/web/src/utils/recordingReviewUrl.ts b/web/src/utils/recordingReviewUrl.ts new file mode 100644 index 0000000000..0bc8960d07 --- /dev/null +++ b/web/src/utils/recordingReviewUrl.ts @@ -0,0 +1,56 @@ +import { baseUrl } from "@/api/baseUrl.ts"; + +export const RECORDING_REVIEW_LINK_PARAM = "timestamp"; + +export type RecordingReviewLinkState = { + camera: string; + timestamp: number; +}; + +export function parseRecordingReviewLink( + value: string | null, +): RecordingReviewLinkState | undefined { + if (!value) { + return undefined; + } + + const separatorIndex = value.lastIndexOf("_"); + + if (separatorIndex <= 0 || separatorIndex == value.length - 1) { + return undefined; + } + + const camera = value.slice(0, separatorIndex); + const timestamp = value.slice(separatorIndex + 1); + + if (!camera || !timestamp) { + return undefined; + } + + const parsedTimestamp = Number(timestamp); + const now = Math.floor(Date.now() / 1000); + + if (!Number.isFinite(parsedTimestamp) || parsedTimestamp <= 0) { + return undefined; + } + + return { + camera, + // clamp future timestamps to now + timestamp: Math.min(Math.floor(parsedTimestamp), now), + }; +} + +export function createRecordingReviewUrl( + pathname: string, + state: RecordingReviewLinkState, +): string { + const url = new URL(baseUrl); + url.pathname = pathname.startsWith("/") ? pathname : `/${pathname}`; + url.searchParams.set( + RECORDING_REVIEW_LINK_PARAM, + `${state.camera}_${Math.floor(state.timestamp)}`, + ); + + return url.toString(); +} diff --git a/web/src/utils/videoUtil.ts b/web/src/utils/videoUtil.ts index 0b09ac0618..d6ab203e93 100644 --- a/web/src/utils/videoUtil.ts +++ b/web/src/utils/videoUtil.ts @@ -78,3 +78,20 @@ export function calculateSeekPosition( return seekSeconds >= 0 ? seekSeconds : undefined; } + +/** + * Attempts to play the video, and if it fails due to a NotAllowedError (often caused by browser autoplay restrictions), + * it temporarily mutes the video and tries to play again. + * @param video - The HTMLVideoElement to play + */ +export function playWithTemporaryMuteFallback(video: HTMLVideoElement) { + return video.play().catch((error: { name?: string }) => { + if (error.name === "NotAllowedError" && !video.muted) { + video.muted = true; + + return video.play().catch(() => undefined); + } + + throw error; + }); +} diff --git a/web/src/views/recording/RecordingView.tsx b/web/src/views/recording/RecordingView.tsx index 6a09b7df29..75b6519fcb 100644 --- a/web/src/views/recording/RecordingView.tsx +++ b/web/src/views/recording/RecordingView.tsx @@ -42,7 +42,7 @@ import { isTablet, } from "react-device-detect"; import { IoMdArrowRoundBack } from "react-icons/io"; -import { useNavigate } from "react-router-dom"; +import { useLocation, useNavigate } from "react-router-dom"; import { Toaster } from "@/components/ui/sonner"; import useSWR from "swr"; import { TimeRange, TimelineType } from "@/types/timeline"; @@ -77,6 +77,9 @@ import { GenAISummaryDialog, GenAISummaryChip, } from "@/components/overlay/chip/GenAISummaryChip"; +import ShareTimestampDialog from "@/components/overlay/ShareTimestampDialog"; +import { shareOrCopy } from "@/utils/browserUtil"; +import { createRecordingReviewUrl } from "@/utils/recordingReviewUrl"; const DATA_REFRESH_TIME = 600000; // 10 minutes @@ -104,9 +107,10 @@ export function RecordingView({ updateFilter, refreshData, }: RecordingViewProps) { - const { t } = useTranslation(["views/events"]); + const { t } = useTranslation(["views/events", "components/dialog"]); const { data: config } = useSWR("config"); const navigate = useNavigate(); + const location = useLocation(); const contentRef = useRef(null); // recordings summary @@ -205,6 +209,16 @@ export function RecordingView({ const [debugReplayMode, setDebugReplayMode] = useState("none"); const [debugReplayRange, setDebugReplayRange] = useState(); + const [shareTimestampOpen, setShareTimestampOpen] = useState(false); + const [shareTimestampAtOpen, setShareTimestampAtOpen] = useState( + Math.floor(startTime), + ); + const [shareTimestampOption, setShareTimestampOption] = useState< + "current" | "custom" + >("current"); + const [customShareTimestamp, setCustomShareTimestamp] = useState( + Math.floor(startTime), + ); // move to next clip @@ -317,6 +331,34 @@ export function RecordingView({ [currentTimeRange, updateSelectedSegment], ); + const onShareReviewLink = useCallback( + (timestamp: number) => { + const reviewUrl = createRecordingReviewUrl(location.pathname, { + camera: mainCamera, + timestamp: Math.floor(timestamp), + }); + + shareOrCopy( + reviewUrl, + t("recording.shareTimestamp.shareTitle", { + ns: "components/dialog", + camera: mainCamera, + }), + ); + }, + [location.pathname, mainCamera, t], + ); + + const handleBack = useCallback(() => { + // if we came from a direct share link, there is no history to go back to, so navigate to the homepage instead + if (recording?.navigationSource === "shared-link") { + navigate("/"); + return; + } + + navigate(-1); + }, [navigate, recording?.navigationSource]); + useEffect(() => { if (!scrubbing) { if (Math.abs(currentTime - playerTime) > 10) { @@ -567,7 +609,7 @@ export function RecordingView({ className="flex items-center gap-2.5 rounded-lg" aria-label={t("label.back", { ns: "common" })} size="sm" - onClick={() => navigate(-1)} + onClick={handleBack} > {isDesktop && ( @@ -663,8 +705,28 @@ export function RecordingView({ setMotionOnly={() => {}} /> )} + {isDesktop && ( + + )} {isDesktop && ( { + const initialTimestamp = Math.floor(currentTime); + + setShareTimestampAtOpen(initialTimestamp); + setShareTimestampOption("current"); + setCustomShareTimestamp(initialTimestamp); + setShareTimestampOpen(true); + }} onDebugReplayClick={() => { const now = new Date(timeRange.before * 1000); now.setHours(now.getHours() - 1); @@ -744,6 +806,7 @@ export function RecordingView({ mainControllerRef.current?.pause(); } }} + onShareTimestamp={onShareReviewLink} onUpdateFilter={updateFilter} setRange={setExportRange} setMode={setExportMode} From 043c746a8b670ec5d00ff683d8da24e8c9215fb0 Mon Sep 17 00:00:00 2001 From: icidi <9525979+icidi@users.noreply.github.com> Date: Mon, 20 Apr 2026 20:35:48 +0800 Subject: [PATCH 011/291] Improve readability by removing trailing digits caused by floating number conversion (#22934) --- frigate/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frigate/storage.py b/frigate/storage.py index 8cc199a1b3..585a5d87f1 100644 --- a/frigate/storage.py +++ b/frigate/storage.py @@ -197,7 +197,7 @@ class StorageMaintainer(threading.Thread): # check if need to delete retained segments if deleted_segments_size < hourly_bandwidth: logger.error( - f"Could not clear {hourly_bandwidth} MB, currently {deleted_segments_size} MB have been cleared. Retained recordings must be deleted." + f"Could not clear {hourly_bandwidth} MB, currently {deleted_segments_size:.2f} MB have been cleared. Retained recordings must be deleted." ) recordings = ( Recordings.select( @@ -225,7 +225,7 @@ class StorageMaintainer(threading.Thread): # this file was not found so we must assume no space was cleaned up pass else: - logger.info(f"Cleaned up {deleted_segments_size} MB of recordings") + logger.info(f"Cleaned up {deleted_segments_size:.2f} MB of recordings") logger.debug(f"Expiring {len(deleted_recordings)} recordings") # delete up to 100,000 at a time From 1a5d15ba813d4462ce8d3d7cd2257d22d6b0734f Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:19:09 -0500 Subject: [PATCH 012/291] Miscellaneous fixes (#22924) * apply annotation offset to frigate+ submission frame time * fix broken docs links with hash fragments that resolve wrong on reload * undo * use recording snapshot for frigate+ frame submission from VideoControls rather than a canvas grab/paint, which may not always align with an ffmpeg snapshot due to keyframes * add more docs links - display docs link for main sections on collapsible fields * dialog button consistency --- web/public/locales/en/components/player.json | 3 +- .../config-form/section-configs/ffmpeg.ts | 2 + .../config-form/section-configs/lpr.ts | 4 +- .../config-form/section-configs/onvif.ts | 5 +++ .../config-form/section-configs/review.ts | 4 ++ .../theme/templates/ObjectFieldTemplate.tsx | 30 +++++++++++++- .../components/overlay/DebugReplayDialog.tsx | 11 ++--- web/src/components/overlay/ExportDialog.tsx | 11 ++--- .../overlay/ShareTimestampDialog.tsx | 15 +++---- .../overlay/detail/TrackingDetails.tsx | 13 +++++- web/src/components/player/HlsVideoPlayer.tsx | 9 ++++ web/src/components/player/VideoControls.tsx | 41 ++++++++++++++++--- .../player/dynamic/DynamicVideoPlayer.tsx | 16 ++++++++ web/src/views/settings/TriggerView.tsx | 13 +++++- 14 files changed, 147 insertions(+), 30 deletions(-) diff --git a/web/public/locales/en/components/player.json b/web/public/locales/en/components/player.json index 3b50ff5ed5..6ceef7e0cd 100644 --- a/web/public/locales/en/components/player.json +++ b/web/public/locales/en/components/player.json @@ -4,7 +4,8 @@ "noPreviewFoundFor": "No Preview Found for {{cameraName}}", "submitFrigatePlus": { "title": "Submit this frame to Frigate+?", - "submit": "Submit" + "submit": "Submit", + "previewError": "Could not load snapshot preview. The recording may not be available at this time." }, "livePlayerRequiredIOSVersion": "iOS 17.1 or greater is required for this live stream type.", "streamOffline": { diff --git a/web/src/components/config-form/section-configs/ffmpeg.ts b/web/src/components/config-form/section-configs/ffmpeg.ts index d0d426a6a1..97326c33f3 100644 --- a/web/src/components/config-form/section-configs/ffmpeg.ts +++ b/web/src/components/config-form/section-configs/ffmpeg.ts @@ -31,6 +31,8 @@ const ffmpeg: SectionConfigOverrides = { "inputs.output_args": "/configuration/ffmpeg_presets#output-args-presets", "output_args.record": "/configuration/ffmpeg_presets#output-args-presets", "inputs.roles": "/configuration/cameras/#setting-up-camera-inputs", + apple_compatibility: + "/configuration/camera_specific#h265-cameras-via-safari", }, restartRequired: [], fieldOrder: [ diff --git a/web/src/components/config-form/section-configs/lpr.ts b/web/src/components/config-form/section-configs/lpr.ts index 0567c6cf48..c56966142a 100644 --- a/web/src/components/config-form/section-configs/lpr.ts +++ b/web/src/components/config-form/section-configs/lpr.ts @@ -27,10 +27,12 @@ const lpr: SectionConfigOverrides = { ], fieldDocs: { enhancement: "/configuration/license_plate_recognition#enhancement", + debug_save_plates: + "/configuration/license_plate_recognition/#how-do-i-debug-lpr-issues", }, restartRequired: [], fieldOrder: ["enabled", "min_area", "enhancement", "expire_time"], - hiddenFields: [], + hiddenFields: ["expire_time"], advancedFields: ["expire_time", "enhancement"], overrideFields: ["enabled", "min_area", "enhancement"], }, diff --git a/web/src/components/config-form/section-configs/onvif.ts b/web/src/components/config-form/section-configs/onvif.ts index c08cd7a58a..acfb68f167 100644 --- a/web/src/components/config-form/section-configs/onvif.ts +++ b/web/src/components/config-form/section-configs/onvif.ts @@ -3,6 +3,11 @@ import type { SectionConfigOverrides } from "./types"; const onvif: SectionConfigOverrides = { base: { sectionDocs: "/configuration/cameras#setting-up-camera-ptz-controls", + fieldDocs: { + autotracking: "/configuration/autotracking", + "autotracking.calibrate_on_startup": + "/configuration/autotracking#calibration", + }, fieldOrder: [ "host", "port", diff --git a/web/src/components/config-form/section-configs/review.ts b/web/src/components/config-form/section-configs/review.ts index ce0d7b9118..1069d82bf2 100644 --- a/web/src/components/config-form/section-configs/review.ts +++ b/web/src/components/config-form/section-configs/review.ts @@ -45,6 +45,10 @@ const review: SectionConfigOverrides = { fieldDocs: { "alerts.labels": "/configuration/review/#alerts-and-detections", "detections.labels": "/configuration/review/#alerts-and-detections", + genai: "/configuration/genai/genai_review", + "genai.image_source": "/configuration/genai/genai_review#image-source", + "genai.additional_concerns": + "/configuration/genai/genai_review#additional-concerns", }, restartRequired: [], fieldOrder: ["alerts", "detections", "genai", "genai.enabled"], diff --git a/web/src/components/config-form/theme/templates/ObjectFieldTemplate.tsx b/web/src/components/config-form/theme/templates/ObjectFieldTemplate.tsx index 65c5c5541b..4b588d62b1 100644 --- a/web/src/components/config-form/theme/templates/ObjectFieldTemplate.tsx +++ b/web/src/components/config-form/theme/templates/ObjectFieldTemplate.tsx @@ -9,11 +9,13 @@ import { import { Children, useState, useEffect, useRef } from "react"; import type { ReactNode } from "react"; import RestartRequiredIndicator from "@/components/indicators/RestartRequiredIndicator"; -import { LuChevronDown, LuChevronRight } from "react-icons/lu"; +import { LuChevronDown, LuChevronRight, LuExternalLink } from "react-icons/lu"; import { useTranslation } from "react-i18next"; +import { Link } from "react-router-dom"; import { cn } from "@/lib/utils"; import { getTranslatedLabel } from "@/utils/i18n"; import { requiresRestartForFieldPath } from "@/utils/configUtil"; +import { useDocDomain } from "@/hooks/use-doc-domain"; import { ConfigFormContext } from "@/types/configForm"; import { buildTranslationPath, @@ -178,6 +180,7 @@ export function ObjectFieldTemplate(props: ObjectFieldTemplateProps) { "views/settings", "common", ]); + const { getLocaleDocUrl } = useDocDomain(); const objectRequiresRestart = requiresRestartForFieldPath( fieldPath, restartRequired, @@ -300,6 +303,17 @@ export function ObjectFieldTemplate(props: ObjectFieldTemplateProps) { schemaDescription; inferredDescription = inferredDescription ?? fallbackDescription; + const pathStringSegments = + path?.filter((segment): segment is string => typeof segment === "string") ?? + []; + const fieldDocsKey = translationPath || pathStringSegments.join("."); + const fieldDocsPath = fieldDocsKey + ? formContext?.fieldDocs?.[fieldDocsKey] + : undefined; + const fieldDocsUrl = fieldDocsPath + ? getLocaleDocUrl(fieldDocsPath) + : undefined; + const renderGroupedFields = (items: (typeof properties)[number][]) => { if (!items.length) { return null; @@ -466,6 +480,20 @@ export function ObjectFieldTemplate(props: ObjectFieldTemplateProps) { {inferredDescription}

)} + {fieldDocsUrl && ( +
+ e.stopPropagation()} + > + {t("readTheDocumentation", { ns: "common" })} + + +
+ )}
{isOpen ? ( diff --git a/web/src/components/overlay/DebugReplayDialog.tsx b/web/src/components/overlay/DebugReplayDialog.tsx index 9c3efb4f56..2a9e09d085 100644 --- a/web/src/components/overlay/DebugReplayDialog.tsx +++ b/web/src/components/overlay/DebugReplayDialog.tsx @@ -113,18 +113,19 @@ export function DebugReplayContent({ {isDesktop && } -
{t("button.cancel", { ns: "common" })} -
+ {activeTab === "export" ? ( + )} {t("markAsReviewed")} diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 49d0e70bfb..dc09fe4f5c 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -389,7 +389,7 @@ export default function LiveCameraView({ return "mse"; }, [lowBandwidth, mic, webRTC, isRestreamed]); - useKeyboardListener(["m"], (key, modifiers) => { + useKeyboardListener(["m", "Escape"], (key, modifiers) => { if (!modifiers.down) { return true; } @@ -407,6 +407,12 @@ export default function LiveCameraView({ return true; } break; + case "Escape": + if (!fullscreen) { + navigate(-1); + return true; + } + break; } return false; From 4a1b7a162978ad049103b4020db48ee86681a8da Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 23 Apr 2026 08:16:22 -0500 Subject: [PATCH 030/291] enforce python-level timeout on ffprobe subprocesses (#22984) --- frigate/util/services.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/frigate/util/services.py b/frigate/util/services.py index 4f89db0927..159b9b6834 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -726,7 +726,20 @@ def ffprobe_stream(ffmpeg, path: str, detailed: bool = False) -> sp.CompletedPro if detailed and format_entries: cmd.extend(["-show_entries", f"format={format_entries}"]) cmd.extend(["-loglevel", "error", clean_path]) - return sp.run(cmd, capture_output=True) + try: + return sp.run(cmd, capture_output=True, timeout=6) + except sp.TimeoutExpired as e: + logger.info( + "ffprobe timed out while probing %s (transport=%s)", + clean_camera_user_pass(path), + rtsp_transport or "default", + ) + return sp.CompletedProcess( + args=cmd, + returncode=1, + stdout=e.stdout or b"", + stderr=(e.stderr or b"") + b"\nffprobe timed out", + ) result = run() @@ -832,11 +845,23 @@ async def get_video_properties( "-show_streams", url, ] + proc = None try: proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) - stdout, _ = await proc.communicate() + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=6) + except asyncio.TimeoutError: + logger.info( + "ffprobe timed out while probing %s (transport=%s)", + clean_camera_user_pass(url), + rtsp_transport or "default", + ) + proc.kill() + await proc.wait() + return False, 0, 0, None, -1 + if proc.returncode != 0: return False, 0, 0, None, -1 From 1a6d04fde7f5d1ade87de145969ed803ae96846a Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 23 Apr 2026 08:53:48 -0500 Subject: [PATCH 031/291] use object-anchored snapshot crops for classification wizard examples (#22985) --- frigate/util/classification.py | 148 +++++++++++++++++++++------------ 1 file changed, 96 insertions(+), 52 deletions(-) diff --git a/frigate/util/classification.py b/frigate/util/classification.py index ada3ee1f71..66bacdeb04 100644 --- a/frigate/util/classification.py +++ b/frigate/util/classification.py @@ -24,8 +24,12 @@ from frigate.log import redirect_output_to_logger, suppress_stderr_during from frigate.models import Event, Recordings, ReviewSegment from frigate.types import ModelStatusTypesEnum from frigate.util.downloader import ModelDownloader -from frigate.util.file import get_event_thumbnail_bytes -from frigate.util.image import get_image_from_recording +from frigate.util.file import get_event_thumbnail_bytes, load_event_snapshot_image +from frigate.util.image import ( + calculate_region, + get_image_from_recording, + relative_box_to_absolute, +) from frigate.util.process import FrigateProcess BATCH_SIZE = 16 @@ -713,7 +717,7 @@ def collect_object_classification_examples( This function: 1. Queries events for the specified label 2. Selects 100 balanced events across different cameras and times - 3. Retrieves thumbnails for selected events (with 33% center crop applied) + 3. Crops each event's clean snapshot around the object bounding box 4. Selects 24 most visually distinct thumbnails 5. Saves to dataset directory @@ -832,66 +836,106 @@ def _select_balanced_events( def _extract_event_thumbnails(events: list[Event], output_dir: str) -> list[str]: """ - Extract thumbnails from events and save to disk. + Extract a training image for each event. + + Preferred path: load the full-frame clean snapshot and crop around the + stored bounding box with the same calculate_region(..., max(w, h), 1.0) + call the live ObjectClassificationProcessor uses, so wizard examples + are framed like inference-time inputs. + + Fallback: if no clean snapshot exists (snapshots disabled, or only a + legacy annotated JPG is on disk), center-crop the stored thumbnail + using a step ladder sized from the box/region area ratio. Args: events: List of Event objects - output_dir: Directory to save thumbnails + output_dir: Directory to save crops Returns: - List of paths to successfully extracted thumbnail images + List of paths to successfully extracted images """ - thumbnail_paths = [] + image_paths = [] for idx, event in enumerate(events): try: - thumbnail_bytes = get_event_thumbnail_bytes(event) + img = _load_event_classification_crop(event) + if img is None: + continue - if thumbnail_bytes: - nparr = np.frombuffer(thumbnail_bytes, np.uint8) - img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) - - if img is not None: - height, width = img.shape[:2] - - crop_size = 1.0 - if event.data and "box" in event.data and "region" in event.data: - box = event.data["box"] - region = event.data["region"] - - if len(box) == 4 and len(region) == 4: - box_w, box_h = box[2], box[3] - region_w, region_h = region[2], region[3] - - box_area = (box_w * box_h) / (region_w * region_h) - - if box_area < 0.05: - crop_size = 0.4 - elif box_area < 0.10: - crop_size = 0.5 - elif box_area < 0.20: - crop_size = 0.65 - elif box_area < 0.35: - crop_size = 0.80 - else: - crop_size = 0.95 - - crop_width = int(width * crop_size) - crop_height = int(height * crop_size) - - x1 = (width - crop_width) // 2 - y1 = (height - crop_height) // 2 - x2 = x1 + crop_width - y2 = y1 + crop_height - - cropped = img[y1:y2, x1:x2] - resized = cv2.resize(cropped, (224, 224)) - output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg") - cv2.imwrite(output_path, resized) - thumbnail_paths.append(output_path) + resized = cv2.resize(img, (224, 224)) + output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg") + cv2.imwrite(output_path, resized) + image_paths.append(output_path) except Exception as e: - logger.debug(f"Failed to extract thumbnail for event {event.id}: {e}") + logger.debug(f"Failed to extract image for event {event.id}: {e}") continue - return thumbnail_paths + return image_paths + + +def _load_event_classification_crop(event: Event) -> np.ndarray | None: + """Prefer a snapshot-based object crop; fall back to a center-cropped thumbnail.""" + if event.data and "box" in event.data: + snapshot, _ = load_event_snapshot_image(event, clean_only=True) + if snapshot is not None: + abs_box = relative_box_to_absolute(snapshot.shape, event.data["box"]) + if abs_box is not None: + xmin, ymin, xmax, ymax = abs_box + box_w = xmax - xmin + box_h = ymax - ymin + if box_w > 0 and box_h > 0: + x1, y1, x2, y2 = calculate_region( + snapshot.shape, + xmin, + ymin, + xmax, + ymax, + max(box_w, box_h), + 1.0, + ) + cropped = snapshot[y1:y2, x1:x2] + if cropped.size > 0: + return cropped + + thumbnail_bytes = get_event_thumbnail_bytes(event) + if not thumbnail_bytes: + return None + + nparr = np.frombuffer(thumbnail_bytes, np.uint8) + img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) + if img is None or img.size == 0: + return None + + height, width = img.shape[:2] + crop_size = 1.0 + + if event.data and "box" in event.data and "region" in event.data: + box = event.data["box"] + region = event.data["region"] + + if len(box) == 4 and len(region) == 4: + box_w, box_h = box[2], box[3] + region_w, region_h = region[2], region[3] + box_area = (box_w * box_h) / (region_w * region_h) + + if box_area < 0.05: + crop_size = 0.4 + elif box_area < 0.10: + crop_size = 0.5 + elif box_area < 0.20: + crop_size = 0.65 + elif box_area < 0.35: + crop_size = 0.80 + else: + crop_size = 0.95 + + crop_width = int(width * crop_size) + crop_height = int(height * crop_size) + x1 = (width - crop_width) // 2 + y1 = (height - crop_height) // 2 + cropped = img[y1 : y1 + crop_height, x1 : x1 + crop_width] + if cropped.size == 0: + return None + + return cropped From 77831304a7a9a7c02cd6881d48061c2b73b60ae3 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:27:49 -0500 Subject: [PATCH 032/291] Camera access fixes (#22987) * only send monitoring notifications to users with camera access * check access to similarity search event id camera * require admin role for storage usage endpoint * check camera access for jsmpeg and birdseye cameras * tests * formatting --- frigate/api/event.py | 9 +++ frigate/api/record.py | 2 +- frigate/comms/webpush.py | 8 +++ frigate/output/birdseye.py | 10 +++- frigate/output/camera.py | 18 +++++- frigate/output/output.py | 5 +- frigate/output/ws_auth.py | 43 ++++++++++++++ frigate/test/http_api/test_http_app.py | 20 +++++++ frigate/test/http_api/test_http_event.py | 19 +++++++ .../test/test_chat_find_similar_objects.py | 7 ++- frigate/test/test_output_ws_auth.py | 57 +++++++++++++++++++ .../test/test_webpush_camera_monitoring.py | 29 ++++++++++ 12 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 frigate/output/ws_auth.py create mode 100644 frigate/test/test_output_ws_auth.py create mode 100644 frigate/test/test_webpush_camera_monitoring.py diff --git a/frigate/api/event.py b/frigate/api/event.py index a7d1cffc87..fc7c58c375 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -754,6 +754,15 @@ def events_search( status_code=404, ) + if search_event.camera not in allowed_cameras: + return JSONResponse( + content={ + "success": False, + "message": "Event not found", + }, + status_code=404, + ) + thumb_result = context.search_thumbnail(search_event) thumb_ids = {result[0]: result[1] for result in thumb_result} search_results = { diff --git a/frigate/api/record.py b/frigate/api/record.py index 4ab4b0af16..f6366813b6 100644 --- a/frigate/api/record.py +++ b/frigate/api/record.py @@ -35,7 +35,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=[Tags.recordings]) -@router.get("/recordings/storage", dependencies=[Depends(allow_any_authenticated())]) +@router.get("/recordings/storage", dependencies=[Depends(require_role(["admin"]))]) def get_recordings_storage_usage(request: Request): recording_stats = request.app.stats_emitter.get_latest_stats()["service"][ "storage" diff --git a/frigate/comms/webpush.py b/frigate/comms/webpush.py index e4ed832682..a9e237e706 100644 --- a/frigate/comms/webpush.py +++ b/frigate/comms/webpush.py @@ -549,6 +549,14 @@ class WebPushClient(Communicator): logger.debug(f"Sending camera monitoring push notification for {camera_name}") for user in self.web_pushers: + if not self._user_has_camera_access(user, camera): + logger.debug( + "Skipping notification for user %s - no access to camera %s", + user, + camera, + ) + continue + self.send_push_notification( user=user, payload=payload, diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py index 8b0fea6d7b..ff846008ca 100644 --- a/frigate/output/birdseye.py +++ b/frigate/output/birdseye.py @@ -19,6 +19,7 @@ import numpy as np from frigate.comms.inter_process import InterProcessRequestor from frigate.config import BirdseyeModeEnum, FfmpegConfig, FrigateConfig from frigate.const import BASE_DIR, BIRDSEYE_PIPE, INSTALL_DIR, UPDATE_BIRDSEYE_LAYOUT +from frigate.output.ws_auth import ws_has_camera_access from frigate.util.image import ( SharedMemoryFrameManager, copy_yuv_to_position, @@ -236,12 +237,14 @@ class BroadcastThread(threading.Thread): converter: FFMpegConverter, websocket_server: Any, stop_event: MpEvent, + config: FrigateConfig, ): super().__init__() self.camera = camera self.converter = converter self.websocket_server = websocket_server self.stop_event = stop_event + self.config = config def run(self) -> None: while not self.stop_event.is_set(): @@ -256,6 +259,7 @@ class BroadcastThread(threading.Thread): if ( not ws.terminated and ws.environ["PATH_INFO"] == f"/{self.camera}" + and ws_has_camera_access(ws, self.camera, self.config) ): try: ws.send(buf, binary=True) @@ -806,7 +810,11 @@ class Birdseye: config.birdseye.restream, ) self.broadcaster = BroadcastThread( - "birdseye", self.converter, websocket_server, stop_event + "birdseye", + self.converter, + websocket_server, + stop_event, + config, ) self.birdseye_manager = BirdsEyeFrameManager(self.config, stop_event) self.frame_manager = SharedMemoryFrameManager() diff --git a/frigate/output/camera.py b/frigate/output/camera.py index 917e38dd1d..88d16ed4b4 100644 --- a/frigate/output/camera.py +++ b/frigate/output/camera.py @@ -7,7 +7,8 @@ import threading from multiprocessing.synchronize import Event as MpEvent from typing import Any -from frigate.config import CameraConfig, FfmpegConfig +from frigate.config import CameraConfig, FfmpegConfig, FrigateConfig +from frigate.output.ws_auth import ws_has_camera_access logger = logging.getLogger(__name__) @@ -102,12 +103,14 @@ class BroadcastThread(threading.Thread): converter: FFMpegConverter, websocket_server: Any, stop_event: MpEvent, + config: FrigateConfig, ): super().__init__() self.camera = camera self.converter = converter self.websocket_server = websocket_server self.stop_event = stop_event + self.config = config def run(self) -> None: while not self.stop_event.is_set(): @@ -122,6 +125,7 @@ class BroadcastThread(threading.Thread): if ( not ws.terminated and ws.environ["PATH_INFO"] == f"/{self.camera}" + and ws_has_camera_access(ws, self.camera, self.config) ): try: ws.send(buf, binary=True) @@ -135,7 +139,11 @@ class BroadcastThread(threading.Thread): class JsmpegCamera: def __init__( - self, config: CameraConfig, stop_event: MpEvent, websocket_server: Any + self, + config: CameraConfig, + frigate_config: FrigateConfig, + stop_event: MpEvent, + websocket_server: Any, ) -> None: self.config = config self.input: queue.Queue[bytes] = queue.Queue(maxsize=config.detect.fps) @@ -154,7 +162,11 @@ class JsmpegCamera: config.live.quality, ) self.broadcaster = BroadcastThread( - config.name or "", self.converter, websocket_server, stop_event + config.name or "", + self.converter, + websocket_server, + stop_event, + frigate_config, ) self.converter.start() diff --git a/frigate/output/output.py b/frigate/output/output.py index 22bcbb31ff..265c226215 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -32,6 +32,7 @@ from frigate.const import ( from frigate.output.birdseye import Birdseye from frigate.output.camera import JsmpegCamera from frigate.output.preview import PreviewRecorder +from frigate.output.ws_auth import ws_has_camera_access from frigate.util.image import SharedMemoryFrameManager, get_blank_yuv_frame from frigate.util.process import FrigateProcess @@ -102,7 +103,7 @@ class OutputProcess(FrigateProcess): ) -> None: camera_config = self.config.cameras[camera] jsmpeg_cameras[camera] = JsmpegCamera( - camera_config, self.stop_event, websocket_server + camera_config, self.config, self.stop_event, websocket_server ) preview_recorders[camera] = PreviewRecorder(camera_config) preview_write_times[camera] = 0 @@ -262,6 +263,7 @@ class OutputProcess(FrigateProcess): # send camera frame to ffmpeg process if websockets are connected if any( ws.environ["PATH_INFO"].endswith(camera) + and ws_has_camera_access(ws, camera, self.config) for ws in websocket_server.manager ): # write to the converter for the camera if clients are listening to the specific camera @@ -275,6 +277,7 @@ class OutputProcess(FrigateProcess): self.config.birdseye.restream or any( ws.environ["PATH_INFO"].endswith("birdseye") + and ws_has_camera_access(ws, "birdseye", self.config) for ws in websocket_server.manager ) ) diff --git a/frigate/output/ws_auth.py b/frigate/output/ws_auth.py new file mode 100644 index 0000000000..33ec4e4980 --- /dev/null +++ b/frigate/output/ws_auth.py @@ -0,0 +1,43 @@ +"""Authorization helpers for JSMPEG websocket clients.""" + +from typing import Any + +from frigate.config import FrigateConfig +from frigate.models import User + + +def _get_valid_ws_roles(ws: Any, config: FrigateConfig) -> list[str]: + role_header = ws.environ.get("HTTP_REMOTE_ROLE", "") + roles = [ + role.strip() + for role in role_header.split(config.proxy.separator) + if role.strip() + ] + return [role for role in roles if role in config.auth.roles] + + +def ws_has_camera_access(ws: Any, camera_name: str, config: FrigateConfig) -> bool: + """Return True when a websocket client is authorized for the camera path.""" + roles = _get_valid_ws_roles(ws, config) + + if not roles: + return False + + roles_dict = config.auth.roles + + # Birdseye is a composite stream, so only users with unrestricted access + # should receive it. + if camera_name == "birdseye": + return any(role == "admin" or not roles_dict.get(role) for role in roles) + + all_camera_names = set(config.cameras.keys()) + + for role in roles: + if role == "admin" or not roles_dict.get(role): + return True + + allowed_cameras = User.get_allowed_cameras(role, roles_dict, all_camera_names) + if camera_name in allowed_cameras: + return True + + return False diff --git a/frigate/test/http_api/test_http_app.py b/frigate/test/http_api/test_http_app.py index bf8e9c72a9..2be0e65da8 100644 --- a/frigate/test/http_api/test_http_app.py +++ b/frigate/test/http_api/test_http_app.py @@ -23,6 +23,26 @@ class TestHttpApp(BaseTestHttp): response_json = response.json() assert response_json == self.test_stats + def test_recordings_storage_requires_admin(self): + stats = Mock(spec=StatsEmitter) + stats.get_latest_stats.return_value = self.test_stats + app = super().create_app(stats) + app.storage_maintainer = Mock() + app.storage_maintainer.calculate_camera_usages.return_value = { + "front_door": {"usage": 2.0}, + } + + with AuthTestClient(app) as client: + response = client.get( + "/recordings/storage", + headers={"remote-user": "viewer", "remote-role": "viewer"}, + ) + assert response.status_code == 403 + + response = client.get("/recordings/storage") + assert response.status_code == 200 + assert response.json()["front_door"]["usage_percent"] == 25.0 + def test_config_set_in_memory_replaces_objects_track_list(self): self.minimal_config["cameras"]["front_door"]["objects"] = { "track": ["person", "car"], diff --git a/frigate/test/http_api/test_http_event.py b/frigate/test/http_api/test_http_event.py index bc7f388e15..8aca6577d9 100644 --- a/frigate/test/http_api/test_http_event.py +++ b/frigate/test/http_api/test_http_event.py @@ -219,6 +219,25 @@ class TestHttpApp(BaseTestHttp): assert len(events) == 1 assert events[0]["id"] == event_id + def test_similarity_search_hides_unauthorized_anchor_event(self): + mock_embeddings = Mock() + self.app.frigate_config.semantic_search.enabled = True + self.app.embeddings = mock_embeddings + + with AuthTestClient(self.app) as client: + super().insert_mock_event("hidden.anchor", camera="back_door") + response = client.get( + "/events/search", + params={ + "search_type": "similarity", + "event_id": "hidden.anchor", + }, + ) + + assert response.status_code == 404 + assert response.json()["message"] == "Event not found" + mock_embeddings.search_thumbnail.assert_not_called() + def test_get_good_event(self): id = "123456.random" diff --git a/frigate/test/test_chat_find_similar_objects.py b/frigate/test/test_chat_find_similar_objects.py index 38055658e1..73fd3b27db 100644 --- a/frigate/test/test_chat_find_similar_objects.py +++ b/frigate/test/test_chat_find_similar_objects.py @@ -145,9 +145,12 @@ class TestExecuteFindSimilarObjects(unittest.TestCase): embeddings=embeddings, frigate_config=SimpleNamespace( semantic_search=SimpleNamespace(enabled=semantic_enabled), + cameras={"driveway": object()}, + auth=SimpleNamespace(roles={"admin": [], "viewer": ["driveway"]}), + proxy=SimpleNamespace(separator=","), ), ) - return SimpleNamespace(app=app) + return SimpleNamespace(app=app, headers={}) def test_semantic_search_disabled_returns_error(self): req = self._make_request(semantic_enabled=False) @@ -180,7 +183,7 @@ class TestExecuteFindSimilarObjects(unittest.TestCase): _execute_find_similar_objects( req, {"event_id": "anchor", "cameras": ["nonexistent_cam"]}, - allowed_cameras=["nonexistent_cam"], + allowed_cameras=["driveway"], ) ) self.assertEqual(result["results"], []) diff --git a/frigate/test/test_output_ws_auth.py b/frigate/test/test_output_ws_auth.py new file mode 100644 index 0000000000..ea4834ef13 --- /dev/null +++ b/frigate/test/test_output_ws_auth.py @@ -0,0 +1,57 @@ +"""Tests for JSMPEG websocket authorization.""" + +import unittest +from types import SimpleNamespace + +from frigate.config import FrigateConfig +from frigate.output.ws_auth import ws_has_camera_access + + +class TestWsHasCameraAccess(unittest.TestCase): + def setUp(self): + self.config = FrigateConfig( + mqtt={"host": "mqtt"}, + auth={"roles": {"limited_user": ["front_door"]}}, + cameras={ + "front_door": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": {"height": 1080, "width": 1920, "fps": 5}, + }, + "back_door": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.2:554/video", "roles": ["detect"]} + ] + }, + "detect": {"height": 1080, "width": 1920, "fps": 5}, + }, + }, + ) + + def _make_ws(self, role: str): + return SimpleNamespace(environ={"HTTP_REMOTE_ROLE": role}) + + def test_restricted_role_only_gets_allowed_camera(self): + ws = self._make_ws("limited_user") + self.assertTrue(ws_has_camera_access(ws, "front_door", self.config)) + self.assertFalse(ws_has_camera_access(ws, "back_door", self.config)) + + def test_unrestricted_role_can_access_any_camera(self): + ws = self._make_ws("viewer") + self.assertTrue(ws_has_camera_access(ws, "front_door", self.config)) + self.assertTrue(ws_has_camera_access(ws, "back_door", self.config)) + + def test_birdseye_requires_unrestricted_access(self): + self.assertTrue( + ws_has_camera_access(self._make_ws("admin"), "birdseye", self.config) + ) + self.assertTrue( + ws_has_camera_access(self._make_ws("viewer"), "birdseye", self.config) + ) + self.assertFalse( + ws_has_camera_access(self._make_ws("limited_user"), "birdseye", self.config) + ) diff --git a/frigate/test/test_webpush_camera_monitoring.py b/frigate/test/test_webpush_camera_monitoring.py new file mode 100644 index 0000000000..fa9172ad20 --- /dev/null +++ b/frigate/test/test_webpush_camera_monitoring.py @@ -0,0 +1,29 @@ +"""Tests for camera monitoring notification authorization.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock + +from frigate.comms.webpush import WebPushClient + + +class TestCameraMonitoringNotifications(unittest.TestCase): + def test_send_camera_monitoring_filters_by_camera_access(self): + client = WebPushClient.__new__(WebPushClient) + client.config = SimpleNamespace( + cameras={"front_door": SimpleNamespace(friendly_name=None)} + ) + client.web_pushers = {"allowed": [], "denied": []} + client.user_cameras = {"allowed": {"front_door"}, "denied": set()} + client.check_registrations = MagicMock() + client.cleanup_registrations = MagicMock() + client.send_push_notification = MagicMock() + + client.send_camera_monitoring( + {"camera": "front_door", "message": "Monitoring condition met"} + ) + + self.assertEqual(client.send_push_notification.call_count, 1) + self.assertEqual( + client.send_push_notification.call_args.kwargs["user"], "allowed" + ) From fe269b77b87f7a918ac5480056e08487ebac9cb7 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 24 Apr 2026 10:14:28 -0600 Subject: [PATCH 033/291] Optimize face recognition (#22993) * Improve mean generation for faces to remove outlier embeddings * Create testing scripts folder * Fix mypy --- frigate/data_processing/common/face/model.py | 59 +- benchmark.py => testing-scripts/benchmark.py | 0 .../benchmark_motion.py | 0 testing-scripts/face_dataset.py | 783 ++++++++++++++++++ .../process_clip.py | 0 5 files changed, 840 insertions(+), 2 deletions(-) rename benchmark.py => testing-scripts/benchmark.py (100%) rename benchmark_motion.py => testing-scripts/benchmark_motion.py (100%) create mode 100644 testing-scripts/face_dataset.py rename process_clip.py => testing-scripts/process_clip.py (100%) diff --git a/frigate/data_processing/common/face/model.py b/frigate/data_processing/common/face/model.py index 45e8b8939e..87293f7f02 100644 --- a/frigate/data_processing/common/face/model.py +++ b/frigate/data_processing/common/face/model.py @@ -133,6 +133,61 @@ class FaceRecognizer(ABC): return 0.0 +def build_class_mean( + embs: list[np.ndarray], + trim: float = 0.15, + outlier_threshold: float = 0.30, + min_keep_frac: float = 0.7, + max_iters: int = 3, +) -> np.ndarray: + """Build a class-mean embedding with two-layer outlier protection. + + Layer 1 (iterative, vector-wise): drop whole embeddings whose cosine + similarity to the current class mean is below ``outlier_threshold``. + Catches mislabeled or corrupted training samples (wrong face in the + folder, full-frame screenshots, extreme crops) that per-dimension + trimming cannot detect. + + Layer 2 (per-dimension): ``scipy.stats.trim_mean`` on the retained set + to smooth per-component noise (lighting, expression, alignment jitter). + + Collections with fewer than 5 images bypass outlier rejection — too few + samples to establish a reliable class center. + """ + arr = np.stack(embs, axis=0) + + if len(arr) < 5: + return np.asarray(stats.trim_mean(arr, trim, axis=0)) + + keep = np.ones(len(arr), dtype=bool) + floor = max(5, int(np.ceil(min_keep_frac * len(arr)))) + + for _ in range(max_iters): + mean = stats.trim_mean(arr[keep], trim, axis=0) + m_norm = mean / (np.linalg.norm(mean) + 1e-9) + e_norms = arr / (np.linalg.norm(arr, axis=1, keepdims=True) + 1e-9) + cos = e_norms @ m_norm + new_keep = cos >= outlier_threshold + + if new_keep.sum() < floor: + top = np.argsort(-cos)[:floor] + new_keep = np.zeros(len(arr), dtype=bool) + new_keep[top] = True + + if np.array_equal(new_keep, keep): + break + keep = new_keep + + dropped = int((~keep).sum()) + + if dropped: + logger.debug( + f"Vector-wise outlier filter dropped {dropped}/{len(arr)} embeddings" + ) + + return np.asarray(stats.trim_mean(arr[keep], trim, axis=0)) + + def similarity_to_confidence( cosine_similarity: float, median: float = 0.3, @@ -229,7 +284,7 @@ class FaceNetRecognizer(FaceRecognizer): for name, embs in face_embeddings_map.items(): if embs: - self.mean_embs[name] = stats.trim_mean(embs, 0.15) + self.mean_embs[name] = build_class_mean(embs) logger.debug("Finished building ArcFace model") @@ -340,7 +395,7 @@ class ArcFaceRecognizer(FaceRecognizer): for name, embs in face_embeddings_map.items(): if embs: - self.mean_embs[name] = stats.trim_mean(embs, 0.15) + self.mean_embs[name] = build_class_mean(embs) logger.debug("Finished building ArcFace model") diff --git a/benchmark.py b/testing-scripts/benchmark.py similarity index 100% rename from benchmark.py rename to testing-scripts/benchmark.py diff --git a/benchmark_motion.py b/testing-scripts/benchmark_motion.py similarity index 100% rename from benchmark_motion.py rename to testing-scripts/benchmark_motion.py diff --git a/testing-scripts/face_dataset.py b/testing-scripts/face_dataset.py new file mode 100644 index 0000000000..0c9e451d1b --- /dev/null +++ b/testing-scripts/face_dataset.py @@ -0,0 +1,783 @@ +""" +Face recognition investigation script. + +Standalone replica of Frigate's ArcFace pipeline (see +frigate/data_processing/common/face/model.py and +frigate/embeddings/onnx/face_embedding.py) for analyzing a face collection +outside the running service. Useful for: + + - Diagnosing why a person's collection produces false positives + - Finding outlier/contaminating training images + - Inspecting the effect of the shipped vector-wise outlier filter + +Layout: + - Core pipeline: LandmarkAligner, ArcFaceEmbedder, arcface_preprocess, + similarity_to_confidence, blur_reduction — all mirroring the production + code exactly + - Default run: summarize positive and negative sets against a baseline + trim_mean class representation + - Optional diagnostics (flags): vector-outlier filter behavior, degenerate + "tiny crop" embedding clustering, and multi-identity contamination + +Usage: + python3 face_investigate.py \\ + --positive \\ + --negative \\ + [--model-cache /path/to/model_cache] \\ + [--vector-outlier] [--degenerate] [--contamination] + +The positive folder should contain training images for a single identity +(same layout as FACE_DIR//*.webp). The negative folder should contain +runtime crops to test against — a mix of true matches and misfires. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from dataclasses import dataclass +from typing import Iterable + +import cv2 +import numpy as np +import onnxruntime as ort +from PIL import Image +from scipy import stats + +ARCFACE_INPUT_SIZE = 112 + + +# --------------------------------------------------------------------------- +# Replicated Frigate pipeline +# --------------------------------------------------------------------------- + + +def _process_image_frigate(image: np.ndarray) -> Image.Image: + """Mirror BaseEmbedding._process_image for an ndarray input. + + NOTE: Frigate passes the output of `cv2.imread` (BGR) directly in. PIL's + `Image.fromarray` does NOT reorder channels, so the embedder effectively + receives a BGR-ordered tensor. We replicate that faithfully here. (Tested + — swapping to RGB produces near-identical embeddings; this model is + robust to channel order.) + """ + return Image.fromarray(image) + + +def arcface_preprocess(image_bgr: np.ndarray) -> np.ndarray: + """Mirror ArcfaceEmbedding._preprocess_inputs.""" + pil = _process_image_frigate(image_bgr) + + width, height = pil.size + if width != ARCFACE_INPUT_SIZE or height != ARCFACE_INPUT_SIZE: + if width > height: + new_height = int(((height / width) * ARCFACE_INPUT_SIZE) // 4 * 4) + pil = pil.resize((ARCFACE_INPUT_SIZE, new_height)) + else: + new_width = int(((width / height) * ARCFACE_INPUT_SIZE) // 4 * 4) + pil = pil.resize((new_width, ARCFACE_INPUT_SIZE)) + + og = np.array(pil).astype(np.float32) + og_h, og_w, channels = og.shape + + frame = np.zeros( + (ARCFACE_INPUT_SIZE, ARCFACE_INPUT_SIZE, channels), dtype=np.float32 + ) + x_center = (ARCFACE_INPUT_SIZE - og_w) // 2 + y_center = (ARCFACE_INPUT_SIZE - og_h) // 2 + frame[y_center : y_center + og_h, x_center : x_center + og_w] = og + + frame = (frame / 127.5) - 1.0 + frame = np.transpose(frame, (2, 0, 1)) + frame = np.expand_dims(frame, axis=0) + return frame + + +class LandmarkAligner: + """Mirror FaceRecognizer.align_face.""" + + def __init__(self, landmark_model_path: str): + if not os.path.exists(landmark_model_path): + raise FileNotFoundError(landmark_model_path) + self.detector = cv2.face.createFacemarkLBF() + self.detector.loadModel(landmark_model_path) + + def align( + self, image: np.ndarray, out_w: int, out_h: int + ) -> tuple[np.ndarray, dict]: + land_image = ( + cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) if image.ndim == 3 else image + ) + _, lands = self.detector.fit( + land_image, np.array([(0, 0, land_image.shape[1], land_image.shape[0])]) + ) + landmarks = lands[0][0] + + leftEyePts = landmarks[42:48] + rightEyePts = landmarks[36:42] + leftEyeCenter = leftEyePts.mean(axis=0).astype("int") + rightEyeCenter = rightEyePts.mean(axis=0).astype("int") + + dY = rightEyeCenter[1] - leftEyeCenter[1] + dX = rightEyeCenter[0] - leftEyeCenter[0] + angle = np.degrees(np.arctan2(dY, dX)) - 180 + dist = float(np.sqrt((dX**2) + (dY**2))) + + desiredRightEyeX = 1.0 - 0.35 + desiredDist = (desiredRightEyeX - 0.35) * out_w + scale = desiredDist / dist if dist > 0 else 1.0 + + eyesCenter = ( + int((leftEyeCenter[0] + rightEyeCenter[0]) // 2), + int((leftEyeCenter[1] + rightEyeCenter[1]) // 2), + ) + M = cv2.getRotationMatrix2D(eyesCenter, angle, scale) + tX = out_w * 0.5 + tY = out_h * 0.35 + M[0, 2] += tX - eyesCenter[0] + M[1, 2] += tY - eyesCenter[1] + + aligned = cv2.warpAffine( + image, M, (out_w, out_h), flags=cv2.INTER_CUBIC + ) + info = dict( + angle=float(angle), + eye_dist_px=dist, + scale=float(scale), + landmarks=landmarks, + ) + return aligned, info + + +class ArcFaceEmbedder: + def __init__(self, model_path: str): + self.session = ort.InferenceSession( + model_path, providers=["CPUExecutionProvider"] + ) + self.input_name = self.session.get_inputs()[0].name + + def embed(self, image_bgr: np.ndarray) -> np.ndarray: + tensor = arcface_preprocess(image_bgr) + out = self.session.run(None, {self.input_name: tensor})[0] + return out.squeeze() + + +def similarity_to_confidence( + cos_sim: float, + median: float = 0.3, + range_width: float = 0.6, + slope_factor: float = 12, +) -> float: + slope = slope_factor / range_width + return float(1.0 / (1.0 + np.exp(-slope * (cos_sim - median)))) + + +def laplacian_variance(image: np.ndarray) -> float: + return float(cv2.Laplacian(image, cv2.CV_64F).var()) + + +def blur_reduction(variance: float) -> float: + if variance < 120: + return 0.06 + elif variance < 160: + return 0.04 + elif variance < 200: + return 0.02 + elif variance < 250: + return 0.01 + return 0.0 + + +def cosine(a: np.ndarray, b: np.ndarray) -> float: + denom = np.linalg.norm(a) * np.linalg.norm(b) + if denom == 0: + return 0.0 + return float(np.dot(a, b) / denom) + + +def l2(v: np.ndarray) -> np.ndarray: + return v / (np.linalg.norm(v) + 1e-9) + + +# --------------------------------------------------------------------------- +# Sample loading +# --------------------------------------------------------------------------- + + +@dataclass +class FaceSample: + path: str + shape: tuple[int, int] + embedding: np.ndarray + blur_var: float + align_info: dict + + +def load_folder( + folder: str, aligner: LandmarkAligner, embedder: ArcFaceEmbedder +) -> list[FaceSample]: + samples: list[FaceSample] = [] + names = sorted(os.listdir(folder)) + for name in names: + if name.startswith("."): + continue + path = os.path.join(folder, name) + if not os.path.isfile(path): + continue + img = cv2.imread(path) + if img is None: + print(f" [skip unreadable] {name}") + continue + aligned, info = aligner.align(img, img.shape[1], img.shape[0]) + emb = embedder.embed(aligned) + samples.append( + FaceSample( + path=path, + shape=(img.shape[1], img.shape[0]), + embedding=emb, + blur_var=laplacian_variance(img), + align_info=info, + ) + ) + return samples + + +def trimmed_mean(embs: Iterable[np.ndarray], trim: float = 0.15) -> np.ndarray: + arr = np.stack(list(embs), axis=0) + return stats.trim_mean(arr, trim, axis=0) + + +# --------------------------------------------------------------------------- +# Baseline analyses (always run) +# --------------------------------------------------------------------------- + + +def summarize_positive(samples: list[FaceSample], mean_emb: np.ndarray) -> None: + """Summary of training set: per-sample cos to class mean, intra-class stats. + + Outliers with cos far below the rest are likely degrading the mean — + they'd be the first candidates the shipped vector-outlier filter drops. + """ + print("\n" + "=" * 78) + print(f"POSITIVE SET ANALYSIS ({len(samples)} images)") + print("=" * 78) + + rows = [] + for s in samples: + cs = cosine(s.embedding, mean_emb) + conf = similarity_to_confidence(cs) + red = blur_reduction(s.blur_var) + rows.append( + dict( + name=os.path.basename(s.path), + shape=f"{s.shape[0]}x{s.shape[1]}", + eye_px=s.align_info["eye_dist_px"], + angle=s.align_info["angle"] + 180, + blur=s.blur_var, + cos=cs, + conf=conf, + red=red, + adj_conf=max(0.0, conf - red), + ) + ) + + rows.sort(key=lambda r: r["cos"]) + sims = np.array([r["cos"] for r in rows]) + print( + f"\nCosine-to-trimmed-mean: mean={sims.mean():.3f} std={sims.std():.3f} " + f"min={sims.min():.3f} max={sims.max():.3f}" + ) + + print("\n-- Worst matches (bottom 10, most likely hurting the mean) --") + print( + f"{'cos':>6} {'conf':>6} {'blur':>7} {'eyes':>6} " + f"{'angle':>6} {'shape':>9} name" + ) + for r in rows[:10]: + print( + f"{r['cos']:6.3f} {r['conf']:6.3f} {r['blur']:7.1f} " + f"{r['eye_px']:6.1f} {r['angle']:6.1f} {r['shape']:>9} {r['name']}" + ) + + print("\n-- Best matches (top 5) --") + for r in rows[-5:][::-1]: + print( + f"{r['cos']:6.3f} {r['conf']:6.3f} {r['blur']:7.1f} " + f"{r['eye_px']:6.1f} {r['angle']:6.1f} {r['shape']:>9} {r['name']}" + ) + + # Pairwise analysis — flags embeddings poorly correlated with the rest + print("\n-- Pairwise intra-class similarity (mean cos vs. other positives) --") + embs = np.stack([s.embedding for s in samples], axis=0) + norms = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9) + sim_matrix = norms @ norms.T + np.fill_diagonal(sim_matrix, np.nan) + mean_pairwise = np.nanmean(sim_matrix, axis=1) + names = [os.path.basename(s.path) for s in samples] + ordered = sorted(zip(names, mean_pairwise), key=lambda t: t[1]) + print(f"{'mean_cos':>9} name") + for nm, mp in ordered[:10]: + print(f"{mp:9.3f} {nm}") + print(f"\n overall mean pairwise cos: {np.nanmean(sim_matrix):.3f}") + print(f" median pairwise cos: {np.nanmedian(sim_matrix):.3f}") + + +def summarize_negative( + neg_samples: list[FaceSample], + mean_emb: np.ndarray, + pos_samples: list[FaceSample], +) -> None: + """Score each negative against the class mean, then show its top-3 + nearest positives. High-scoring negatives that match specific outlier + positives hint at training-set contamination. + """ + print("\n" + "=" * 78) + print(f"NEGATIVE SET ANALYSIS ({len(neg_samples)} images)") + print("=" * 78) + print( + f"\n{'cos':>6} {'conf':>6} {'red':>5} {'adj':>5} " + f"{'blur':>7} {'eyes':>6} {'shape':>9} name" + ) + for s in neg_samples: + cs = cosine(s.embedding, mean_emb) + conf = similarity_to_confidence(cs) + red = blur_reduction(s.blur_var) + print( + f"{cs:6.3f} {conf:6.3f} {red:5.2f} {max(0, conf - red):5.2f} " + f"{s.blur_var:7.1f} {s.align_info['eye_dist_px']:6.1f} " + f"{s.shape[0]}x{s.shape[1]:<5} {os.path.basename(s.path)}" + ) + + print("\n-- For each negative, top-3 most similar positives --") + pos_embs = np.stack([p.embedding for p in pos_samples]) + pos_norm = pos_embs / (np.linalg.norm(pos_embs, axis=1, keepdims=True) + 1e-9) + for s in neg_samples: + v = s.embedding / (np.linalg.norm(s.embedding) + 1e-9) + sims = pos_norm @ v + idx = np.argsort(-sims)[:3] + print(f"\n {os.path.basename(s.path)}:") + for i in idx: + print( + f" {sims[i]:6.3f} {os.path.basename(pos_samples[i].path)} " + f"blur={pos_samples[i].blur_var:.1f} " + f"eyes={pos_samples[i].align_info['eye_dist_px']:.1f}" + ) + + +# --------------------------------------------------------------------------- +# Optional diagnostics +# --------------------------------------------------------------------------- + + +def vector_outlier_test( + pos: list[FaceSample], neg: list[FaceSample], base_trim: float = 0.15 +) -> None: + """Measure the shipped vector-wise outlier filter at various thresholds. + + The production filter at `build_class_mean` in + frigate/data_processing/common/face/model.py uses T=0.30. This test + sweeps T so you can see which images would be dropped on a new collection + and how that affects the negative scores. + + Algorithm: iteratively recompute trim_mean on the kept set, drop any + embedding with cos < T to that mean, repeat until converged. Floor at + 50% of the collection to avoid collapse. + """ + print("\n" + "=" * 78) + print("VECTOR-WISE OUTLIER PRE-FILTER — layered on trim_mean(0.15)") + print("=" * 78) + + all_embs = np.stack([s.embedding for s in pos]) + + def iterative_mean( + embs: np.ndarray, + threshold: float, + iters: int = 3, + min_keep_frac: float = 0.5, + ) -> tuple[np.ndarray, np.ndarray]: + keep = np.ones(len(embs), dtype=bool) + floor = max(5, int(np.ceil(min_keep_frac * len(embs)))) + for _ in range(iters): + m = stats.trim_mean(embs[keep], base_trim, axis=0) + m_norm = m / (np.linalg.norm(m) + 1e-9) + e_norms = embs / (np.linalg.norm(embs, axis=1, keepdims=True) + 1e-9) + cos_to_mean = e_norms @ m_norm + new_keep = cos_to_mean >= threshold + if new_keep.sum() < floor: + top_idx = np.argsort(-cos_to_mean)[:floor] + new_keep = np.zeros_like(new_keep) + new_keep[top_idx] = True + if np.array_equal(new_keep, keep): + break + keep = new_keep + final = stats.trim_mean(embs[keep], base_trim, axis=0) + return final, keep + + provisional = stats.trim_mean(all_embs, base_trim, axis=0) + p_norm = provisional / (np.linalg.norm(provisional) + 1e-9) + e_norms_all = all_embs / (np.linalg.norm(all_embs, axis=1, keepdims=True) + 1e-9) + cos_to_prov = e_norms_all @ p_norm + print("\nDistribution of cos(positive, provisional trim_mean):") + print( + f" min={cos_to_prov.min():.3f} p10={np.percentile(cos_to_prov, 10):.3f} " + f"p25={np.percentile(cos_to_prov, 25):.3f} " + f"median={np.median(cos_to_prov):.3f} " + f"p75={np.percentile(cos_to_prov, 75):.3f} max={cos_to_prov.max():.3f}" + ) + + baseline_mean = stats.trim_mean(all_embs, base_trim, axis=0) + baseline_pos = np.array([cosine(p.embedding, baseline_mean) for p in pos]) + baseline_neg = ( + np.array([cosine(n.embedding, baseline_mean) for n in neg]) + if neg + else np.array([]) + ) + baseline_conf_neg = np.array( + [similarity_to_confidence(c) for c in baseline_neg] + ) + + print( + f"\nBaseline (trim_mean only, {len(pos)} images):" + f"\n pos cos min={baseline_pos.min():.3f} " + f"mean={baseline_pos.mean():.3f} max={baseline_pos.max():.3f}" + ) + if len(neg): + print( + f" neg cos min={baseline_neg.min():.3f} " + f"mean={baseline_neg.mean():.3f} max={baseline_neg.max():.3f}" + ) + print( + f" neg conf min={baseline_conf_neg.min():.3f} " + f"mean={baseline_conf_neg.mean():.3f} max={baseline_conf_neg.max():.3f}" + ) + print( + f" margin (pos.min - neg.max): " + f"{baseline_pos.min() - baseline_neg.max():+.3f}" + ) + + print("\nIterative (refine mean → drop vectors with cos5} {'kept':>6} {'pos min':>7} {'pos mean':>8} " + f"{'neg max':>7} {'neg mean':>8} {'neg conf.max':>12} {'margin':>7}" + ) + for T in [0.15, 0.20, 0.25, 0.28, 0.30, 0.33, 0.36, 0.40]: + mean, keep = iterative_mean(all_embs, T) + pos_sims = np.array([cosine(p.embedding, mean) for p in pos]) + neg_sims = ( + np.array([cosine(n.embedding, mean) for n in neg]) + if neg + else np.array([]) + ) + neg_conf = np.array([similarity_to_confidence(c) for c in neg_sims]) + margin = pos_sims.min() - (neg_sims.max() if len(neg_sims) else 0) + print( + f"{T:5.2f} {int(keep.sum()):>3}/{len(pos):<2} " + f"{pos_sims.min():7.3f} {pos_sims.mean():8.3f} " + f"{neg_sims.max() if len(neg_sims) else float('nan'):7.3f} " + f"{neg_sims.mean() if len(neg_sims) else float('nan'):8.3f} " + f"{neg_conf.max() if len(neg_conf) else float('nan'):12.3f} " + f"{margin:+7.3f}" + ) + + # Show which images get dropped at the shipped threshold + neighbors + for T_show in (0.25, 0.30, 0.33): + _, keep = iterative_mean(all_embs, T_show) + print( + f"\nAt T={T_show}, the {int((~keep).sum())} dropped positives are:" + ) + final_mean = stats.trim_mean(all_embs[keep], base_trim, axis=0) + m_n = final_mean / (np.linalg.norm(final_mean) + 1e-9) + for i, (p, k) in enumerate(zip(pos, keep)): + if not k: + e_n = p.embedding / (np.linalg.norm(p.embedding) + 1e-9) + cos_final = float(e_n @ m_n) + print( + f" cos_to_clean_mean={cos_final:6.3f} " + f"shape={p.shape[0]}x{p.shape[1]} " + f"eyes={p.align_info['eye_dist_px']:6.1f} " + f"blur={p.blur_var:7.1f} " + f"{os.path.basename(p.path)}" + ) + + +def degenerate_embedding_test( + pos: list[FaceSample], neg: list[FaceSample] +) -> None: + """Detect whether negatives and low-quality positives share a degenerate + 'tiny/noisy face' region of the embedding space. + + Signal: if neg-to-neg cos is higher than pos-to-pos cos, the negatives + aren't really per-identity embeddings — they're dominated by upsample / + low-resolution artifacts that all map to a similar corner of embedding + space regardless of who the face belongs to. + + Also rebuilds the mean using only high-intra-similarity positives to + show whether a cleaner training set separates the negatives. + """ + print("\n" + "=" * 78) + print("DEGENERATE-EMBEDDING TEST") + print("=" * 78) + + pos_embs = np.stack([l2(s.embedding) for s in pos]) + neg_embs = np.stack([l2(s.embedding) for s in neg]) + + nn = neg_embs @ neg_embs.T + np.fill_diagonal(nn, np.nan) + pp = pos_embs @ pos_embs.T + np.fill_diagonal(pp, np.nan) + pn = pos_embs @ neg_embs.T + + print( + f"\n neg<->neg mean cos : {np.nanmean(nn):.3f} " + f"(how tightly negatives cluster together)" + ) + print( + f" pos<->pos mean cos : {np.nanmean(pp):.3f} " + f"(how tightly positives cluster)" + ) + print( + f" pos<->neg mean cos : {pn.mean():.3f} " + f"(cross-class — should be low for a clean class)" + ) + if np.nanmean(nn) > np.nanmean(pp): + print( + "\n >> neg<->neg > pos<->pos: negatives cluster more tightly than\n" + " positives. This is the degenerate-embedding signature —\n" + " upsampled tiny crops share a common 'face-like blob' region\n" + " regardless of identity." + ) + + mean_intra = np.nanmean(pp, axis=1) + for thresh in (0.30, 0.33, 0.36): + keep = mean_intra >= thresh + if keep.sum() < 5: + continue + clean_embs = [pos[i].embedding for i in range(len(pos)) if keep[i]] + clean_mean = stats.trim_mean(np.stack(clean_embs), 0.15, axis=0) + neg_scores = np.array([cosine(n.embedding, clean_mean) for n in neg]) + neg_confs = np.array([similarity_to_confidence(c) for c in neg_scores]) + pos_scores = np.array( + [ + cosine(pos[i].embedding, clean_mean) + for i in range(len(pos)) + if keep[i] + ] + ) + print( + f"\n mean_intra >= {thresh}: keeping {int(keep.sum())}/{len(pos)} positives" + ) + print( + f" pos cos vs mean : min={pos_scores.min():.3f} " + f"mean={pos_scores.mean():.3f} max={pos_scores.max():.3f}" + ) + print( + f" neg cos vs mean : min={neg_scores.min():.3f} " + f"mean={neg_scores.mean():.3f} max={neg_scores.max():.3f}" + ) + print( + f" neg conf : min={neg_confs.min():.3f} " + f"mean={neg_confs.mean():.3f} max={neg_confs.max():.3f}" + ) + print( + f" margin (pos.min - neg.max): " + f"{pos_scores.min() - neg_scores.max():+.3f}" + ) + + +def contamination_analysis( + pos: list[FaceSample], neg: list[FaceSample] +) -> None: + """Check whether the positive collection contains a second identity. + + Two signals: + (a) Per-positive: if an image is closer to at least one negative than + to the rest of the positive class, it's likely a mislabeled face. + (b) 2-means split of the positive embeddings: if one cluster center + lands close to the negative mean, that cluster is a contaminating + sub-identity that's pulling the class mean toward the negatives. + """ + print("\n" + "=" * 78) + print("CONTAMINATION ANALYSIS") + print("=" * 78) + + pos_embs = np.stack([l2(s.embedding) for s in pos]) + neg_embs = np.stack([l2(s.embedding) for s in neg]) + pos_names = [os.path.basename(s.path) for s in pos] + + pos_pos = pos_embs @ pos_embs.T + np.fill_diagonal(pos_pos, np.nan) + pos_neg = pos_embs @ neg_embs.T + + mean_intra = np.nanmean(pos_pos, axis=1) + max_to_neg = pos_neg.max(axis=1) + mean_to_neg = pos_neg.mean(axis=1) + + print( + "\nPositives closer to a negative than to their own class avg" + "\n(these are candidates for mislabeled images):" + ) + print( + f"\n{'max_neg':>7} {'mean_neg':>8} {'mean_intra':>10} " + f"{'delta':>6} name" + ) + rows = list(zip(pos_names, max_to_neg, mean_to_neg, mean_intra)) + rows.sort(key=lambda r: -(r[1] - r[3])) + for nm, mxn, mnn, mi in rows[:15]: + delta = mxn - mi + marker = " <<" if delta > 0 else "" + print(f"{mxn:7.3f} {mnn:8.3f} {mi:10.3f} {delta:6.3f} {nm}{marker}") + + # 2-means in cosine space (no sklearn dependency). + print("\n2-means split of positive embeddings (cosine space):") + rng = np.random.default_rng(0) + best = None + for _ in range(5): + idx = rng.choice(len(pos_embs), 2, replace=False) + centers = pos_embs[idx].copy() + for _ in range(50): + sims = pos_embs @ centers.T + labels = np.argmax(sims, axis=1) + new_centers = np.stack( + [ + l2(pos_embs[labels == k].mean(axis=0)) + if np.any(labels == k) + else centers[k] + for k in range(2) + ] + ) + if np.allclose(new_centers, centers): + break + centers = new_centers + tight = float(np.mean([sims[i, labels[i]] for i in range(len(labels))])) + if best is None or tight > best[0]: + best = (tight, labels.copy(), centers.copy()) + + _, labels, centers = best + sizes = [int((labels == k).sum()) for k in range(2)] + neg_mean = l2(neg_embs.mean(axis=0)) + print( + f" cluster 0: size={sizes[0]:>2} " + f"center<->other_center_cos={float(centers[0] @ centers[1]):.3f} " + f"center<->neg_mean_cos={float(centers[0] @ neg_mean):.3f}" + ) + print( + f" cluster 1: size={sizes[1]:>2} " + f"center<->neg_mean_cos={float(centers[1] @ neg_mean):.3f}" + ) + + neg_aligned = 0 if centers[0] @ neg_mean > centers[1] @ neg_mean else 1 + print( + f"\n cluster {neg_aligned} is more similar to the negatives — " + f"its members are the contamination candidates:" + ) + for i, lbl in enumerate(labels): + if lbl == neg_aligned: + print( + f" max_to_neg={max_to_neg[i]:.3f} " + f"mean_intra={mean_intra[i]:.3f} {pos_names[i]}" + ) + + keep_mask = labels != neg_aligned + if keep_mask.sum() >= 3: + clean_embs = [pos[i].embedding for i in range(len(pos)) if keep_mask[i]] + clean_mean = stats.trim_mean(np.stack(clean_embs), 0.15, axis=0) + print( + f"\n Rebuilding class mean from the OTHER cluster " + f"({keep_mask.sum()} images):" + ) + print(f" {'cos':>6} {'conf':>6} name") + for n in neg: + cs = cosine(n.embedding, clean_mean) + cf = similarity_to_confidence(cs) + print(f" {cs:6.3f} {cf:6.3f} {os.path.basename(n.path)}") + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Analyze a face recognition collection outside Frigate.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + ap.add_argument("--positive", required=True, help="Training folder for one identity") + ap.add_argument( + "--negative", + default=None, + help="Runtime-crop folder to score against (optional)", + ) + ap.add_argument( + "--model-cache", + default="/config/model_cache", + help="Directory containing facedet/arcface.onnx and facedet/landmarkdet.yaml", + ) + ap.add_argument( + "--trim", + type=float, + default=0.15, + help="trim_mean proportion (Frigate uses 0.15)", + ) + ap.add_argument( + "--vector-outlier", + action="store_true", + help="Sweep the vector-wise outlier filter threshold", + ) + ap.add_argument( + "--degenerate", + action="store_true", + help="Test whether negatives share a degenerate embedding region", + ) + ap.add_argument( + "--contamination", + action="store_true", + help="Check whether the positive folder contains a second identity", + ) + args = ap.parse_args() + + arcface_path = os.path.join(args.model_cache, "facedet", "arcface.onnx") + landmark_path = os.path.join(args.model_cache, "facedet", "landmarkdet.yaml") + for p in (arcface_path, landmark_path): + if not os.path.exists(p): + print(f"ERROR: model file not found: {p}") + return 1 + + print(f"Loading ArcFace from {arcface_path}") + embedder = ArcFaceEmbedder(arcface_path) + print(f"Loading landmark model from {landmark_path}") + aligner = LandmarkAligner(landmark_path) + + print(f"\nLoading positives from {args.positive} ...") + pos = load_folder(args.positive, aligner, embedder) + print(f" {len(pos)} positives loaded") + + neg: list[FaceSample] = [] + if args.negative: + print(f"\nLoading negatives from {args.negative} ...") + neg = load_folder(args.negative, aligner, embedder) + print(f" {len(neg)} negatives loaded") + + if not pos: + print("no positive samples — aborting") + return 1 + + mean_emb = trimmed_mean([s.embedding for s in pos], trim=args.trim) + summarize_positive(pos, mean_emb) + if neg: + summarize_negative(neg, mean_emb, pos) + + if args.vector_outlier: + vector_outlier_test(pos, neg, args.trim) + if args.degenerate and neg: + degenerate_embedding_test(pos, neg) + if args.contamination and neg: + contamination_analysis(pos, neg) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/process_clip.py b/testing-scripts/process_clip.py similarity index 100% rename from process_clip.py rename to testing-scripts/process_clip.py From 434ef358a21dd654ed9b1671027ac249853888fe Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 24 Apr 2026 11:42:30 -0500 Subject: [PATCH 034/291] add analyze keyframes testing script (#22994) --- .../analyze_recording_keyframes.py | 376 ++++++++++++++++++ 1 file changed, 376 insertions(+) create mode 100644 testing-scripts/analyze_recording_keyframes.py diff --git a/testing-scripts/analyze_recording_keyframes.py b/testing-scripts/analyze_recording_keyframes.py new file mode 100644 index 0000000000..982cac82f3 --- /dev/null +++ b/testing-scripts/analyze_recording_keyframes.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +"""Analyze keyframe and timestamp structure of Frigate recording segments. + +This is a diagnostic tool for investigating seek precision / GOP behavior on +recorded segments. It does not modify anything. + +ffprobe is only available inside the Frigate container, at + /usr/lib/ffmpeg/$DEFAULT_FFMPEG_VERSION/bin/ffprobe +This script auto-resolves that path from the DEFAULT_FFMPEG_VERSION env var +(or falls back to scanning /usr/lib/ffmpeg/*/bin/ffprobe). Pass --ffprobe to +override if needed. + +All recording segments on the filesystem are in UTC. The --timestamp flag +expects a UTC Unix timestamp. + +Typical use: + # Inside the Frigate container (or wherever recordings are mounted) + python3 analyze_recording_keyframes.py + + # Analyze 10 most recent segments + python3 analyze_recording_keyframes.py --count 10 + + # Locate the segment that contains a specific UTC Unix timestamp and + # show it plus surrounding segments + python3 analyze_recording_keyframes.py --timestamp 1713471234.567 + + # Custom recordings directory + python3 analyze_recording_keyframes.py --recordings-dir /media/frigate/recordings + + # Override the ffprobe path explicitly + python3 analyze_recording_keyframes.py --ffprobe /usr/lib/ffmpeg/7.0/bin/ffprobe +""" + +import argparse +import datetime +import json +import os +import subprocess +import sys +from pathlib import Path +from statistics import mean, median, stdev + + +def resolve_ffprobe_path(override: str | None) -> str: + """Resolve the ffprobe binary path. + + Inside the Frigate container, ffprobe lives at + /usr/lib/ffmpeg/{DEFAULT_FFMPEG_VERSION}/bin/ffprobe — the exact version + depends on the image build and is exposed as an env var. + """ + if override: + return override + version = os.environ.get("DEFAULT_FFMPEG_VERSION", "") + if version: + path = f"/usr/lib/ffmpeg/{version}/bin/ffprobe" + if Path(path).is_file(): + return path + # Fall back to scanning the Frigate ffmpeg install root. + for candidate in sorted(Path("/usr/lib/ffmpeg").glob("*/bin/ffprobe")): + if candidate.is_file(): + return str(candidate) + print( + "Could not locate ffprobe. Pass --ffprobe or set " + "DEFAULT_FFMPEG_VERSION.", + file=sys.stderr, + ) + sys.exit(1) + + +def find_recent_segments(recordings_dir: Path, camera: str, count: int) -> list[Path]: + """Return the N most recent .mp4 segments for the given camera. + + Expected layout: ////..mp4 + """ + pattern = f"*/*/{camera}/*.mp4" + segments = sorted(recordings_dir.glob(pattern)) + return segments[-count:] + + +def find_segments_near_timestamp( + recordings_dir: Path, camera: str, target_ts: float, count: int +) -> tuple[list[Path], Path | None]: + """Return `count` segments centered on the one containing `target_ts`. + + Also returns the specific segment that should contain the timestamp, so + callers can highlight it in output. + """ + pattern = f"*/*/{camera}/*.mp4" + with_ts: list[tuple[float, Path]] = [] + for seg in sorted(recordings_dir.glob(pattern)): + ts = filename_to_timestamp(seg) + if ts is not None: + with_ts.append((ts, seg)) + + if not with_ts: + return [], None + + # Largest filename_ts that is <= target_ts — that's the segment that + # should contain the timestamp (Frigate catalogs segments by filename). + target_idx = -1 + for i, (ts, _) in enumerate(with_ts): + if ts <= target_ts: + target_idx = i + else: + break + + if target_idx < 0: + # target_ts is before the earliest segment we have — just return the + # first `count` segments so the user can see what's available. + window = with_ts[:count] + return [seg for _, seg in window], None + + half = count // 2 + start = max(0, target_idx - half) + end = min(len(with_ts), start + count) + start = max(0, end - count) + + window = with_ts[start:end] + return [seg for _, seg in window], with_ts[target_idx][1] + + +def filename_to_timestamp(segment: Path) -> float | None: + """Parse the wall-clock time from Frigate's segment path layout.""" + try: + date = segment.parent.parent.parent.name # YYYY-MM-DD + hour = segment.parent.parent.name # HH + mm_ss = segment.stem # MM.SS + minute, second = mm_ss.split(".") + dt = datetime.datetime.strptime( + f"{date} {hour}:{minute}:{second}", + "%Y-%m-%d %H:%M:%S", + ).replace(tzinfo=datetime.timezone.utc) + return dt.timestamp() + except (ValueError, IndexError): + return None + + +def run_ffprobe(ffprobe: str, args: list[str]) -> dict: + """Run ffprobe and return parsed JSON, or empty dict on failure.""" + result = subprocess.run( + [ffprobe, "-v", "error", *args, "-of", "json"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + print(f" ffprobe error: {result.stderr.strip()}", file=sys.stderr) + return {} + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return {} + + +def get_format_info(ffprobe: str, segment: Path) -> tuple[dict, dict]: + """Return (format_dict, stream_dict) for the first video stream.""" + data = run_ffprobe( + ffprobe, + [ + "-show_entries", + "format=duration,start_time", + "-show_entries", + "stream=codec_name,profile,r_frame_rate,width,height", + "-select_streams", + "v:0", + str(segment), + ], + ) + fmt = data.get("format", {}) + streams = data.get("streams") or [{}] + return fmt, streams[0] + + +def get_video_packets(ffprobe: str, segment: Path) -> list[dict]: + """Return video packets with pts_time and flags.""" + data = run_ffprobe( + ffprobe, + [ + "-select_streams", + "v", + "-show_entries", + "packet=pts_time,dts_time,flags", + str(segment), + ], + ) + return data.get("packets", []) + + +def analyze(ffprobe: str, segment: Path, highlight: bool = False) -> None: + marker = " <-- contains target timestamp" if highlight else "" + print(f"\n=== {segment} ==={marker}") + + fmt, stream = get_format_info(ffprobe, segment) + duration = float(fmt.get("duration", 0) or 0) + start_time = float(fmt.get("start_time", 0) or 0) + codec = stream.get("codec_name", "?") + profile = stream.get("profile", "?") + width = stream.get("width", "?") + height = stream.get("height", "?") + fps = stream.get("r_frame_rate", "?/1") + + filename_ts = filename_to_timestamp(segment) + filename_iso = ( + datetime.datetime.fromtimestamp( + filename_ts, tz=datetime.timezone.utc + ).isoformat() + if filename_ts is not None + else "?" + ) + + print(f" Codec: {codec} ({profile}) {width}x{height} {fps}") + print(f" Filename time: {filename_ts} ({filename_iso})") + print(f" Format duration: {duration:.3f}s") + print(f" Format start: {start_time:.3f}s (PTS offset of first packet)") + + packets = get_video_packets(ffprobe, segment) + if not packets: + print(" (no video packets)") + return + + keyframe_times: list[float] = [] + first_pts: float | None = None + last_pts: float | None = None + + for pkt in packets: + pts_str = pkt.get("pts_time") + if pts_str is None or pts_str == "N/A": + continue + pts = float(pts_str) + if first_pts is None: + first_pts = pts + last_pts = pts + if "K" in pkt.get("flags", ""): + keyframe_times.append(pts) + + total_packets = len(packets) + kf_count = len(keyframe_times) + + print(f" Video packets: {total_packets}") + print(f" Keyframes: {kf_count}") + if first_pts is not None and last_pts is not None: + print( + f" Packet PTS: first={first_pts:.3f}s last={last_pts:.3f}s " + f"span={last_pts - first_pts:.3f}s" + ) + + if keyframe_times: + print( + f" Keyframe PTS: first={keyframe_times[0]:.3f}s " + f"last={keyframe_times[-1]:.3f}s" + ) + formatted = ", ".join(f"{t:.3f}" for t in keyframe_times) + print(f" Keyframe times: [{formatted}]") + + if len(keyframe_times) >= 2: + gaps = [b - a for a, b in zip(keyframe_times, keyframe_times[1:])] + avg_fps_estimate = ( + total_packets / (last_pts - first_pts) + if last_pts and first_pts is not None and last_pts > first_pts + else 0 + ) + print( + f" GOP gaps (s): min={min(gaps):.3f} max={max(gaps):.3f} " + f"mean={mean(gaps):.3f} median={median(gaps):.3f}" + ) + if len(gaps) > 1: + print(f" stdev={stdev(gaps):.3f}") + print( + f" Est. mean GOP: ~{mean(gaps) * avg_fps_estimate:.1f} frames" + if avg_fps_estimate + else "" + ) + if max(gaps) > 5: + print( + " !! Max GOP > 5s — consistent with adaptive/smart codec " + "(even if 'Smart Codec' is off in the UI, some cameras still " + "produce irregular GOPs under specific encoder profiles)" + ) + elif kf_count == 1: + print(" !! Only one keyframe in segment — very long GOP") + + # Report how well filename time aligns with first-packet PTS. + # (Filename time is what Frigate uses as recording.start_time in the DB.) + if filename_ts is not None and first_pts is not None: + print( + f" Notes: first packet PTS is {first_pts:.3f}s into the file; " + f"Frigate treats filename time as PTS=0 for seek math." + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("camera", help="Camera name (matches the recordings subfolder)") + parser.add_argument( + "--count", + type=int, + default=5, + help="Number of most recent segments to analyze (default: 5)", + ) + parser.add_argument( + "--recordings-dir", + default="/media/frigate/recordings", + help="Path to the recordings directory (default: /media/frigate/recordings)", + ) + parser.add_argument( + "--ffprobe", + default=None, + help=( + "Full path to the ffprobe binary. Defaults to the Frigate-bundled " + "binary at /usr/lib/ffmpeg/$DEFAULT_FFMPEG_VERSION/bin/ffprobe." + ), + ) + parser.add_argument( + "--timestamp", + type=float, + default=None, + help=( + "Unix timestamp (UTC seconds, decimals allowed) to locate. The " + "script finds the segment that should contain this time and " + "analyzes it plus surrounding segments (count controls the " + "window). All on-disk segments are stored in UTC, so pass a UTC " + "Unix timestamp." + ), + ) + args = parser.parse_args() + + ffprobe = resolve_ffprobe_path(args.ffprobe) + + recordings_dir = Path(args.recordings_dir) + if not recordings_dir.is_dir(): + print( + f"Recordings directory not found: {recordings_dir}", + file=sys.stderr, + ) + sys.exit(1) + + target_segment: Path | None = None + if args.timestamp is not None: + segments, target_segment = find_segments_near_timestamp( + recordings_dir, args.camera, args.timestamp, args.count + ) + target_iso = datetime.datetime.fromtimestamp( + args.timestamp, tz=datetime.timezone.utc + ).isoformat() + mode = f"around timestamp {args.timestamp} ({target_iso})" + else: + segments = find_recent_segments(recordings_dir, args.camera, args.count) + mode = "most recent" + + if not segments: + print( + f"No segments found for camera '{args.camera}' under {recordings_dir}", + file=sys.stderr, + ) + sys.exit(1) + + if args.timestamp is not None and target_segment is None: + print( + f"!! Target timestamp {args.timestamp} is before the earliest " + f"segment on disk; showing the earliest available segments instead.", + file=sys.stderr, + ) + + print( + f"Analyzing {len(segments)} {mode} segment(s) for camera " + f"'{args.camera}' under {recordings_dir} (ffprobe: {ffprobe})" + ) + for segment in segments: + analyze(ffprobe, segment, highlight=(segment == target_segment)) + + +if __name__ == "__main__": + main() From d8f70b7fedd23cbc2f0bf4054a7b8adee1da2726 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:24:24 -0500 Subject: [PATCH 035/291] Fix dismissable layer regression (#22995) * reset several dropdown and context menus to non-modal * add specific e2e test to confirm pointer events bug --- web/e2e/specs/face-library.spec.ts | 150 ++++++++++++++++++ web/src/components/card/ExportCard.tsx | 2 +- web/src/components/card/ReviewCard.tsx | 2 +- web/src/components/menu/LiveContextMenu.tsx | 2 +- .../components/menu/SearchResultActions.tsx | 4 +- .../components/overlay/ActionsDropdown.tsx | 2 +- .../overlay/ClassificationSelectionDialog.tsx | 2 +- .../overlay/FaceSelectionDialog.tsx | 2 +- web/src/pages/FaceLibrary.tsx | 2 +- .../classification/ModelSelectionView.tsx | 2 +- .../classification/ModelTrainingView.tsx | 2 +- 11 files changed, 161 insertions(+), 11 deletions(-) diff --git a/web/e2e/specs/face-library.spec.ts b/web/e2e/specs/face-library.spec.ts index ca21642bd2..74f12fce2a 100644 --- a/web/e2e/specs/face-library.spec.ts +++ b/web/e2e/specs/face-library.spec.ts @@ -358,6 +358,156 @@ test.describe("FaceSelectionDialog @high", () => { await frigateApp.page.keyboard.press("Escape"); await expect(menu).not.toBeVisible({ timeout: 3_000 }); }); + + test("classifying the last image in a group leaves body interactive", async ({ + frigateApp, + }) => { + // Regression guard for the stuck body pointer-events bug when the + // last image in a grouped-recognition detail Dialog is classified. + // Tracked upstream at radix-ui/primitives#3445. + // + // Root cause: when the user clicks a FaceSelectionDialog menu item, + // the modal DropdownMenu enters its exit animation (Radix's Presence + // keeps it in the DOM with data-state="closed" until animationend). + // While that is in flight the classify axios resolves, SWR removes + // the image from /api/faces, the parent's map no longer renders the + // grouped card, and React unmounts the subtree — including the still- + // animating DropdownMenu's Presence container. DismissableLayer's + // shared modal-layer stack can't reconcile the interrupted exit, so + // the `body { pointer-events: none }` entry it put on mount is never + // popped and the rest of the UI becomes unclickable. + // + // The fix is `modal={false}` on the FaceSelectionDialog's + // DropdownMenu (desktop path only). With modal=false the DropdownMenu + // never puts an entry on DismissableLayer's body-pointer-events stack + // in the first place, so there's nothing to leak when its Presence is + // torn down mid-animation. The Radix-community-documented workaround + // for #3445. + // + // The bug only reproduces when the mock resolves fast enough that + // the parent unmounts before the dropdown's exit animation finishes. + // Measured window via a 3x sweep on the pre-fix build: 0–200 ms + // triggers it; 300 ms+ no longer reproduces. Production LAN networks + // sit comfortably inside the bad window, while `npm run dev` seems + // to mask it via React StrictMode's double-effect scheduling. + const EVENT_ID = "1775487131.3863528-race"; + const initialFaces = withGroupedTrainingAttempt(basicFacesMock(), { + eventId: EVENT_ID, + attempts: [ + { timestamp: 1775487131.3863528, label: "unknown", score: 0.95 }, + ], + }); + + let classified = false; + + await frigateApp.installDefaults({ + faces: initialFaces, + events: [ + { + id: EVENT_ID, + label: "person", + sub_label: null, + camera: "front_door", + start_time: 1775487131.3863528, + end_time: 1775487161.3863528, + false_positive: false, + zones: ["front_yard"], + thumbnail: null, + has_clip: true, + has_snapshot: true, + retain_indefinitely: false, + plus_id: null, + model_hash: "abc123", + detector_type: "cpu", + model_type: "ssd", + data: { + top_score: 0.92, + score: 0.92, + region: [0.1, 0.1, 0.5, 0.8], + box: [0.2, 0.15, 0.45, 0.75], + area: 0.18, + ratio: 0.6, + type: "object", + path_data: [], + }, + }, + ], + }); + + // Re-route /api/faces to flip to the "train empty" payload once the + // classify POST has been received. Registered AFTER installDefaults so + // Playwright's LIFO route matching hits this handler first. + await frigateApp.page.route("**/api/faces", async (route) => { + const payload = classified ? basicFacesMock() : initialFaces; + await route.fulfill({ json: payload }); + }); + + // Hold the classify POST briefly. The race opens when the parent + // unmounts before the dropdown's exit animation finishes (~200ms + // in Radix). 100ms keeps us comfortably inside that window and + // reliably triggered the bug in a 3x sweep across 0/50/100/200ms + // on the pre-fix build. CLASSIFY_DELAY_MS overrides for local sweeps. + const delayMs = Number( + (globalThis as { process?: { env?: Record } }).process + ?.env?.CLASSIFY_DELAY_MS ?? "100", + ); + await frigateApp.page.route( + "**/api/faces/train/*/classify", + async (route) => { + classified = true; + if (delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + await route.fulfill({ json: { success: true } }); + }, + ); + + await frigateApp.goto("/faces"); + + // Open the grouped detail Dialog. + const groupedImage = frigateApp.page + .locator('img[src*="clips/faces/train/"]') + .first(); + await expect(groupedImage).toBeVisible({ timeout: 5_000 }); + await groupedImage.locator("xpath=..").click(); + const dialog = frigateApp.page + .getByRole("dialog") + .filter({ has: frigateApp.page.locator('img[src*="clips/faces/train/"]') }) + .first(); + await expect(dialog).toBeVisible({ timeout: 5_000 }); + + // Single attempt → single `+` trigger. + const triggers = dialog.locator('[aria-haspopup="menu"]'); + await expect(triggers).toHaveCount(1); + await triggers.first().click(); + + const menu = frigateApp.page + .locator('[role="menu"], [data-radix-menu-content]') + .first(); + await expect(menu).toBeVisible({ timeout: 5_000 }); + await menu.getByRole("menuitem", { name: /^alice$/i }).click(); + + // The Dialog must leave the tree cleanly, and body must recover. + await expect(dialog).not.toBeVisible({ timeout: 5_000 }); + + // Give Radix's exit animation + cleanup a comfortable margin on top of + // the ~300ms simulated network delay. + await waitForBodyInteractive(frigateApp.page, 5_000); + await expectBodyInteractive(frigateApp.page); + + // User-visible confirmation: click something outside the dialog + // and assert it actually responds. + const librarySelector = frigateApp.page + .getByRole("button") + .filter({ hasText: /\(\d+\)/ }) + .first(); + await librarySelector.click(); + await expect( + frigateApp.page + .locator('[role="menu"], [data-radix-menu-content]') + .first(), + ).toBeVisible({ timeout: 3_000 }); + }); }); test.describe("Face Library — mobile @high @mobile", () => { diff --git a/web/src/components/card/ExportCard.tsx b/web/src/components/card/ExportCard.tsx index 893f251f8f..966aab4dcc 100644 --- a/web/src/components/card/ExportCard.tsx +++ b/web/src/components/card/ExportCard.tsx @@ -266,7 +266,7 @@ export function ExportCard({ )} {!exportedRecording.in_progress && !selectionMode && (
- + - + {content} diff --git a/web/src/components/menu/LiveContextMenu.tsx b/web/src/components/menu/LiveContextMenu.tsx index 8ed78e348c..982895200d 100644 --- a/web/src/components/menu/LiveContextMenu.tsx +++ b/web/src/components/menu/LiveContextMenu.tsx @@ -272,7 +272,7 @@ export default function LiveContextMenu({ return (
- + {children}
diff --git a/web/src/components/menu/SearchResultActions.tsx b/web/src/components/menu/SearchResultActions.tsx index aa2562b428..2a8cca5a8e 100644 --- a/web/src/components/menu/SearchResultActions.tsx +++ b/web/src/components/menu/SearchResultActions.tsx @@ -258,13 +258,13 @@ export default function SearchResultActions({ {isContextMenu ? ( - + {children} {menuItems} ) : ( <> - + diff --git a/web/src/components/overlay/ActionsDropdown.tsx b/web/src/components/overlay/ActionsDropdown.tsx index 9f9596d0ac..7f841be4fb 100644 --- a/web/src/components/overlay/ActionsDropdown.tsx +++ b/web/src/components/overlay/ActionsDropdown.tsx @@ -22,7 +22,7 @@ export default function ActionsDropdown({ const { t } = useTranslation(["components/dialog", "views/replay", "common"]); return ( - +
- + e.stopPropagation()}> diff --git a/web/src/views/classification/ModelTrainingView.tsx b/web/src/views/classification/ModelTrainingView.tsx index dd4f3c9c00..23fd6f3746 100644 --- a/web/src/views/classification/ModelTrainingView.tsx +++ b/web/src/views/classification/ModelTrainingView.tsx @@ -698,7 +698,7 @@ function LibrarySelector({ regexErrorMessage={t("description.invalidName")} /> - +
- + e.stopPropagation()}> diff --git a/web/src/views/classification/ModelTrainingView.tsx b/web/src/views/classification/ModelTrainingView.tsx index 23fd6f3746..dd4f3c9c00 100644 --- a/web/src/views/classification/ModelTrainingView.tsx +++ b/web/src/views/classification/ModelTrainingView.tsx @@ -698,7 +698,7 @@ function LibrarySelector({ regexErrorMessage={t("description.invalidName")} /> - +