Compare commits

..
Author SHA1 Message Date
Josh Hawkins dd08ebe2a9 remove motion search range check from the UI 2026-10-18 13:14:54 -05:00
Josh Hawkins babe15b9e8 drop motion search stall detection and range cap 2026-10-18 13:14:54 -05:00
Josh Hawkins 9d17e55052 treat timed-out VOD segments as motion search errors
With rw_timeout, the HLS demuxer skipped a timed-out segment and exited cleanly, so a run reported success with a hole in it, and when the keyframe probe and the decode lost different segments the equal frame counts paired frames with the wrong timestamps. The keyframe path now requires the probe to cover the whole run without a gap over 5 s, a decode whose stderr shows a skipped segment fails the run, and a cadence decode more than 5 s short of the run's duration fails it too, so these runs land in segments_with_errors. The keyframe probe polls should_stop and is killed on cancel, probe_video_dimensions gets the read timeout, and a user's own cancelled job no longer blocks their next search while it still counts toward the global cap.
2026-10-18 12:50:33 -05:00
Josh Hawkins 1683d4a0cc keep wedged or shared-identity motion searches from locking out users
A stalled VOD read blocked the runner forever, so a wedged job held its user and global slot until restart, and on installs without per-user identity (internal port, or auth disabled without a proxy user header) everyone shared one per-user slot. VOD ffmpeg and ffprobe inputs now use a 30 s rw_timeout, a cancel also drops queued parallel runs, a cancelled job stops counting after 60 s, and the per-user limit only applies when the request carries a real username. A failed runner start also stops its requestor.
2026-10-18 12:28:17 -05:00
Josh Hawkins 1eeb293f65 release the motion search slot if the runner fails to start 2026-10-18 12:15:35 -05:00
Josh Hawkins 535650092e show translated errors for motion search limits 2026-10-18 12:14:10 -05:00
Josh Hawkins f850269620 limit concurrent motion search jobs and search range
Any viewer could start unlimited motion search jobs over an unbounded time range, and finished jobs were never dropped from memory. Each user now gets one active job (409), at most 3 run at once (429), the range is capped at 48 hours, and finished jobs are evicted 10 minutes after they end. Jobs are no longer filed in the shared job manager, which kept every finished job forever.
2026-10-18 12:10:42 -05:00
23 changed files with 875 additions and 793 deletions
+4 -6
View File
@@ -122,14 +122,12 @@ updates:
# Intermittent failure in the UI settings import spec, cause not found
- dependency-name: "zod"
update-types: ["version-update:semver-major"]
# Every later version clamps the pan when the player hides or drops
# pinch gestures near the edges, 4.2.0 included
# Needs an on-device pinch test before moving
- dependency-name: "react-zoom-pan-pinch"
versions: [">3.4.4"]
# 0.54 pins a vulnerable dompurify; 0.55+ breaks YAML validation until
# monaco-worker-manager releases its PR 4
update-types: ["version-update:semver-major"]
# Build fails until monaco-worker-manager releases its PR 4
- dependency-name: "monaco-editor"
versions: [">=0.54.0"]
versions: [">=0.55.0"]
# Later versions need Node 22 and extract false keys
- dependency-name: "i18next-cli"
versions: [">1.5.11"]
+1 -1
View File
@@ -1,2 +1,2 @@
numpy
openvino >= 2026.4.1
openvino >= 2026.2.0
+4 -4
View File
@@ -23,7 +23,7 @@ pandas == 3.0.*
peewee == 4.5.*
peewee_migrate == 2.3.*
psutil == 7.2.*
pydantic == 2.14.*
pydantic == 2.13.*
git+https://github.com/fbcotter/py3nvml#egg=py3nvml
tzdata == 2026.5
pyzmq == 27.2.*
@@ -42,16 +42,16 @@ opencv-python-headless == 4.11.0.*
opencv-contrib-python == 4.11.0.*
scipy == 1.17.*
# OpenVino & ONNX
openvino == 2026.4.*
openvino == 2025.4.*
onnxruntime == 1.30.*
# Embeddings
transformers == 5.19.*
# Generative AI
google-genai == 2.31.*
google-genai == 2.29.*
ollama == 0.6.*
openai == 3.26.*
# push notifications
py-vapid == 1.9.6
py-vapid == 1.9.4
pywebpush == 2.5.*
# alpr
pyclipper == 1.4.*
+1 -1
View File
@@ -1 +1 @@
scikit-build == 0.19.*
scikit-build == 0.18.*
+50 -11
View File
@@ -2,9 +2,32 @@
# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
ARG DEBIAN_FRONTEND=noninteractive
ARG ROCM=1
ARG HSA_OVERRIDE_GFX_VERSION
ARG HSA_OVERRIDE
#######################################################################
FROM wget AS rocm
ARG ROCM
RUN apt update -qq && \
apt install -y wget gpg && \
wget -O rocm.deb https://repo.radeon.com/amdgpu-install/7.2.3/ubuntu/jammy/amdgpu-install_7.2.3.70203-1_all.deb && \
apt install -y ./rocm.deb && \
apt update && \
apt install -qq -y rocm
RUN mkdir -p /opt/rocm-dist/opt/rocm-$ROCM/lib
RUN cd /opt/rocm-$ROCM/lib && \
cp -dpr libMIOpen*.so* libamd*.so* libhip*.so* libhsa*.so* libmigraphx*.so* librocm*.so* librocblas*.so* libroctracer*.so* librocsolver*.so* librocfft*.so* librocprofiler*.so* libroctx*.so* librocroller.so* /opt/rocm-dist/opt/rocm-$ROCM/lib/ && \
mkdir -p /opt/rocm-dist/opt/rocm-$ROCM/lib/migraphx/lib && \
cp -dpr migraphx/lib/* /opt/rocm-dist/opt/rocm-$ROCM/lib/migraphx/lib
RUN cd /opt/rocm-dist/opt/ && ln -s rocm-$ROCM rocm
RUN mkdir -p /opt/rocm-dist/etc/ld.so.conf.d/
RUN echo /opt/rocm/lib|tee /opt/rocm-dist/etc/ld.so.conf.d/rocm.conf
#######################################################################
FROM deps AS deps-prelim
@@ -19,35 +42,51 @@ RUN echo "deb http://deb.debian.org/debian trixie main" > /etc/apt/sources.list.
rm -f /etc/apt/sources.list.d/trixie.list && \
rm -rf /var/lib/apt/lists/*
WORKDIR /opt/frigate
COPY --from=rootfs / /
RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
&& sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \
&& python3 get-pip.py "pip" --break-system-packages
RUN python3 -m pip config set global.break-system-packages true
# ROCm installs from AMD's wheels ahead of Frigate's code so code changes
# reuse this layer, and is pruned in the same layer so the removed files never
# reach the image. /opt/rocm links to the runtime for rocminfo and HIP headers.
COPY docker/rocm/requirements-wheels-rocm.txt /requirements.txt
RUN --mount=type=bind,source=docker/rocm/prune_rocm.sh,target=/deps/prune_rocm.sh \
pip3 uninstall -y onnxruntime \
&& pip3 install --no-cache-dir -r /requirements.txt \
&& ROCM_SDK_CORE="$(python3 -c 'import _rocm_sdk_core, os; print(os.path.dirname(_rocm_sdk_core.__file__))')" \
&& /deps/prune_rocm.sh "$(dirname "${ROCM_SDK_CORE}")" \
&& ln -s "${ROCM_SDK_CORE}" /opt/rocm
RUN pip3 uninstall -y onnxruntime \
&& pip3 install -r /requirements.txt
WORKDIR /opt/frigate
COPY --from=rootfs / /
#######################################################################
FROM scratch AS rocm-dist
ARG ROCM
# Copy HIP headers required for MIOpen JIT (BuildHip) / HIPRTC at runtime
COPY --from=rocm /opt/rocm-${ROCM}/include/ /opt/rocm-${ROCM}/include/
COPY --from=rocm /opt/rocm-$ROCM/bin/rocminfo /opt/rocm-$ROCM/bin/migraphx-driver /opt/rocm-$ROCM/bin/
# Copy MIOpen database files for gfx10xx, gfx11xx, and gfx12xx only (RDNA2/RDNA3/RDNA4)
COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx10* /opt/rocm-$ROCM/share/miopen/db/
COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx11* /opt/rocm-$ROCM/share/miopen/db/
COPY --from=rocm /opt/rocm-$ROCM/share/miopen/db/*gfx12* /opt/rocm-$ROCM/share/miopen/db/
# Copy rocBLAS library files for gfx10xx, gfx11xx, and gfx12xx only
COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*gfx10* /opt/rocm-$ROCM/lib/rocblas/library/
COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*gfx11* /opt/rocm-$ROCM/lib/rocblas/library/
COPY --from=rocm /opt/rocm-$ROCM/lib/rocblas/library/*gfx12* /opt/rocm-$ROCM/lib/rocblas/library/
COPY --from=rocm /opt/rocm-dist/ /
#######################################################################
FROM deps-prelim AS rocm-prelim-hsa-override0
ENV MIGRAPHX_DISABLE_MIOPEN_FUSION=1
ENV MIGRAPHX_DISABLE_SCHEDULE_PASS=1
ENV MIGRAPHX_DISABLE_REDUCE_FUSION=1
ENV MIGRAPHX_ENABLE_HIPRTC_WORKAROUNDS=1
ENV MIOPEN_CUSTOM_CACHE_DIR=/config/model_cache/migraphx
ENV MIOPEN_USER_DB_PATH=/config/model_cache/migraphx
ENV AMD_COMGR_CACHE=1
ENV AMD_COMGR_CACHE_DIR=/config/model_cache/migraphx
COPY --from=rocm-dist / /
RUN ldconfig
#######################################################################
FROM rocm-prelim-hsa-override0 as rocm-prelim-hsa-override1
-63
View File
@@ -1,63 +0,0 @@
#!/bin/bash
# Remove the parts of AMD's ROCm wheels that MIGraphX inference never loads:
# compilers, debuggers, profilers, management tools, and the math libraries
# (sparse, solver, FFT, random, RCCL, image processing) outside MIGraphX's
# dependencies. Runs in the same layer as the install so the files never
# reach the image.
set -euo pipefail
site_packages="$1"
core="${site_packages}/_rocm_sdk_core"
libraries="${site_packages}/_rocm_sdk_libraries"
# rocminfo stays for checking that the GPU is visible in the container
find "${core}/bin" -mindepth 1 -maxdepth 1 \
! -name rocminfo ! -name rocm_agent_enumerator -exec rm -rf {} +
# hiprtc compiles MIGraphX's kernels through comgr, which needs libLLVM,
# libclang-cpp, the clang resource headers, and the amdgcn device libraries
rm -rf "${core}/lib/llvm/bin" "${core}/lib/llvm/include"
find "${core}/lib/llvm/lib" -mindepth 1 -maxdepth 1 \
! -name 'libLLVM.so*' ! -name 'libclang-cpp.so*' ! -name clang \
-exec rm -rf {} +
rm -rf "${core}"/lib/llvm/lib/clang/*/lib
rm -rf \
"${core}/lib/host-math" \
"${core}"/lib/python3* \
"${core}/lib/rocprofiler-sdk" \
"${core}"/lib/librocprofiler-sdk*.so* \
"${core}"/lib/librocprof-trace-decoder.so* \
"${core}"/lib/librocm-dbgapi.so* \
"${core}"/lib/librocm-debug-agent.so* \
"${core}"/lib/libamd_smi*.so* \
"${core}"/lib/libgoamdsmi_shim64.so* \
"${core}"/lib/librocdecode.so* \
"${core}"/lib/librocjpeg.so* \
"${core}"/lib/librocdxg.so* \
"${core}"/lib/librocjitsu_*.so* \
"${core}"/lib/libhsa_hotswap_rocjitsu.so* \
"${core}"/lib/*.a
rm -rf \
"${libraries}"/lib/librpp.so* \
"${libraries}"/lib/librocsparse.so* \
"${libraries}"/lib/libhipsparse.so* \
"${libraries}"/lib/libhipsparselt.so* \
"${libraries}/lib/rocsparse" \
"${libraries}"/lib/librocsolver.so* \
"${libraries}"/lib/libhipsolver.so* \
"${libraries}"/lib/libhipblas.so* \
"${libraries}"/lib/librocrand.so* \
"${libraries}"/lib/libhiprand.so* \
"${libraries}"/lib/librocfft.so* \
"${libraries}"/lib/libhipfft.so* \
"${libraries}"/lib/libhipfftw.so* \
"${libraries}"/lib/librccl.so* \
"${libraries}"/.kpack/sparse_lib_*.kpack \
"${libraries}"/.kpack/solver_lib_*.kpack \
"${libraries}"/.kpack/rand_lib_*.kpack \
"${libraries}"/.kpack/fft_lib_*.kpack \
"${libraries}"/.kpack/rccl_lib_*.kpack
+1 -19
View File
@@ -1,19 +1 @@
# ROCm 10.1.0, MIGraphX, and its ONNX Runtime plugin provider from AMD's wheels.
onnxruntime == 1.29.*
onnxruntime-ep-migraphx @ https://stable.repo.amd.com/rocm/onnxruntime/whl-next/onnxruntime-ep-migraphx/onnxruntime_ep_migraphx-1.0.0%2Brocm10.1.0-cp311-cp311-manylinux_2_28_x86_64.whl
migraphx-libs @ https://stable.repo.amd.com/rocm/migraphx/whl-next/migraphx-libs/migraphx_libs-2.18.0%2Brocm10.1.0-py3-none-manylinux_2_28_x86_64.whl
rocm-sdk-core @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-core/rocm_sdk_core-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-libraries @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-libraries/rocm_sdk_libraries-10.1.0-py3-none-linux_x86_64.whl
# Kernels for each supported GPU target. MIGraphX has no gfx1031-gfx1036 kernels,
# so those GPUs run as gfx1030 with HSA_OVERRIDE_GFX_VERSION=10.3.0
rocm-sdk-device-gfx1030 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1030/rocm_sdk_device_gfx1030-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1100 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1100/rocm_sdk_device_gfx1100-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1101 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1101/rocm_sdk_device_gfx1101-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1102 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1102/rocm_sdk_device_gfx1102-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1103 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1103/rocm_sdk_device_gfx1103-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1150 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1150/rocm_sdk_device_gfx1150-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1151 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1151/rocm_sdk_device_gfx1151-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1152 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1152/rocm_sdk_device_gfx1152-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1153 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1153/rocm_sdk_device_gfx1153-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1200 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1200/rocm_sdk_device_gfx1200-10.1.0-py3-none-linux_x86_64.whl
rocm-sdk-device-gfx1201 @ https://stable.repo.amd.com/rocm/whl-next/rocm-sdk-device-gfx1201/rocm_sdk_device_gfx1201-10.1.0-py3-none-linux_x86_64.whl
onnxruntime-migraphx @ https://github.com/NickM-27/frigate-onnxruntime-rocm/releases/download/v7.2.3-1/onnxruntime_migraphx-1.24.4-cp311-cp311-linux_x86_64.whl
+11
View File
@@ -1,3 +1,6 @@
variable "ROCM" {
default = "7.2.3"
}
variable "HSA_OVERRIDE_GFX_VERSION" {
default = ""
}
@@ -5,6 +8,12 @@ variable "HSA_OVERRIDE" {
default = "1"
}
target wget {
dockerfile = "docker/main/Dockerfile"
platforms = ["linux/amd64"]
target = "wget"
}
target deps {
dockerfile = "docker/main/Dockerfile"
platforms = ["linux/amd64"]
@@ -21,10 +30,12 @@ target rocm {
dockerfile = "docker/rocm/Dockerfile"
contexts = {
deps = "target:deps",
wget = "target:wget",
rootfs = "target:rootfs"
}
platforms = ["linux/amd64"]
args = {
ROCM = ROCM,
HSA_OVERRIDE_GFX_VERSION = HSA_OVERRIDE_GFX_VERSION,
HSA_OVERRIDE = HSA_OVERRIDE
}
+18 -26
View File
@@ -394,22 +394,6 @@ Note that the labelmap uses a subset of the complete COCO label set that has onl
Support for AMD GPUs is provided using the [ONNX detector](#onnx). In order to utilize the AMD GPU for object detection use a frigate docker image with `-rocm` suffix, for example `ghcr.io/blakeblackshear/frigate:stable-rocm`.
### Supported GPUs {#supported-gpus-rocm}
The `-rocm` image includes ROCm support for these GPU chipsets, which work without any chipset override:
| Chipset | Example GPUs |
| -------------------------------------------- | ------------------------------------------------------------------------ |
| gfx1030 (RDNA2) | Radeon RX 6800 / 6800 XT / 6900 XT / 6950 XT, Radeon PRO W6800 |
| gfx1100, gfx1101, gfx1102 (RDNA3) | Radeon RX 7900 / 7800 / 7700 / 7600 series, Radeon PRO W7000 series |
| gfx1103 (RDNA3 integrated) | Radeon 780M / 760M in Ryzen 7040, 8040, 8000G, and 200 series processors |
| gfx1150, gfx1151, gfx1152, gfx1153 (RDNA3.5) | Integrated GPUs in Ryzen AI 300 / 400 series and Ryzen AI Max processors |
| gfx1200, gfx1201 (RDNA4) | Radeon RX 9060 / 9070 series, Radeon AI PRO R9700 |
Other RDNA2 GPUs (gfx1031 through gfx1036), such as the Radeon RX 6700 / 6600 / 6500 series and the Radeon 680M / 660M integrated GPUs, run on the gfx1030 support by [overriding the chipset](#docker-settings-for-overriding-the-gpu-chipset) to `10.3.0`.
Older GPUs, including RDNA1 (Radeon RX 5000 series) and Vega based dedicated and integrated GPUs, are not supported.
### Docker settings for GPU access
ROCm needs access to the `/dev/kfd` and `/dev/dri` devices. When docker or frigate is not run under root then also `video` (and possibly `render` and `ssl/_ssl`) groups should be added.
@@ -436,10 +420,19 @@ For reference on recommended settings see [running ROCm/pytorch in Docker](https
### Docker settings for overriding the GPU chipset
A GPU whose chipset isn't in the [supported GPUs](#supported-gpus-rocm) list can run on the support for a closely related chipset by setting the `HSA_OVERRIDE_GFX_VERSION` environment variable. RDNA2 GPUs other than gfx1030 (gfx1031 through gfx1036) need this, using `10.3.0`:
Your GPU might work just fine without any special configuration but in many cases they need manual settings. AMD/ROCm software stack comes with a limited set of GPU drivers and for newer or missing models you will have to override the chipset version to an older/generic version to get things working.
Also AMD/ROCm does not "officially" support integrated GPUs. It still does work with most of them just fine but requires special settings. One has to configure the `HSA_OVERRIDE_GFX_VERSION` environment variable. See the [ROCm bug report](https://github.com/ROCm/ROCm/issues/1743) for context and examples.
For the rocm frigate build there is some automatic detection:
- gfx1031 -> 10.3.0
- gfx1103 -> 11.0.0
If you have something else you might need to override the `HSA_OVERRIDE_GFX_VERSION` at Docker launch. Suppose the version you want is `10.0.0`, then you should configure it from command line as:
```bash
$ docker run -e HSA_OVERRIDE_GFX_VERSION=10.3.0 \
$ docker run -e HSA_OVERRIDE_GFX_VERSION=10.0.0 \
...
```
@@ -450,17 +443,16 @@ services:
frigate:
...
environment:
HSA_OVERRIDE_GFX_VERSION: "10.3.0"
HSA_OVERRIDE_GFX_VERSION: "10.0.0"
```
GPUs in the supported list don't need an override. An override that was set for the Radeon 780M / 760M (gfx1103) on an earlier version of Frigate is no longer needed and can be removed.
Figuring out what version you need can be complicated as you can't tell the chipset name and driver from the AMD brand name.
You can't tell the chipset from the AMD brand name, so check it in the container:
- first make sure that the ROCm environment is running properly by running `/opt/rocm/bin/rocminfo` in the Frigate container. It should list both the CPU and the GPU with their properties
- find the chipset version you have (gfxNNN) from the output of `rocminfo` (see below)
- if the chipset isn't in the supported list, set `HSA_OVERRIDE_GFX_VERSION` to the closest supported chipset, written as major.minor.stepping (for example gfx1030 is `10.3.0`)
- if things are not working check the Frigate docker logs
- first make sure that rocm environment is running properly by running `/opt/rocm/bin/rocminfo` in the frigate container -- it should list both the CPU and the GPU with their properties
- find the chipset version you have (gfxNNN) from the output of the `rocminfo` (see below)
- use a search engine to query what `HSA_OVERRIDE_GFX_VERSION` you need for the given gfx name ("gfxNNN ROCm HSA_OVERRIDE_GFX_VERSION")
- override the `HSA_OVERRIDE_GFX_VERSION` with relevant value
- if things are not working check the frigate docker logs
#### Figuring out if AMD/ROCm is working and found your GPU
+2 -2
View File
@@ -72,7 +72,7 @@ Frigate supports multiple different detectors that work on different types of ha
**AMD**
- [ROCm](#rocm---amd-gpu): ROCm can run on AMD discrete and recent integrated GPUs to provide efficient object detection
- [ROCm](#rocm---amd-gpu): ROCm can run on AMD Discrete GPUs to provide efficient object detection
- [Supports limited model architectures](../../configuration/object_detectors#amdrocm-gpu-detector)
- Runs best on discrete AMD GPUs
- <CommunityBadge /> [XDNA2 (Ryzen AI)](#amd-xdna2): AMD XDNA2 NPU (sub-watt power AI/ML processor separate to the GPU) inside Strix and other "AI" branded AMD platforms
@@ -240,7 +240,7 @@ Apple Silicon can not run within a container, so a ZMQ proxy is utilized to comm
### ROCm - AMD GPU
With the [ROCm](../configuration/object_detectors.md#amdrocm-gpu-detector) detector Frigate can take advantage of many AMD discrete and integrated GPUs, see the [supported GPUs](../configuration/object_detectors.md#supported-gpus-rocm).
With the [ROCm](../configuration/object_detectors.md#amdrocm-gpu-detector) detector Frigate can take advantage of many discrete AMD GPUs.
| Name | YOLOv9 Inference Time | YOLO-NAS Inference Time | RF-DETR Inference Time |
| -------------- | --------------------------- | ------------------------- | ---------------------- |
+46 -12
View File
@@ -9,7 +9,10 @@ from pydantic import BaseModel, Field
from frigate.api.auth import require_camera_access
from frigate.api.defs.tags import Tags
from frigate.config import FrigateConfig
from frigate.jobs.motion_search import (
MotionSearchCapacityError,
MotionSearchUserBusyError,
cancel_motion_search_job,
get_motion_search_job,
start_motion_search_job,
@@ -95,6 +98,20 @@ class MotionSearchStatusResponse(BaseModel):
progress: float | None = None
def _per_user_identity(request: Request, config: FrigateConfig) -> str | None:
"""Return the requesting user, or None when callers share one identity.
Internal-port requests are all "anonymous", and with auth disabled and no
proxy user header every browser is "viewer".
"""
username = request.headers.get("remote-user")
if not username or username == "anonymous":
return None
if config.auth.enabled or config.proxy.header_map.user:
return username
return None
@router.post(
"/{camera_name}/search/motion",
response_model=MotionSearchStartResponse,
@@ -138,18 +155,35 @@ async def start_motion_search(
status_code=400,
)
# Start the job using the jobs module
job_id = start_motion_search_job(
config=config,
camera_name=camera_name,
start_time=body.start_time,
end_time=body.end_time,
polygon_points=body.polygon_points,
threshold=body.threshold,
min_area=body.min_area,
parallel=body.parallel,
max_results=body.max_results,
)
try:
job_id = start_motion_search_job(
config=config,
camera_name=camera_name,
start_time=body.start_time,
end_time=body.end_time,
polygon_points=body.polygon_points,
threshold=body.threshold,
min_area=body.min_area,
parallel=body.parallel,
max_results=body.max_results,
username=_per_user_identity(request, config),
)
except MotionSearchUserBusyError:
return JSONResponse(
content={
"success": False,
"message": "A motion search is already running for this user",
},
status_code=409,
)
except MotionSearchCapacityError:
return JSONResponse(
content={
"success": False,
"message": "Too many motion searches are running",
},
status_code=429,
)
return JSONResponse(
content={
+11 -71
View File
@@ -10,7 +10,6 @@ from typing import Any
import numpy as np
import onnxruntime as ort
from frigate.const import MODEL_CACHE_DIR
from frigate.util.model import get_ort_providers
from frigate.util.rknn_converter import auto_convert_model, is_rknn_compatible
@@ -56,9 +55,6 @@ _PROVIDER_LABELS = {
LIGHTER_ANE_EP_NAME = "LighterANE"
LIGHTER_ANE_LIBRARY = "/usr/lib/lighter/liblighter_ane_ep.so"
# AMD's MIGraphX plugin execution provider, installed in the ROCm image
MIGRAPHX_EP_NAME = "MIGraphXExecutionProvider"
def get_lighter_ane_devices() -> list[Any]:
"""Get the Neural Engine devices lighter's provider offers, registering it once.
@@ -87,42 +83,6 @@ def get_lighter_ane_devices() -> list[Any]:
return devices
def get_migraphx_devices() -> list[Any]:
"""Get the AMD GPU devices the MIGraphX provider offers, registering it once.
Returns:
The provider's ONNX Runtime devices, or an empty list outside the ROCm
image or without an AMD GPU
"""
# Import here to avoid circular imports
from frigate.detectors.hardware import detect_amd_gpu
try:
import onnxruntime_ep_migraphx
except ImportError:
return []
# ONNX Runtime finds the provider's GPUs in the same /sys/class/drm entries,
# so an AMD GPU here means the provider lists it once registered
if detect_amd_gpu() is None:
return []
devices = [d for d in ort.get_ep_devices() if d.ep_name == MIGRAPHX_EP_NAME]
if not devices:
try:
ort.register_execution_provider_library(
MIGRAPHX_EP_NAME, onnxruntime_ep_migraphx.get_library_path()
)
except Exception as e:
logger.warning("Failed to load the MIGraphX provider: %s", e)
return []
devices = [d for d in ort.get_ep_devices() if d.ep_name == MIGRAPHX_EP_NAME]
return devices
def is_arm64_platform() -> bool:
"""Check if we're running on an ARM platform."""
machine = platform.machine().lower()
@@ -796,41 +756,12 @@ def get_optimized_runner(
ONNXModelRunner(session, model_type=model_type),
)
if (
device != "CPU"
and not ONNXModelRunner.is_migraphx_complex_model(model_type)
and (migraphx_devices := get_migraphx_devices())
):
migraphx_cache_dir = os.path.join(MODEL_CACHE_DIR, "migraphx/ort")
os.makedirs(migraphx_cache_dir, exist_ok=True)
sess_options = get_ort_session_options(model_type) or ort.SessionOptions()
# only the first AMD GPU is used, matching detect_amd_gpu
sess_options.add_provider_for_devices(
migraphx_devices[:1], {"cache_dir": migraphx_cache_dir}
)
try:
session = ort.InferenceSession(model_path, sess_options=sess_options)
except Exception as e:
logger.warning(
"Failed to load %s with MIGraphX, using the default providers: %s",
model_path,
e,
)
else:
return _record_runner(
model_path,
model_type,
ONNXModelRunner(session, model_type=model_type),
)
providers, options = get_ort_providers(device == "CPU", device, **kwargs)
if providers[0] == "CPUExecutionProvider":
# In the default image, ONNXRuntime is used so we will only get CPUExecutionProvider
# In other images we will get CUDA which is preferred over OpenVINO, while
# MIGraphX in the ROCm image is already handled above
# There is currently no way to prioritize OpenVINO over CUDA in these images
# In other images we will get CUDA / ROCm which are preferred over OpenVINO
# There is currently no way to prioritize OpenVINO over CUDA / ROCm in these images
if device != "CPU" and is_openvino_gpu_npu_available():
return _record_runner(
model_path,
@@ -859,6 +790,15 @@ def get_optimized_runner(
),
)
if (
providers
and providers[0] == "MIGraphXExecutionProvider"
and ONNXModelRunner.is_migraphx_complex_model(model_type)
):
# Don't use MIGraphX for models that are not supported
providers.pop(0)
options.pop(0)
return _record_runner(
model_path,
model_type,
+76 -19
View File
@@ -8,7 +8,7 @@ from collections.abc import Callable, Generator, Iterable
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
from dataclasses import asdict, dataclass, field
from datetime import datetime
from typing import Any, cast
from typing import Any
import cv2
import numpy as np
@@ -17,10 +17,6 @@ from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.const import STREAM_TYPE_MAIN, UPDATE_JOB_STATE
from frigate.jobs.job import Job
from frigate.jobs.manager import (
get_job_by_id,
set_current_job,
)
from frigate.jobs.motion_search_batch import (
build_segment_time_map,
coalesce_runs,
@@ -53,6 +49,12 @@ PROGRESS_BROADCAST_INTERVAL = 1.0
# Output frame rate for the fixed-cadence fallback used on long-GOP cameras
# (where keyframe sampling is too sparse). Keyframe cameras ignore this.
FALLBACK_SAMPLE_FPS = 2.0
# Each parallel job runs up to 4 decode workers, so this bounds total ffmpeg load.
MAX_ACTIVE_JOBS = 3
# Finished jobs stay pollable for this long before eviction (seconds).
FINISHED_JOB_TTL = 600.0
# A cancelled job stops holding a slot after this long, even if its thread is wedged.
CANCEL_GRACE_SECONDS = 60.0
@dataclass
@@ -98,6 +100,8 @@ class MotionSearchJob(Job):
min_area: float = 5.0
parallel: bool = False
max_results: int = 25
username: str | None = None
cancelled_at: float | None = None
# Track progress
total_frames_processed: int = 0
@@ -112,6 +116,8 @@ class MotionSearchJob(Job):
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary for WebSocket transmission."""
d = asdict(self)
d.pop("username")
d.pop("cancelled_at")
if self.metrics:
d["metrics"] = self.metrics.to_dict()
return d
@@ -579,7 +585,7 @@ class MotionSearchRunner(threading.Thread):
# Decide keyframe vs fixed-cadence sampling once from the first run's GOP
# (keyframe structure is a per-camera constant).
first_pts = probe_vod_keyframe_pts(ffprobe_path, first_url)
first_pts = probe_vod_keyframe_pts(ffprobe_path, first_url, self._should_stop)
self.use_keyframe = keyframe_sampling_eligible(first_pts)
logger.debug(
@@ -641,7 +647,9 @@ class MotionSearchRunner(threading.Thread):
time_map = build_segment_time_map(run)
if self.use_keyframe:
kf_pts = probe_vod_keyframe_pts(self.ffprobe_path, vod_url)
kf_pts = probe_vod_keyframe_pts(
self.ffprobe_path, vod_url, self._should_stop
)
frames = list(
iter_vod_frames(
self.ffmpeg_path,
@@ -777,13 +785,15 @@ class MotionSearchRunner(threading.Thread):
run_results, frames = completed.pop(next_idx)
if self._merge_run(runs[next_idx], run_results, frames, state):
self.internal_stop_event.set()
for pending in futures:
pending.cancel()
break
next_idx += 1
if self.internal_stop_event.is_set():
break
# Queued runs would otherwise still execute when the pool exits.
for pending in futures:
pending.cancel()
else:
for run in runs:
if self._should_stop():
@@ -833,11 +843,36 @@ class MotionSearchRunner(threading.Thread):
return deduplicated
# Module-level state for managing per-camera jobs
class MotionSearchUserBusyError(Exception):
"""The user already has an active motion search job."""
class MotionSearchCapacityError(Exception):
"""The global limit of active motion search jobs is reached."""
_motion_search_jobs: dict[str, tuple[MotionSearchJob, threading.Event]] = {}
_jobs_lock = threading.Lock()
def _evict_finished_jobs(now: float) -> None:
"""Drop jobs that finished more than FINISHED_JOB_TTL ago. Caller holds the lock."""
expired = [
job_id
for job_id, (job, _) in _motion_search_jobs.items()
if job.end_time is not None and now - job.end_time > FINISHED_JOB_TTL
]
for job_id in expired:
del _motion_search_jobs[job_id]
def _is_active(job: MotionSearchJob, now: float) -> bool:
"""True while the job's thread may still be running and holding a slot."""
if job.end_time is not None:
return False
return job.cancelled_at is None or now - job.cancelled_at < CANCEL_GRACE_SECONDS
def stop_all_motion_search_jobs() -> None:
"""Cancel all running motion search jobs for clean shutdown."""
with _jobs_lock:
@@ -857,10 +892,13 @@ def start_motion_search_job(
min_area: float = 5.0,
parallel: bool = False,
max_results: int = 25,
username: str | None = None,
) -> str:
"""Start a new motion search job.
Returns the job ID.
Returns the job ID. Raises MotionSearchUserBusyError when ``username`` already
has an active job and MotionSearchCapacityError when MAX_ACTIVE_JOBS are active.
Pass ``username=None`` when callers share an identity to skip the per-user check.
"""
job = MotionSearchJob(
camera=camera_name,
@@ -871,17 +909,38 @@ def start_motion_search_job(
min_area=min_area,
parallel=parallel,
max_results=max_results,
username=username,
)
cancel_event = threading.Event()
with _jobs_lock:
now = datetime.now().timestamp()
_evict_finished_jobs(now)
# A cancelled job keeps its thread until the runner sets end_time, so it
# still counts toward the global cap. Its user may start a new search.
active = [j for j, _ in _motion_search_jobs.values() if _is_active(j, now)]
if username is not None and any(
j.username == username and j.status != JobStatusTypesEnum.cancelled
for j in active
):
raise MotionSearchUserBusyError()
if len(active) >= MAX_ACTIVE_JOBS:
raise MotionSearchCapacityError()
_motion_search_jobs[job.id] = (job, cancel_event)
set_current_job(job)
runner = MotionSearchRunner(job, config, cancel_event)
runner.start()
runner: MotionSearchRunner | None = None
try:
runner = MotionSearchRunner(job, config, cancel_event)
runner.start()
except Exception:
if runner is not None:
runner.requestor.stop()
with _jobs_lock:
del _motion_search_jobs[job.id]
raise
logger.debug(
"Started motion search job %s for camera %s: "
@@ -904,10 +963,7 @@ def get_motion_search_job(job_id: str) -> MotionSearchJob | None:
"""Get a motion search job by ID."""
with _jobs_lock:
job_entry = _motion_search_jobs.get(job_id)
if job_entry:
return job_entry[0]
# Check completed jobs via manager
return cast(MotionSearchJob | None, get_job_by_id("motion_search", job_id))
return job_entry[0] if job_entry else None
def cancel_motion_search_job(job_id: str) -> bool:
@@ -928,6 +984,7 @@ def cancel_motion_search_job(job_id: str) -> bool:
cancel_event.set()
job.status = JobStatusTypesEnum.cancelled
job.cancelled_at = datetime.now().timestamp()
job_payload = job.to_dict()
logger.info("Cancelled motion search job %s", job_id)
+23 -6
View File
@@ -10,6 +10,7 @@ import json
import logging
import subprocess as sp
import tempfile
import time
from collections.abc import Callable, Generator
from typing import IO
@@ -264,11 +265,16 @@ def iter_vod_frames(
)
def probe_vod_keyframe_pts(ffprobe_path: str, vod_url: str) -> list[float]:
KEYFRAME_PROBE_TIMEOUT = 120.0
def probe_vod_keyframe_pts(
ffprobe_path: str, vod_url: str, should_stop: Callable[[], bool]
) -> list[float]:
"""Return keyframe presentation timestamps (VOD stream time) in order.
Reads packet flags via ffprobe over the VOD URL (no decode). Returns [] on
any failure so the caller can fall back.
any failure, timeout, or stop request so the caller can fall back.
"""
cmd = [
ffprobe_path,
@@ -286,16 +292,27 @@ def probe_vod_keyframe_pts(ffprobe_path: str, vod_url: str) -> list[float]:
"json",
]
try:
completed = sp.run(cmd, capture_output=True, text=True, timeout=120)
except (OSError, sp.SubprocessError):
proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.DEVNULL, text=True)
except OSError:
logger.warning("ffprobe failed for VOD keyframe probe")
return []
if completed.returncode != 0 or not completed.stdout:
deadline = time.monotonic() + KEYFRAME_PROBE_TIMEOUT
while True:
try:
stdout, _ = proc.communicate(timeout=0.5)
break
except sp.TimeoutExpired:
if should_stop() or time.monotonic() > deadline:
proc.kill()
proc.communicate()
return []
if proc.returncode != 0 or not stdout:
return []
try:
packets = json.loads(completed.stdout).get("packets", [])
packets = json.loads(stdout).get("packets", [])
except json.JSONDecodeError:
return []
@@ -0,0 +1,121 @@
"""Tests for motion search job limits."""
from datetime import datetime
from unittest.mock import patch
from frigate.jobs import motion_search
from frigate.models import Event, Recordings, ReviewSegment
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
URL = "/front_door/search/motion"
BODY = {
"start_time": 1000,
"end_time": 2000,
"polygon_points": [[0, 0], [1, 0], [1, 1]],
}
class TestHttpMotionSearch(BaseTestHttp):
def setUp(self):
super().setUp([Event, Recordings, ReviewSegment])
self.app = self.create_app()
motion_search._motion_search_jobs.clear()
self.addCleanup(motion_search._motion_search_jobs.clear)
runner_patch = patch("frigate.jobs.motion_search.MotionSearchRunner")
self.runner = runner_patch.start()
self.addCleanup(runner_patch.stop)
def start(self, client: AuthTestClient, user: str, body: dict = BODY):
return client.post(URL, json=body, headers={"remote-user": user})
def test_second_job_for_same_user_rejected(self):
with AuthTestClient(self.app) as client:
self.assertEqual(self.start(client, "alice").status_code, 200)
self.assertEqual(self.start(client, "alice").status_code, 409)
self.assertEqual(self.start(client, "bob").status_code, 200)
def test_user_can_start_again_after_job_finishes(self):
with AuthTestClient(self.app) as client:
job_id = self.start(client, "alice").json()["job_id"]
job = motion_search.get_motion_search_job(job_id)
job.end_time = datetime.now().timestamp()
self.assertEqual(self.start(client, "alice").status_code, 200)
self.assertIsNotNone(motion_search.get_motion_search_job(job_id))
def test_cancelled_job_frees_user_but_holds_global_slot_until_grace(self):
with AuthTestClient(self.app) as client:
job_id = self.start(client, "alice").json()["job_id"]
with patch("frigate.jobs.motion_search.InterProcessRequestor"):
self.assertTrue(motion_search.cancel_motion_search_job(job_id))
job = motion_search.get_motion_search_job(job_id)
self.assertIsNotNone(job.cancelled_at)
self.assertIsNone(job.end_time)
self.assertEqual(self.start(client, "alice").status_code, 200)
self.assertEqual(self.start(client, "bob").status_code, 200)
self.assertEqual(self.start(client, "carol").status_code, 429)
job.cancelled_at -= motion_search.CANCEL_GRACE_SECONDS + 1
self.assertEqual(self.start(client, "carol").status_code, 200)
def test_start_failure_releases_slot(self):
self.runner.return_value.start.side_effect = RuntimeError("boom")
with AuthTestClient(self.app) as client:
with self.assertRaises(RuntimeError):
self.start(client, "alice")
self.assertEqual(motion_search._motion_search_jobs, {})
self.runner.return_value.requestor.stop.assert_called_once()
self.runner.return_value.start.side_effect = None
self.assertEqual(self.start(client, "alice").status_code, 200)
def test_shared_identity_skips_per_user_limit(self):
with AuthTestClient(self.app) as client:
self.assertEqual(self.start(client, "anonymous").status_code, 200)
self.assertEqual(self.start(client, "anonymous").status_code, 200)
motion_search._motion_search_jobs.clear()
self.minimal_config["auth"] = {"enabled": False}
app = self.create_app()
with AuthTestClient(app) as client:
self.assertEqual(self.start(client, "viewer").status_code, 200)
self.assertEqual(self.start(client, "viewer").status_code, 200)
def test_proxy_user_header_keeps_per_user_limit_without_auth(self):
self.minimal_config["auth"] = {"enabled": False}
self.minimal_config["proxy"] = {"header_map": {"user": "x-forwarded-user"}}
app = self.create_app()
with AuthTestClient(app) as client:
self.assertEqual(self.start(client, "alice").status_code, 200)
self.assertEqual(self.start(client, "alice").status_code, 409)
def test_global_cap_enforced(self):
with AuthTestClient(self.app) as client:
for i in range(motion_search.MAX_ACTIVE_JOBS):
self.assertEqual(self.start(client, f"user{i}").status_code, 200)
self.assertEqual(self.start(client, "late").status_code, 429)
def test_finished_jobs_evicted_after_ttl(self):
with AuthTestClient(self.app) as client:
old_id = self.start(client, "alice").json()["job_id"]
recent_id = self.start(client, "bob").json()["job_id"]
running_id = self.start(client, "carol").json()["job_id"]
now = 10_000.0
motion_search.get_motion_search_job(old_id).end_time = (
now - motion_search.FINISHED_JOB_TTL - 1
)
motion_search.get_motion_search_job(recent_id).end_time = now - 1
motion_search._evict_finished_jobs(now)
self.assertIsNone(motion_search.get_motion_search_job(old_id))
self.assertIsNotNone(motion_search.get_motion_search_job(recent_id))
self.assertIsNotNone(motion_search.get_motion_search_job(running_id))
def test_username_and_cancel_time_not_broadcast(self):
job = motion_search.MotionSearchJob(username="alice", cancelled_at=1.0)
payload = job.to_dict()
self.assertNotIn("username", payload)
self.assertNotIn("cancelled_at", payload)
+29 -5
View File
@@ -1,5 +1,6 @@
"""Tests for the motion search hardware-accelerated decode helpers."""
import subprocess as sp
import unittest
from types import SimpleNamespace
from unittest import mock
@@ -140,6 +141,11 @@ class TestVodDecodeCommand(unittest.TestCase):
class TestProbeVodKeyframePts(unittest.TestCase):
def _proc(self, stdout="", returncode=0):
proc = mock.Mock(returncode=returncode)
proc.communicate.return_value = (stdout, None)
return proc
def test_parses_keyframe_packets(self):
sample = (
'{"packets":['
@@ -147,19 +153,37 @@ class TestProbeVodKeyframePts(unittest.TestCase):
'{"pts_time":"1.000000","flags":"___"},'
'{"pts_time":"2.000000","flags":"K__"}]}'
)
completed = mock.Mock(stdout=sample, returncode=0)
with mock.patch(
"frigate.jobs.motion_search_decode.sp.run", return_value=completed
"frigate.jobs.motion_search_decode.sp.Popen",
return_value=self._proc(sample),
):
pts = probe_vod_keyframe_pts("ffprobe", "http://x/index.m3u8")
pts = probe_vod_keyframe_pts(
"ffprobe", "http://x/index.m3u8", lambda: False
)
self.assertEqual(pts, [0.0, 2.0])
def test_returns_empty_on_failure(self):
with mock.patch(
"frigate.jobs.motion_search_decode.sp.run",
"frigate.jobs.motion_search_decode.sp.Popen",
side_effect=OSError("boom"),
):
self.assertEqual(probe_vod_keyframe_pts("ffprobe", "http://x"), [])
self.assertEqual(
probe_vod_keyframe_pts("ffprobe", "http://x", lambda: False), []
)
def test_stop_request_kills_probe(self):
proc = self._proc()
proc.communicate.side_effect = [
sp.TimeoutExpired("ffprobe", 0.5),
("", None),
]
with mock.patch(
"frigate.jobs.motion_search_decode.sp.Popen", return_value=proc
):
self.assertEqual(
probe_vod_keyframe_pts("ffprobe", "http://x", lambda: True), []
)
proc.kill.assert_called_once()
class TestProbeVideoDimensions(unittest.TestCase):
+1 -215
View File
@@ -1,13 +1,12 @@
"""Tests for the device each model runner reports after loading."""
import os
import sys
import tempfile
import threading
import unittest
from unittest.mock import MagicMock, patch
from frigate.detectors import detection_runners, hardware
from frigate.detectors import detection_runners
from frigate.detectors.detection_runners import (
CudaGraphRunner,
ONNXModelRunner,
@@ -18,7 +17,6 @@ from frigate.detectors.detection_runners import (
record_loaded_device,
snapshot_loaded_devices,
)
from frigate.util import model as model_util
class TestRunnerDeviceName(unittest.TestCase):
@@ -319,215 +317,3 @@ class TestLighterANE(unittest.TestCase):
get_optimized_runner("/models/arcface.onnx", "CPU", "arcface")
ane.assert_not_called()
class TestMIGraphX(unittest.TestCase):
"""AMD's MIGraphX plugin provider, selected through its devices in the ROCm image."""
def setUp(self):
loaded_devices.clear()
self.root = tempfile.TemporaryDirectory()
self.addCleanup(self.root.cleanup)
for patcher in (
patch.object(hardware, "detect_amd_gpu", return_value=MagicMock()),
patch.object(detection_runners, "MODEL_CACHE_DIR", self.root.name),
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
patch.object(detection_runners, "get_lighter_ane_devices", return_value=[]),
):
patcher.start()
self.addCleanup(patcher.stop)
self.plugin = MagicMock()
self.plugin.get_library_path.return_value = "/site/libmigraphx-ep.so"
def _device(self) -> MagicMock:
device = MagicMock()
device.ep_name = "MIGraphXExecutionProvider"
return device
def test_no_devices_outside_the_rocm_image(self):
with (
patch.dict(sys.modules, {"onnxruntime_ep_migraphx": None}),
patch.object(
detection_runners.ort, "register_execution_provider_library"
) as register,
):
self.assertEqual(detection_runners.get_migraphx_devices(), [])
self.assertEqual(detection_runners.get_migraphx_devices(), [])
register.assert_not_called()
def test_no_devices_without_an_amd_gpu(self):
with (
patch.dict(sys.modules, {"onnxruntime_ep_migraphx": self.plugin}),
patch.object(hardware, "detect_amd_gpu", return_value=None),
patch.object(detection_runners.ort, "get_ep_devices") as get_ep_devices,
patch.object(
detection_runners.ort, "register_execution_provider_library"
) as register,
):
self.assertEqual(detection_runners.get_migraphx_devices(), [])
get_ep_devices.assert_not_called()
register.assert_not_called()
def test_the_provider_is_registered_once(self):
device = self._device()
other = MagicMock()
other.ep_name = "CPUExecutionProvider"
with (
patch.dict(sys.modules, {"onnxruntime_ep_migraphx": self.plugin}),
patch.object(
detection_runners.ort,
"get_ep_devices",
side_effect=[[other], [other, device], [other, device]],
),
patch.object(
detection_runners.ort, "register_execution_provider_library"
) as register,
):
self.assertEqual(detection_runners.get_migraphx_devices(), [device])
self.assertEqual(detection_runners.get_migraphx_devices(), [device])
register.assert_called_once_with(
"MIGraphXExecutionProvider", "/site/libmigraphx-ep.so"
)
def test_a_provider_that_will_not_load_is_skipped(self):
with (
patch.dict(sys.modules, {"onnxruntime_ep_migraphx": self.plugin}),
patch.object(detection_runners.ort, "get_ep_devices", return_value=[]),
patch.object(
detection_runners.ort,
"register_execution_provider_library",
side_effect=RuntimeError("libamdhip64.so.7: cannot open shared object"),
),
self.assertLogs(detection_runners.logger, level="WARNING"),
):
self.assertEqual(detection_runners.get_migraphx_devices(), [])
def test_the_session_runs_on_the_first_gpu(self):
device = self._device()
session = MagicMock()
session.get_providers.return_value = [
"MIGraphXExecutionProvider",
"CPUExecutionProvider",
]
options = MagicMock()
with (
patch.object(
detection_runners,
"get_migraphx_devices",
return_value=[device, self._device()],
),
patch.object(
detection_runners, "get_ort_session_options", return_value=options
),
patch.object(
detection_runners.ort, "InferenceSession", return_value=session
) as inference_session,
):
runner = get_optimized_runner("/models/yolo.onnx", "AUTO", "yolo-generic")
cache_dir = os.path.join(self.root.name, "migraphx/ort")
options.add_provider_for_devices.assert_called_once_with(
[device], {"cache_dir": cache_dir}
)
inference_session.assert_called_once_with(
"/models/yolo.onnx", sess_options=options
)
self.assertTrue(os.path.isdir(cache_dir))
self.assertIsInstance(runner, ONNXModelRunner)
self.assertEqual(
loaded_devices["/models/yolo.onnx"], ("yolo-generic", "MIGraphX")
)
def test_a_model_migraphx_cannot_load_uses_the_default_providers(self):
session = MagicMock()
session.get_providers.return_value = ["CPUExecutionProvider"]
with (
patch.object(
detection_runners,
"get_migraphx_devices",
return_value=[self._device()],
),
patch.object(
detection_runners,
"get_ort_providers",
return_value=(["CPUExecutionProvider"], [{}]),
),
patch.object(
detection_runners, "is_openvino_gpu_npu_available", return_value=False
),
patch.object(
detection_runners.ort,
"InferenceSession",
side_effect=[RuntimeError("compile failed"), session],
) as inference_session,
patch.object(
detection_runners, "get_ort_session_options", return_value=MagicMock()
),
self.assertLogs(detection_runners.logger, level="WARNING"),
):
runner = get_optimized_runner("/models/arcface.onnx", "AUTO", "arcface")
self.assertEqual(inference_session.call_count, 2)
self.assertIsInstance(runner, ONNXModelRunner)
self.assertEqual(loaded_devices["/models/arcface.onnx"], ("arcface", "CPU"))
def test_unsupported_models_and_cpu_models_skip_migraphx(self):
for model_path, device, model_type in [
("/models/paddleocr.onnx", "AUTO", "paddleocr"),
("/models/dfine.onnx", "AUTO", "dfine"),
("/models/yolo.onnx", "CPU", "yolo-generic"),
]:
session = MagicMock()
session.get_providers.return_value = ["CPUExecutionProvider"]
with (
self.subTest(model_type=model_type, device=device),
patch.object(
detection_runners,
"get_migraphx_devices",
return_value=[self._device()],
) as migraphx,
patch.object(
detection_runners,
"get_ort_providers",
return_value=(["CPUExecutionProvider"], [{}]),
),
patch.object(
detection_runners,
"is_openvino_gpu_npu_available",
return_value=False,
),
patch.object(
detection_runners.ort, "InferenceSession", return_value=session
),
patch.object(
detection_runners, "get_ort_session_options", return_value=None
),
):
get_optimized_runner(model_path, device, model_type)
migraphx.assert_not_called()
self.assertEqual(loaded_devices[model_path], (model_type, "CPU"))
def test_migraphx_is_never_requested_by_name(self):
with patch.object(
model_util.ort,
"get_available_providers",
return_value=[
"AzureExecutionProvider",
"CPUExecutionProvider",
"MIGraphXExecutionProvider",
],
):
providers, options = model_util.get_ort_providers()
self.assertEqual(providers, ["CPUExecutionProvider"])
self.assertEqual(options, [{"enable_cpu_mem_arena": False}])
+9 -3
View File
@@ -385,9 +385,15 @@ def get_ort_providers(
}
)
elif provider == "MIGraphXExecutionProvider":
# MIGraphX is a plugin provider, which ONNX Runtime can't create by
# name, so get_optimized_runner selects it through its devices
continue
migraphx_cache_dir = os.path.join(MODEL_CACHE_DIR, "migraphx")
os.makedirs(migraphx_cache_dir, exist_ok=True)
providers.append(provider)
options.append(
{
"migraphx_model_cache_dir": migraphx_cache_dir,
}
)
elif provider == "CPUExecutionProvider":
providers.append(provider)
options.append(
+426 -305
View File
File diff suppressed because it is too large Load Diff
+23 -22
View File
@@ -23,28 +23,28 @@
"dependencies": {
"@cycjimmy/jsmpeg-player": "^6.1.2",
"@hookform/resolvers": "^5.9.1",
"@radix-ui/react-alert-dialog": "^1.2.1",
"@radix-ui/react-aspect-ratio": "^1.2.0",
"@radix-ui/react-checkbox": "^1.4.1",
"@radix-ui/react-collapsible": "^1.2.1",
"@radix-ui/react-context-menu": "^2.4.1",
"@radix-ui/react-alert-dialog": "^1.1.24",
"@radix-ui/react-aspect-ratio": "^1.1.16",
"@radix-ui/react-checkbox": "^1.3.12",
"@radix-ui/react-collapsible": "^1.1.21",
"@radix-ui/react-context-menu": "^2.3.8",
"@radix-ui/react-dialog": "^1.2.0",
"@radix-ui/react-dropdown-menu": "^2.2.1",
"@radix-ui/react-hover-card": "^1.2.1",
"@radix-ui/react-label": "^2.1.17",
"@radix-ui/react-popover": "^1.3.1",
"@radix-ui/react-progress": "^1.2.1",
"@radix-ui/react-radio-group": "^1.5.1",
"@radix-ui/react-scroll-area": "^1.4.0",
"@radix-ui/react-select": "^2.4.1",
"@radix-ui/react-separator": "^1.2.0",
"@radix-ui/react-slider": "^1.6.1",
"@radix-ui/react-slot": "1.5.0",
"@radix-ui/react-switch": "^1.4.1",
"@radix-ui/react-tabs": "^1.2.1",
"@radix-ui/react-toggle": "^1.2.1",
"@radix-ui/react-toggle-group": "^1.2.1",
"@radix-ui/react-tooltip": "^1.4.1",
"@radix-ui/react-dropdown-menu": "^2.1.25",
"@radix-ui/react-hover-card": "^1.1.24",
"@radix-ui/react-label": "^2.1.16",
"@radix-ui/react-popover": "^1.2.0",
"@radix-ui/react-progress": "^1.1.17",
"@radix-ui/react-radio-group": "^1.4.8",
"@radix-ui/react-scroll-area": "^1.3.0",
"@radix-ui/react-select": "^2.3.8",
"@radix-ui/react-separator": "^1.1.16",
"@radix-ui/react-slider": "^1.5.0",
"@radix-ui/react-slot": "1.4.0",
"@radix-ui/react-switch": "^1.3.8",
"@radix-ui/react-tabs": "^1.1.22",
"@radix-ui/react-toggle": "^1.1.19",
"@radix-ui/react-toggle-group": "^1.1.20",
"@radix-ui/react-tooltip": "^1.3.0",
"@rjsf/core": "^6.10.1",
"@rjsf/shadcn": "^6.10.1",
"@rjsf/utils": "^6.10.1",
@@ -127,6 +127,7 @@
},
"overrides": {
"@radix-ui/react-compose-refs": "1.1.2",
"@radix-ui/react-slot": "1.5.0"
"@radix-ui/react-popper": "1.2.8",
"@radix-ui/react-slot": "1.4.0"
}
}
@@ -56,6 +56,8 @@
"noROI": "Please draw a region of interest",
"noTimeRange": "Please select a time range",
"invalidTimeRange": "End time must be after start time",
"alreadyRunning": "You already have a motion search running. Wait for it to finish or cancel it.",
"tooManySearches": "Too many motion searches are running. Try again when one finishes.",
"searchFailed": "Search failed: {{message}}",
"polygonTooSmall": "Polygon must have at least 3 points",
"unknown": "Unknown error"
@@ -210,7 +210,7 @@ export function VideoPreview({
}, [playerRef]);
const onProgressHover = useCallback(
(event: React.MouseEvent<HTMLElement>) => {
(event: React.MouseEvent<HTMLDivElement>) => {
if (!sliderRef.current) {
return;
}
@@ -401,7 +401,7 @@ export function InProgressPreview({
);
const onProgressHover = useCallback(
(event: React.MouseEvent<HTMLElement>) => {
(event: React.MouseEvent<HTMLDivElement>) => {
if (!sliderRef.current || !previewFrames) {
return;
}
@@ -892,6 +892,20 @@ export default function MotionSearchView({
setIsSearching(false);
}
} catch (error) {
const status = axios.isAxiosError(error)
? error.response?.status
: undefined;
if (status === 409 || status === 429) {
toast.error(
t(
status === 409 ? "errors.alreadyRunning" : "errors.tooManySearches",
),
);
setIsSearching(false);
return;
}
let errorMessage = t("errors.unknown");
if (axios.isAxiosError<{ message?: string; detail?: string }>(error)) {