mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-17 19:13:02 +03:00
Improve System Health pane (#24188)
* build out system health pane * tweaks * fixes * fix notice link so it opens the correct camera * tweak language
This commit is contained in:
committed by
Nicolas Mowen
parent
6d33b31bc6
commit
70ce193e09
@@ -0,0 +1,37 @@
|
||||
"""Tests for credential scrubbing in the ffprobe API."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
|
||||
|
||||
class TestHttpFfprobe(BaseTestHttp):
|
||||
def setUp(self):
|
||||
super().setUp([])
|
||||
self.app = self.create_app()
|
||||
|
||||
def client(self) -> TestClient:
|
||||
return AuthTestClient(self.app)
|
||||
|
||||
def test_failed_probe_scrubs_credentials_from_stderr(self):
|
||||
failed = SimpleNamespace(
|
||||
returncode=1,
|
||||
stdout=b"",
|
||||
stderr=(
|
||||
b"[tcp @ 0x1] Connection to tcp://10.0.0.1:554 failed\n"
|
||||
b"rtsp://admin:hunter2@10.0.0.1:554/video: Connection refused\n"
|
||||
),
|
||||
)
|
||||
|
||||
with patch("frigate.api.camera.ffprobe_stream", return_value=failed):
|
||||
with self.client() as client:
|
||||
response = client.get("/ffprobe", params={"paths": "camera:front_door"})
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
lines = response.json()[0]["stderr"]
|
||||
self.assertEqual(len(lines), 2)
|
||||
self.assertNotIn("hunter2", " ".join(lines))
|
||||
self.assertEqual(lines[1], "rtsp://*:*@10.0.0.1:554/video: Connection refused")
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Tests that the tflite face model reports a runtime device."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.detectors.detection_runners import loaded_devices, snapshot_loaded_devices
|
||||
|
||||
try:
|
||||
from frigate.embeddings.onnx import face_embedding
|
||||
except ImportError: # tflite runtime is not installed everywhere
|
||||
face_embedding = None
|
||||
|
||||
|
||||
@unittest.skipIf(face_embedding is None, "tflite runtime not available")
|
||||
class TestFaceNetDevice(unittest.TestCase):
|
||||
def setUp(self):
|
||||
loaded_devices.clear()
|
||||
|
||||
def test_facenet_records_cpu(self):
|
||||
with (
|
||||
patch.object(face_embedding, "Interpreter", return_value=MagicMock()),
|
||||
patch.object(face_embedding.os.path, "exists", return_value=True),
|
||||
):
|
||||
embedding = face_embedding.FaceNetEmbedding()
|
||||
|
||||
recorded = list(snapshot_loaded_devices().values())
|
||||
self.assertEqual(recorded, [("facenet", "CPU")])
|
||||
self.assertIsNotNone(embedding.runner)
|
||||
@@ -100,3 +100,58 @@ class TestModelDownloadNotice(unittest.TestCase):
|
||||
[n["scope"] for n in self._notice_calls(sibling.requestor)],
|
||||
["paddleocr-onnx/det.onnx"],
|
||||
)
|
||||
|
||||
|
||||
class TestModelDownloadState(unittest.TestCase):
|
||||
"""A failed download marks the model state as error, not stuck downloading."""
|
||||
|
||||
def setUp(self):
|
||||
self.download_path = tempfile.mkdtemp()
|
||||
downloader.last_download_error.clear()
|
||||
|
||||
def _state_calls(self, requestor: MagicMock) -> list[dict]:
|
||||
return [
|
||||
call.args[1]
|
||||
for call in requestor.send_data.call_args_list
|
||||
if call.args[0] == "update_model_state"
|
||||
]
|
||||
|
||||
def _downloader(self, download_func) -> ModelDownloader:
|
||||
with patch("frigate.util.downloader.InterProcessRequestor"):
|
||||
return ModelDownloader(
|
||||
"facedet", self.download_path, ["facedet.onnx"], download_func
|
||||
)
|
||||
|
||||
def test_raising_download_marks_error(self):
|
||||
def failing(path: str) -> None:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
model_downloader = self._downloader(failing)
|
||||
|
||||
with self.assertRaises(RuntimeError):
|
||||
model_downloader._download_models()
|
||||
|
||||
states = self._state_calls(model_downloader.requestor)
|
||||
self.assertEqual(states[-1]["state"], "error")
|
||||
self.assertEqual(states[-1]["model"], "facedet-facedet.onnx")
|
||||
|
||||
def test_missing_file_marks_error(self):
|
||||
def swallowing(path: str) -> None:
|
||||
pass
|
||||
|
||||
model_downloader = self._downloader(swallowing)
|
||||
model_downloader._download_models()
|
||||
|
||||
states = self._state_calls(model_downloader.requestor)
|
||||
self.assertEqual([s["state"] for s in states], ["error"])
|
||||
|
||||
def test_success_marks_downloaded(self):
|
||||
def succeeding(path: str) -> None:
|
||||
with open(path, "w") as f:
|
||||
f.write("model")
|
||||
|
||||
model_downloader = self._downloader(succeeding)
|
||||
model_downloader._download_models()
|
||||
|
||||
states = self._state_calls(model_downloader.requestor)
|
||||
self.assertEqual([s["state"] for s in states], ["downloaded"])
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Tests for the device each model runner reports after loading."""
|
||||
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.detectors import detection_runners
|
||||
from frigate.detectors.detection_runners import (
|
||||
CudaGraphRunner,
|
||||
ONNXModelRunner,
|
||||
OpenVINOModelRunner,
|
||||
RKNNModelRunner,
|
||||
get_optimized_runner,
|
||||
loaded_devices,
|
||||
record_loaded_device,
|
||||
snapshot_loaded_devices,
|
||||
)
|
||||
|
||||
|
||||
class TestRunnerDeviceName(unittest.TestCase):
|
||||
def _onnx(self, providers: list[str]) -> ONNXModelRunner:
|
||||
session = MagicMock()
|
||||
session.get_providers.return_value = providers
|
||||
return ONNXModelRunner(session, "arcface")
|
||||
|
||||
def test_onnx_provider_names(self):
|
||||
self.assertEqual(
|
||||
self._onnx(["CUDAExecutionProvider", "CPUExecutionProvider"]).device_name,
|
||||
"CUDA",
|
||||
)
|
||||
self.assertEqual(
|
||||
self._onnx(["TensorrtExecutionProvider"]).device_name, "TensorRT"
|
||||
)
|
||||
self.assertEqual(
|
||||
self._onnx(["MIGraphXExecutionProvider"]).device_name, "MIGraphX"
|
||||
)
|
||||
self.assertEqual(
|
||||
self._onnx(["OpenVINOExecutionProvider"]).device_name, "OpenVINO"
|
||||
)
|
||||
self.assertEqual(self._onnx(["CPUExecutionProvider"]).device_name, "CPU")
|
||||
self.assertEqual(self._onnx(["ROCMExecutionProvider"]).device_name, "ROCM")
|
||||
self.assertEqual(self._onnx([]).device_name, "CPU")
|
||||
|
||||
def test_cuda_graph_runner(self):
|
||||
runner = CudaGraphRunner(MagicMock(), 0)
|
||||
self.assertEqual(runner.device_name, "CUDA")
|
||||
|
||||
def test_openvino_reports_compiled_device(self):
|
||||
runner = OpenVINOModelRunner.__new__(OpenVINOModelRunner)
|
||||
runner.compiled_device = "GPU"
|
||||
runner.compiled_model = MagicMock()
|
||||
self.assertEqual(runner.device_name, "OpenVINO GPU")
|
||||
|
||||
def test_openvino_auto_resolves_execution_device(self):
|
||||
runner = OpenVINOModelRunner.__new__(OpenVINOModelRunner)
|
||||
runner.compiled_device = "AUTO"
|
||||
runner.compiled_model = MagicMock()
|
||||
runner.compiled_model.get_property.return_value = ["GPU.0"]
|
||||
self.assertEqual(runner.device_name, "OpenVINO GPU.0")
|
||||
runner.compiled_model.get_property.side_effect = RuntimeError("no prop")
|
||||
self.assertEqual(runner.device_name, "OpenVINO AUTO")
|
||||
|
||||
def test_rknn(self):
|
||||
runner = RKNNModelRunner.__new__(RKNNModelRunner)
|
||||
self.assertEqual(runner.device_name, "RKNN")
|
||||
|
||||
|
||||
class TestLoadRegistry(unittest.TestCase):
|
||||
def setUp(self):
|
||||
loaded_devices.clear()
|
||||
|
||||
def test_get_optimized_runner_records_device(self):
|
||||
session = MagicMock()
|
||||
session.get_providers.return_value = ["CPUExecutionProvider"]
|
||||
|
||||
with (
|
||||
patch.object(detection_runners, "is_rknn_compatible", return_value=False),
|
||||
patch.object(
|
||||
detection_runners,
|
||||
"get_ort_providers",
|
||||
return_value=(["CPUExecutionProvider"], [{}]),
|
||||
),
|
||||
patch.object(
|
||||
detection_runners, "is_openvino_gpu_npu_available", return_value=False
|
||||
),
|
||||
patch.object(
|
||||
detection_runners.ort, "InferenceSession", return_value=session
|
||||
),
|
||||
patch.object(
|
||||
detection_runners, "get_ort_session_options", return_value=None
|
||||
),
|
||||
):
|
||||
runner = get_optimized_runner("/models/arcface.onnx", "GPU", "arcface")
|
||||
|
||||
self.assertIsInstance(runner, ONNXModelRunner)
|
||||
self.assertEqual(loaded_devices["/models/arcface.onnx"], ("arcface", "CPU"))
|
||||
|
||||
|
||||
class TestLoadedDeviceSnapshot(unittest.TestCase):
|
||||
def setUp(self):
|
||||
loaded_devices.clear()
|
||||
|
||||
def test_record_and_snapshot_copy(self):
|
||||
record_loaded_device("/m/a.onnx", "arcface", "CUDA")
|
||||
|
||||
snapshot = snapshot_loaded_devices()
|
||||
self.assertEqual(snapshot, {"/m/a.onnx": ("arcface", "CUDA")})
|
||||
|
||||
# a copy, so a load on another thread cannot disturb a fold in progress
|
||||
record_loaded_device("/m/b.onnx", "jina_v1", "CPU")
|
||||
self.assertNotIn("/m/b.onnx", snapshot)
|
||||
self.assertIn("/m/b.onnx", loaded_devices)
|
||||
|
||||
def test_snapshot_survives_concurrent_inserts(self):
|
||||
stop = threading.Event()
|
||||
|
||||
def writer() -> None:
|
||||
i = 0
|
||||
while not stop.is_set():
|
||||
record_loaded_device(f"/m/{i}.onnx", "paddleocr", "CPU")
|
||||
i += 1
|
||||
|
||||
thread = threading.Thread(target=writer, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
for _ in range(200):
|
||||
for _entry in snapshot_loaded_devices().values():
|
||||
pass
|
||||
finally:
|
||||
stop.set()
|
||||
thread.join(timeout=5)
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Tests for folding loaded model devices per enrichment and emitting them."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.embeddings.types import ENRICHMENT_FOR_MODEL_TYPE, fold_runtime_devices
|
||||
from frigate.stats.util import embeddings_stats
|
||||
|
||||
|
||||
class TestFoldRuntimeDevices(unittest.TestCase):
|
||||
def test_every_enrichment_model_type_is_mapped(self):
|
||||
self.assertEqual(
|
||||
set(ENRICHMENT_FOR_MODEL_TYPE),
|
||||
{
|
||||
"arcface",
|
||||
"facenet",
|
||||
"jina_v1",
|
||||
"jina_v2",
|
||||
"paddleocr",
|
||||
"yolov9_license_plate",
|
||||
},
|
||||
)
|
||||
|
||||
def test_non_cpu_wins_within_an_enrichment(self):
|
||||
# jina v1 pins its text model to the CPU by design
|
||||
loaded = {
|
||||
"/m/jina_v1_text.onnx": ("jina_v1", "CPU"),
|
||||
"/m/jina_v1_vision.onnx": ("jina_v1", "OpenVINO GPU"),
|
||||
}
|
||||
self.assertEqual(
|
||||
fold_runtime_devices(loaded), {"semantic_search": "OpenVINO GPU"}
|
||||
)
|
||||
|
||||
def test_all_cpu_stays_cpu(self):
|
||||
loaded = {
|
||||
"/m/jina_v1_text.onnx": ("jina_v1", "CPU"),
|
||||
"/m/jina_v1_vision.onnx": ("jina_v1", "CPU"),
|
||||
}
|
||||
self.assertEqual(fold_runtime_devices(loaded), {"semantic_search": "CPU"})
|
||||
|
||||
def test_detector_models_are_ignored(self):
|
||||
loaded = {"/m/yolo.onnx": ("yolov9", "CUDA")}
|
||||
self.assertEqual(fold_runtime_devices(loaded), {})
|
||||
|
||||
def test_lpr_and_face(self):
|
||||
loaded = {
|
||||
"/m/det.onnx": ("paddleocr", "CUDA"),
|
||||
"/m/rec.onnx": ("paddleocr", "CUDA"),
|
||||
"/m/arcface.onnx": ("arcface", "CPU"),
|
||||
}
|
||||
self.assertEqual(
|
||||
fold_runtime_devices(loaded), {"lpr": "CUDA", "face_recognition": "CPU"}
|
||||
)
|
||||
|
||||
|
||||
def _metrics(devices: dict[str, str]) -> SimpleNamespace:
|
||||
value = SimpleNamespace(value=0.0)
|
||||
return SimpleNamespace(
|
||||
image_embeddings_speed=value,
|
||||
image_embeddings_eps=value,
|
||||
text_embeddings_speed=value,
|
||||
text_embeddings_eps=value,
|
||||
face_rec_speed=value,
|
||||
face_rec_fps=value,
|
||||
alpr_speed=value,
|
||||
alpr_pps=value,
|
||||
yolov9_lpr_speed=value,
|
||||
yolov9_lpr_pps=value,
|
||||
review_desc_speed=value,
|
||||
review_desc_dps=value,
|
||||
object_desc_speed=value,
|
||||
object_desc_dps=value,
|
||||
classification_speeds={},
|
||||
classification_cps={},
|
||||
runtime_devices=devices,
|
||||
)
|
||||
|
||||
|
||||
class TestEmbeddingsStats(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.config = FrigateConfig(
|
||||
**{
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"face_recognition": {"enabled": True},
|
||||
"cameras": {
|
||||
"front_door": {
|
||||
"ffmpeg": {
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video",
|
||||
"roles": ["detect"],
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
def test_devices_emitted_when_present(self):
|
||||
stats = embeddings_stats(self.config, _metrics({"face_recognition": "CUDA"}))
|
||||
self.assertEqual(stats["devices"], {"face_recognition": "CUDA"})
|
||||
self.assertIn("face_recognition_speed", stats)
|
||||
|
||||
def test_devices_omitted_when_empty(self):
|
||||
stats = embeddings_stats(self.config, _metrics({}))
|
||||
self.assertNotIn("devices", stats)
|
||||
Reference in New Issue
Block a user