mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-17 19:13:02 +03:00
Add a notice registry and System Health tab (#24178)
* add a notice registry and System Health tab Problems Frigate detects on its own (ffmpeg crash loops, stuck detectors, failed model downloads, recordings deleted before their retention period) only ever existed as log lines. This adds a `NoticeRegistry` in the main process backed by two tables, an `update_notice` IPC topic so producers in other processes can reach it through the dispatcher, an admin-only API and websocket topic, and a Health tab that lists them. Kinds declare their own mode, severity, and category in one place: state notices are resolved by their producer, event notices are dismissed by the user. * treat a prerelease as behind its final release * fix notices clearing early * rename menu items and update docs * don't resolve the update notice on a failed version lookup
This commit is contained in:
committed by
Nicolas Mowen
parent
af60d2db48
commit
f3a31e2fb4
@@ -144,7 +144,9 @@ class BaseTestHttp(unittest.TestCase):
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def create_app(self, stats=None, event_metadata_publisher=None):
|
||||
def create_app(
|
||||
self, stats=None, event_metadata_publisher=None, notice_registry=None
|
||||
):
|
||||
from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user
|
||||
|
||||
app = create_fastapi_app(
|
||||
@@ -159,6 +161,7 @@ class BaseTestHttp(unittest.TestCase):
|
||||
None,
|
||||
DebugReplayManager(),
|
||||
enforce_default_admin=False,
|
||||
notice_registry=notice_registry,
|
||||
)
|
||||
|
||||
# Default test mocks for authentication
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Tests for the notices API."""
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from frigate.models import Notice, NoticeStats
|
||||
from frigate.notices.registry import NoticeRegistry
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
|
||||
|
||||
class TestHttpNotices(BaseTestHttp):
|
||||
def setUp(self):
|
||||
super().setUp([Notice, NoticeStats])
|
||||
self.registry = NoticeRegistry()
|
||||
self.app = self.create_app(notice_registry=self.registry)
|
||||
|
||||
def client(self) -> TestClient:
|
||||
return AuthTestClient(self.app)
|
||||
|
||||
def test_get_notices_orders_by_severity(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
|
||||
with self.client() as client:
|
||||
response = client.get("/notices")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(
|
||||
[n["id"] for n in response.json()], ["retention_unmet", "detector_stuck"]
|
||||
)
|
||||
self.assertEqual(response.json()[0]["severity"], "error")
|
||||
|
||||
def test_get_stats(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
|
||||
with self.client() as client:
|
||||
response = client.get("/notices/stats")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.json()[0]["kind"], "detector_stuck")
|
||||
self.assertEqual(response.json()[0]["occurrences"], 1)
|
||||
|
||||
def test_dismiss_event(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
|
||||
with self.client() as client:
|
||||
response = client.post("/notices/detector_stuck/dismiss")
|
||||
listed = client.get("/notices").json()
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(listed, [])
|
||||
|
||||
def test_dismiss_unknown_is_404(self):
|
||||
with self.client() as client:
|
||||
response = client.post("/notices/nope/dismiss")
|
||||
|
||||
self.assertEqual(response.status_code, 404)
|
||||
|
||||
def test_dismiss_state_is_400(self):
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
|
||||
with self.client() as client:
|
||||
response = client.post("/notices/retention_unmet/dismiss")
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
|
||||
def test_requires_admin(self):
|
||||
with TestClient(self.app) as client:
|
||||
response = client.get(
|
||||
"/notices", headers={"remote-user": "viewer", "remote-role": "viewer"}
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 403)
|
||||
@@ -6,62 +6,72 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.const import STREAM_TYPE_MAIN, STREAM_TYPE_SUB
|
||||
from frigate.video.ffmpeg import CameraWatchdog
|
||||
from frigate.video.ffmpeg import (
|
||||
CRASH_LOOP_RECOVERY_S,
|
||||
CRASH_LOOP_RESTARTS,
|
||||
CameraWatchdog,
|
||||
)
|
||||
|
||||
|
||||
def build_watchdog(
|
||||
sub_enabled: bool = True, output_args: dict | None = None
|
||||
) -> CameraWatchdog:
|
||||
config = FrigateConfig(
|
||||
**{
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"cameras": {
|
||||
"front_door": {
|
||||
"ffmpeg": {
|
||||
"output_args": output_args or {},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video",
|
||||
"roles": ["record"],
|
||||
},
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video2",
|
||||
"roles": ["detect", "record_sub"],
|
||||
},
|
||||
],
|
||||
},
|
||||
"record": {
|
||||
"enabled": True,
|
||||
"sub": {"enabled": sub_enabled},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
camera_config = config.cameras["front_door"]
|
||||
|
||||
with (
|
||||
patch("frigate.video.ffmpeg.LogPipe"),
|
||||
patch("frigate.video.ffmpeg.InterProcessRequestor"),
|
||||
patch("frigate.video.ffmpeg.RecordingsDataSubscriber"),
|
||||
patch("frigate.video.ffmpeg.CameraConfigUpdateSubscriber"),
|
||||
):
|
||||
watchdog = CameraWatchdog(
|
||||
camera_config,
|
||||
1,
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
)
|
||||
|
||||
watchdog.requestor = MagicMock()
|
||||
return watchdog
|
||||
|
||||
|
||||
class TestCameraWatchdogStreamHealth(unittest.TestCase):
|
||||
def _build_watchdog(
|
||||
self, sub_enabled: bool = True, output_args: dict | None = None
|
||||
) -> CameraWatchdog:
|
||||
config = FrigateConfig(
|
||||
**{
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"cameras": {
|
||||
"front_door": {
|
||||
"ffmpeg": {
|
||||
"output_args": output_args or {},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video",
|
||||
"roles": ["record"],
|
||||
},
|
||||
{
|
||||
"path": "rtsp://10.0.0.1:554/video2",
|
||||
"roles": ["detect", "record_sub"],
|
||||
},
|
||||
],
|
||||
},
|
||||
"record": {
|
||||
"enabled": True,
|
||||
"sub": {"enabled": sub_enabled},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
camera_config = config.cameras["front_door"]
|
||||
|
||||
with (
|
||||
patch("frigate.video.ffmpeg.LogPipe"),
|
||||
patch("frigate.video.ffmpeg.InterProcessRequestor"),
|
||||
patch("frigate.video.ffmpeg.RecordingsDataSubscriber"),
|
||||
patch("frigate.video.ffmpeg.CameraConfigUpdateSubscriber"),
|
||||
):
|
||||
watchdog = CameraWatchdog(
|
||||
camera_config,
|
||||
1,
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
)
|
||||
|
||||
watchdog.requestor = MagicMock()
|
||||
return watchdog
|
||||
return build_watchdog(sub_enabled, output_args)
|
||||
|
||||
def test_stale_sub_does_not_mark_main_stale(self):
|
||||
watchdog = self._build_watchdog()
|
||||
@@ -218,3 +228,75 @@ class TestCameraWatchdogStreamHealth(unittest.TestCase):
|
||||
|
||||
assert watchdog.record_stale_threshold[STREAM_TYPE_MAIN] == 120
|
||||
assert watchdog.record_stale_threshold[STREAM_TYPE_SUB] == 150
|
||||
|
||||
|
||||
class TestCameraWatchdogCrashLoop(unittest.TestCase):
|
||||
"""The crash loop notice follows restarts in the trailing hour."""
|
||||
|
||||
def _build_watchdog(self) -> CameraWatchdog:
|
||||
return build_watchdog()
|
||||
|
||||
def _notice_payloads(self, watchdog: CameraWatchdog) -> list[dict]:
|
||||
return [
|
||||
call.args[1]
|
||||
for call in watchdog.requestor.send_data.call_args_list
|
||||
if call.args[0] == "update_notice"
|
||||
]
|
||||
|
||||
def test_below_threshold_raises_nothing(self):
|
||||
watchdog = self._build_watchdog()
|
||||
now = datetime.now().timestamp()
|
||||
watchdog.reconnect_timestamps.extend([now] * (CRASH_LOOP_RESTARTS - 1))
|
||||
watchdog.capture_thread = MagicMock()
|
||||
watchdog.capture_thread.is_alive.return_value = False
|
||||
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
|
||||
self.assertEqual(self._notice_payloads(watchdog), [])
|
||||
|
||||
def test_threshold_raises_once_per_restart_count(self):
|
||||
watchdog = self._build_watchdog()
|
||||
now = datetime.now().timestamp()
|
||||
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
|
||||
watchdog.capture_thread = MagicMock()
|
||||
watchdog.capture_thread.is_alive.return_value = False
|
||||
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
|
||||
payloads = self._notice_payloads(watchdog)
|
||||
self.assertEqual(len(payloads), 1)
|
||||
self.assertEqual(payloads[0]["action"], "raise")
|
||||
self.assertEqual(payloads[0]["kind"], "ffmpeg_crash_loop")
|
||||
self.assertEqual(payloads[0]["scope"], "front_door")
|
||||
self.assertEqual(payloads[0]["params"], {"restarts": CRASH_LOOP_RESTARTS})
|
||||
|
||||
def test_recovery_window_resolves(self):
|
||||
watchdog = self._build_watchdog()
|
||||
now = datetime.now().timestamp()
|
||||
# the last restart is at `now`, so recovery is measured from there
|
||||
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
|
||||
watchdog.capture_thread = MagicMock()
|
||||
watchdog.capture_thread.is_alive.return_value = False
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
|
||||
watchdog.capture_thread.is_alive.return_value = True
|
||||
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S - 1)
|
||||
self.assertEqual(len(self._notice_payloads(watchdog)), 1)
|
||||
|
||||
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S + 1)
|
||||
|
||||
payloads = self._notice_payloads(watchdog)
|
||||
self.assertEqual(len(payloads), 2)
|
||||
self.assertEqual(payloads[1]["action"], "resolve")
|
||||
self.assertFalse(watchdog.crash_loop_raised)
|
||||
|
||||
def test_disable_resolves(self):
|
||||
watchdog = self._build_watchdog()
|
||||
watchdog.crash_loop_raised = True
|
||||
|
||||
watchdog._resolve_crash_loop()
|
||||
|
||||
payloads = self._notice_payloads(watchdog)
|
||||
self.assertEqual(payloads[-1]["action"], "resolve")
|
||||
self.assertFalse(watchdog.crash_loop_raised)
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.app import FrigateApp
|
||||
from frigate.comms.dispatcher import Dispatcher
|
||||
from frigate.comms.mqtt import MqttClient
|
||||
from frigate.comms.runtime_state import RuntimeStatePersistence
|
||||
from frigate.config import BirdseyeModeEnum
|
||||
|
||||
@@ -510,3 +511,92 @@ class TestStartupAppliesConfigLayersBeforeWorkersStart(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestNoticeWiring(unittest.TestCase):
|
||||
"""The dispatcher forwards update_notice requests and publishes the list."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.registry = MagicMock()
|
||||
self.registry.active.return_value = [{"id": "retention_unmet"}]
|
||||
config = MagicMock()
|
||||
config.cameras = {}
|
||||
|
||||
with (
|
||||
patch("frigate.comms.dispatcher.CameraActivityManager"),
|
||||
patch("frigate.comms.dispatcher.AudioActivityManager"),
|
||||
):
|
||||
self.dispatcher = Dispatcher(
|
||||
config,
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
{},
|
||||
[],
|
||||
notice_registry=self.registry,
|
||||
)
|
||||
|
||||
def test_registry_listener_is_subscribed(self) -> None:
|
||||
self.registry.subscribe.assert_called_once_with(
|
||||
self.dispatcher._publish_notices
|
||||
)
|
||||
|
||||
def test_raise_request_reaches_registry(self) -> None:
|
||||
self.dispatcher._receive(
|
||||
"update_notice",
|
||||
{
|
||||
"action": "raise",
|
||||
"kind": "ffmpeg_crash_loop",
|
||||
"scope": "front_door",
|
||||
"params": {"restarts": 5},
|
||||
},
|
||||
)
|
||||
|
||||
self.registry.raise_notice.assert_called_once_with(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
)
|
||||
|
||||
def test_resolve_request_reaches_registry(self) -> None:
|
||||
self.dispatcher._receive(
|
||||
"update_notice",
|
||||
{"action": "resolve", "kind": "retention_unmet", "scope": None},
|
||||
)
|
||||
|
||||
self.registry.resolve.assert_called_once_with("retention_unmet", None)
|
||||
|
||||
def test_malformed_request_does_not_raise(self) -> None:
|
||||
self.registry.raise_notice.side_effect = RuntimeError("boom")
|
||||
|
||||
self.dispatcher._receive("update_notice", "not a dict")
|
||||
self.dispatcher._receive(
|
||||
"update_notice", {"action": "raise", "kind": "x", "params": {}}
|
||||
)
|
||||
|
||||
def test_publish_local_skips_mqtt(self) -> None:
|
||||
mqtt = MagicMock(spec=MqttClient)
|
||||
other = MagicMock()
|
||||
self.dispatcher.comms = [mqtt, other]
|
||||
|
||||
self.dispatcher.publish_local("notices", "[]")
|
||||
|
||||
mqtt.publish.assert_not_called()
|
||||
other.publish.assert_called_once_with("notices", "[]", False)
|
||||
|
||||
def test_listener_publishes_active_list(self) -> None:
|
||||
self.dispatcher.publish_local = MagicMock()
|
||||
|
||||
self.dispatcher._publish_notices()
|
||||
|
||||
self.dispatcher.publish_local.assert_called_once_with(
|
||||
"notices", '[{"id": "retention_unmet"}]'
|
||||
)
|
||||
|
||||
def test_snapshot_includes_notices(self) -> None:
|
||||
publisher = MagicMock()
|
||||
self.dispatcher._build_camera_activity_snapshot = MagicMock(
|
||||
return_value=({}, {})
|
||||
)
|
||||
self.dispatcher.web_push_client = None
|
||||
|
||||
self.dispatcher.publish_runtime_snapshot(publisher)
|
||||
|
||||
publisher.assert_any_call("notices", '[{"id": "retention_unmet"}]', False)
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Tests for the model download failure notice."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.util import downloader
|
||||
from frigate.util.downloader import ModelDownloader
|
||||
|
||||
|
||||
class TestModelDownloadNotice(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.download_path = tempfile.mkdtemp()
|
||||
downloader.last_download_error.clear()
|
||||
|
||||
def _notice_calls(self, requestor: MagicMock) -> list[dict]:
|
||||
return [
|
||||
call.args[1]
|
||||
for call in requestor.send_data.call_args_list
|
||||
if call.args[0] == "update_notice"
|
||||
]
|
||||
|
||||
def _downloader(self, download_func) -> ModelDownloader:
|
||||
with patch("frigate.util.downloader.InterProcessRequestor"):
|
||||
return ModelDownloader(
|
||||
"facedet", self.download_path, ["facedet.onnx"], download_func
|
||||
)
|
||||
|
||||
def test_raising_download_reports_and_reraises(self):
|
||||
def failing(path: str) -> None:
|
||||
raise RuntimeError("HTTP 503 from upstream\nmore detail")
|
||||
|
||||
model_downloader = self._downloader(failing)
|
||||
|
||||
with self.assertRaises(RuntimeError):
|
||||
model_downloader._download_models()
|
||||
|
||||
notices = self._notice_calls(model_downloader.requestor)
|
||||
self.assertEqual(len(notices), 1)
|
||||
self.assertEqual(notices[0]["action"], "raise")
|
||||
self.assertEqual(notices[0]["kind"], "model_download_failed")
|
||||
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
|
||||
self.assertEqual(
|
||||
notices[0]["params"],
|
||||
{"file": "facedet.onnx", "error": "HTTP 503 from upstream"},
|
||||
)
|
||||
|
||||
def test_swallowing_download_that_leaves_no_file_reports(self):
|
||||
target = os.path.join(self.download_path, "facedet.onnx")
|
||||
|
||||
def swallowing(path: str) -> None:
|
||||
# mirrors the processors: download_from_url fails, they only log
|
||||
with patch(
|
||||
"frigate.util.downloader.requests.get", side_effect=OSError("dns")
|
||||
):
|
||||
try:
|
||||
ModelDownloader.download_from_url("http://x/facedet.onnx", path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
model_downloader = self._downloader(swallowing)
|
||||
model_downloader._download_models()
|
||||
|
||||
notices = self._notice_calls(model_downloader.requestor)
|
||||
self.assertEqual(len(notices), 1)
|
||||
self.assertEqual(notices[0]["action"], "raise")
|
||||
self.assertEqual(notices[0]["params"], {"file": "facedet.onnx", "error": "dns"})
|
||||
self.assertFalse(os.path.exists(target))
|
||||
|
||||
def test_success_resolves(self):
|
||||
def succeeding(path: str) -> None:
|
||||
with open(path, "w") as f:
|
||||
f.write("model")
|
||||
|
||||
model_downloader = self._downloader(succeeding)
|
||||
model_downloader._download_models()
|
||||
|
||||
notices = self._notice_calls(model_downloader.requestor)
|
||||
self.assertEqual(len(notices), 1)
|
||||
self.assertEqual(notices[0]["action"], "resolve")
|
||||
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
|
||||
|
||||
def test_a_sibling_downloader_only_resolves_its_own_files(self):
|
||||
"""PaddleOCR, facedet and jina each spread one model_name over several
|
||||
downloaders, so a resolve must never reach a sibling's file."""
|
||||
|
||||
def succeeding(path: str) -> None:
|
||||
with open(path, "w") as f:
|
||||
f.write("model")
|
||||
|
||||
with patch("frigate.util.downloader.InterProcessRequestor"):
|
||||
sibling = ModelDownloader(
|
||||
"paddleocr-onnx", self.download_path, ["det.onnx"], succeeding
|
||||
)
|
||||
|
||||
sibling._download_models()
|
||||
|
||||
self.assertEqual(
|
||||
[n["scope"] for n in self._notice_calls(sibling.requestor)],
|
||||
["paddleocr-onnx/det.onnx"],
|
||||
)
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Tests for the notice registry and its kind definitions."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from peewee_migrate import Router
|
||||
from playhouse.sqlite_ext import SqliteExtDatabase
|
||||
from playhouse.sqliteq import SqliteQueueDatabase
|
||||
|
||||
from frigate.models import Notice, NoticeStats
|
||||
from frigate.notices.registry import DismissResult, NoticeRegistry
|
||||
from frigate.notices.types import (
|
||||
NOTICE_KINDS,
|
||||
NoticeMode,
|
||||
NoticeSeverity,
|
||||
notice_id,
|
||||
)
|
||||
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
|
||||
|
||||
|
||||
class TestNoticeKinds(unittest.TestCase):
|
||||
def test_every_kind_is_keyed_by_its_own_key(self):
|
||||
for key, definition in NOTICE_KINDS.items():
|
||||
self.assertEqual(key, definition.key)
|
||||
|
||||
def test_first_producers_are_defined(self):
|
||||
self.assertEqual(NOTICE_KINDS["ffmpeg_crash_loop"].mode, NoticeMode.state)
|
||||
self.assertEqual(
|
||||
NOTICE_KINDS["ffmpeg_crash_loop"].severity, NoticeSeverity.error
|
||||
)
|
||||
self.assertEqual(NOTICE_KINDS["detector_stuck"].mode, NoticeMode.event)
|
||||
self.assertEqual(
|
||||
NOTICE_KINDS["detector_stuck"].severity, NoticeSeverity.warning
|
||||
)
|
||||
self.assertEqual(NOTICE_KINDS["model_download_failed"].mode, NoticeMode.event)
|
||||
self.assertEqual(NOTICE_KINDS["retention_unmet"].mode, NoticeMode.state)
|
||||
self.assertEqual(NOTICE_KINDS["update_available"].mode, NoticeMode.state)
|
||||
self.assertEqual(NOTICE_KINDS["update_available"].severity, NoticeSeverity.info)
|
||||
|
||||
def test_notice_id_with_and_without_scope(self):
|
||||
self.assertEqual(notice_id("retention_unmet", None), "retention_unmet")
|
||||
self.assertEqual(
|
||||
notice_id("ffmpeg_crash_loop", "front_door"),
|
||||
"ffmpeg_crash_loop:front_door",
|
||||
)
|
||||
|
||||
|
||||
class TestNoticeRegistry(unittest.TestCase):
|
||||
def setUp(self):
|
||||
migrate_db = SqliteExtDatabase("test.db")
|
||||
del logging.getLogger("peewee_migrate").handlers[:]
|
||||
router = Router(migrate_db)
|
||||
router.run()
|
||||
migrate_db.close()
|
||||
self.db = SqliteQueueDatabase(TEST_DB)
|
||||
self.db.bind([Notice, NoticeStats])
|
||||
self.registry = NoticeRegistry()
|
||||
self.listener = MagicMock()
|
||||
self.registry.subscribe(self.listener)
|
||||
|
||||
def tearDown(self):
|
||||
if not self.db.is_closed():
|
||||
self.db.close()
|
||||
|
||||
try:
|
||||
for file in TEST_DB_CLEANUPS:
|
||||
os.remove(file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_raise_inserts_and_counts_one_occurrence(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
)
|
||||
|
||||
active = self.registry.active()
|
||||
self.assertEqual(len(active), 1)
|
||||
self.assertEqual(active[0]["id"], "ffmpeg_crash_loop:front_door")
|
||||
self.assertEqual(active[0]["severity"], "error")
|
||||
self.assertEqual(active[0]["mode"], "state")
|
||||
self.assertEqual(active[0]["category"], "camera")
|
||||
self.assertEqual(active[0]["scope"], "front_door")
|
||||
self.assertEqual(active[0]["params"], {"restarts": 5})
|
||||
self.assertEqual(active[0]["count"], 1)
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_called_once()
|
||||
|
||||
def test_state_re_raise_with_new_params_updates_without_counting(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
)
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 6}
|
||||
)
|
||||
|
||||
active = self.registry.active()
|
||||
self.assertEqual(active[0]["count"], 1)
|
||||
self.assertEqual(active[0]["params"], {"restarts": 6})
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_called_once()
|
||||
|
||||
def test_state_re_raise_with_same_params_is_a_no_op(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
)
|
||||
before = self.registry.active()[0]["last_seen"]
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
)
|
||||
|
||||
self.assertEqual(self.registry.active()[0]["last_seen"], before)
|
||||
self.listener.assert_not_called()
|
||||
|
||||
def test_event_re_raise_counts_and_undismisses(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.assertEqual(self.registry.dismiss("detector_stuck"), DismissResult.ok)
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
# no scope here on purpose: a global event id is still valid
|
||||
|
||||
active = self.registry.active()
|
||||
self.assertEqual(len(active), 1)
|
||||
self.assertEqual(active[0]["count"], 2)
|
||||
self.assertIsNone(active[0]["dismissed_at"])
|
||||
stats = self.registry.stats()[0]
|
||||
self.assertEqual(stats["occurrences"], 2)
|
||||
self.assertEqual(stats["dismissals"], 1)
|
||||
|
||||
def test_resolve_deletes_and_keeps_stats(self):
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.resolve("retention_unmet")
|
||||
self.registry.resolve("retention_unmet")
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_called_once()
|
||||
|
||||
def test_state_notice_cannot_be_dismissed(self):
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
|
||||
self.assertEqual(
|
||||
self.registry.dismiss("retention_unmet"), DismissResult.not_dismissable
|
||||
)
|
||||
self.assertEqual(len(self.registry.active()), 1)
|
||||
|
||||
def test_dismiss_unknown_id(self):
|
||||
self.assertEqual(self.registry.dismiss("nope"), DismissResult.not_found)
|
||||
|
||||
def test_dismissed_hidden_unless_requested(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.registry.dismiss("detector_stuck")
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
self.assertEqual(len(self.registry.active(include_dismissed=True)), 1)
|
||||
|
||||
def test_startup_hides_state_notices_until_confirmed(self):
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
self.registry.raise_notice(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
)
|
||||
|
||||
self.registry.mark_state_notices_unconfirmed()
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["detector_stuck:ov"])
|
||||
|
||||
# the producer confirms the same episode: visible again, no new occurrence
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["retention_unmet", "detector_stuck:ov"])
|
||||
self.assertEqual(
|
||||
[s for s in self.registry.stats() if s["kind"] == "retention_unmet"][0][
|
||||
"occurrences"
|
||||
],
|
||||
1,
|
||||
)
|
||||
|
||||
def test_unconfirmed_across_two_startups_is_deleted(self):
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
|
||||
self.registry.mark_state_notices_unconfirmed()
|
||||
self.registry.mark_state_notices_unconfirmed()
|
||||
|
||||
self.assertEqual(self.registry.active(include_dismissed=True), [])
|
||||
|
||||
def test_replay_camera_is_ignored(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="_replay_front_door", params={}
|
||||
)
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
self.assertEqual(self.registry.stats(), [])
|
||||
|
||||
def test_unknown_kind_is_dropped(self):
|
||||
self.registry.raise_notice("not_a_kind", params={})
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
self.listener.assert_not_called()
|
||||
|
||||
def test_active_sorts_by_severity_then_recency(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["retention_unmet", "detector_stuck"])
|
||||
|
||||
def test_listener_exception_does_not_propagate(self):
|
||||
self.registry.subscribe(MagicMock(side_effect=RuntimeError("boom")))
|
||||
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
|
||||
self.assertEqual(len(self.registry.active()), 1)
|
||||
|
||||
def test_resolve_camera_drops_only_that_camera(self):
|
||||
self.registry.raise_notice("ffmpeg_crash_loop", scope="front_door", params={})
|
||||
self.registry.raise_notice("ffmpeg_crash_loop", scope="garage", params={})
|
||||
self.registry.raise_notice(
|
||||
"model_download_failed", scope="front_door", params={}
|
||||
)
|
||||
|
||||
self.registry.resolve_camera("front_door")
|
||||
|
||||
ids = sorted(n["id"] for n in self.registry.active())
|
||||
self.assertEqual(
|
||||
ids, ["ffmpeg_crash_loop:garage", "model_download_failed:front_door"]
|
||||
)
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Tests for the daily latest version refresh and the update notice."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.stats import emitter
|
||||
from frigate.stats.util import is_newer_version
|
||||
|
||||
|
||||
class TestIsNewerVersion(unittest.TestCase):
|
||||
def test_newer(self):
|
||||
self.assertTrue(is_newer_version("0.18.1-abcdef", "0.19.0"))
|
||||
self.assertTrue(is_newer_version("0.19.0-abcdef", "0.19.1"))
|
||||
self.assertTrue(is_newer_version("0.19.0-abcdef", "1.0.0"))
|
||||
|
||||
def test_not_newer(self):
|
||||
self.assertFalse(is_newer_version("0.19.0-abcdef", "0.19.0"))
|
||||
self.assertFalse(is_newer_version("0.19.1-abcdef", "0.19.0"))
|
||||
self.assertFalse(is_newer_version("0.19.0-abcdef", "0.19.0-beta2"))
|
||||
|
||||
def test_prerelease_is_behind_its_final_release(self):
|
||||
self.assertTrue(is_newer_version("0.19.0-beta2", "0.19.0"))
|
||||
self.assertTrue(is_newer_version("0.19.0-rc1", "0.19.0"))
|
||||
self.assertTrue(is_newer_version("0.19.0-RC1", "0.19.0"))
|
||||
|
||||
def test_prerelease_of_a_later_line_is_not_behind(self):
|
||||
self.assertFalse(is_newer_version("0.20.0-beta1", "0.19.0"))
|
||||
self.assertFalse(is_newer_version("0.19.0-beta2", "0.19.0-beta2"))
|
||||
|
||||
def test_unparseable_is_never_newer(self):
|
||||
self.assertFalse(is_newer_version("0.19.0-abcdef", "disabled"))
|
||||
self.assertFalse(is_newer_version("0.19.0-abcdef", "unknown"))
|
||||
self.assertFalse(is_newer_version("dev", "0.19.0"))
|
||||
|
||||
|
||||
class TestUpdateNotice(unittest.TestCase):
|
||||
def _emitter(self, latest: str) -> emitter.StatsEmitter:
|
||||
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
|
||||
stats_emitter.config = MagicMock()
|
||||
stats_emitter.stats_tracking = {"latest_frigate_version": latest}
|
||||
stats_emitter.notice_registry = MagicMock()
|
||||
return stats_emitter
|
||||
|
||||
def test_newer_release_raises(self):
|
||||
stats_emitter = self._emitter("0.19.0")
|
||||
|
||||
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
|
||||
stats_emitter._check_update_notice()
|
||||
|
||||
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
|
||||
"update_available", params={"version": "0.19.0"}
|
||||
)
|
||||
stats_emitter.notice_registry.resolve.assert_not_called()
|
||||
|
||||
def test_current_or_disabled_resolves(self):
|
||||
for latest in ("0.18.1", "disabled"):
|
||||
stats_emitter = self._emitter(latest)
|
||||
|
||||
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
|
||||
stats_emitter._check_update_notice()
|
||||
|
||||
stats_emitter.notice_registry.raise_notice.assert_not_called()
|
||||
stats_emitter.notice_registry.resolve.assert_called_once_with(
|
||||
"update_available"
|
||||
)
|
||||
|
||||
def test_a_failed_lookup_leaves_the_notice_alone(self):
|
||||
stats_emitter = self._emitter("unknown")
|
||||
|
||||
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
|
||||
stats_emitter._check_update_notice()
|
||||
|
||||
stats_emitter.notice_registry.raise_notice.assert_not_called()
|
||||
stats_emitter.notice_registry.resolve.assert_not_called()
|
||||
|
||||
def test_refresh_updates_tracking_on_a_thread_then_checks(self):
|
||||
stats_emitter = self._emitter("0.18.0")
|
||||
|
||||
with (
|
||||
patch.object(emitter, "get_latest_version", return_value="0.19.0"),
|
||||
patch.object(emitter, "VERSION", "0.18.1-abcdef"),
|
||||
patch.object(emitter.threading, "Thread") as thread,
|
||||
):
|
||||
stats_emitter._refresh_latest_version()
|
||||
thread.return_value.start.assert_called_once()
|
||||
# run the thread body inline
|
||||
thread.call_args.kwargs["target"]()
|
||||
|
||||
self.assertEqual(
|
||||
stats_emitter.stats_tracking["latest_frigate_version"], "0.19.0"
|
||||
)
|
||||
stats_emitter.notice_registry.raise_notice.assert_called_once()
|
||||
|
||||
def test_failed_refresh_keeps_the_last_known_version(self):
|
||||
stats_emitter = self._emitter("0.20.0")
|
||||
|
||||
with (
|
||||
patch.object(emitter, "get_latest_version", return_value="unknown"),
|
||||
patch.object(emitter, "VERSION", "0.19.0-abcdef"),
|
||||
patch.object(emitter.threading, "Thread") as thread,
|
||||
):
|
||||
stats_emitter._refresh_latest_version()
|
||||
thread.call_args.kwargs["target"]()
|
||||
|
||||
self.assertEqual(
|
||||
stats_emitter.stats_tracking["latest_frigate_version"], "0.20.0"
|
||||
)
|
||||
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
|
||||
"update_available", params={"version": "0.20.0"}
|
||||
)
|
||||
stats_emitter.notice_registry.resolve.assert_not_called()
|
||||
|
||||
def test_disabled_version_check_still_clears_the_notice(self):
|
||||
stats_emitter = self._emitter("0.20.0")
|
||||
|
||||
with (
|
||||
patch.object(emitter, "get_latest_version", return_value="disabled"),
|
||||
patch.object(emitter, "VERSION", "0.19.0-abcdef"),
|
||||
patch.object(emitter.threading, "Thread") as thread,
|
||||
):
|
||||
stats_emitter._refresh_latest_version()
|
||||
thread.call_args.kwargs["target"]()
|
||||
|
||||
self.assertEqual(
|
||||
stats_emitter.stats_tracking["latest_frigate_version"], "disabled"
|
||||
)
|
||||
stats_emitter.notice_registry.resolve.assert_called_once_with(
|
||||
"update_available"
|
||||
)
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from peewee import DoesNotExist
|
||||
from peewee_migrate import Router
|
||||
@@ -556,6 +556,34 @@ class TestHttp(unittest.TestCase):
|
||||
assert Recordings.get(Recordings.id == rec_k2_id)
|
||||
assert Recordings.get(Recordings.id == rec_k3_id)
|
||||
|
||||
def test_unmet_retention_raises_notice(self):
|
||||
registry = MagicMock()
|
||||
config = FrigateConfig(**self.minimal_config)
|
||||
storage = StorageMaintainer(config, MagicMock(), registry)
|
||||
storage.camera_storage_stats = {
|
||||
"front_door": {"needs_refresh": False, "usage": 10, "bandwidth": 10}
|
||||
}
|
||||
|
||||
with patch.object(storage, "expected_hourly_bandwidth", return_value=100.0):
|
||||
storage.reduce_storage_consumption()
|
||||
|
||||
registry.raise_notice.assert_called_once()
|
||||
kind = registry.raise_notice.call_args.args[0]
|
||||
params = registry.raise_notice.call_args.kwargs["params"]
|
||||
self.assertEqual(kind, "retention_unmet")
|
||||
self.assertEqual(params["needed_mb"], 100.0)
|
||||
self.assertEqual(params["cleared_mb"], 0.0)
|
||||
|
||||
def test_clean_run_resolves_notice(self):
|
||||
registry = MagicMock()
|
||||
config = FrigateConfig(**self.minimal_config)
|
||||
storage = StorageMaintainer(config, MagicMock(), registry)
|
||||
|
||||
with patch.object(storage, "check_storage_needs_cleanup", return_value=False):
|
||||
storage._maintain_once()
|
||||
|
||||
registry.resolve.assert_called_once_with("retention_unmet")
|
||||
|
||||
|
||||
def _insert_mock_event(
|
||||
id: str,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Tests for the detector stuck notice raised by the Frigate watchdog."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.watchdog import FrigateWatchdog
|
||||
|
||||
|
||||
class TestDetectorStuckNotice(unittest.TestCase):
|
||||
def _stuck_detector(self) -> MagicMock:
|
||||
detector = MagicMock()
|
||||
detector.detection_start.value = 1.0
|
||||
detector.detect_process.is_alive.return_value = True
|
||||
return detector
|
||||
|
||||
def test_stuck_restart_raises_notice(self):
|
||||
registry = MagicMock()
|
||||
detector = self._stuck_detector()
|
||||
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
|
||||
|
||||
with patch("frigate.watchdog.datetime") as mock_datetime:
|
||||
mock_datetime.datetime.now.return_value.timestamp.return_value = 100.0
|
||||
watchdog._check_detectors()
|
||||
|
||||
detector.start_or_restart.assert_called_once()
|
||||
registry.raise_notice.assert_called_once_with(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
)
|
||||
|
||||
def test_healthy_detector_raises_nothing(self):
|
||||
registry = MagicMock()
|
||||
detector = self._stuck_detector()
|
||||
detector.detection_start.value = 0.0
|
||||
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
|
||||
|
||||
watchdog._check_detectors()
|
||||
|
||||
registry.raise_notice.assert_not_called()
|
||||
@@ -7,6 +7,7 @@ from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from frigate.comms.ws import (
|
||||
_WS_BLOCKED_TOPICS,
|
||||
WebSocketClient,
|
||||
_classify_outbound,
|
||||
_collect_zone_names,
|
||||
@@ -106,6 +107,12 @@ class TestClassifyOutbound(unittest.TestCase):
|
||||
def test_birdseye_layout_is_unrestricted_only(self):
|
||||
self.assertEqual(self._classify("birdseye_layout"), ("unrestricted_only", None))
|
||||
|
||||
def test_notices_is_admin_only(self):
|
||||
self.assertEqual(self._classify("notices"), ("admin_only", None))
|
||||
|
||||
def test_update_notice_is_blocked_inbound(self):
|
||||
self.assertIn("update_notice", _WS_BLOCKED_TOPICS)
|
||||
|
||||
# --- Camera-prefixed ---
|
||||
|
||||
def test_camera_state_topic_resolves_to_camera(self):
|
||||
@@ -295,6 +302,19 @@ class TestMaterializeForWs(unittest.TestCase):
|
||||
full = json.dumps({"topic": topic, "payload": payload})
|
||||
return _materialize_for_ws(ws, topic, full, scope, parsed, self.config)
|
||||
|
||||
def test_admin_only_topic_reaches_admin_not_viewer(self):
|
||||
admin = SimpleNamespace(environ={"HTTP_REMOTE_ROLE": "admin"})
|
||||
viewer = SimpleNamespace(environ={"HTTP_REMOTE_ROLE": "viewer"})
|
||||
scope = ("admin_only", None)
|
||||
|
||||
self.assertEqual(
|
||||
_materialize_for_ws(admin, "notices", "msg", scope, None, self.config),
|
||||
"msg",
|
||||
)
|
||||
self.assertIsNone(
|
||||
_materialize_for_ws(viewer, "notices", "msg", scope, None, self.config)
|
||||
)
|
||||
|
||||
# --- Globals: every authenticated client sees them ---
|
||||
|
||||
def test_globals_reach_admin(self):
|
||||
|
||||
Reference in New Issue
Block a user