mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-17 19:13:02 +03:00
Refactor Notices and System Health pane (#24243)
* refactor notices * show startup message for enrichments in health pane * tweaks
This commit is contained in:
committed by
Nicolas Mowen
parent
5cef6823a6
commit
7bc32fd4c9
@@ -1,9 +1,16 @@
|
||||
"""Tests for authentication endpoints."""
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
import unittest
|
||||
from unittest.mock import ANY, patch
|
||||
|
||||
from frigate.api.auth import hash_password
|
||||
from frigate.api.auth import (
|
||||
FAILED_LOGIN_BURST_GAP_S,
|
||||
MAX_NOTICE_USERNAME,
|
||||
MAX_OPEN_BURSTS,
|
||||
FailedLoginTracker,
|
||||
hash_password,
|
||||
)
|
||||
from frigate.const import JWT_SECRET_ENV_VAR
|
||||
from frigate.models import User
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
@@ -43,3 +50,116 @@ class TestHttpAuth(BaseTestHttp):
|
||||
)
|
||||
assert response.status_code == 401
|
||||
assert any("Login failed" in m and "admin" in m for m in logs.output)
|
||||
|
||||
|
||||
@patch.dict(os.environ, {JWT_SECRET_ENV_VAR: "test-secret"})
|
||||
class TestFailedLoginNotices(BaseTestHttp):
|
||||
def setUp(self):
|
||||
super().setUp([User])
|
||||
self.app = super().create_app()
|
||||
User.insert(
|
||||
username="admin",
|
||||
password_hash=hash_password("correct-horse-battery", iterations=1000),
|
||||
role="admin",
|
||||
notification_tokens=[],
|
||||
).execute()
|
||||
tracker_patch = patch("frigate.api.auth.failed_logins")
|
||||
self.failed_logins = tracker_patch.start()
|
||||
self.addCleanup(tracker_patch.stop)
|
||||
|
||||
def tearDown(self):
|
||||
User.delete().execute()
|
||||
super().tearDown()
|
||||
|
||||
def _login(self, user: str, password: str) -> int:
|
||||
with AuthTestClient(self.app) as client:
|
||||
response = client.post("/login", json={"user": user, "password": password})
|
||||
|
||||
return response.status_code
|
||||
|
||||
def test_unknown_user_is_recorded(self):
|
||||
self.assertEqual(self._login("ghost", "irrelevant"), 401)
|
||||
self.failed_logins.record.assert_called_once_with("ghost", ANY)
|
||||
|
||||
def test_bad_password_is_recorded_as_known(self):
|
||||
self.assertEqual(self._login("admin", "wrong"), 401)
|
||||
self.failed_logins.record.assert_called_once_with("admin", ANY, known=True)
|
||||
|
||||
def test_success_is_not_recorded(self):
|
||||
self.assertEqual(self._login("admin", "correct-horse-battery"), 200)
|
||||
self.failed_logins.record.assert_not_called()
|
||||
|
||||
|
||||
class TestFailedLoginTracker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tracker = FailedLoginTracker()
|
||||
raise_patch = patch("frigate.api.auth.raise_notice")
|
||||
self.raise_notice = raise_patch.start()
|
||||
self.addCleanup(raise_patch.stop)
|
||||
|
||||
def _scopes(self) -> list[str]:
|
||||
return [call.kwargs["scope"] for call in self.raise_notice.call_args_list]
|
||||
|
||||
def _fill(self, now: float) -> None:
|
||||
for index in range(MAX_OPEN_BURSTS):
|
||||
self.tracker.record(f"ghost{index}", now)
|
||||
|
||||
def test_attempts_inside_the_gap_share_a_burst(self):
|
||||
for now in (1000.0, 1010.0, 1299.0):
|
||||
self.tracker.record("admin", now)
|
||||
|
||||
self.raise_notice.assert_called_with(
|
||||
"failed_login", scope="admin:1000", params={"user": "admin"}
|
||||
)
|
||||
self.assertEqual(self._scopes(), ["admin:1000"] * 3)
|
||||
|
||||
def test_each_user_gets_their_own_burst(self):
|
||||
self.tracker.record("admin", 1000.0)
|
||||
self.tracker.record("ghost", 1010.0)
|
||||
self.tracker.record("admin", 1020.0)
|
||||
|
||||
self.assertEqual(self._scopes(), ["admin:1000", "ghost:1010", "admin:1000"])
|
||||
|
||||
def test_a_quiet_gap_opens_a_new_burst(self):
|
||||
self.tracker.record("admin", 1000.0)
|
||||
self.tracker.record("admin", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
|
||||
|
||||
self.assertEqual(self._scopes(), ["admin:1000", "admin:1300"])
|
||||
|
||||
def test_the_gap_runs_from_the_latest_attempt(self):
|
||||
for now in (1000.0, 1200.0, 1450.0):
|
||||
self.tracker.record("admin", now)
|
||||
|
||||
self.assertEqual(self._scopes(), ["admin:1000"] * 3)
|
||||
|
||||
def test_quiet_users_are_forgotten(self):
|
||||
self.tracker.record("ghost", 1000.0)
|
||||
self.tracker.record("admin", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
|
||||
|
||||
self.assertEqual(list(self.tracker._bursts), ["admin"])
|
||||
|
||||
def test_a_long_username_is_cut(self):
|
||||
self.tracker.record("x" * 500, 1000.0)
|
||||
|
||||
user = self.raise_notice.call_args.kwargs["params"]["user"]
|
||||
self.assertEqual(len(user), MAX_NOTICE_USERNAME)
|
||||
|
||||
def test_unknown_users_past_the_cap_get_no_notice(self):
|
||||
self._fill(1000.0)
|
||||
self.tracker.record("one-too-many", 1001.0)
|
||||
self.tracker.record("ghost0", 1002.0)
|
||||
|
||||
self.assertEqual(self.raise_notice.call_count, MAX_OPEN_BURSTS + 1)
|
||||
self.assertEqual(self._scopes()[-1], "ghost0:1000")
|
||||
|
||||
def test_known_users_skip_the_cap(self):
|
||||
self._fill(1000.0)
|
||||
self.tracker.record("admin", 1001.0, known=True)
|
||||
|
||||
self.assertEqual(self._scopes()[-1], "admin:1001")
|
||||
|
||||
def test_quiet_bursts_free_their_slots(self):
|
||||
self._fill(1000.0)
|
||||
self.tracker.record("late", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
|
||||
|
||||
self.assertEqual(self._scopes()[-1], "late:1300")
|
||||
|
||||
@@ -6,6 +6,8 @@ from frigate.models import Notice, NoticeStats
|
||||
from frigate.notices.registry import NoticeRegistry
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
|
||||
CONFIG_CHECK = "config:detect:fps-greater-than-five:camera.garage"
|
||||
|
||||
|
||||
class TestHttpNotices(BaseTestHttp):
|
||||
def setUp(self):
|
||||
@@ -17,20 +19,32 @@ class TestHttpNotices(BaseTestHttp):
|
||||
return AuthTestClient(self.app)
|
||||
|
||||
def test_get_notices_orders_by_severity(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
self.registry.raise_notice(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
)
|
||||
self.registry.raise_notice(
|
||||
"model_download_failed",
|
||||
scope="yolo/model.onnx",
|
||||
params={"file": "model.onnx", "error": "timeout"},
|
||||
)
|
||||
|
||||
with self.client() as client:
|
||||
response = client.get("/notices")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(
|
||||
[n["id"] for n in response.json()], ["retention_unmet", "detector_stuck"]
|
||||
[n["id"] for n in response.json()],
|
||||
["model_download_failed:yolo/model.onnx", "detector_stuck:ov"],
|
||||
)
|
||||
self.assertEqual(response.json()[0]["severity"], "error")
|
||||
self.assertEqual(
|
||||
[n["link"] for n in response.json()], [None, "/system#general"]
|
||||
)
|
||||
|
||||
def test_get_stats(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.registry.raise_notice(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
)
|
||||
|
||||
with self.client() as client:
|
||||
response = client.get("/notices/stats")
|
||||
@@ -39,15 +53,33 @@ class TestHttpNotices(BaseTestHttp):
|
||||
self.assertEqual(response.json()[0]["kind"], "detector_stuck")
|
||||
self.assertEqual(response.json()[0]["occurrences"], 1)
|
||||
|
||||
def test_dismiss_event(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
def test_dismiss_hides_and_history_keeps_it(self):
|
||||
self.registry.raise_notice(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
)
|
||||
|
||||
with self.client() as client:
|
||||
response = client.post("/notices/detector_stuck/dismiss")
|
||||
response = client.post("/notices/detector_stuck:ov/dismiss")
|
||||
listed = client.get("/notices").json()
|
||||
history = client.get("/notices", params={"include_dismissed": True}).json()
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(listed, [])
|
||||
self.assertEqual([n["id"] for n in history], ["detector_stuck:ov"])
|
||||
self.assertIsNotNone(history[0]["dismissed_at"])
|
||||
|
||||
def test_dismiss_an_id_with_a_slash(self):
|
||||
self.registry.raise_notice(
|
||||
"model_download_failed", scope="yolo/model.onnx", params={}
|
||||
)
|
||||
|
||||
with self.client() as client:
|
||||
response = client.post(
|
||||
"/notices/model_download_failed:yolo/model.onnx/dismiss"
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
|
||||
def test_dismiss_unknown_is_404(self):
|
||||
with self.client() as client:
|
||||
@@ -55,14 +87,6 @@ class TestHttpNotices(BaseTestHttp):
|
||||
|
||||
self.assertEqual(response.status_code, 404)
|
||||
|
||||
def test_dismiss_state_is_400(self):
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
|
||||
with self.client() as client:
|
||||
response = client.post("/notices/retention_unmet/dismiss")
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
|
||||
def test_requires_admin(self):
|
||||
with TestClient(self.app) as client:
|
||||
response = client.get(
|
||||
@@ -70,3 +94,51 @@ class TestHttpNotices(BaseTestHttp):
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 403)
|
||||
|
||||
def test_dismissed_checks_are_listed_once(self):
|
||||
with self.client() as client:
|
||||
first = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
|
||||
again = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
|
||||
listed = client.get("/notices/dismissed_checks").json()
|
||||
history = client.get("/notices", params={"include_dismissed": True}).json()
|
||||
|
||||
self.assertEqual(first.status_code, 200)
|
||||
self.assertEqual(again.status_code, 200)
|
||||
self.assertEqual([check["id"] for check in listed], [CONFIG_CHECK])
|
||||
self.assertIsNotNone(listed[0]["dismissed_at"])
|
||||
# the Health tab builds the row itself, so it never comes back as a notice
|
||||
self.assertEqual(history, [])
|
||||
|
||||
def test_dismissed_checks_require_admin(self):
|
||||
with TestClient(self.app) as client:
|
||||
response = client.get(
|
||||
"/notices/dismissed_checks",
|
||||
headers={"remote-user": "viewer", "remote-role": "viewer"},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 403)
|
||||
|
||||
def test_purge_dismissed_clears_notices_and_checks(self):
|
||||
self.registry.raise_notice(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
)
|
||||
self.registry.dismiss("detector_stuck:ov")
|
||||
self.registry.dismiss(CONFIG_CHECK)
|
||||
|
||||
with self.client() as client:
|
||||
response = client.delete("/notices/dismissed")
|
||||
history = client.get("/notices", params={"include_dismissed": True}).json()
|
||||
checks = client.get("/notices/dismissed_checks").json()
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(history, [])
|
||||
self.assertEqual(checks, [])
|
||||
|
||||
def test_purge_dismissed_requires_admin(self):
|
||||
with TestClient(self.app) as client:
|
||||
response = client.delete(
|
||||
"/notices/dismissed",
|
||||
headers={"remote-user": "viewer", "remote-role": "viewer"},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 403)
|
||||
|
||||
@@ -6,11 +6,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.const import STREAM_TYPE_MAIN, STREAM_TYPE_SUB
|
||||
from frigate.video.ffmpeg import (
|
||||
CRASH_LOOP_RECOVERY_S,
|
||||
CRASH_LOOP_RESTARTS,
|
||||
CameraWatchdog,
|
||||
)
|
||||
from frigate.video.ffmpeg import CameraWatchdog
|
||||
|
||||
|
||||
def build_watchdog(
|
||||
@@ -228,75 +224,3 @@ class TestCameraWatchdogStreamHealth(unittest.TestCase):
|
||||
|
||||
assert watchdog.record_stale_threshold[STREAM_TYPE_MAIN] == 120
|
||||
assert watchdog.record_stale_threshold[STREAM_TYPE_SUB] == 150
|
||||
|
||||
|
||||
class TestCameraWatchdogCrashLoop(unittest.TestCase):
|
||||
"""The crash loop notice follows restarts in the trailing hour."""
|
||||
|
||||
def _build_watchdog(self) -> CameraWatchdog:
|
||||
return build_watchdog()
|
||||
|
||||
def _notice_payloads(self, watchdog: CameraWatchdog) -> list[dict]:
|
||||
return [
|
||||
call.args[1]
|
||||
for call in watchdog.requestor.send_data.call_args_list
|
||||
if call.args[0] == "update_notice"
|
||||
]
|
||||
|
||||
def test_below_threshold_raises_nothing(self):
|
||||
watchdog = self._build_watchdog()
|
||||
now = datetime.now().timestamp()
|
||||
watchdog.reconnect_timestamps.extend([now] * (CRASH_LOOP_RESTARTS - 1))
|
||||
watchdog.capture_thread = MagicMock()
|
||||
watchdog.capture_thread.is_alive.return_value = False
|
||||
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
|
||||
self.assertEqual(self._notice_payloads(watchdog), [])
|
||||
|
||||
def test_threshold_raises_once_per_restart_count(self):
|
||||
watchdog = self._build_watchdog()
|
||||
now = datetime.now().timestamp()
|
||||
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
|
||||
watchdog.capture_thread = MagicMock()
|
||||
watchdog.capture_thread.is_alive.return_value = False
|
||||
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
|
||||
payloads = self._notice_payloads(watchdog)
|
||||
self.assertEqual(len(payloads), 1)
|
||||
self.assertEqual(payloads[0]["action"], "raise")
|
||||
self.assertEqual(payloads[0]["kind"], "ffmpeg_crash_loop")
|
||||
self.assertEqual(payloads[0]["scope"], "front_door")
|
||||
self.assertEqual(payloads[0]["params"], {"restarts": CRASH_LOOP_RESTARTS})
|
||||
|
||||
def test_recovery_window_resolves(self):
|
||||
watchdog = self._build_watchdog()
|
||||
now = datetime.now().timestamp()
|
||||
# the last restart is at `now`, so recovery is measured from there
|
||||
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
|
||||
watchdog.capture_thread = MagicMock()
|
||||
watchdog.capture_thread.is_alive.return_value = False
|
||||
watchdog._update_crash_loop_notice(now)
|
||||
|
||||
watchdog.capture_thread.is_alive.return_value = True
|
||||
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S - 1)
|
||||
self.assertEqual(len(self._notice_payloads(watchdog)), 1)
|
||||
|
||||
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S + 1)
|
||||
|
||||
payloads = self._notice_payloads(watchdog)
|
||||
self.assertEqual(len(payloads), 2)
|
||||
self.assertEqual(payloads[1]["action"], "resolve")
|
||||
self.assertFalse(watchdog.crash_loop_raised)
|
||||
|
||||
def test_disable_resolves(self):
|
||||
watchdog = self._build_watchdog()
|
||||
watchdog.crash_loop_raised = True
|
||||
|
||||
watchdog._resolve_crash_loop()
|
||||
|
||||
payloads = self._notice_payloads(watchdog)
|
||||
self.assertEqual(payloads[-1]["action"], "resolve")
|
||||
self.assertFalse(watchdog.crash_loop_raised)
|
||||
|
||||
@@ -518,7 +518,7 @@ class TestNoticeWiring(unittest.TestCase):
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.registry = MagicMock()
|
||||
self.registry.active.return_value = [{"id": "retention_unmet"}]
|
||||
self.registry.active.return_value = [{"id": "detector_stuck:ov"}]
|
||||
config = MagicMock()
|
||||
config.cameras = {}
|
||||
|
||||
@@ -540,37 +540,28 @@ class TestNoticeWiring(unittest.TestCase):
|
||||
self.dispatcher._publish_notices
|
||||
)
|
||||
|
||||
def test_raise_request_reaches_registry(self) -> None:
|
||||
self.dispatcher._receive(
|
||||
"update_notice",
|
||||
{
|
||||
"action": "raise",
|
||||
"kind": "ffmpeg_crash_loop",
|
||||
"scope": "front_door",
|
||||
"params": {"restarts": 5},
|
||||
},
|
||||
)
|
||||
def test_update_request_reaches_registry(self) -> None:
|
||||
update = {
|
||||
"action": "raise",
|
||||
"kind": "model_download_failed",
|
||||
"scope": "yolo/model.onnx",
|
||||
"params": {"file": "model.onnx", "error": "timeout"},
|
||||
}
|
||||
|
||||
self.registry.raise_notice.assert_called_once_with(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
)
|
||||
self.dispatcher._receive("update_notice", update)
|
||||
|
||||
def test_resolve_request_reaches_registry(self) -> None:
|
||||
self.dispatcher._receive(
|
||||
"update_notice",
|
||||
{"action": "resolve", "kind": "retention_unmet", "scope": None},
|
||||
)
|
||||
|
||||
self.registry.resolve.assert_called_once_with("retention_unmet", None)
|
||||
self.registry.apply.assert_called_once_with(update)
|
||||
|
||||
def test_malformed_request_does_not_raise(self) -> None:
|
||||
self.registry.raise_notice.side_effect = RuntimeError("boom")
|
||||
self.registry.apply.side_effect = RuntimeError("boom")
|
||||
|
||||
self.dispatcher._receive("update_notice", "not a dict")
|
||||
self.dispatcher._receive(
|
||||
"update_notice", {"action": "raise", "kind": "x", "params": {}}
|
||||
)
|
||||
|
||||
self.registry.apply.assert_called_once()
|
||||
|
||||
def test_publish_local_skips_mqtt(self) -> None:
|
||||
mqtt = MagicMock(spec=MqttClient)
|
||||
other = MagicMock()
|
||||
@@ -587,7 +578,7 @@ class TestNoticeWiring(unittest.TestCase):
|
||||
self.dispatcher._publish_notices()
|
||||
|
||||
self.dispatcher.publish_local.assert_called_once_with(
|
||||
"notices", '[{"id": "retention_unmet"}]'
|
||||
"notices", '[{"id": "detector_stuck:ov"}]'
|
||||
)
|
||||
|
||||
def test_snapshot_includes_notices(self) -> None:
|
||||
@@ -599,4 +590,4 @@ class TestNoticeWiring(unittest.TestCase):
|
||||
|
||||
self.dispatcher.publish_runtime_snapshot(publisher)
|
||||
|
||||
publisher.assert_any_call("notices", '[{"id": "retention_unmet"}]', False)
|
||||
publisher.assert_any_call("notices", '[{"id": "detector_stuck:ov"}]', False)
|
||||
|
||||
@@ -13,13 +13,12 @@ class TestModelDownloadNotice(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.download_path = tempfile.mkdtemp()
|
||||
downloader.last_download_error.clear()
|
||||
|
||||
def _notice_calls(self, requestor: MagicMock) -> list[dict]:
|
||||
return [
|
||||
call.args[1]
|
||||
for call in requestor.send_data.call_args_list
|
||||
if call.args[0] == "update_notice"
|
||||
]
|
||||
raise_patch = patch("frigate.util.downloader.raise_notice")
|
||||
resolve_patch = patch("frigate.util.downloader.resolve_notice")
|
||||
self.raise_notice = raise_patch.start()
|
||||
self.resolve_notice = resolve_patch.start()
|
||||
self.addCleanup(raise_patch.stop)
|
||||
self.addCleanup(resolve_patch.stop)
|
||||
|
||||
def _downloader(self, download_func) -> ModelDownloader:
|
||||
with patch("frigate.util.downloader.InterProcessRequestor"):
|
||||
@@ -36,15 +35,16 @@ class TestModelDownloadNotice(unittest.TestCase):
|
||||
with self.assertRaises(RuntimeError):
|
||||
model_downloader._download_models()
|
||||
|
||||
notices = self._notice_calls(model_downloader.requestor)
|
||||
self.assertEqual(len(notices), 1)
|
||||
self.assertEqual(notices[0]["action"], "raise")
|
||||
self.assertEqual(notices[0]["kind"], "model_download_failed")
|
||||
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
|
||||
self.assertEqual(
|
||||
notices[0]["params"],
|
||||
{"file": "facedet.onnx", "error": "HTTP 503 from upstream"},
|
||||
self.raise_notice.assert_called_once_with(
|
||||
"model_download_failed",
|
||||
scope="facedet/facedet.onnx",
|
||||
params={
|
||||
"file": "facedet.onnx",
|
||||
"model": "facedet",
|
||||
"error": "HTTP 503 from upstream",
|
||||
},
|
||||
)
|
||||
self.resolve_notice.assert_not_called()
|
||||
|
||||
def test_swallowing_download_that_leaves_no_file_reports(self):
|
||||
target = os.path.join(self.download_path, "facedet.onnx")
|
||||
@@ -62,10 +62,11 @@ class TestModelDownloadNotice(unittest.TestCase):
|
||||
model_downloader = self._downloader(swallowing)
|
||||
model_downloader._download_models()
|
||||
|
||||
notices = self._notice_calls(model_downloader.requestor)
|
||||
self.assertEqual(len(notices), 1)
|
||||
self.assertEqual(notices[0]["action"], "raise")
|
||||
self.assertEqual(notices[0]["params"], {"file": "facedet.onnx", "error": "dns"})
|
||||
self.raise_notice.assert_called_once()
|
||||
self.assertEqual(
|
||||
self.raise_notice.call_args.kwargs["params"],
|
||||
{"file": "facedet.onnx", "model": "facedet", "error": "dns"},
|
||||
)
|
||||
self.assertFalse(os.path.exists(target))
|
||||
|
||||
def test_success_resolves(self):
|
||||
@@ -76,10 +77,10 @@ class TestModelDownloadNotice(unittest.TestCase):
|
||||
model_downloader = self._downloader(succeeding)
|
||||
model_downloader._download_models()
|
||||
|
||||
notices = self._notice_calls(model_downloader.requestor)
|
||||
self.assertEqual(len(notices), 1)
|
||||
self.assertEqual(notices[0]["action"], "resolve")
|
||||
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
|
||||
self.raise_notice.assert_not_called()
|
||||
self.resolve_notice.assert_called_once_with(
|
||||
"model_download_failed", "facedet/facedet.onnx"
|
||||
)
|
||||
|
||||
def test_a_sibling_downloader_only_resolves_its_own_files(self):
|
||||
"""PaddleOCR, facedet and jina each spread one model_name over several
|
||||
@@ -96,9 +97,8 @@ class TestModelDownloadNotice(unittest.TestCase):
|
||||
|
||||
sibling._download_models()
|
||||
|
||||
self.assertEqual(
|
||||
[n["scope"] for n in self._notice_calls(sibling.requestor)],
|
||||
["paddleocr-onnx/det.onnx"],
|
||||
self.resolve_notice.assert_called_once_with(
|
||||
"model_download_failed", "paddleocr-onnx/det.onnx"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Tests for raising and resolving notices through frigate.notices."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from frigate import notices
|
||||
|
||||
|
||||
class TestNoticeFacade(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
notices.install_registry(None)
|
||||
|
||||
def test_the_installed_registry_applies_each_update(self):
|
||||
registry = MagicMock()
|
||||
notices.install_registry(registry)
|
||||
|
||||
notices.raise_notice("detector_stuck", scope="ov", params={"detector": "ov"})
|
||||
notices.resolve_notice("detector_stuck", "ov")
|
||||
notices.resolve_kind("update_available")
|
||||
|
||||
self.assertEqual(
|
||||
[call.args[0] for call in registry.apply.call_args_list],
|
||||
[
|
||||
{
|
||||
"action": "raise",
|
||||
"kind": "detector_stuck",
|
||||
"scope": "ov",
|
||||
"params": {"detector": "ov"},
|
||||
},
|
||||
{"action": "resolve", "kind": "detector_stuck", "scope": "ov"},
|
||||
{"action": "resolve_kind", "kind": "update_available"},
|
||||
],
|
||||
)
|
||||
|
||||
def test_a_child_process_sends_over_ipc(self):
|
||||
with (
|
||||
patch.object(notices.mp, "parent_process", return_value=MagicMock()),
|
||||
patch.object(notices, "InterProcessRequestor") as requestor,
|
||||
patch.object(notices, "_requestor", None),
|
||||
):
|
||||
notices.raise_notice("detector_stuck", scope="ov")
|
||||
notices.resolve_notice("detector_stuck", "ov")
|
||||
|
||||
# one requestor serves every call in the process
|
||||
requestor.assert_called_once_with()
|
||||
send_data = requestor.return_value.send_data
|
||||
self.assertEqual(send_data.call_count, 2)
|
||||
self.assertEqual(
|
||||
send_data.call_args_list[0].args,
|
||||
(
|
||||
"update_notice",
|
||||
{
|
||||
"action": "raise",
|
||||
"kind": "detector_stuck",
|
||||
"scope": "ov",
|
||||
"params": {},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
def test_the_main_process_without_a_registry_drops_updates(self):
|
||||
with (
|
||||
patch.object(notices.mp, "parent_process", return_value=None),
|
||||
patch.object(notices, "InterProcessRequestor") as requestor,
|
||||
):
|
||||
notices.raise_notice("detector_stuck", scope="ov")
|
||||
|
||||
requestor.assert_not_called()
|
||||
|
||||
def test_a_failing_registry_is_logged_not_raised(self):
|
||||
registry = MagicMock()
|
||||
registry.apply.side_effect = RuntimeError("db")
|
||||
notices.install_registry(registry)
|
||||
|
||||
with self.assertLogs("frigate.notices", level="ERROR"):
|
||||
notices.raise_notice("detector_stuck", scope="ov")
|
||||
|
||||
def test_flush_reaches_only_an_installed_registry(self):
|
||||
notices.flush_notices()
|
||||
|
||||
registry = MagicMock()
|
||||
notices.install_registry(registry)
|
||||
notices.flush_notices()
|
||||
|
||||
registry.flush.assert_called_once_with()
|
||||
@@ -3,51 +3,53 @@
|
||||
import logging
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from peewee_migrate import Router
|
||||
from playhouse.sqlite_ext import SqliteExtDatabase
|
||||
from playhouse.sqliteq import SqliteQueueDatabase
|
||||
|
||||
from frigate.models import Notice, NoticeStats
|
||||
from frigate.notices.registry import DismissResult, NoticeRegistry
|
||||
from frigate.notices.types import (
|
||||
NOTICE_KINDS,
|
||||
NoticeMode,
|
||||
NoticeSeverity,
|
||||
notice_id,
|
||||
)
|
||||
from frigate.notices.registry import NoticeRegistry
|
||||
from frigate.notices.types import NOTICE_KINDS, NoticeKind, NoticeSeverity, notice_id
|
||||
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
|
||||
|
||||
# kinds that switch on the lifecycle knobs, so the tests do not depend on the
|
||||
# values the real catalog picks
|
||||
BATCHED = NoticeKind(
|
||||
"batched", NoticeSeverity.warning, "system", batch_repeats=True, reopen_at_count=3
|
||||
)
|
||||
PRUNED = NoticeKind("pruned", NoticeSeverity.info, "system", keep_latest=2)
|
||||
TEST_KINDS = {kind.key: kind for kind in (BATCHED, PRUNED)}
|
||||
|
||||
|
||||
class TestNoticeKinds(unittest.TestCase):
|
||||
def test_every_kind_is_keyed_by_its_own_key(self):
|
||||
for key, definition in NOTICE_KINDS.items():
|
||||
self.assertEqual(key, definition.key)
|
||||
|
||||
def test_first_producers_are_defined(self):
|
||||
self.assertEqual(NOTICE_KINDS["ffmpeg_crash_loop"].mode, NoticeMode.state)
|
||||
self.assertEqual(
|
||||
NOTICE_KINDS["ffmpeg_crash_loop"].severity, NoticeSeverity.error
|
||||
)
|
||||
self.assertEqual(NOTICE_KINDS["detector_stuck"].mode, NoticeMode.event)
|
||||
self.assertEqual(
|
||||
NOTICE_KINDS["detector_stuck"].severity, NoticeSeverity.warning
|
||||
)
|
||||
self.assertEqual(NOTICE_KINDS["model_download_failed"].mode, NoticeMode.event)
|
||||
self.assertEqual(NOTICE_KINDS["retention_unmet"].mode, NoticeMode.state)
|
||||
self.assertEqual(NOTICE_KINDS["update_available"].mode, NoticeMode.state)
|
||||
self.assertEqual(NOTICE_KINDS["update_available"].severity, NoticeSeverity.info)
|
||||
|
||||
def test_notice_id_with_and_without_scope(self):
|
||||
self.assertEqual(notice_id("retention_unmet", None), "retention_unmet")
|
||||
self.assertEqual(notice_id("failed_login", None), "failed_login")
|
||||
self.assertEqual(
|
||||
notice_id("ffmpeg_crash_loop", "front_door"),
|
||||
"ffmpeg_crash_loop:front_door",
|
||||
notice_id("skipped_detections", "front_door"),
|
||||
"skipped_detections:front_door",
|
||||
)
|
||||
|
||||
def test_link_is_filled_in_from_params(self):
|
||||
kind = NoticeKind(
|
||||
"k", NoticeSeverity.info, "system", link="https://x/v{version}"
|
||||
)
|
||||
|
||||
class TestNoticeRegistry(unittest.TestCase):
|
||||
self.assertEqual(kind.link_for({"version": "0.19.1"}), "https://x/v0.19.1")
|
||||
self.assertIsNone(kind.link_for({}))
|
||||
|
||||
def test_a_kind_without_a_link(self):
|
||||
kind = NoticeKind("k", NoticeSeverity.info, "system")
|
||||
|
||||
self.assertIsNone(kind.link_for({"version": "0.19.1"}))
|
||||
|
||||
|
||||
class RegistryTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
migrate_db = SqliteExtDatabase("test.db")
|
||||
del logging.getLogger("peewee_migrate").handlers[:]
|
||||
@@ -70,132 +72,128 @@ class TestNoticeRegistry(unittest.TestCase):
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _raise_at(self, timestamp: float, kind: str, **kwargs) -> None:
|
||||
with patch("frigate.notices.registry.datetime") as mock_datetime:
|
||||
mock_datetime.now.return_value.timestamp.return_value = timestamp
|
||||
self.registry.raise_notice(kind, **kwargs)
|
||||
|
||||
|
||||
class TestNoticeRegistry(RegistryTestCase):
|
||||
def test_raise_inserts_and_counts_one_occurrence(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
"skipped_detections", scope="front_door", params={"pct": 12.5}
|
||||
)
|
||||
|
||||
active = self.registry.active()
|
||||
self.assertEqual(len(active), 1)
|
||||
self.assertEqual(active[0]["id"], "ffmpeg_crash_loop:front_door")
|
||||
self.assertEqual(active[0]["severity"], "error")
|
||||
self.assertEqual(active[0]["mode"], "state")
|
||||
self.assertEqual(active[0]["id"], "skipped_detections:front_door")
|
||||
self.assertEqual(active[0]["severity"], "warning")
|
||||
self.assertEqual(active[0]["category"], "camera")
|
||||
self.assertEqual(active[0]["scope"], "front_door")
|
||||
self.assertEqual(active[0]["params"], {"restarts": 5})
|
||||
self.assertEqual(active[0]["params"], {"pct": 12.5})
|
||||
self.assertEqual(active[0]["link"], "/system#cameras")
|
||||
self.assertEqual(active[0]["count"], 1)
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_called_once()
|
||||
|
||||
def test_state_re_raise_with_new_params_updates_without_counting(self):
|
||||
def test_re_raise_counts_and_updates_params(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
"skipped_detections", scope="front_door", params={"pct": 12.5}
|
||||
)
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 6}
|
||||
"skipped_detections", scope="front_door", params={"pct": 8.0}
|
||||
)
|
||||
|
||||
active = self.registry.active()
|
||||
self.assertEqual(active[0]["count"], 1)
|
||||
self.assertEqual(active[0]["params"], {"restarts": 6})
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_called_once()
|
||||
self.assertEqual(active[0]["count"], 2)
|
||||
self.assertEqual(active[0]["params"], {"pct": 8.0})
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 2)
|
||||
|
||||
def test_state_re_raise_with_same_params_is_a_no_op(self):
|
||||
def test_repeat_of_a_non_counting_kind_is_a_no_op(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
"update_available", scope="0.19.1", params={"version": "0.19.1"}
|
||||
)
|
||||
before = self.registry.active()[0]["last_seen"]
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
|
||||
"update_available", scope="0.19.1", params={"version": "0.19.1"}
|
||||
)
|
||||
|
||||
self.assertEqual(self.registry.active()[0]["last_seen"], before)
|
||||
self.assertEqual(self.registry.active()[0]["count"], 1)
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_not_called()
|
||||
|
||||
def test_event_re_raise_counts_and_undismisses(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.assertEqual(self.registry.dismiss("detector_stuck"), DismissResult.ok)
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
def test_dismissal_survives_a_re_raise(self):
|
||||
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
|
||||
self.assertTrue(self.registry.dismiss("skipped_detections:front_door"))
|
||||
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
# no scope here on purpose: a global event id is still valid
|
||||
|
||||
active = self.registry.active()
|
||||
self.assertEqual(len(active), 1)
|
||||
self.assertEqual(active[0]["count"], 2)
|
||||
self.assertIsNone(active[0]["dismissed_at"])
|
||||
stats = self.registry.stats()[0]
|
||||
self.assertEqual(stats["occurrences"], 2)
|
||||
self.assertEqual(stats["dismissals"], 1)
|
||||
|
||||
def test_resolve_deletes_and_keeps_stats(self):
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.resolve("retention_unmet")
|
||||
self.registry.resolve("retention_unmet")
|
||||
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_called_once()
|
||||
history = self.registry.active(include_dismissed=True)
|
||||
self.assertEqual(history[0]["count"], 2)
|
||||
self.assertIsNotNone(history[0]["dismissed_at"])
|
||||
|
||||
def test_state_notice_cannot_be_dismissed(self):
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
def test_dismiss_counts_once(self):
|
||||
self.registry.raise_notice("detector_stuck", scope="ov", params={})
|
||||
|
||||
self.assertEqual(
|
||||
self.registry.dismiss("retention_unmet"), DismissResult.not_dismissable
|
||||
)
|
||||
self.assertEqual(len(self.registry.active()), 1)
|
||||
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
|
||||
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
|
||||
|
||||
self.assertEqual(self.registry.stats()[0]["dismissals"], 1)
|
||||
|
||||
def test_dismiss_unknown_id(self):
|
||||
self.assertEqual(self.registry.dismiss("nope"), DismissResult.not_found)
|
||||
self.assertFalse(self.registry.dismiss("nope"))
|
||||
|
||||
def test_dismissed_hidden_unless_requested(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.registry.dismiss("detector_stuck")
|
||||
self.registry.raise_notice("detector_stuck", scope="ov", params={})
|
||||
self.registry.dismiss("detector_stuck:ov")
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
self.assertEqual(len(self.registry.active(include_dismissed=True)), 1)
|
||||
|
||||
def test_startup_hides_state_notices_until_confirmed(self):
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
def test_resolve_deletes_and_keeps_stats(self):
|
||||
self.registry.raise_notice(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
"model_download_failed", scope="yolo/model.onnx", params={}
|
||||
)
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.mark_state_notices_unconfirmed()
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["detector_stuck:ov"])
|
||||
|
||||
# the producer confirms the same episode: visible again, no new occurrence
|
||||
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["retention_unmet", "detector_stuck:ov"])
|
||||
self.assertEqual(
|
||||
[s for s in self.registry.stats() if s["kind"] == "retention_unmet"][0][
|
||||
"occurrences"
|
||||
],
|
||||
1,
|
||||
)
|
||||
|
||||
def test_unconfirmed_across_two_startups_is_deleted(self):
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
|
||||
self.registry.mark_state_notices_unconfirmed()
|
||||
self.registry.mark_state_notices_unconfirmed()
|
||||
self.registry.resolve("model_download_failed", "yolo/model.onnx")
|
||||
self.registry.resolve("model_download_failed", "yolo/model.onnx")
|
||||
|
||||
self.assertEqual(self.registry.active(include_dismissed=True), [])
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
|
||||
self.listener.assert_called_once()
|
||||
|
||||
def test_resolve_kind_drops_every_scope_of_that_kind(self):
|
||||
self.registry.raise_notice("detector_stuck", params={})
|
||||
self.registry.raise_notice("detector_stuck", scope="ov", params={})
|
||||
self.registry.raise_notice("skipped_detections", scope="garage", params={})
|
||||
|
||||
self.registry.resolve_kind("detector_stuck")
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["skipped_detections:garage"])
|
||||
|
||||
def test_active_fills_in_each_link(self):
|
||||
self.registry.raise_notice(
|
||||
"update_available", scope="0.19.1", params={"version": "0.19.1"}
|
||||
)
|
||||
self.registry.raise_notice(
|
||||
"model_download_failed", scope="yolo/model.onnx", params={}
|
||||
)
|
||||
|
||||
links = {n["kind"]: n["link"] for n in self.registry.active()}
|
||||
self.assertEqual(
|
||||
links,
|
||||
{
|
||||
"update_available": "https://github.com/blakeblackshear/frigate/releases/tag/v0.19.1",
|
||||
"model_download_failed": None,
|
||||
},
|
||||
)
|
||||
|
||||
def test_replay_camera_is_ignored(self):
|
||||
self.registry.raise_notice(
|
||||
"ffmpeg_crash_loop", scope="_replay_front_door", params={}
|
||||
"skipped_detections", scope="_replay_front_door", params={}
|
||||
)
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
@@ -208,22 +206,26 @@ class TestNoticeRegistry(unittest.TestCase):
|
||||
self.listener.assert_not_called()
|
||||
|
||||
def test_active_sorts_by_severity_then_recency(self):
|
||||
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
self.registry.raise_notice("detector_stuck", scope="ov", params={})
|
||||
self.registry.raise_notice(
|
||||
"model_download_failed", scope="yolo/model.onnx", params={}
|
||||
)
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["retention_unmet", "detector_stuck"])
|
||||
self.assertEqual(
|
||||
ids, ["model_download_failed:yolo/model.onnx", "detector_stuck:ov"]
|
||||
)
|
||||
|
||||
def test_listener_exception_does_not_propagate(self):
|
||||
self.registry.subscribe(MagicMock(side_effect=RuntimeError("boom")))
|
||||
|
||||
self.registry.raise_notice("retention_unmet", params={})
|
||||
self.registry.raise_notice("detector_stuck", scope="ov", params={})
|
||||
|
||||
self.assertEqual(len(self.registry.active()), 1)
|
||||
|
||||
def test_resolve_camera_drops_only_that_camera(self):
|
||||
self.registry.raise_notice("ffmpeg_crash_loop", scope="front_door", params={})
|
||||
self.registry.raise_notice("ffmpeg_crash_loop", scope="garage", params={})
|
||||
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
|
||||
self.registry.raise_notice("skipped_detections", scope="garage", params={})
|
||||
self.registry.raise_notice(
|
||||
"model_download_failed", scope="front_door", params={}
|
||||
)
|
||||
@@ -232,5 +234,190 @@ class TestNoticeRegistry(unittest.TestCase):
|
||||
|
||||
ids = sorted(n["id"] for n in self.registry.active())
|
||||
self.assertEqual(
|
||||
ids, ["ffmpeg_crash_loop:garage", "model_download_failed:front_door"]
|
||||
ids, ["model_download_failed:front_door", "skipped_detections:garage"]
|
||||
)
|
||||
|
||||
def test_resolve_camera_drops_that_cameras_check_dismissals(self):
|
||||
for check_id in (
|
||||
"stream:front_door:0:probe",
|
||||
"config:detect:fps-greater-than-five:camera.front_door",
|
||||
"config:detect:fps-greater-than-five:global",
|
||||
"stream:garage:0:probe",
|
||||
):
|
||||
self.assertTrue(self.registry.dismiss(check_id))
|
||||
|
||||
self.registry.resolve_camera("front_door")
|
||||
|
||||
ids = sorted(check["id"] for check in self.registry.dismissed_checks())
|
||||
self.assertEqual(
|
||||
ids,
|
||||
["config:detect:fps-greater-than-five:global", "stream:garage:0:probe"],
|
||||
)
|
||||
|
||||
def test_camera_named_global_keeps_global_check_dismissals(self):
|
||||
self.registry.dismiss("config:detect:fps-greater-than-five:global")
|
||||
self.registry.dismiss("config:detect:fps-greater-than-five:camera.global")
|
||||
|
||||
self.registry.resolve_camera("global")
|
||||
|
||||
ids = [check["id"] for check in self.registry.dismissed_checks()]
|
||||
self.assertEqual(ids, ["config:detect:fps-greater-than-five:global"])
|
||||
|
||||
def test_purge_dismissed_keeps_active_notices_and_counts(self):
|
||||
self.registry.raise_notice("detector_stuck", scope="ov", params={})
|
||||
self.registry.raise_notice("skipped_detections", scope="garage", params={})
|
||||
self.registry.dismiss("detector_stuck:ov")
|
||||
self.registry.dismiss("config:detect:fps-greater-than-five:camera.garage")
|
||||
|
||||
self.assertEqual(self.registry.purge_dismissed(), 2)
|
||||
|
||||
ids = [n["id"] for n in self.registry.active(include_dismissed=True)]
|
||||
self.assertEqual(ids, ["skipped_detections:garage"])
|
||||
self.assertEqual(self.registry.dismissed_checks(), [])
|
||||
dismissals = {s["kind"]: s["dismissals"] for s in self.registry.stats()}
|
||||
self.assertEqual(dismissals["detector_stuck"], 1)
|
||||
|
||||
|
||||
class TestApply(RegistryTestCase):
|
||||
def test_each_action_reaches_its_method(self):
|
||||
self.registry.apply(
|
||||
{
|
||||
"action": "raise",
|
||||
"kind": "detector_stuck",
|
||||
"scope": "ov",
|
||||
"params": {"detector": "ov"},
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
[n["id"] for n in self.registry.active()], ["detector_stuck:ov"]
|
||||
)
|
||||
|
||||
self.registry.apply(
|
||||
{"action": "resolve", "kind": "detector_stuck", "scope": "ov"}
|
||||
)
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
|
||||
for scope in ("a", "b"):
|
||||
self.registry.apply(
|
||||
{"action": "raise", "kind": "detector_stuck", "scope": scope}
|
||||
)
|
||||
self.registry.apply({"action": "resolve_kind", "kind": "detector_stuck"})
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
|
||||
def test_malformed_updates_are_logged_and_ignored(self):
|
||||
with self.assertLogs("frigate.notices.registry", level="WARNING") as logs:
|
||||
self.registry.apply({"action": "raise"})
|
||||
self.registry.apply({"action": "explode", "kind": "detector_stuck"})
|
||||
|
||||
self.assertEqual(len(logs.output), 2)
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
|
||||
|
||||
@patch.dict(NOTICE_KINDS, TEST_KINDS)
|
||||
class TestLifecycleKnobs(RegistryTestCase):
|
||||
def test_batched_repeats_wait_for_the_flush(self):
|
||||
self.registry.raise_notice("batched", params={"n": 1})
|
||||
self.listener.reset_mock()
|
||||
|
||||
for n in (2, 3, 4):
|
||||
self.registry.raise_notice("batched", params={"n": n})
|
||||
|
||||
self.assertEqual(self.registry.active()[0]["count"], 1)
|
||||
self.listener.assert_not_called()
|
||||
|
||||
self.registry.flush()
|
||||
|
||||
notice = self.registry.active()[0]
|
||||
self.assertEqual(notice["count"], 4)
|
||||
self.assertEqual(notice["params"], {"n": 4})
|
||||
self.assertEqual(self.registry.stats()[0]["occurrences"], 4)
|
||||
self.listener.assert_called_once()
|
||||
|
||||
def test_flush_with_nothing_held_is_a_no_op(self):
|
||||
self.registry.raise_notice("batched")
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.flush()
|
||||
|
||||
self.listener.assert_not_called()
|
||||
|
||||
def test_repeats_of_a_resolved_notice_are_dropped(self):
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.resolve("batched")
|
||||
self.listener.reset_mock()
|
||||
|
||||
self.registry.flush()
|
||||
|
||||
self.assertEqual(self.registry.active(include_dismissed=True), [])
|
||||
self.listener.assert_not_called()
|
||||
|
||||
def test_a_new_row_starts_without_stale_repeats(self):
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.resolve("batched")
|
||||
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.flush()
|
||||
|
||||
self.assertEqual(self.registry.active()[0]["count"], 1)
|
||||
|
||||
def test_a_dismissed_notice_reopens_at_the_count(self):
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.dismiss("batched")
|
||||
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.flush()
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.flush()
|
||||
|
||||
notice = self.registry.active()[0]
|
||||
self.assertEqual(notice["count"], BATCHED.reopen_at_count)
|
||||
self.assertIsNone(notice["dismissed_at"])
|
||||
|
||||
def test_a_notice_dismissed_past_the_count_stays_dismissed(self):
|
||||
for _ in range(3):
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.flush()
|
||||
self.registry.dismiss("batched")
|
||||
|
||||
self.registry.raise_notice("batched")
|
||||
self.registry.flush()
|
||||
|
||||
self.assertEqual(self.registry.active(), [])
|
||||
self.assertEqual(self.registry.active(include_dismissed=True)[0]["count"], 4)
|
||||
|
||||
def test_keep_latest_drops_the_oldest_rows(self):
|
||||
for index, scope in enumerate(("a", "b", "c")):
|
||||
self._raise_at(1000.0 + index, "pruned", scope=scope)
|
||||
|
||||
ids = sorted(n["id"] for n in self.registry.active())
|
||||
self.assertEqual(ids, ["pruned:b", "pruned:c"])
|
||||
|
||||
|
||||
class TestCatalogLifecycles(RegistryTestCase):
|
||||
def test_a_new_release_replaces_the_old_update_notice(self):
|
||||
for index, version in enumerate(("0.19.0", "0.19.1")):
|
||||
self._raise_at(
|
||||
1000.0 + index,
|
||||
"update_available",
|
||||
scope=version,
|
||||
params={"version": version},
|
||||
)
|
||||
|
||||
ids = [n["id"] for n in self.registry.active()]
|
||||
self.assertEqual(ids, ["update_available:0.19.1"])
|
||||
|
||||
def test_a_dismissed_failed_login_burst_reopens_at_five_attempts(self):
|
||||
self.registry.raise_notice("failed_login", scope="1000")
|
||||
self.registry.dismiss("failed_login:1000")
|
||||
|
||||
for _ in range(4):
|
||||
self.registry.raise_notice("failed_login", scope="1000")
|
||||
self.registry.flush()
|
||||
|
||||
burst = self.registry.active()[0]
|
||||
self.assertEqual(burst["count"], 5)
|
||||
self.assertIsNone(burst["dismissed_at"])
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for the notice raised while /dev/shm is too small."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from frigate.stats import emitter
|
||||
|
||||
TOO_SMALL = {"total": 64.0, "min_shm": 180}
|
||||
BIG_ENOUGH = {"total": 512.0, "min_shm": 180}
|
||||
|
||||
|
||||
class TestShmNotice(unittest.TestCase):
|
||||
def setUp(self):
|
||||
raise_patch = patch.object(emitter, "raise_notice")
|
||||
resolve_patch = patch.object(emitter, "resolve_notice")
|
||||
self.raise_notice = raise_patch.start()
|
||||
self.resolve_notice = resolve_patch.start()
|
||||
self.addCleanup(raise_patch.stop)
|
||||
self.addCleanup(resolve_patch.stop)
|
||||
|
||||
self.emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
|
||||
self.emitter._shm_checked = False
|
||||
self.emitter._shm_params = None
|
||||
|
||||
def test_too_small_raises_once(self):
|
||||
for _ in range(3):
|
||||
self.emitter._update_shm_notice(TOO_SMALL)
|
||||
|
||||
self.raise_notice.assert_called_once_with(
|
||||
"shm_too_low", params={"total": 64.0, "min": 180}
|
||||
)
|
||||
self.resolve_notice.assert_not_called()
|
||||
|
||||
def test_first_tick_clears_a_row_the_last_run_left(self):
|
||||
self.emitter._update_shm_notice(BIG_ENOUGH)
|
||||
self.emitter._update_shm_notice(BIG_ENOUGH)
|
||||
|
||||
self.resolve_notice.assert_called_once_with("shm_too_low")
|
||||
self.raise_notice.assert_not_called()
|
||||
|
||||
def test_a_new_requirement_updates_the_notice(self):
|
||||
self.emitter._update_shm_notice(TOO_SMALL)
|
||||
self.emitter._update_shm_notice({"total": 64.0, "min_shm": 200})
|
||||
|
||||
self.assertEqual(self.raise_notice.call_count, 2)
|
||||
self.raise_notice.assert_called_with(
|
||||
"shm_too_low", params={"total": 64.0, "min": 200}
|
||||
)
|
||||
|
||||
def test_growing_shm_resolves_the_notice(self):
|
||||
self.emitter._update_shm_notice(TOO_SMALL)
|
||||
self.emitter._update_shm_notice(BIG_ENOUGH)
|
||||
|
||||
self.resolve_notice.assert_called_once_with("shm_too_low")
|
||||
|
||||
def test_missing_shm_is_not_a_problem(self):
|
||||
self.emitter._update_shm_notice({})
|
||||
|
||||
self.raise_notice.assert_not_called()
|
||||
self.resolve_notice.assert_called_once_with("shm_too_low")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Tests for the skipped detection rate and the notice it can raise."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from frigate.stats import emitter
|
||||
from frigate.stats.emitter import SkippedDetectionsTracker
|
||||
from frigate.stats.util import skipped_percent
|
||||
|
||||
|
||||
class TestSkippedPercent(unittest.TestCase):
|
||||
def test_share_of_all_frames(self):
|
||||
self.assertEqual(skipped_percent(0.5, 5.0, True), 10.0)
|
||||
|
||||
def test_offline_camera_reports_zero(self):
|
||||
self.assertEqual(skipped_percent(2.0, 0.0, True), 0.0)
|
||||
|
||||
def test_disabled_camera_reports_zero(self):
|
||||
self.assertEqual(skipped_percent(2.0, 5.0, False), 0.0)
|
||||
|
||||
|
||||
class TestSkippedDetectionsTracker(unittest.TestCase):
|
||||
def _cameras(self, pct: float) -> dict:
|
||||
return {"front_door": {"skipped_pct": pct}}
|
||||
|
||||
def test_below_threshold_never_qualifies(self):
|
||||
tracker = SkippedDetectionsTracker()
|
||||
|
||||
for tick in range(10):
|
||||
self.assertEqual(tracker.update(self._cameras(4.9), tick * 15.0), [])
|
||||
|
||||
def test_qualifies_once_after_the_hold(self):
|
||||
tracker = SkippedDetectionsTracker()
|
||||
|
||||
results = [
|
||||
tracker.update(self._cameras(10.0), tick * 15.0) for tick in range(8)
|
||||
]
|
||||
|
||||
# ticks at 0, 15, 30, and 45 s are inside the hold; 60 s qualifies
|
||||
self.assertEqual(results[:4], [[], [], [], []])
|
||||
self.assertEqual(results[4], ["front_door"])
|
||||
self.assertEqual(results[5:], [[], [], []])
|
||||
|
||||
def test_a_dip_restarts_the_hold(self):
|
||||
tracker = SkippedDetectionsTracker()
|
||||
|
||||
for now, pct in ((0.0, 10.0), (15.0, 10.0), (30.0, 10.0), (45.0, 2.0)):
|
||||
self.assertEqual(tracker.update(self._cameras(pct), now), [])
|
||||
|
||||
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
|
||||
self.assertEqual(tracker.update(self._cameras(10.0), 120.0), ["front_door"])
|
||||
|
||||
def test_recovery_then_relapse_is_a_new_episode(self):
|
||||
tracker = SkippedDetectionsTracker()
|
||||
tracker.update(self._cameras(10.0), 0.0)
|
||||
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), ["front_door"])
|
||||
|
||||
tracker.update(self._cameras(0.0), 75.0)
|
||||
tracker.update(self._cameras(10.0), 90.0)
|
||||
|
||||
self.assertEqual(tracker.update(self._cameras(10.0), 150.0), ["front_door"])
|
||||
|
||||
def test_cameras_are_tracked_separately(self):
|
||||
tracker = SkippedDetectionsTracker()
|
||||
tracker.update({"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 0.0}}, 0.0)
|
||||
|
||||
qualified = tracker.update(
|
||||
{"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 10.0}}, 60.0
|
||||
)
|
||||
|
||||
self.assertEqual(qualified, ["a"])
|
||||
|
||||
def test_a_removed_camera_starts_a_new_episode(self):
|
||||
tracker = SkippedDetectionsTracker()
|
||||
tracker.update(self._cameras(10.0), 0.0)
|
||||
tracker.update({}, 15.0)
|
||||
tracker.update(self._cameras(10.0), 30.0)
|
||||
|
||||
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
|
||||
self.assertEqual(tracker.update(self._cameras(10.0), 90.0), ["front_door"])
|
||||
|
||||
|
||||
class TestEmitterNotices(unittest.TestCase):
|
||||
def setUp(self):
|
||||
raise_patch = patch.object(emitter, "raise_notice")
|
||||
resolve_patch = patch.object(emitter, "resolve_notice")
|
||||
flush_patch = patch.object(emitter, "flush_notices")
|
||||
self.raise_notice = raise_patch.start()
|
||||
resolve_patch.start()
|
||||
self.flush_notices = flush_patch.start()
|
||||
self.addCleanup(raise_patch.stop)
|
||||
self.addCleanup(resolve_patch.stop)
|
||||
self.addCleanup(flush_patch.stop)
|
||||
|
||||
def _emitter(self) -> emitter.StatsEmitter:
|
||||
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
|
||||
stats_emitter.skipped_detections = SkippedDetectionsTracker()
|
||||
stats_emitter._shm_checked = False
|
||||
stats_emitter._shm_params = None
|
||||
return stats_emitter
|
||||
|
||||
def _stats(self, uptime: int, pct: float) -> dict:
|
||||
return {
|
||||
"service": {"uptime": uptime, "storage": {"/dev/shm": {}}},
|
||||
"cameras": {"front_door": {"skipped_pct": pct}},
|
||||
}
|
||||
|
||||
def test_qualified_camera_raises_a_notice(self):
|
||||
stats_emitter = self._emitter()
|
||||
|
||||
stats_emitter._update_notices(self._stats(300, 12.5), 0.0)
|
||||
stats_emitter._update_notices(self._stats(360, 12.5), 60.0)
|
||||
|
||||
self.raise_notice.assert_called_once_with(
|
||||
"skipped_detections", scope="front_door", params={"pct": 12.5}
|
||||
)
|
||||
|
||||
def test_startup_window_is_ignored(self):
|
||||
stats_emitter = self._emitter()
|
||||
|
||||
stats_emitter._update_notices(self._stats(30, 50.0), 0.0)
|
||||
stats_emitter._update_notices(self._stats(90, 50.0), 60.0)
|
||||
|
||||
self.raise_notice.assert_not_called()
|
||||
|
||||
def test_every_tick_flushes_held_repeats(self):
|
||||
self._emitter()._update_notices(self._stats(30, 0.0), 0.0)
|
||||
|
||||
self.flush_notices.assert_called_once_with()
|
||||
@@ -34,44 +34,46 @@ class TestIsNewerVersion(unittest.TestCase):
|
||||
|
||||
|
||||
class TestUpdateNotice(unittest.TestCase):
|
||||
def setUp(self):
|
||||
raise_patch = patch.object(emitter, "raise_notice")
|
||||
resolve_patch = patch.object(emitter, "resolve_kind")
|
||||
self.raise_notice = raise_patch.start()
|
||||
self.resolve_kind = resolve_patch.start()
|
||||
self.addCleanup(raise_patch.stop)
|
||||
self.addCleanup(resolve_patch.stop)
|
||||
|
||||
def _emitter(self, latest: str) -> emitter.StatsEmitter:
|
||||
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
|
||||
stats_emitter.config = MagicMock()
|
||||
stats_emitter.stats_tracking = {"latest_frigate_version": latest}
|
||||
stats_emitter.notice_registry = MagicMock()
|
||||
return stats_emitter
|
||||
|
||||
def test_newer_release_raises(self):
|
||||
stats_emitter = self._emitter("0.19.0")
|
||||
|
||||
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
|
||||
stats_emitter._check_update_notice()
|
||||
self._emitter("0.19.0")._check_update_notice()
|
||||
|
||||
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
|
||||
"update_available", params={"version": "0.19.0"}
|
||||
self.raise_notice.assert_called_once_with(
|
||||
"update_available", scope="0.19.0", params={"version": "0.19.0"}
|
||||
)
|
||||
stats_emitter.notice_registry.resolve.assert_not_called()
|
||||
self.resolve_kind.assert_not_called()
|
||||
|
||||
def test_current_or_disabled_resolves(self):
|
||||
for latest in ("0.18.1", "disabled"):
|
||||
stats_emitter = self._emitter(latest)
|
||||
with self.subTest(latest=latest):
|
||||
self.resolve_kind.reset_mock()
|
||||
|
||||
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
|
||||
stats_emitter._check_update_notice()
|
||||
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
|
||||
self._emitter(latest)._check_update_notice()
|
||||
|
||||
stats_emitter.notice_registry.raise_notice.assert_not_called()
|
||||
stats_emitter.notice_registry.resolve.assert_called_once_with(
|
||||
"update_available"
|
||||
)
|
||||
self.raise_notice.assert_not_called()
|
||||
self.resolve_kind.assert_called_once_with("update_available")
|
||||
|
||||
def test_a_failed_lookup_leaves_the_notice_alone(self):
|
||||
stats_emitter = self._emitter("unknown")
|
||||
|
||||
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
|
||||
stats_emitter._check_update_notice()
|
||||
self._emitter("unknown")._check_update_notice()
|
||||
|
||||
stats_emitter.notice_registry.raise_notice.assert_not_called()
|
||||
stats_emitter.notice_registry.resolve.assert_not_called()
|
||||
self.raise_notice.assert_not_called()
|
||||
self.resolve_kind.assert_not_called()
|
||||
|
||||
def test_refresh_updates_tracking_on_a_thread_then_checks(self):
|
||||
stats_emitter = self._emitter("0.18.0")
|
||||
@@ -89,7 +91,7 @@ class TestUpdateNotice(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
stats_emitter.stats_tracking["latest_frigate_version"], "0.19.0"
|
||||
)
|
||||
stats_emitter.notice_registry.raise_notice.assert_called_once()
|
||||
self.raise_notice.assert_called_once()
|
||||
|
||||
def test_failed_refresh_keeps_the_last_known_version(self):
|
||||
stats_emitter = self._emitter("0.20.0")
|
||||
@@ -105,10 +107,10 @@ class TestUpdateNotice(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
stats_emitter.stats_tracking["latest_frigate_version"], "0.20.0"
|
||||
)
|
||||
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
|
||||
"update_available", params={"version": "0.20.0"}
|
||||
self.raise_notice.assert_called_once_with(
|
||||
"update_available", scope="0.20.0", params={"version": "0.20.0"}
|
||||
)
|
||||
stats_emitter.notice_registry.resolve.assert_not_called()
|
||||
self.resolve_kind.assert_not_called()
|
||||
|
||||
def test_disabled_version_check_still_clears_the_notice(self):
|
||||
stats_emitter = self._emitter("0.20.0")
|
||||
@@ -124,6 +126,4 @@ class TestUpdateNotice(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
stats_emitter.stats_tracking["latest_frigate_version"], "disabled"
|
||||
)
|
||||
stats_emitter.notice_registry.resolve.assert_called_once_with(
|
||||
"update_available"
|
||||
)
|
||||
self.resolve_kind.assert_called_once_with("update_available")
|
||||
|
||||
@@ -556,10 +556,9 @@ class TestHttp(unittest.TestCase):
|
||||
assert Recordings.get(Recordings.id == rec_k2_id)
|
||||
assert Recordings.get(Recordings.id == rec_k3_id)
|
||||
|
||||
def test_unmet_retention_raises_notice(self):
|
||||
registry = MagicMock()
|
||||
def test_unmet_retention_sets_the_live_flag(self):
|
||||
config = FrigateConfig(**self.minimal_config)
|
||||
storage = StorageMaintainer(config, MagicMock(), registry)
|
||||
storage = StorageMaintainer(config, MagicMock())
|
||||
storage.camera_storage_stats = {
|
||||
"front_door": {"needs_refresh": False, "usage": 10, "bandwidth": 10}
|
||||
}
|
||||
@@ -567,22 +566,17 @@ class TestHttp(unittest.TestCase):
|
||||
with patch.object(storage, "expected_hourly_bandwidth", return_value=100.0):
|
||||
storage.reduce_storage_consumption()
|
||||
|
||||
registry.raise_notice.assert_called_once()
|
||||
kind = registry.raise_notice.call_args.args[0]
|
||||
params = registry.raise_notice.call_args.kwargs["params"]
|
||||
self.assertEqual(kind, "retention_unmet")
|
||||
self.assertEqual(params["needed_mb"], 100.0)
|
||||
self.assertEqual(params["cleared_mb"], 0.0)
|
||||
self.assertTrue(storage.retention_unmet)
|
||||
|
||||
def test_clean_run_resolves_notice(self):
|
||||
registry = MagicMock()
|
||||
def test_clean_run_clears_the_live_flag(self):
|
||||
config = FrigateConfig(**self.minimal_config)
|
||||
storage = StorageMaintainer(config, MagicMock(), registry)
|
||||
storage = StorageMaintainer(config, MagicMock())
|
||||
storage.retention_unmet = True
|
||||
|
||||
with patch.object(storage, "check_storage_needs_cleanup", return_value=False):
|
||||
storage._maintain_once()
|
||||
|
||||
registry.resolve.assert_called_once_with("retention_unmet")
|
||||
self.assertFalse(storage.retention_unmet)
|
||||
|
||||
|
||||
def _insert_mock_event(
|
||||
|
||||
@@ -14,25 +14,27 @@ class TestDetectorStuckNotice(unittest.TestCase):
|
||||
return detector
|
||||
|
||||
def test_stuck_restart_raises_notice(self):
|
||||
registry = MagicMock()
|
||||
detector = self._stuck_detector()
|
||||
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
|
||||
watchdog = FrigateWatchdog({"ov": detector}, MagicMock())
|
||||
|
||||
with patch("frigate.watchdog.datetime") as mock_datetime:
|
||||
with (
|
||||
patch("frigate.watchdog.datetime") as mock_datetime,
|
||||
patch("frigate.watchdog.raise_notice") as raise_notice,
|
||||
):
|
||||
mock_datetime.datetime.now.return_value.timestamp.return_value = 100.0
|
||||
watchdog._check_detectors()
|
||||
|
||||
detector.start_or_restart.assert_called_once()
|
||||
registry.raise_notice.assert_called_once_with(
|
||||
raise_notice.assert_called_once_with(
|
||||
"detector_stuck", scope="ov", params={"detector": "ov"}
|
||||
)
|
||||
|
||||
def test_healthy_detector_raises_nothing(self):
|
||||
registry = MagicMock()
|
||||
detector = self._stuck_detector()
|
||||
detector.detection_start.value = 0.0
|
||||
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
|
||||
watchdog = FrigateWatchdog({"ov": detector}, MagicMock())
|
||||
|
||||
watchdog._check_detectors()
|
||||
with patch("frigate.watchdog.raise_notice") as raise_notice:
|
||||
watchdog._check_detectors()
|
||||
|
||||
registry.raise_notice.assert_not_called()
|
||||
raise_notice.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user