administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-10-16 11:42:47 +03:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Notices and status bar improvements (#24459)

Browse Source 
* Notice and status bar improvements

Status bar problems added in the same pass got the same `Date.now()` id and overwrote each other, so usually only one showed. Messages now fall back to their text as the id. The desktop status bar shows the most severe message with a count of the rest that opens a popover listing all of them, and the mobile drawer stacks them vertically instead of placing them side by side.

Dismissing a notice hid it for good, so a detector that restarted again after a dismissal was never shown. Dismiss is replaced by acknowledge, which hides a notice until it happens again, and mute, which hides it permanently. Kinds that never repeat (config and stream checks, the update notice) can only be muted. `reopen_at_count` is removed since acknowledge covers the failed login case.

* move camera CPU warnings to notices

High ffmpeg and detect CPU warnings sat in the status bar with no way to dismiss them. They're now `ffmpeg_high_cpu` and `detect_high_cpu` notices, raised per episode by the same tracker as skipped detections. Also stop failed login attempts held from before an acknowledgement from reopening the notice.

* fix mypy and handle missing cpu stats in notices
This commit is contained in:
Josh Hawkins
2026-09-24 15:39:18 -06:00
committed by GitHub
parent c959df32c9
commit 9664d9ceae
30 changed files with 1307 additions and 707 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frigate/api/notices.py
+50 -22
View File
@@ -14,17 +14,18 @@ router = APIRouter(tags=[Tags.notices])
@router.get("/notices", dependencies=[Depends(require_role(["admin"]))])
def get_notices(request: Request, include_dismissed: bool = False) -> JSONResponse:
def get_notices(request: Request, include_hidden: bool = False) -> JSONResponse:
"""Get notices, most severe first.
Args:
include_dismissed: Also return dismissed notices, for the history view
include_hidden: Also return acknowledged and muted notices, for the
hidden list
Returns:
The notices
"""
return JSONResponse(
content=request.app.notice_registry.active(include_dismissed=include_dismissed)
content=request.app.notice_registry.active(include_hidden=include_hidden)
)
@@ -34,37 +35,64 @@ def get_notice_stats(request: Request) -> JSONResponse:
return JSONResponse(content=request.app.notice_registry.stats())
@router.get(
"/notices/dismissed_checks", dependencies=[Depends(require_role(["admin"]))]
)
def get_dismissed_checks(request: Request) -> JSONResponse:
"""Get the dismissed config and stream check rows, newest first."""
return JSONResponse(content=request.app.notice_registry.dismissed_checks())
@router.get("/notices/muted_checks", dependencies=[Depends(require_role(["admin"]))])
def get_muted_checks(request: Request) -> JSONResponse:
"""Get the muted config and stream check rows, newest first."""
return JSONResponse(content=request.app.notice_registry.muted_checks())
@router.delete("/notices/dismissed", dependencies=[Depends(require_role(["admin"]))])
def purge_dismissed(request: Request) -> JSONResponse:
"""Delete every dismissed notice and check row so each can show again."""
request.app.notice_registry.purge_dismissed()
return JSONResponse(
content={"success": True, "message": "Dismissed notices cleared"}
)
@router.delete("/notices/hidden", dependencies=[Depends(require_role(["admin"]))])
def unhide_all_notices(request: Request) -> JSONResponse:
"""Show every acknowledged and muted notice and check row again."""
request.app.notice_registry.unhide_all()
return JSONResponse(content={"success": True, "message": "Notices shown again"})
# model notice ids contain a slash, so the id is a path parameter
@router.post(
"/notices/{notice_id:path}/dismiss",
"/notices/{notice_id:path}/acknowledge",
dependencies=[Depends(require_role(["admin"]))],
)
def dismiss_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide a notice or a config or stream check row.
def acknowledge_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide a notice until it happens again.
It stays hidden if the same problem happens again.
Config and stream check rows and the update notice never repeat, so they
can only be muted.
"""
if not request.app.notice_registry.dismiss(notice_id):
if not request.app.notice_registry.acknowledge(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
return JSONResponse(content={"success": True, "message": "Notice dismissed"})
return JSONResponse(content={"success": True, "message": "Notice acknowledged"})
@router.post(
"/notices/{notice_id:path}/mute",
dependencies=[Depends(require_role(["admin"]))],
)
def mute_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide a notice or a config or stream check row for good."""
if not request.app.notice_registry.mute(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
return JSONResponse(content={"success": True, "message": "Notice muted"})
@router.delete(
"/notices/{notice_id:path}/hidden",
dependencies=[Depends(require_role(["admin"]))],
)
def unhide_notice(request: Request, notice_id: str) -> JSONResponse:
"""Show an acknowledged or muted notice or check row again."""
if not request.app.notice_registry.unhide(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
return JSONResponse(content={"success": True, "message": "Notice shown again"})
frigate/models.py
+8 -3
View File
@@ -195,15 +195,20 @@ class Notice(Model):
first_seen = DateTimeField()
last_seen = DateTimeField()
count = IntegerField(default=1)
dismissed_at = DateTimeField(null=True)
# hidden until the next occurrence
acknowledged_at = DateTimeField(null=True)
# hidden for good
muted_at = DateTimeField(null=True)
class NoticeStats(Model):
kind = CharField(null=False, primary_key=True, max_length=50)
occurrences = IntegerField(default=0)
dismissals = IntegerField(default=0)
acknowledgements = IntegerField(default=0)
mutes = IntegerField(default=0)
first_seen = DateTimeField()
last_seen = DateTimeField()
# watermarks for a future analytics reporter; unused until then
reported_occurrences = IntegerField(default=0)
reported_dismissals = IntegerField(default=0)
reported_acknowledgements = IntegerField(default=0)
reported_mutes = IntegerField(default=0)
frigate/notices/registry.py
+110 -43
View File
@@ -5,7 +5,7 @@ import threading
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from typing import Any, cast
from frigate.const import REPLAY_CAMERA_PREFIX
from frigate.models import Notice, NoticeStats
@@ -77,9 +77,8 @@ class NoticeRegistry:
) -> None:
"""Insert a notice or count another occurrence of it.
A dismissed notice stays dismissed when it is raised again, unless its
kind sets reopen_at_count. A kind that should come back after a
dismissal gives each episode its own scope.
Another occurrence shows an acknowledged notice again. A muted notice
stays hidden.
"""
definition = NOTICE_KINDS.get(kind)
@@ -112,7 +111,6 @@ class NoticeRegistry:
first_seen=now,
last_seen=now,
count=1,
dismissed_at=None,
)
self._bump_occurrences(kind, 1, now)
@@ -175,7 +173,7 @@ class NoticeRegistry:
self._notify()
def resolve_camera(self, camera: str) -> None:
"""Drop the notices and check dismissals of a camera being deleted."""
"""Drop the notices and check mutes of a camera being deleted."""
camera_kinds = [
key
for key, definition in NOTICE_KINDS.items()
@@ -193,7 +191,7 @@ class NoticeRegistry:
)
# a stream id names its camera first; a config id ends with camera.<name>
for check in self.dismissed_checks():
for check in self.muted_checks():
check_id = check["id"]
if check_id.startswith(f"stream:{camera}:") or (
@@ -205,26 +203,50 @@ class NoticeRegistry:
if deleted:
self._notify()
def purge_dismissed(self) -> int:
"""Delete every dismissed row so each can show again. Returns how many."""
with self._lock:
return int(
Notice.delete().where(Notice.dismissed_at.is_null(False)).execute()
)
def acknowledge(self, row_id: str) -> bool:
"""Hide a notice until it happens again.
def dismiss(self, row_id: str) -> bool:
"""Hide a notice or check row for good. Returns False for an unknown id."""
Returns False for an unknown id or a kind that never repeats, such as
a check row or the update notice.
"""
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
# a check row gets a notice row only once it is dismissed
return False
definition = NOTICE_KINDS.get(existing.kind)
if definition is None or not definition.counts_repeats:
return False
if existing.acknowledged_at is not None or existing.muted_at is not None:
return True
Notice.update(acknowledged_at=datetime.now().timestamp()).where(
Notice.id == row_id
).execute()
NoticeStats.update(acknowledgements=NoticeStats.acknowledgements + 1).where(
NoticeStats.kind == existing.kind
).execute()
self._notify()
return True
def mute(self, row_id: str) -> bool:
"""Hide a notice or check row for good. Returns False for an unknown id."""
now = datetime.now().timestamp()
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
# a check row gets a notice row only once it is muted
kind, _, scope = row_id.partition(":")
if kind not in CHECK_KINDS or not scope:
return False
now = datetime.now().timestamp()
Notice.create(
id=row_id,
kind=kind,
@@ -233,40 +255,78 @@ class NoticeRegistry:
first_seen=now,
last_seen=now,
count=1,
dismissed_at=now,
muted_at=now,
)
return True
if existing.dismissed_at is not None:
if existing.muted_at is not None:
return True
if existing.kind not in NOTICE_KINDS:
return False
Notice.update(dismissed_at=datetime.now().timestamp()).where(
Notice.update(acknowledged_at=None, muted_at=now).where(
Notice.id == row_id
).execute()
NoticeStats.update(dismissals=NoticeStats.dismissals + 1).where(
NoticeStats.update(mutes=NoticeStats.mutes + 1).where(
NoticeStats.kind == existing.kind
).execute()
self._notify()
return True
def dismissed_checks(self) -> list[dict[str, Any]]:
"""Dismissed config and stream check rows, newest first."""
def unhide(self, row_id: str) -> bool:
"""Show an acknowledged or muted row again. Returns False for an unknown id."""
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
return False
if existing.kind in CHECK_KINDS:
Notice.delete_by_id(row_id)
return True
Notice.update(acknowledged_at=None, muted_at=None).where(
Notice.id == row_id
).execute()
self._notify()
return True
def unhide_all(self) -> None:
"""Show every acknowledged and muted row again."""
with self._lock:
for check in self.muted_checks():
Notice.delete_by_id(check["id"])
shown = (
Notice.update(acknowledged_at=None, muted_at=None)
.where(
Notice.acknowledged_at.is_null(False)
| Notice.muted_at.is_null(False)
)
.execute()
)
if shown:
self._notify()
def muted_checks(self) -> list[dict[str, Any]]:
"""Muted config and stream check rows, newest first."""
rows = (
Notice.select()
.where(Notice.kind.in_(list(CHECK_KINDS)))
.order_by(Notice.dismissed_at.desc())
.order_by(Notice.muted_at.desc())
)
return [{"id": row.id, "dismissed_at": row.dismissed_at} for row in rows]
return [{"id": row.id, "muted_at": row.muted_at} for row in rows]
def active(self, include_dismissed: bool = False) -> list[dict[str, Any]]:
def active(self, include_hidden: bool = False) -> list[dict[str, Any]]:
"""Notices most severe first, then most recent first.
Args:
include_dismissed: Also return dismissed notices, for the history view
include_hidden: Also return acknowledged and muted notices, for the
hidden list
"""
rows = []
@@ -276,7 +336,9 @@ class NoticeRegistry:
if definition is None:
continue
if row.dismissed_at is not None and not include_dismissed:
hidden = row.acknowledged_at is not None or row.muted_at is not None
if hidden and not include_hidden:
continue
rows.append(
@@ -291,7 +353,9 @@ class NoticeRegistry:
"first_seen": row.first_seen,
"last_seen": row.last_seen,
"count": row.count,
"dismissed_at": row.dismissed_at,
"acknowledgeable": definition.counts_repeats,
"acknowledged_at": row.acknowledged_at,
"muted_at": row.muted_at,
}
)
@@ -309,11 +373,13 @@ class NoticeRegistry:
{
"kind": row.kind,
"occurrences": row.occurrences,
"dismissals": row.dismissals,
"acknowledgements": row.acknowledgements,
"mutes": row.mutes,
"first_seen": row.first_seen,
"last_seen": row.last_seen,
"reported_occurrences": row.reported_occurrences,
"reported_dismissals": row.reported_dismissals,
"reported_acknowledgements": row.reported_acknowledgements,
"reported_mutes": row.reported_mutes,
}
for row in NoticeStats.select()
if row.kind in NOTICE_KINDS
@@ -327,18 +393,18 @@ class NoticeRegistry:
last_seen: float,
params: dict[str, Any],
) -> None:
# called with the lock held
fields: dict[str, Any] = {
"count": row.count + count,
"last_seen": last_seen,
"params": params,
}
reopen_at = NOTICE_KINDS[kind].reopen_at_count
# called with the lock held; held repeats from before an acknowledgement
# still count but leave the notice hidden
still_acknowledged = row.acknowledged_at is not None and (
last_seen <= cast(float, row.acknowledged_at)
)
if reopen_at is not None and row.count < reopen_at <= row.count + count:
fields["dismissed_at"] = None
Notice.update(**fields).where(Notice.id == row.id).execute()
Notice.update(
count=row.count + count,
last_seen=last_seen,
params=params,
acknowledged_at=row.acknowledged_at if still_acknowledged else None,
).where(Notice.id == row.id).execute()
self._bump_occurrences(kind, count, last_seen)
def _prune(self, kind: str, keep: int) -> None:
@@ -362,7 +428,8 @@ class NoticeRegistry:
NoticeStats.create(
kind=kind,
occurrences=count,
dismissals=0,
acknowledgements=0,
mutes=0,
first_seen=now,
last_seen=now,
)
frigate/notices/types.py
+10 -6
View File
@@ -28,9 +28,9 @@ class NoticeKind:
category: camera, detector, model, or system; a camera scope is a
camera name, and the UI shows it
link: app route or absolute URL for the row, filled in from params
counts_repeats: whether raising an existing notice counts another occurrence
counts_repeats: whether raising an existing notice counts another
occurrence, which also shows an acknowledged notice again
batch_repeats: whether repeats wait in memory for the next flush
reopen_at_count: count at which a dismissed notice shows again
keep_latest: rows of this kind to keep; a new row drops the oldest
reportable: whether a future analytics reporter may send this kind's counts
"""
@@ -41,7 +41,6 @@ class NoticeKind:
link: str | None = None
counts_repeats: bool = True
batch_repeats: bool = False
reopen_at_count: int | None = None
keep_latest: int | None = None
reportable: bool = True
@@ -67,6 +66,12 @@ _KINDS = (
"camera",
link="/system#cameras",
),
NoticeKind(
"ffmpeg_high_cpu", NoticeSeverity.warning, "camera", link="/system#cameras"
),
NoticeKind(
"detect_high_cpu", NoticeSeverity.warning, "camera", link="/system#cameras"
),
NoticeKind("shm_too_low", NoticeSeverity.warning, "system", link="/system#storage"),
# one row per user per burst; the login log lines carry the address
NoticeKind(
@@ -75,10 +80,9 @@ _KINDS = (
"system",
link="/logs",
batch_repeats=True,
reopen_at_count=5,
keep_latest=100,
),
# one row per release, so a dismissal lasts until the next release
# one row per release, so muting it lasts until the next release
NoticeKind(
"update_available",
NoticeSeverity.info,
@@ -92,7 +96,7 @@ _KINDS = (
NOTICE_KINDS: dict[str, NoticeKind] = {kind.key: kind for kind in _KINDS}
# the Health tab builds config and stream check rows in the browser, so a notice
# row of these kinds only records a dismissal; its other fields are placeholders
# row of these kinds only records a mute; its other fields are placeholders
CHECK_KINDS = frozenset({"config", "stream"})
frigate/stats/emitter.py
+61 -15
View File
@@ -29,38 +29,59 @@ VERSION_REFRESH_S = 24 * 60 * 60
# a camera skipping at least this percent of its frames is falling behind
SKIPPED_DETECTIONS_PCT = 5
# for at least this long before it becomes a notice
SKIPPED_DETECTIONS_HOLD_S = 60
# lifetime CPU averages at or above these are high; they match
# CameraFfmpegThreshold.error and CameraDetectThreshold.error in the UI
FFMPEG_HIGH_CPU_PCT = 20
DETECT_HIGH_CPU_PCT = 40
# a camera stays over a threshold this long before it becomes a notice
EPISODE_HOLD_S = 60
# detectors warm up after a start; the status bar waits this long too
STARTUP_GRACE_S = 120
class SkippedDetectionsTracker:
"""Finds cameras whose skipped share stays high long enough for a notice."""
def cpu_average(cpu_usages: dict[str, Any], pid: int | None) -> float | None:
"""A process's CPU use averaged over its lifetime, or None if unknown."""
usage = cpu_usages.get(str(pid)) if pid else None
def __init__(self) -> None:
try:
return float(usage["cpu_average"]) if usage else None
except (KeyError, TypeError, ValueError):
return None
class EpisodeTracker:
"""Finds cameras whose value stays over a threshold long enough for a notice."""
def __init__(self, threshold: float) -> None:
self._threshold = threshold
self._since: dict[str, float] = {}
self._raised: set[str] = set()
def update(self, cameras: dict[str, dict[str, Any]], now: float) -> list[str]:
"""Return the cameras whose episode qualified on this sample."""
def update(self, values: dict[str, float | None], now: float) -> list[str]:
"""Return the cameras whose episode qualified on this sample.
Args:
values: Each camera's current value, or None when it has none
now: Sample time in seconds
"""
qualified: list[str] = []
# a removed camera that comes back starts a new episode
for camera in self._since.keys() - cameras.keys():
for camera in self._since.keys() - values.keys():
self._since.pop(camera)
self._raised.discard(camera)
for camera, camera_stats in cameras.items():
if camera_stats["skipped_pct"] < SKIPPED_DETECTIONS_PCT:
for camera, value in values.items():
if value is None or value < self._threshold:
self._since.pop(camera, None)
self._raised.discard(camera)
continue
since = self._since.setdefault(camera, now)
if camera not in self._raised and now - since >= SKIPPED_DETECTIONS_HOLD_S:
if camera not in self._raised and now - since >= EPISODE_HOLD_S:
self._raised.add(camera)
qualified.append(camera)
@@ -80,7 +101,9 @@ class StatsEmitter(threading.Thread):
self.stop_event = stop_event
self.hardware_stats = HardwareStats(config)
self.stats_history: list[dict[str, Any]] = []
self.skipped_detections = SkippedDetectionsTracker()
self.skipped_detections = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
self.ffmpeg_cpu = EpisodeTracker(FFMPEG_HIGH_CPU_PCT)
self.detect_cpu = EpisodeTracker(DETECT_HIGH_CPU_PCT)
# the shm notice's params as last sent, so only a change is written
self._shm_checked = False
@@ -227,15 +250,38 @@ class StatsEmitter(threading.Thread):
def _update_notices(self, stats: dict[str, Any], now: float) -> None:
"""Update notices based on current stats or time."""
# skipped detections
cameras = stats["cameras"]
# absent when CPU collection timed out or failed on this tick
cpu_usages = stats.get("cpu_usages", {})
if stats["service"]["uptime"] >= STARTUP_GRACE_S:
for camera in self.skipped_detections.update(stats["cameras"], now):
# skipped detections
skipped = {
camera: camera_stats["skipped_pct"]
for camera, camera_stats in cameras.items()
}
for camera in self.skipped_detections.update(skipped, now):
raise_notice(
"skipped_detections",
scope=camera,
params={"pct": stats["cameras"][camera]["skipped_pct"]},
params={"pct": skipped[camera]},
)
# high ffmpeg and detect CPU
for tracker, kind, pid_key in (
(self.ffmpeg_cpu, "ffmpeg_high_cpu", "ffmpeg_pid"),
(self.detect_cpu, "detect_high_cpu", "pid"),
):
averages = {
camera: cpu_average(cpu_usages, camera_stats.get(pid_key))
for camera, camera_stats in cameras.items()
}
for camera in tracker.update(averages, now):
raise_notice(kind, scope=camera, params={"cpu": averages[camera]})
# shm too small for the cameras
self._update_shm_notice(stats["service"]["storage"]["/dev/shm"])
frigate/test/http_api/test_http_notices.py
+61 -45
View File
@@ -53,92 +53,108 @@ class TestHttpNotices(BaseTestHttp):
self.assertEqual(response.json()[0]["kind"], "detector_stuck")
self.assertEqual(response.json()[0]["occurrences"], 1)
def test_dismiss_hides_and_history_keeps_it(self):
def test_acknowledge_hides_and_the_hidden_list_keeps_it(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
with self.client() as client:
response = client.post("/notices/detector_stuck:ov/dismiss")
response = client.post("/notices/detector_stuck:ov/acknowledge")
listed = client.get("/notices").json()
history = client.get("/notices", params={"include_dismissed": True}).json()
hidden = client.get("/notices", params={"include_hidden": True}).json()
self.assertEqual(response.status_code, 200)
self.assertEqual(listed, [])
self.assertEqual([n["id"] for n in history], ["detector_stuck:ov"])
self.assertIsNotNone(history[0]["dismissed_at"])
self.assertEqual([n["id"] for n in hidden], ["detector_stuck:ov"])
self.assertIsNotNone(hidden[0]["acknowledged_at"])
def test_dismiss_an_id_with_a_slash(self):
def test_acknowledging_a_check_is_404(self):
with self.client() as client:
response = client.post(f"/notices/{CONFIG_CHECK}/acknowledge")
self.assertEqual(response.status_code, 404)
def test_mute_an_id_with_a_slash(self):
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
with self.client() as client:
response = client.post(
"/notices/model_download_failed:yolo/model.onnx/dismiss"
"/notices/model_download_failed:yolo/model.onnx/mute"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(self.registry.active(), [])
def test_dismiss_unknown_is_404(self):
def test_unknown_ids_are_404(self):
with self.client() as client:
response = client.post("/notices/nope/dismiss")
responses = [
client.post("/notices/nope/acknowledge"),
client.post("/notices/nope/mute"),
client.delete("/notices/nope/hidden"),
]
self.assertEqual(response.status_code, 404)
self.assertEqual([r.status_code for r in responses], [404, 404, 404])
def test_requires_admin(self):
viewer = {"remote-user": "viewer", "remote-role": "viewer"}
with TestClient(self.app) as client:
response = client.get(
"/notices", headers={"remote-user": "viewer", "remote-role": "viewer"}
)
responses = [
client.get("/notices", headers=viewer),
client.get("/notices/muted_checks", headers=viewer),
client.post("/notices/detector_stuck:ov/acknowledge", headers=viewer),
client.post("/notices/detector_stuck:ov/mute", headers=viewer),
client.delete("/notices/detector_stuck:ov/hidden", headers=viewer),
client.delete("/notices/hidden", headers=viewer),
]
self.assertEqual(response.status_code, 403)
self.assertEqual([r.status_code for r in responses], [403] * 6)
def test_dismissed_checks_are_listed_once(self):
def test_muted_checks_are_listed_once(self):
with self.client() as client:
first = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
again = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
listed = client.get("/notices/dismissed_checks").json()
history = client.get("/notices", params={"include_dismissed": True}).json()
first = client.post(f"/notices/{CONFIG_CHECK}/mute")
again = client.post(f"/notices/{CONFIG_CHECK}/mute")
listed = client.get("/notices/muted_checks").json()
hidden = client.get("/notices", params={"include_hidden": True}).json()
self.assertEqual(first.status_code, 200)
self.assertEqual(again.status_code, 200)
self.assertEqual([check["id"] for check in listed], [CONFIG_CHECK])
self.assertIsNotNone(listed[0]["dismissed_at"])
self.assertIsNotNone(listed[0]["muted_at"])
# the Health tab builds the row itself, so it never comes back as a notice
self.assertEqual(history, [])
self.assertEqual(hidden, [])
def test_dismissed_checks_require_admin(self):
with TestClient(self.app) as client:
response = client.get(
"/notices/dismissed_checks",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
self.assertEqual(response.status_code, 403)
def test_purge_dismissed_clears_notices_and_checks(self):
def test_unhide_one_row(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss(CONFIG_CHECK)
self.registry.mute("detector_stuck:ov")
self.registry.mute(CONFIG_CHECK)
with self.client() as client:
response = client.delete("/notices/dismissed")
history = client.get("/notices", params={"include_dismissed": True}).json()
checks = client.get("/notices/dismissed_checks").json()
notice = client.delete("/notices/detector_stuck:ov/hidden")
check = client.delete(f"/notices/{CONFIG_CHECK}/hidden")
listed = client.get("/notices").json()
checks = client.get("/notices/muted_checks").json()
self.assertEqual(response.status_code, 200)
self.assertEqual(history, [])
self.assertEqual([notice.status_code, check.status_code], [200, 200])
self.assertEqual([n["id"] for n in listed], ["detector_stuck:ov"])
self.assertEqual(checks, [])
def test_purge_dismissed_requires_admin(self):
with TestClient(self.app) as client:
response = client.delete(
"/notices/dismissed",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
def test_unhide_all_shows_notices_and_drops_check_mutes(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.acknowledge("detector_stuck:ov")
self.registry.mute(CONFIG_CHECK)
self.assertEqual(response.status_code, 403)
with self.client() as client:
response = client.delete("/notices/hidden")
listed = client.get("/notices").json()
checks = client.get("/notices/muted_checks").json()
self.assertEqual(response.status_code, 200)
self.assertEqual([n["id"] for n in listed], ["detector_stuck:ov"])
self.assertEqual(checks, [])
frigate/test/test_notice_registry.py
+120 -58
View File
@@ -16,9 +16,7 @@ from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
# kinds that switch on the lifecycle knobs, so the tests do not depend on the
# values the real catalog picks
BATCHED = NoticeKind(
"batched", NoticeSeverity.warning, "system", batch_repeats=True, reopen_at_count=3
)
BATCHED = NoticeKind("batched", NoticeSeverity.warning, "system", batch_repeats=True)
PRUNED = NoticeKind("pruned", NoticeSeverity.info, "system", keep_latest=2)
TEST_KINDS = {kind.key: kind for kind in (BATCHED, PRUNED)}
@@ -77,6 +75,11 @@ class RegistryTestCase(unittest.TestCase):
mock_datetime.now.return_value.timestamp.return_value = timestamp
self.registry.raise_notice(kind, **kwargs)
def _acknowledge_at(self, timestamp: float, row_id: str) -> None:
with patch("frigate.notices.registry.datetime") as mock_datetime:
mock_datetime.now.return_value.timestamp.return_value = timestamp
self.registry.acknowledge(row_id)
class TestNoticeRegistry(RegistryTestCase):
def test_raise_inserts_and_counts_one_occurrence(self):
@@ -122,34 +125,98 @@ class TestNoticeRegistry(RegistryTestCase):
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_not_called()
def test_dismissal_survives_a_re_raise(self):
def test_acknowledged_notice_returns_on_a_re_raise(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
self.assertEqual(self.registry.active(), [])
self.registry.raise_notice("detector_stuck", scope="ov", params={})
notice = self.registry.active()[0]
self.assertEqual(notice["count"], 2)
self.assertIsNone(notice["acknowledged_at"])
def test_muted_notice_stays_hidden_on_a_re_raise(self):
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertTrue(self.registry.dismiss("skipped_detections:front_door"))
self.assertTrue(self.registry.mute("skipped_detections:front_door"))
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertEqual(self.registry.active(), [])
history = self.registry.active(include_dismissed=True)
self.assertEqual(history[0]["count"], 2)
self.assertIsNotNone(history[0]["dismissed_at"])
hidden = self.registry.active(include_hidden=True)
self.assertEqual(hidden[0]["count"], 2)
self.assertIsNotNone(hidden[0]["muted_at"])
def test_dismiss_counts_once(self):
def test_muting_an_acknowledged_notice_replaces_the_acknowledgement(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.acknowledge("detector_stuck:ov")
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
self.assertTrue(self.registry.mute("detector_stuck:ov"))
self.assertEqual(self.registry.stats()[0]["dismissals"], 1)
notice = self.registry.active(include_hidden=True)[0]
self.assertIsNone(notice["acknowledged_at"])
self.assertIsNotNone(notice["muted_at"])
def test_dismiss_unknown_id(self):
self.assertFalse(self.registry.dismiss("nope"))
def test_dismissed_hidden_unless_requested(self):
def test_acknowledging_a_muted_notice_keeps_it_muted(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.dismiss("detector_stuck:ov")
self.registry.mute("detector_stuck:ov")
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
notice = self.registry.active(include_hidden=True)[0]
self.assertIsNone(notice["acknowledged_at"])
self.assertIsNotNone(notice["muted_at"])
def test_acknowledge_and_mute_each_count_once(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("shm_too_low", params={})
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
self.assertTrue(self.registry.acknowledge("detector_stuck:ov"))
self.assertTrue(self.registry.mute("shm_too_low"))
self.assertTrue(self.registry.mute("shm_too_low"))
stats = {s["kind"]: s for s in self.registry.stats()}
self.assertEqual(stats["detector_stuck"]["acknowledgements"], 1)
self.assertEqual(stats["detector_stuck"]["mutes"], 0)
self.assertEqual(stats["shm_too_low"]["mutes"], 1)
def test_unknown_ids_are_rejected(self):
self.assertFalse(self.registry.acknowledge("nope"))
self.assertFalse(self.registry.mute("nope"))
self.assertFalse(self.registry.unhide("nope"))
def test_kinds_that_never_repeat_cannot_be_acknowledged(self):
self.registry.raise_notice(
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
self.assertFalse(self.registry.acknowledge("update_available:0.19.1"))
self.assertFalse(
self.registry.acknowledge("config:detect:fps-greater-than-five:global")
)
self.assertEqual(self.registry.active()[0]["acknowledgeable"], False)
def test_hidden_notices_listed_only_when_requested(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.acknowledge("detector_stuck:ov")
self.assertEqual(self.registry.active(), [])
self.assertEqual(len(self.registry.active(include_dismissed=True)), 1)
self.assertEqual(len(self.registry.active(include_hidden=True)), 1)
def test_unhide_shows_a_notice_and_drops_a_check_mute(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.mute("detector_stuck:ov")
self.registry.mute("config:detect:fps-greater-than-five:global")
self.assertTrue(self.registry.unhide("detector_stuck:ov"))
self.assertTrue(
self.registry.unhide("config:detect:fps-greater-than-five:global")
)
notice = self.registry.active()[0]
self.assertIsNone(notice["muted_at"])
self.assertEqual(self.registry.muted_checks(), [])
def test_resolve_deletes_and_keeps_stats(self):
self.registry.raise_notice(
@@ -160,7 +227,7 @@ class TestNoticeRegistry(RegistryTestCase):
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.assertEqual(self.registry.active(include_hidden=True), [])
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
@@ -237,45 +304,47 @@ class TestNoticeRegistry(RegistryTestCase):
ids, ["model_download_failed:front_door", "skipped_detections:garage"]
)
def test_resolve_camera_drops_that_cameras_check_dismissals(self):
def test_resolve_camera_drops_that_cameras_check_mutes(self):
for check_id in (
"stream:front_door:0:probe",
"config:detect:fps-greater-than-five:camera.front_door",
"config:detect:fps-greater-than-five:global",
"stream:garage:0:probe",
):
self.assertTrue(self.registry.dismiss(check_id))
self.assertTrue(self.registry.mute(check_id))
self.registry.resolve_camera("front_door")
ids = sorted(check["id"] for check in self.registry.dismissed_checks())
ids = sorted(check["id"] for check in self.registry.muted_checks())
self.assertEqual(
ids,
["config:detect:fps-greater-than-five:global", "stream:garage:0:probe"],
)
def test_camera_named_global_keeps_global_check_dismissals(self):
self.registry.dismiss("config:detect:fps-greater-than-five:global")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.global")
def test_camera_named_global_keeps_global_check_mutes(self):
self.registry.mute("config:detect:fps-greater-than-five:global")
self.registry.mute("config:detect:fps-greater-than-five:camera.global")
self.registry.resolve_camera("global")
ids = [check["id"] for check in self.registry.dismissed_checks()]
ids = [check["id"] for check in self.registry.muted_checks()]
self.assertEqual(ids, ["config:detect:fps-greater-than-five:global"])
def test_purge_dismissed_keeps_active_notices_and_counts(self):
def test_unhide_all_shows_every_notice_and_keeps_counts(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.garage")
self.registry.acknowledge("detector_stuck:ov")
self.registry.mute("skipped_detections:garage")
self.registry.mute("config:detect:fps-greater-than-five:camera.garage")
self.assertEqual(self.registry.purge_dismissed(), 2)
self.registry.unhide_all()
ids = [n["id"] for n in self.registry.active(include_dismissed=True)]
self.assertEqual(ids, ["skipped_detections:garage"])
self.assertEqual(self.registry.dismissed_checks(), [])
dismissals = {s["kind"]: s["dismissals"] for s in self.registry.stats()}
self.assertEqual(dismissals["detector_stuck"], 1)
ids = sorted(n["id"] for n in self.registry.active())
self.assertEqual(ids, ["detector_stuck:ov", "skipped_detections:garage"])
self.assertEqual(self.registry.muted_checks(), [])
stats = {s["kind"]: s for s in self.registry.stats()}
self.assertEqual(stats["detector_stuck"]["acknowledgements"], 1)
self.assertEqual(stats["skipped_detections"]["mutes"], 1)
class TestApply(RegistryTestCase):
@@ -349,7 +418,7 @@ class TestLifecycleKnobs(RegistryTestCase):
self.registry.flush()
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.assertEqual(self.registry.active(include_hidden=True), [])
self.listener.assert_not_called()
def test_a_new_row_starts_without_stale_repeats(self):
@@ -362,32 +431,26 @@ class TestLifecycleKnobs(RegistryTestCase):
self.assertEqual(self.registry.active()[0]["count"], 1)
def test_a_dismissed_notice_reopens_at_the_count(self):
self.registry.raise_notice("batched")
self.registry.dismiss("batched")
def test_an_acknowledged_notice_returns_when_a_later_repeat_flushes(self):
self._raise_at(1000.0, "batched")
self._acknowledge_at(1010.0, "batched")
self.registry.raise_notice("batched")
self.registry.flush()
self._raise_at(1020.0, "batched")
self.assertEqual(self.registry.active(), [])
self.registry.raise_notice("batched")
self.registry.flush()
notice = self.registry.active()[0]
self.assertEqual(notice["count"], BATCHED.reopen_at_count)
self.assertIsNone(notice["dismissed_at"])
self.assertEqual(self.registry.active()[0]["count"], 2)
def test_a_notice_dismissed_past_the_count_stays_dismissed(self):
for _ in range(3):
self.registry.raise_notice("batched")
self.registry.flush()
self.registry.dismiss("batched")
def test_repeats_held_from_before_an_acknowledgement_stay_hidden(self):
self._raise_at(1000.0, "batched")
self._raise_at(1005.0, "batched")
self._acknowledge_at(1010.0, "batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active(), [])
self.assertEqual(self.registry.active(include_dismissed=True)[0]["count"], 4)
self.assertEqual(self.registry.active(include_hidden=True)[0]["count"], 2)
def test_keep_latest_drops_the_oldest_rows(self):
for index, scope in enumerate(("a", "b", "c")):
@@ -410,14 +473,13 @@ class TestCatalogLifecycles(RegistryTestCase):
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["update_available:0.19.1"])
def test_a_dismissed_failed_login_burst_reopens_at_five_attempts(self):
def test_an_acknowledged_failed_login_burst_returns_on_the_next_attempt(self):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.dismiss("failed_login:1000")
self.registry.acknowledge("failed_login:1000")
for _ in range(4):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.raise_notice("failed_login", scope="1000")
self.registry.flush()
burst = self.registry.active()[0]
self.assertEqual(burst["count"], 5)
self.assertIsNone(burst["dismissed_at"])
self.assertEqual(burst["count"], 2)
self.assertIsNone(burst["acknowledged_at"])
frigate/test/test_skipped_detections.py
+86 -18
View File
@@ -1,10 +1,14 @@
"""Tests for the skipped detection rate and the notice it can raise."""
"""Tests for the per-camera episode notices: skipped detections and high CPU."""
import unittest
from unittest.mock import patch
from unittest.mock import call, patch
from frigate.stats import emitter
from frigate.stats.emitter import SkippedDetectionsTracker
from frigate.stats.emitter import (
SKIPPED_DETECTIONS_PCT,
EpisodeTracker,
cpu_average,
)
from frigate.stats.util import skipped_percent
@@ -19,18 +23,18 @@ class TestSkippedPercent(unittest.TestCase):
self.assertEqual(skipped_percent(2.0, 5.0, False), 0.0)
class TestSkippedDetectionsTracker(unittest.TestCase):
class TestEpisodeTracker(unittest.TestCase):
def _cameras(self, pct: float) -> dict:
return {"front_door": {"skipped_pct": pct}}
return {"front_door": pct}
def test_below_threshold_never_qualifies(self):
tracker = SkippedDetectionsTracker()
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
for tick in range(10):
self.assertEqual(tracker.update(self._cameras(4.9), tick * 15.0), [])
def test_qualifies_once_after_the_hold(self):
tracker = SkippedDetectionsTracker()
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
results = [
tracker.update(self._cameras(10.0), tick * 15.0) for tick in range(8)
@@ -42,7 +46,7 @@ class TestSkippedDetectionsTracker(unittest.TestCase):
self.assertEqual(results[5:], [[], [], []])
def test_a_dip_restarts_the_hold(self):
tracker = SkippedDetectionsTracker()
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
for now, pct in ((0.0, 10.0), (15.0, 10.0), (30.0, 10.0), (45.0, 2.0)):
self.assertEqual(tracker.update(self._cameras(pct), now), [])
@@ -51,7 +55,7 @@ class TestSkippedDetectionsTracker(unittest.TestCase):
self.assertEqual(tracker.update(self._cameras(10.0), 120.0), ["front_door"])
def test_recovery_then_relapse_is_a_new_episode(self):
tracker = SkippedDetectionsTracker()
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker.update(self._cameras(10.0), 0.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), ["front_door"])
@@ -61,17 +65,15 @@ class TestSkippedDetectionsTracker(unittest.TestCase):
self.assertEqual(tracker.update(self._cameras(10.0), 150.0), ["front_door"])
def test_cameras_are_tracked_separately(self):
tracker = SkippedDetectionsTracker()
tracker.update({"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 0.0}}, 0.0)
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker.update({"a": 10.0, "b": 0.0}, 0.0)
qualified = tracker.update(
{"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 10.0}}, 60.0
)
qualified = tracker.update({"a": 10.0, "b": 10.0}, 60.0)
self.assertEqual(qualified, ["a"])
def test_a_removed_camera_starts_a_new_episode(self):
tracker = SkippedDetectionsTracker()
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker.update(self._cameras(10.0), 0.0)
tracker.update({}, 15.0)
tracker.update(self._cameras(10.0), 30.0)
@@ -79,6 +81,25 @@ class TestSkippedDetectionsTracker(unittest.TestCase):
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
self.assertEqual(tracker.update(self._cameras(10.0), 90.0), ["front_door"])
def test_a_missing_value_ends_the_episode(self):
tracker = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
tracker.update(self._cameras(10.0), 0.0)
tracker.update({"front_door": None}, 30.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
class TestCpuAverage(unittest.TestCase):
def test_reads_the_lifetime_average(self):
usages = {"42": {"cpu": "80.0", "cpu_average": "25.0"}}
self.assertEqual(cpu_average(usages, 42), 25.0)
def test_unknown_process_has_no_average(self):
self.assertIsNone(cpu_average({}, 42))
self.assertIsNone(cpu_average({"42": {"cpu_average": "25.0"}}, None))
self.assertIsNone(cpu_average({"42": {"cpu_average": ""}}, 42))
class TestEmitterNotices(unittest.TestCase):
def setUp(self):
@@ -94,15 +115,25 @@ class TestEmitterNotices(unittest.TestCase):
def _emitter(self) -> emitter.StatsEmitter:
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
stats_emitter.skipped_detections = SkippedDetectionsTracker()
stats_emitter.skipped_detections = EpisodeTracker(SKIPPED_DETECTIONS_PCT)
stats_emitter.ffmpeg_cpu = EpisodeTracker(emitter.FFMPEG_HIGH_CPU_PCT)
stats_emitter.detect_cpu = EpisodeTracker(emitter.DETECT_HIGH_CPU_PCT)
stats_emitter._shm_checked = False
stats_emitter._shm_params = None
return stats_emitter
def _stats(self, uptime: int, pct: float) -> dict:
def _stats(
self, uptime: int, pct: float, ffmpeg_cpu: str = "5.0", detect_cpu: str = "5.0"
) -> dict:
return {
"service": {"uptime": uptime, "storage": {"/dev/shm": {}}},
"cameras": {"front_door": {"skipped_pct": pct}},
"cameras": {
"front_door": {"skipped_pct": pct, "ffmpeg_pid": 10, "pid": 11}
},
"cpu_usages": {
"10": {"cpu_average": ffmpeg_cpu},
"11": {"cpu_average": detect_cpu},
},
}
def test_qualified_camera_raises_a_notice(self):
@@ -115,6 +146,43 @@ class TestEmitterNotices(unittest.TestCase):
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
def test_high_cpu_raises_a_notice_per_process(self):
stats_emitter = self._emitter()
for uptime, now in ((300, 0.0), (360, 60.0)):
stats_emitter._update_notices(
self._stats(uptime, 0.0, ffmpeg_cpu="25.0", detect_cpu="45.0"), now
)
self.assertEqual(
self.raise_notice.call_args_list,
[
call("ffmpeg_high_cpu", scope="front_door", params={"cpu": 25.0}),
call("detect_high_cpu", scope="front_door", params={"cpu": 45.0}),
],
)
def test_cpu_below_each_threshold_raises_nothing(self):
stats_emitter = self._emitter()
for uptime, now in ((300, 0.0), (360, 60.0)):
stats_emitter._update_notices(
self._stats(uptime, 0.0, ffmpeg_cpu="19.0", detect_cpu="39.0"), now
)
self.raise_notice.assert_not_called()
def test_missing_cpu_stats_raise_nothing(self):
stats_emitter = self._emitter()
for uptime, now in ((300, 0.0), (360, 60.0)):
stats = self._stats(uptime, 0.0)
del stats["cpu_usages"]
stats_emitter._update_notices(stats, now)
self.raise_notice.assert_not_called()
self.assertEqual(self.flush_notices.call_count, 2)
def test_startup_window_is_ignored(self):
stats_emitter = self._emitter()