administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-09-24 18:26:51 +03:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Refactor Notices and System Health pane (#24243)

Browse Source 
* refactor notices

* show startup message for enrichments in health pane

* tweaks
This commit is contained in:
Josh Hawkins
2026-09-12 07:30:04 -06:00
committed by Nicolas Mowen
parent 5cef6823a6
commit 7bc32fd4c9
50 changed files with 2471 additions and 898 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/docs/troubleshooting/faqs.md
+1 -1
View File
@@ -184,6 +184,6 @@ Filters and masks only hide the incorrect result - they don't teach Frigate what
### Where do I see problems Frigate has detected?
Open System > Health. The Notices list shows problems the backend has noticed on its own, such as a camera whose ffmpeg keeps crashing, a detector that had to be restarted, a model download that failed, or recordings being deleted before their retention period. Entries that describe a one-time event can be dismissed; entries that describe an ongoing condition clear themselves once it is fixed.
Open System > Health. The Notices list keeps a record of problems Frigate has found, and you can dismiss any entry to acknowledge it. Ongoing conditions, such as an offline camera or recordings deleted before their retention period, appear in the status bar for admins until they clear, and the status bar links to the Notices list while it has undismissed entries. On mobile, tap the warning icon in the bottom navigation bar to see them.
The Hardware section below the notices shows whether the detection hardware, hardware acceleration, and enrichment devices in your config were found and are being used, so a GPU that silently fell back to the CPU shows up as a warning. Run stream checks to probe every camera's streams for the same problems the camera wizard reports.
docs/static/frigate-api.yaml
Vendored
+44 -4
View File
@@ -4171,13 +4171,13 @@ paths:
description: |-
**Access:** Admin role required.
Get the active notices, most severe first.
Get notices, most severe first.
Args:
include_dismissed: Also return event notices the user has dismissed
include_dismissed: Also return dismissed notices, for the history view
Returns:
The active notices
The notices
operationId: get_notices_notices_get
parameters:
- name: include_dismissed
@@ -4221,6 +4221,44 @@ paths:
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/dismissed_checks:
get:
tags:
- Notices
summary: Get Dismissed Checks
description: |-
**Access:** Admin role required.
Get the dismissed config and stream check rows, newest first.
operationId: get_dismissed_checks_notices_dismissed_checks_get
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/dismissed:
delete:
tags:
- Notices
summary: Purge Dismissed
description: |-
**Access:** Admin role required.
Delete every dismissed notice and check row so each can show again.
operationId: purge_dismissed_notices_dismissed_delete
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
security:
- frigateAdminAuth: []
x-required-role: admin
/notices/{notice_id}/dismiss:
post:
tags:
@@ -4229,7 +4267,9 @@ paths:
description: |-
**Access:** Admin role required.
Hide an event notice until it is raised again.
Hide a notice or a config or stream check row.
It stays hidden if the same problem happens again.
operationId: dismiss_notice_notices__notice_id__dismiss_post
parameters:
- name: notice_id
frigate/api/auth.py
+56
View File
@@ -8,6 +8,7 @@ import logging
import os
import re
import secrets
import threading
import time
from datetime import datetime
from pathlib import Path
@@ -34,6 +35,7 @@ from frigate.api.media_auth import (
from frigate.config import AuthConfig, ProxyConfig
from frigate.const import CONFIG_DIR, JWT_SECRET_ENV_VAR, PASSWORD_HASH_ALGORITHM
from frigate.models import User
from frigate.notices import raise_notice
logger = logging.getLogger(__name__)
@@ -253,6 +255,58 @@ class RateLimiter:
rateLimiter = RateLimiter()
# a failed login this long after the user's previous one opens a new burst
FAILED_LOGIN_BURST_GAP_S = 300
# the username comes from the request, so it is cut before it reaches a notice
MAX_NOTICE_USERNAME = 64
# unknown usernames are unbounded, so past this many open bursts a new one only
# reaches the log
MAX_OPEN_BURSTS = 100
class FailedLoginTracker:
"""Groups each user's failed logins into bursts, one notice per burst."""
def __init__(self) -> None:
self._lock = threading.Lock()
# user -> (burst start, last attempt), stalest attempt first
self._bursts: dict[str, tuple[int, float]] = {}
def record(self, user: str, now: float, *, known: bool = False) -> None:
"""Count a failed login toward the user's open burst, or open a new one.
Once MAX_OPEN_BURSTS are open, only a known user opens another.
"""
user = user[:MAX_NOTICE_USERNAME]
with self._lock:
# bursts that went quiet are over
while self._bursts:
stalest = next(iter(self._bursts))
if now - self._bursts[stalest][1] < FAILED_LOGIN_BURST_GAP_S:
break
del self._bursts[stalest]
if (
not known
and user not in self._bursts
and len(self._bursts) >= MAX_OPEN_BURSTS
):
return
start, _ = self._bursts.pop(user, (int(now), now))
self._bursts[user] = (start, now)
raise_notice("failed_login", scope=f"{user}:{start}", params={"user": user})
failed_logins = FailedLoginTracker()
def get_remote_addr(request: Request):
# fall back to the direct TCP peer when no proxy chain is present
@@ -880,6 +934,7 @@ def login(request: Request, body: AppPostLoginBody):
db_user: User = User.get_by_id(user)
except DoesNotExist:
logger.warning(f"Login failed for unknown user '{user}' from {remote_addr}")
failed_logins.record(user, time.time())
return JSONResponse(content={"message": "Login failed"}, status_code=401)
password_hash = db_user.password_hash
@@ -911,6 +966,7 @@ def login(request: Request, body: AppPostLoginBody):
logger.warning(
f"Login failed for user '{user}' (invalid password) from {remote_addr}"
)
failed_logins.record(user, time.time(), known=True)
return JSONResponse(content={"message": "Login failed"}, status_code=401)
frigate/api/notices.py
+27 -17
View File
@@ -7,7 +7,6 @@ from fastapi.responses import JSONResponse
from frigate.api.auth import require_role
from frigate.api.defs.tags import Tags
from frigate.notices.registry import DismissResult
logger = logging.getLogger(__name__)
@@ -16,13 +15,13 @@ router = APIRouter(tags=[Tags.notices])
@router.get("/notices", dependencies=[Depends(require_role(["admin"]))])
def get_notices(request: Request, include_dismissed: bool = False) -> JSONResponse:
"""Get the active notices, most severe first.
"""Get notices, most severe first.
Args:
include_dismissed: Also return event notices the user has dismissed
include_dismissed: Also return dismissed notices, for the history view
Returns:
The active notices
The notices
"""
return JSONResponse(
content=request.app.notice_registry.active(include_dismissed=include_dismissed)
@@ -35,26 +34,37 @@ def get_notice_stats(request: Request) -> JSONResponse:
return JSONResponse(content=request.app.notice_registry.stats())
@router.get(
"/notices/dismissed_checks", dependencies=[Depends(require_role(["admin"]))]
)
def get_dismissed_checks(request: Request) -> JSONResponse:
"""Get the dismissed config and stream check rows, newest first."""
return JSONResponse(content=request.app.notice_registry.dismissed_checks())
@router.delete("/notices/dismissed", dependencies=[Depends(require_role(["admin"]))])
def purge_dismissed(request: Request) -> JSONResponse:
"""Delete every dismissed notice and check row so each can show again."""
request.app.notice_registry.purge_dismissed()
return JSONResponse(
content={"success": True, "message": "Dismissed notices cleared"}
)
# model notice ids contain a slash, so the id is a path parameter
@router.post(
"/notices/{notice_id}/dismiss", dependencies=[Depends(require_role(["admin"]))]
"/notices/{notice_id:path}/dismiss",
dependencies=[Depends(require_role(["admin"]))],
)
def dismiss_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide an event notice until it is raised again."""
result = request.app.notice_registry.dismiss(notice_id)
"""Hide a notice or a config or stream check row.
if result == DismissResult.not_found:
It stays hidden if the same problem happens again.
"""
if not request.app.notice_registry.dismiss(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
if result == DismissResult.not_dismissable:
return JSONResponse(
content={
"success": False,
"message": "This notice clears itself when the problem is fixed",
},
status_code=400,
)
return JSONResponse(content={"success": True, "message": "Notice dismissed"})
frigate/app.py
+9 -11
View File
@@ -73,6 +73,7 @@ from frigate.models import (
Trigger,
User,
)
from frigate.notices import install_registry
from frigate.notices.registry import NoticeRegistry
from frigate.object_detection.base import ObjectDetectProcess
from frigate.object_detection.util import detection_frame_size
@@ -310,10 +311,6 @@ class FrigateApp:
]
self.db.bind(models)
self.notice_registry = NoticeRegistry()
# producers confirm state notices again at startup if they still hold
self.notice_registry.mark_state_notices_unconfirmed()
def check_db_data_migrations(self) -> None:
# check if vacuum needs to be run
if not os.path.exists(f"{CONFIG_DIR}/.exports"):
@@ -325,6 +322,10 @@ class FrigateApp:
migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
def install_notice_registry(self) -> None:
self.notice_registry = NoticeRegistry()
install_registry(self.notice_registry)
def init_embeddings_client(self) -> None:
# Create a client for other processes to use
self.embeddings = EmbeddingsContext(self.db)
@@ -510,9 +511,7 @@ class FrigateApp:
self.record_cleanup.start()
def start_storage_maintainer(self) -> None:
self.storage_maintainer = StorageMaintainer(
self.config, self.stop_event, self.notice_registry
)
self.storage_maintainer = StorageMaintainer(self.config, self.stop_event)
self.storage_maintainer.start()
def start_stats_emitter(self) -> None:
@@ -524,16 +523,14 @@ class FrigateApp:
self.embeddings_metrics,
self.detectors,
self.processes,
self.storage_maintainer,
),
self.stop_event,
notice_registry=self.notice_registry,
)
self.stats_emitter.start()
def start_watchdog(self) -> None:
self.frigate_watchdog = FrigateWatchdog(
self.detectors, self.stop_event, self.notice_registry
)
self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event)
# (attribute on self, key in self.processes, factory)
specs: list[tuple[str, str, Callable[[], FrigateProcess]]] = [
@@ -644,6 +641,7 @@ class FrigateApp:
self.init_embeddings_manager()
self.bind_database()
self.check_db_data_migrations()
self.install_notice_registry()
# Clean up any stale replay camera artifacts (filesystem + DB)
cleanup_replay_cameras()
frigate/comms/dispatcher.py
+1 -14
View File
@@ -357,20 +357,7 @@ class Dispatcher:
return
try:
action = payload.get("action")
kind = payload.get("kind")
scope = payload.get("scope")
if not isinstance(kind, str):
logger.warning("Ignoring notice update without a kind")
elif action == "raise":
self.notice_registry.raise_notice(
kind, scope=scope, params=payload.get("params") or {}
)
elif action == "resolve":
self.notice_registry.resolve(kind, scope)
else:
logger.warning("Ignoring notice update with action %s", action)
self.notice_registry.apply(payload)
except Exception:
# a raise here would kill the REP thread for every process
logger.exception("Failed to apply notice update")
frigate/notices/__init__.py
+84
View File
@@ -0,0 +1,84 @@
"""Raise and resolve notices from any Frigate process or thread.
A notice records something worth seeing later, such as an event or a setup
problem only the backend can detect. A condition that comes and goes belongs in
stats and the status bar, and a config problem belongs in a settings message
flagged for health.
"""
import logging
import multiprocessing as mp
import threading
from typing import TYPE_CHECKING, Any
from frigate.comms.inter_process import InterProcessRequestor
from frigate.const import UPDATE_NOTICE
if TYPE_CHECKING:
from frigate.notices.registry import NoticeRegistry
logger = logging.getLogger(__name__)
_registry: "NoticeRegistry | None" = None
# zmq sockets are not thread safe, so every thread in a process shares one
# requestor behind a lock
_requestor: InterProcessRequestor | None = None
_requestor_lock = threading.Lock()
def install_registry(registry: "NoticeRegistry | None") -> None:
"""Send this process's notices straight to the registry.
Only the main process installs one. Children start from the forkserver,
so they never inherit it and send over IPC instead.
"""
global _registry
_registry = registry
def raise_notice(
kind: str, *, scope: str | None = None, params: dict[str, Any] | None = None
) -> None:
"""Record a notice, or count another occurrence of it. Never raises."""
_send({"action": "raise", "kind": kind, "scope": scope, "params": params or {}})
def resolve_notice(kind: str, scope: str | None = None) -> None:
"""Delete a notice once its problem is fixed. Never raises."""
_send({"action": "resolve", "kind": kind, "scope": scope})
def resolve_kind(kind: str) -> None:
"""Delete every notice of a kind. Never raises."""
_send({"action": "resolve_kind", "kind": kind})
def flush_notices() -> None:
"""Write repeats the registry held back. Only the main process has any."""
if _registry is None:
return
try:
_registry.flush()
except Exception:
logger.exception("Failed to flush notices")
def _send(update: dict[str, Any]) -> None:
global _requestor
try:
if _registry is not None:
_registry.apply(update)
elif mp.parent_process() is not None:
with _requestor_lock:
if _requestor is None:
_requestor = InterProcessRequestor()
_requestor.send_data(UPDATE_NOTICE, update)
else:
# the main process before startup installs the registry, or a test
logger.debug("No notice registry for %s", update["kind"])
except Exception:
logger.exception("Failed to update notice %s", update["kind"])
frigate/notices/registry.py
+187 -78
View File
@@ -1,49 +1,44 @@
"""Registry of active notices and per-kind occurrence counts."""
"""Registry of notices and per-kind occurrence counts."""
import logging
import threading
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any
from frigate.const import REPLAY_CAMERA_PREFIX
from frigate.models import Notice, NoticeStats
from frigate.notices.types import (
NOTICE_KINDS,
SEVERITY_ORDER,
NoticeMode,
notice_id,
)
from frigate.notices.types import CHECK_KINDS, NOTICE_KINDS, SEVERITY_ORDER, notice_id
logger = logging.getLogger(__name__)
class DismissResult(str, Enum):
ok = "ok"
not_found = "not_found"
not_dismissable = "not_dismissable"
# last_seen value of a state notice that has not been raised since startup
UNCONFIRMED = 0
@dataclass
class _HeldRepeats:
kind: str
count: int
last_seen: float
params: dict[str, Any]
class NoticeRegistry:
"""Owns the notice tables. Lives in the main process only.
Producers in the main process call it directly. Other processes send an
update_notice request that the dispatcher forwards here. The FastAPI app
runs inside the main process too, so the API's dismiss shares this exact
object, its lock, and its listeners.
Producers call the functions in frigate.notices, which reach this object
directly in the main process and through the dispatcher everywhere else.
It holds no rules for particular kinds; those live in NOTICE_KINDS.
"""
def __init__(self) -> None:
self._lock = threading.Lock()
self._listeners: list[Callable[[], None]] = []
# row id -> repeats of a batch_repeats kind that flush() writes
self._held: dict[str, _HeldRepeats] = {}
def subscribe(self, listener: Callable[[], None]) -> None:
"""Call listener after every change to the active list."""
"""Call listener after every change to the notices."""
self._listeners.append(listener)
def _notify(self) -> None:
@@ -53,6 +48,26 @@ class NoticeRegistry:
except Exception:
logger.exception("Notice listener failed")
def apply(self, update: dict[str, Any]) -> None:
"""Apply one update message built by frigate.notices."""
kind = update.get("kind")
if not isinstance(kind, str):
logger.warning("Ignoring notice update without a kind")
return
match update.get("action"):
case "raise":
self.raise_notice(
kind, scope=update.get("scope"), params=update.get("params")
)
case "resolve":
self.resolve(kind, update.get("scope"))
case "resolve_kind":
self.resolve_kind(kind)
case action:
logger.warning("Ignoring notice update with action %s", action)
def raise_notice(
self,
kind: str,
@@ -60,7 +75,12 @@ class NoticeRegistry:
scope: str | None = None,
params: dict[str, Any] | None = None,
) -> None:
"""Insert or update a notice. See the counting rule in the spec."""
"""Insert a notice or count another occurrence of it.
A dismissed notice stays dismissed when it is raised again, unless its
kind sets reopen_at_count. A kind that should come back after a
dismissal gives each episode its own scope.
"""
definition = NOTICE_KINDS.get(kind)
if definition is None:
@@ -82,6 +102,8 @@ class NoticeRegistry:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
# repeats held for a row that has since been deleted are stale
self._held.pop(row_id, None)
Notice.create(
id=row_id,
kind=kind,
@@ -92,28 +114,48 @@ class NoticeRegistry:
count=1,
dismissed_at=None,
)
self._bump_occurrences(kind, now)
elif definition.mode == NoticeMode.event:
Notice.update(
count=Notice.count + 1,
last_seen=now,
params=params,
dismissed_at=None,
).where(Notice.id == row_id).execute()
self._bump_occurrences(kind, now)
self._bump_occurrences(kind, 1, now)
if definition.keep_latest is not None:
self._prune(kind, definition.keep_latest)
elif not definition.counts_repeats:
return
elif definition.batch_repeats:
held = self._held.setdefault(row_id, _HeldRepeats(kind, 0, now, params))
held.count += 1
held.last_seen = now
held.params = params
return
else:
# one episode counts once; a repeat with nothing new is free
confirmed = existing.last_seen != UNCONFIRMED
if confirmed and existing.params == params:
return
Notice.update(last_seen=now, params=params).where(
Notice.id == row_id
).execute()
self._write_repeats(existing, kind, 1, now, params)
self._notify()
def flush(self) -> None:
"""Write the repeats that batch_repeats kinds held back."""
with self._lock:
held, self._held = self._held, {}
written = False
for row_id, repeats in held.items():
row = Notice.get_or_none(Notice.id == row_id)
# resolved while its repeats waited
if row is None:
continue
self._write_repeats(
row,
repeats.kind,
repeats.count,
repeats.last_seen,
repeats.params,
)
written = True
if written:
self._notify()
def resolve(self, kind: str, scope: str | None = None) -> None:
"""Delete a notice if present. Safe to call when it is absent."""
with self._lock:
@@ -124,8 +166,16 @@ class NoticeRegistry:
if deleted:
self._notify()
def resolve_kind(self, kind: str) -> None:
"""Delete every notice of a kind."""
with self._lock:
deleted = Notice.delete().where(Notice.kind == kind).execute()
if deleted:
self._notify()
def resolve_camera(self, camera: str) -> None:
"""Drop every camera-category notice scoped to a camera being deleted."""
"""Drop the notices and check dismissals of a camera being deleted."""
camera_kinds = [
key
for key, definition in NOTICE_KINDS.items()
@@ -142,33 +192,82 @@ class NoticeRegistry:
.execute()
)
# a stream id names its camera first; a config id ends with camera.<name>
for check in self.dismissed_checks():
check_id = check["id"]
if check_id.startswith(f"stream:{camera}:") or (
check_id.startswith("config:")
and check_id.endswith(f":camera.{camera}")
):
Notice.delete_by_id(check_id)
if deleted:
self._notify()
def dismiss(self, row_id: str) -> DismissResult:
"""Hide an event notice until it is raised again."""
def purge_dismissed(self) -> int:
"""Delete every dismissed row so each can show again. Returns how many."""
with self._lock:
return int(
Notice.delete().where(Notice.dismissed_at.is_null(False)).execute()
)
def dismiss(self, row_id: str) -> bool:
"""Hide a notice or check row for good. Returns False for an unknown id."""
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
return DismissResult.not_found
# a check row gets a notice row only once it is dismissed
kind, _, scope = row_id.partition(":")
definition = NOTICE_KINDS.get(existing.kind)
if kind not in CHECK_KINDS or not scope:
return False
if definition is None or definition.mode == NoticeMode.state:
return DismissResult.not_dismissable
now = datetime.now().timestamp()
Notice.create(
id=row_id,
kind=kind,
scope=scope,
params={},
first_seen=now,
last_seen=now,
count=1,
dismissed_at=now,
)
return True
now = datetime.now().timestamp()
Notice.update(dismissed_at=now).where(Notice.id == row_id).execute()
if existing.dismissed_at is not None:
return True
if existing.kind not in NOTICE_KINDS:
return False
Notice.update(dismissed_at=datetime.now().timestamp()).where(
Notice.id == row_id
).execute()
NoticeStats.update(dismissals=NoticeStats.dismissals + 1).where(
NoticeStats.kind == existing.kind
).execute()
self._notify()
return DismissResult.ok
return True
def dismissed_checks(self) -> list[dict[str, Any]]:
"""Dismissed config and stream check rows, newest first."""
rows = (
Notice.select()
.where(Notice.kind.in_(list(CHECK_KINDS))) # type: ignore[call-arg, arg-type, misc]
.order_by(Notice.dismissed_at.desc())
)
return [{"id": row.id, "dismissed_at": row.dismissed_at} for row in rows]
def active(self, include_dismissed: bool = False) -> list[dict[str, Any]]:
"""The active notices, most severe first, then most recent first."""
"""Notices most severe first, then most recent first.
Args:
include_dismissed: Also return dismissed notices, for the history view
"""
rows = []
for row in Notice.select():
@@ -180,18 +279,15 @@ class NoticeRegistry:
if row.dismissed_at is not None and not include_dismissed:
continue
if definition.mode == NoticeMode.state and row.last_seen == UNCONFIRMED:
continue
rows.append(
{
"id": row.id,
"kind": row.kind,
"mode": definition.mode.value,
"severity": definition.severity.value,
"category": definition.category,
"scope": row.scope,
"params": row.params,
"link": definition.link_for(row.params),
"first_seen": row.first_seen,
"last_seen": row.last_seen,
"count": row.count,
@@ -208,7 +304,7 @@ class NoticeRegistry:
return rows
def stats(self) -> list[dict[str, Any]]:
"""Lifetime counts per kind, for the UI meta line and later analytics."""
"""Lifetime counts per kind, for later analytics."""
return [
{
"kind": row.kind,
@@ -223,41 +319,54 @@ class NoticeRegistry:
if row.kind in NOTICE_KINDS
]
def mark_state_notices_unconfirmed(self) -> None:
"""Hide state notices until their producer raises them again.
def _write_repeats(
self,
row: Notice,
kind: str,
count: int,
last_seen: float,
params: dict[str, Any],
) -> None:
# called with the lock held
fields: dict[str, Any] = {
"count": row.count + count,
"last_seen": last_seen,
"params": params,
}
reopen_at = NOTICE_KINDS[kind].reopen_at_count
Rows already unconfirmed from the previous run are deleted. Hiding
rather than deleting keeps one episode that spans a restart at one
occurrence.
"""
state_kinds = [
key
for key, definition in NOTICE_KINDS.items()
if definition.mode == NoticeMode.state
]
if reopen_at is not None and row.count < reopen_at <= row.count + count:
fields["dismissed_at"] = None
with self._lock:
Notice.delete().where(
Notice.kind.in_(state_kinds), # type: ignore[call-arg, arg-type, misc]
Notice.last_seen == UNCONFIRMED,
).execute()
Notice.update(last_seen=UNCONFIRMED).where(
Notice.kind.in_(state_kinds) # type: ignore[call-arg, arg-type, misc]
).execute()
Notice.update(**fields).where(Notice.id == row.id).execute()
self._bump_occurrences(kind, count, last_seen)
def _bump_occurrences(self, kind: str, now: float) -> None:
def _prune(self, kind: str, keep: int) -> None:
# called with the lock held
newest = (
Notice.select(Notice.id)
.where(Notice.kind == kind)
.order_by(Notice.first_seen.desc())
.limit(keep)
)
Notice.delete().where(
Notice.kind == kind,
Notice.id.not_in(newest), # type: ignore[call-arg, misc]
).execute()
def _bump_occurrences(self, kind: str, count: int, now: float) -> None:
# called with the lock held
stats = NoticeStats.get_or_none(NoticeStats.kind == kind)
if stats is None:
NoticeStats.create(
kind=kind,
occurrences=1,
occurrences=count,
dismissals=0,
first_seen=now,
last_seen=now,
)
else:
NoticeStats.update(
occurrences=NoticeStats.occurrences + 1, last_seen=now
occurrences=NoticeStats.occurrences + count, last_seen=now
).where(NoticeStats.kind == kind).execute()
frigate/notices/types.py
+55 -17
View File
@@ -1,14 +1,8 @@
"""Notice kinds and the definitions that fix their mode, severity, and category."""
"""Notice kinds and the definitions that fix their severity and category."""
from dataclasses import dataclass
from enum import Enum
class NoticeMode(str, Enum):
"""Whether a producer resolves the notice itself or the user dismisses it."""
state = "state"
event = "event"
from typing import Any
class NoticeSeverity(str, Enum):
@@ -30,33 +24,77 @@ class NoticeKind:
Attributes:
key: Stable machine-readable name, also the translation key suffix
mode: state notices are resolved by their producer, event notices by the user
severity: error, warning, or info
category: what the scope names: camera, detector, and model kinds carry a scope
category: camera, detector, model, or system; a camera scope is a
camera name, and the UI shows it
link: app route or absolute URL for the row, filled in from params
counts_repeats: whether raising an existing notice counts another occurrence
batch_repeats: whether repeats wait in memory for the next flush
reopen_at_count: count at which a dismissed notice shows again
keep_latest: rows of this kind to keep; a new row drops the oldest
reportable: whether a future analytics reporter may send this kind's counts
"""
key: str
mode: NoticeMode
severity: NoticeSeverity
category: str
link: str | None = None
counts_repeats: bool = True
batch_repeats: bool = False
reopen_at_count: int | None = None
keep_latest: int | None = None
reportable: bool = True
def link_for(self, params: dict[str, Any]) -> str | None:
"""The link with params filled in, or None if a param is missing."""
if self.link is None:
return None
try:
return self.link.format(**params)
except (KeyError, IndexError, TypeError, ValueError):
return None
SCOPED_CATEGORIES = frozenset({"camera", "detector", "model"})
_KINDS = (
NoticeKind("ffmpeg_crash_loop", NoticeMode.state, NoticeSeverity.error, "camera"),
NoticeKind("detector_stuck", NoticeMode.event, NoticeSeverity.warning, "detector"),
NoticeKind(
"model_download_failed", NoticeMode.event, NoticeSeverity.error, "model"
"detector_stuck", NoticeSeverity.warning, "detector", link="/system#general"
),
NoticeKind("model_download_failed", NoticeSeverity.error, "model"),
NoticeKind(
"skipped_detections",
NoticeSeverity.warning,
"camera",
link="/system#cameras",
),
NoticeKind("shm_too_low", NoticeSeverity.warning, "system", link="/system#storage"),
# one row per user per burst; the login log lines carry the address
NoticeKind(
"failed_login",
NoticeSeverity.warning,
"system",
link="/logs",
batch_repeats=True,
reopen_at_count=5,
keep_latest=100,
),
# one row per release, so a dismissal lasts until the next release
NoticeKind(
"update_available",
NoticeSeverity.info,
"system",
link="https://github.com/blakeblackshear/frigate/releases/tag/v{version}",
counts_repeats=False,
keep_latest=1,
),
NoticeKind("retention_unmet", NoticeMode.state, NoticeSeverity.error, "storage"),
NoticeKind("update_available", NoticeMode.state, NoticeSeverity.info, "system"),
)
NOTICE_KINDS: dict[str, NoticeKind] = {kind.key: kind for kind in _KINDS}
# the Health tab builds config and stream check rows in the browser, so a notice
# row of these kinds only records a dismissal; its other fields are placeholders
CHECK_KINDS = frozenset({"config", "stream"})
def notice_id(kind: str, scope: str | None) -> str:
"""The row id for a kind and optional scope."""
frigate/stats/emitter.py
+91 -10
View File
@@ -11,7 +11,7 @@ from typing import Any
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.const import FREQUENCY_STATS_POINTS
from frigate.notices.registry import NoticeRegistry
from frigate.notices import flush_notices, raise_notice, resolve_kind, resolve_notice
from frigate.stats.hardware import HardwareStats
from frigate.stats.prometheus import update_metrics
from frigate.stats.util import get_latest_version, is_newer_version, stats_snapshot
@@ -26,6 +26,46 @@ MAX_STATS_POINTS = 80
# how often to ask GitHub for the latest release
VERSION_REFRESH_S = 24 * 60 * 60
# a camera skipping at least this percent of its frames is falling behind
SKIPPED_DETECTIONS_PCT = 5
# for at least this long before it becomes a notice
SKIPPED_DETECTIONS_HOLD_S = 60
# detectors warm up after a start; the status bar waits this long too
STARTUP_GRACE_S = 120
class SkippedDetectionsTracker:
"""Finds cameras whose skipped share stays high long enough for a notice."""
def __init__(self) -> None:
self._since: dict[str, float] = {}
self._raised: set[str] = set()
def update(self, cameras: dict[str, dict[str, Any]], now: float) -> list[str]:
"""Return the cameras whose episode qualified on this sample."""
qualified: list[str] = []
# a removed camera that comes back starts a new episode
for camera in self._since.keys() - cameras.keys():
self._since.pop(camera)
self._raised.discard(camera)
for camera, camera_stats in cameras.items():
if camera_stats["skipped_pct"] < SKIPPED_DETECTIONS_PCT:
self._since.pop(camera, None)
self._raised.discard(camera)
continue
since = self._since.setdefault(camera, now)
if camera not in self._raised and now - since >= SKIPPED_DETECTIONS_HOLD_S:
self._raised.add(camera)
qualified.append(camera)
return qualified
class StatsEmitter(threading.Thread):
def __init__(
@@ -33,15 +73,18 @@ class StatsEmitter(threading.Thread):
config: FrigateConfig,
stats_tracking: StatsTrackingTypes,
stop_event: MpEvent,
notice_registry: NoticeRegistry | None = None,
):
super().__init__(name="frigate_stats_emitter")
self.config = config
self.stats_tracking = stats_tracking
self.stop_event = stop_event
self.notice_registry = notice_registry
self.hardware_stats = HardwareStats(config)
self.stats_history: list[dict[str, Any]] = []
self.skipped_detections = SkippedDetectionsTracker()
# the shm notice's params as last sent, so only a change is written
self._shm_checked = False
self._shm_params: dict[str, Any] | None = None
# create communication for stats
self.requestor = InterProcessRequestor()
@@ -134,9 +177,6 @@ class StatsEmitter(threading.Thread):
def _check_update_notice(self) -> None:
"""Raise or resolve the update notice from the tracked latest version."""
if self.notice_registry is None:
return
latest = self.stats_tracking["latest_frigate_version"]
# a failed lookup says nothing about whether an update exists
@@ -144,11 +184,9 @@ class StatsEmitter(threading.Thread):
return
if is_newer_version(VERSION, latest):
self.notice_registry.raise_notice(
"update_available", params={"version": latest}
)
raise_notice("update_available", scope=latest, params={"version": latest})
else:
self.notice_registry.resolve("update_available")
resolve_kind("update_available")
def _refresh_latest_version(self) -> None:
"""Refresh the latest release on a daemon thread so the request never stalls stats."""
@@ -166,6 +204,46 @@ class StatsEmitter(threading.Thread):
target=refresh, name="frigate_version_check", daemon=True
).start()
def _update_shm_notice(self, shm: dict[str, Any]) -> None:
"""Raise the shm notice while /dev/shm is smaller than the cameras need."""
params = (
{"total": shm["total"], "min": shm["min_shm"]}
if shm and shm["total"] < shm["min_shm"]
else None
)
# the first tick always sends, so a row the last run left is raised
# again or cleared
if self._shm_checked and params == self._shm_params:
return
self._shm_checked = True
self._shm_params = params
if params is None:
resolve_notice("shm_too_low")
else:
raise_notice("shm_too_low", params=params)
def _update_notices(self, stats: dict[str, Any], now: float) -> None:
"""Update notices based on current stats or time."""
# skipped detections
if stats["service"]["uptime"] >= STARTUP_GRACE_S:
for camera in self.skipped_detections.update(stats["cameras"], now):
raise_notice(
"skipped_detections",
scope=camera,
params={"pct": stats["cameras"][camera]["skipped_pct"]},
)
# shm too small for the cameras
self._update_shm_notice(stats["service"]["storage"]["/dev/shm"])
# repeats of batched kinds, such as failed logins
flush_notices()
# add any additional notice types here
def run(self) -> None:
time.sleep(10)
self._check_update_notice()
@@ -186,11 +264,14 @@ class StatsEmitter(threading.Thread):
)
self.stats_history.append(stats)
self.stats_history = self.stats_history[-MAX_STATS_POINTS:]
self._update_notices(stats, time.time())
if counter == 0:
self.requestor.send_data("stats", json.dumps(stats))
logger.debug("Finished stats collection")
# write the repeats held back since the last tick
flush_notices()
self.hardware_stats.stop()
logger.info("Exiting stats emitter...")
frigate/stats/util.py
+28 -1
View File
@@ -6,7 +6,7 @@ import shutil
import time
from json import JSONDecodeError
from multiprocessing.managers import DictProxy
from typing import Any
from typing import TYPE_CHECKING, Any
import requests
from requests.exceptions import RequestException
@@ -24,6 +24,9 @@ from frigate.util.services import (
)
from frigate.version import VERSION
if TYPE_CHECKING:
from frigate.storage import StorageMaintainer
logger = logging.getLogger(__name__)
@@ -88,6 +91,7 @@ def stats_init(
embeddings_metrics: DataProcessorMetrics,
detectors: dict[str, ObjectDetectProcess],
processes: dict[str, int],
storage_maintainer: "StorageMaintainer | None" = None,
) -> StatsTrackingTypes:
stats_tracking: StatsTrackingTypes = {
"camera_metrics": camera_metrics,
@@ -97,6 +101,7 @@ def stats_init(
"latest_frigate_version": get_latest_version(config),
"last_updated": int(time.time()),
"processes": processes,
"storage_maintainer": storage_maintainer,
}
return stats_tracking
@@ -216,6 +221,18 @@ def embeddings_stats(
return stats
def skipped_percent(skipped_fps: float, camera_fps: float, enabled: bool) -> float:
"""Percent of a camera's frames dropped before detection.
camera_fps counts the dropped frames too. A disabled camera keeps its last
readings, so it reports zero.
"""
if not enabled or camera_fps <= 0:
return 0.0
return round(skipped_fps / camera_fps * 100, 1)
def stats_snapshot(
config: FrigateConfig,
stats_tracking: StatsTrackingTypes,
@@ -273,6 +290,11 @@ def stats_snapshot(
"camera_fps": round(camera_stats.camera_fps.value, 2),
"process_fps": round(camera_stats.process_fps.value, 2),
"skipped_fps": round(camera_stats.skipped_fps.value, 2),
"skipped_pct": skipped_percent(
camera_stats.skipped_fps.value,
current_fps,
config.cameras[name].enabled,
),
"detection_fps": round(camera_stats.detection_fps.value, 2),
"detection_enabled": config.cameras[name].detect.enabled,
"pid": pid,
@@ -301,12 +323,17 @@ def stats_snapshot(
if bandwidth_stats:
stats["bandwidth_usages"] = bandwidth_stats
storage_maintainer = stats_tracking.get("storage_maintainer")
stats["service"] = {
"uptime": (int(time.time()) - stats_tracking["started"]),
"version": VERSION,
"latest_version": stats_tracking["latest_frigate_version"],
"storage": {},
"last_updated": int(time.time()),
"retention_unmet": bool(
storage_maintainer and storage_maintainer.retention_unmet
),
}
for path in [RECORD_DIR, CLIPS_DIR, CACHE_DIR]:
frigate/storage.py
+8 -21
View File
@@ -20,7 +20,6 @@ from frigate.const import (
STREAM_TYPE_SUB,
)
from frigate.models import Event, Recordings
from frigate.notices.registry import NoticeRegistry
from frigate.util.builtin import clear_and_unlink
logger = logging.getLogger(__name__)
@@ -35,17 +34,14 @@ BANDWIDTH_SAMPLE_TARGET = 50
class StorageMaintainer(threading.Thread):
"""Maintain frigates recording storage."""
def __init__(
self,
config: FrigateConfig,
stop_event: MpEvent,
notice_registry: NoticeRegistry | None = None,
) -> None:
def __init__(self, config: FrigateConfig, stop_event: MpEvent) -> None:
super().__init__(name="storage_maintainer")
self.config = config
self.stop_event = stop_event
self.notice_registry = notice_registry
self.camera_storage_stats: dict[str, dict] = {}
# read by stats; true while maintenance has to delete retained recordings
self.retention_unmet = False
self.config_subscriber = CameraConfigUpdateSubscriber(
self.config,
self.config.cameras,
@@ -350,19 +346,10 @@ class StorageMaintainer(threading.Thread):
# this file was not found so we must assume no space was cleaned up
pass
if self.notice_registry is not None:
self.notice_registry.raise_notice(
"retention_unmet",
params={
"needed_mb": round(float(hourly_bandwidth), 1),
"cleared_mb": round(float(deleted_segments_size), 1),
},
)
self.retention_unmet = True
else:
logger.info(f"Cleaned up {deleted_segments_size:.2f} MB of recordings")
if self.notice_registry is not None:
self.notice_registry.resolve("retention_unmet")
self.retention_unmet = False
logger.debug(f"Expiring {len(deleted_recordings)} recordings")
# delete up to 100,000 at a time
@@ -448,5 +435,5 @@ class StorageMaintainer(threading.Thread):
"Less than 1 hour of recording space left, running storage maintenance..."
)
self.reduce_storage_consumption()
elif self.notice_registry is not None:
self.notice_registry.resolve("retention_unmet")
else:
self.retention_unmet = False
frigate/test/http_api/test_http_auth.py
+122 -2
View File
@@ -1,9 +1,16 @@
"""Tests for authentication endpoints."""
import os
from unittest.mock import patch
import unittest
from unittest.mock import ANY, patch
from frigate.api.auth import hash_password
from frigate.api.auth import (
FAILED_LOGIN_BURST_GAP_S,
MAX_NOTICE_USERNAME,
MAX_OPEN_BURSTS,
FailedLoginTracker,
hash_password,
)
from frigate.const import JWT_SECRET_ENV_VAR
from frigate.models import User
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
@@ -43,3 +50,116 @@ class TestHttpAuth(BaseTestHttp):
)
assert response.status_code == 401
assert any("Login failed" in m and "admin" in m for m in logs.output)
@patch.dict(os.environ, {JWT_SECRET_ENV_VAR: "test-secret"})
class TestFailedLoginNotices(BaseTestHttp):
def setUp(self):
super().setUp([User])
self.app = super().create_app()
User.insert(
username="admin",
password_hash=hash_password("correct-horse-battery", iterations=1000),
role="admin",
notification_tokens=[],
).execute()
tracker_patch = patch("frigate.api.auth.failed_logins")
self.failed_logins = tracker_patch.start()
self.addCleanup(tracker_patch.stop)
def tearDown(self):
User.delete().execute()
super().tearDown()
def _login(self, user: str, password: str) -> int:
with AuthTestClient(self.app) as client:
response = client.post("/login", json={"user": user, "password": password})
return response.status_code
def test_unknown_user_is_recorded(self):
self.assertEqual(self._login("ghost", "irrelevant"), 401)
self.failed_logins.record.assert_called_once_with("ghost", ANY)
def test_bad_password_is_recorded_as_known(self):
self.assertEqual(self._login("admin", "wrong"), 401)
self.failed_logins.record.assert_called_once_with("admin", ANY, known=True)
def test_success_is_not_recorded(self):
self.assertEqual(self._login("admin", "correct-horse-battery"), 200)
self.failed_logins.record.assert_not_called()
class TestFailedLoginTracker(unittest.TestCase):
def setUp(self):
self.tracker = FailedLoginTracker()
raise_patch = patch("frigate.api.auth.raise_notice")
self.raise_notice = raise_patch.start()
self.addCleanup(raise_patch.stop)
def _scopes(self) -> list[str]:
return [call.kwargs["scope"] for call in self.raise_notice.call_args_list]
def _fill(self, now: float) -> None:
for index in range(MAX_OPEN_BURSTS):
self.tracker.record(f"ghost{index}", now)
def test_attempts_inside_the_gap_share_a_burst(self):
for now in (1000.0, 1010.0, 1299.0):
self.tracker.record("admin", now)
self.raise_notice.assert_called_with(
"failed_login", scope="admin:1000", params={"user": "admin"}
)
self.assertEqual(self._scopes(), ["admin:1000"] * 3)
def test_each_user_gets_their_own_burst(self):
self.tracker.record("admin", 1000.0)
self.tracker.record("ghost", 1010.0)
self.tracker.record("admin", 1020.0)
self.assertEqual(self._scopes(), ["admin:1000", "ghost:1010", "admin:1000"])
def test_a_quiet_gap_opens_a_new_burst(self):
self.tracker.record("admin", 1000.0)
self.tracker.record("admin", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
self.assertEqual(self._scopes(), ["admin:1000", "admin:1300"])
def test_the_gap_runs_from_the_latest_attempt(self):
for now in (1000.0, 1200.0, 1450.0):
self.tracker.record("admin", now)
self.assertEqual(self._scopes(), ["admin:1000"] * 3)
def test_quiet_users_are_forgotten(self):
self.tracker.record("ghost", 1000.0)
self.tracker.record("admin", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
self.assertEqual(list(self.tracker._bursts), ["admin"])
def test_a_long_username_is_cut(self):
self.tracker.record("x" * 500, 1000.0)
user = self.raise_notice.call_args.kwargs["params"]["user"]
self.assertEqual(len(user), MAX_NOTICE_USERNAME)
def test_unknown_users_past_the_cap_get_no_notice(self):
self._fill(1000.0)
self.tracker.record("one-too-many", 1001.0)
self.tracker.record("ghost0", 1002.0)
self.assertEqual(self.raise_notice.call_count, MAX_OPEN_BURSTS + 1)
self.assertEqual(self._scopes()[-1], "ghost0:1000")
def test_known_users_skip_the_cap(self):
self._fill(1000.0)
self.tracker.record("admin", 1001.0, known=True)
self.assertEqual(self._scopes()[-1], "admin:1001")
def test_quiet_bursts_free_their_slots(self):
self._fill(1000.0)
self.tracker.record("late", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
self.assertEqual(self._scopes()[-1], "late:1300")
frigate/test/http_api/test_http_notices.py
+87 -15
View File
@@ -6,6 +6,8 @@ from frigate.models import Notice, NoticeStats
from frigate.notices.registry import NoticeRegistry
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
CONFIG_CHECK = "config:detect:fps-greater-than-five:camera.garage"
class TestHttpNotices(BaseTestHttp):
def setUp(self):
@@ -17,20 +19,32 @@ class TestHttpNotices(BaseTestHttp):
return AuthTestClient(self.app)
def test_get_notices_orders_by_severity(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.raise_notice(
"model_download_failed",
scope="yolo/model.onnx",
params={"file": "model.onnx", "error": "timeout"},
)
with self.client() as client:
response = client.get("/notices")
self.assertEqual(response.status_code, 200)
self.assertEqual(
[n["id"] for n in response.json()], ["retention_unmet", "detector_stuck"]
[n["id"] for n in response.json()],
["model_download_failed:yolo/model.onnx", "detector_stuck:ov"],
)
self.assertEqual(response.json()[0]["severity"], "error")
self.assertEqual(
[n["link"] for n in response.json()], [None, "/system#general"]
)
def test_get_stats(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
with self.client() as client:
response = client.get("/notices/stats")
@@ -39,15 +53,33 @@ class TestHttpNotices(BaseTestHttp):
self.assertEqual(response.json()[0]["kind"], "detector_stuck")
self.assertEqual(response.json()[0]["occurrences"], 1)
def test_dismiss_event(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
def test_dismiss_hides_and_history_keeps_it(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
with self.client() as client:
response = client.post("/notices/detector_stuck/dismiss")
response = client.post("/notices/detector_stuck:ov/dismiss")
listed = client.get("/notices").json()
history = client.get("/notices", params={"include_dismissed": True}).json()
self.assertEqual(response.status_code, 200)
self.assertEqual(listed, [])
self.assertEqual([n["id"] for n in history], ["detector_stuck:ov"])
self.assertIsNotNone(history[0]["dismissed_at"])
def test_dismiss_an_id_with_a_slash(self):
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
with self.client() as client:
response = client.post(
"/notices/model_download_failed:yolo/model.onnx/dismiss"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(self.registry.active(), [])
def test_dismiss_unknown_is_404(self):
with self.client() as client:
@@ -55,14 +87,6 @@ class TestHttpNotices(BaseTestHttp):
self.assertEqual(response.status_code, 404)
def test_dismiss_state_is_400(self):
self.registry.raise_notice("retention_unmet", params={})
with self.client() as client:
response = client.post("/notices/retention_unmet/dismiss")
self.assertEqual(response.status_code, 400)
def test_requires_admin(self):
with TestClient(self.app) as client:
response = client.get(
@@ -70,3 +94,51 @@ class TestHttpNotices(BaseTestHttp):
)
self.assertEqual(response.status_code, 403)
def test_dismissed_checks_are_listed_once(self):
with self.client() as client:
first = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
again = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
listed = client.get("/notices/dismissed_checks").json()
history = client.get("/notices", params={"include_dismissed": True}).json()
self.assertEqual(first.status_code, 200)
self.assertEqual(again.status_code, 200)
self.assertEqual([check["id"] for check in listed], [CONFIG_CHECK])
self.assertIsNotNone(listed[0]["dismissed_at"])
# the Health tab builds the row itself, so it never comes back as a notice
self.assertEqual(history, [])
def test_dismissed_checks_require_admin(self):
with TestClient(self.app) as client:
response = client.get(
"/notices/dismissed_checks",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
self.assertEqual(response.status_code, 403)
def test_purge_dismissed_clears_notices_and_checks(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss(CONFIG_CHECK)
with self.client() as client:
response = client.delete("/notices/dismissed")
history = client.get("/notices", params={"include_dismissed": True}).json()
checks = client.get("/notices/dismissed_checks").json()
self.assertEqual(response.status_code, 200)
self.assertEqual(history, [])
self.assertEqual(checks, [])
def test_purge_dismissed_requires_admin(self):
with TestClient(self.app) as client:
response = client.delete(
"/notices/dismissed",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
self.assertEqual(response.status_code, 403)
frigate/test/test_camera_watchdog.py
+1 -77
View File
@@ -6,11 +6,7 @@ from unittest.mock import MagicMock, patch
from frigate.config import FrigateConfig
from frigate.const import STREAM_TYPE_MAIN, STREAM_TYPE_SUB
from frigate.video.ffmpeg import (
CRASH_LOOP_RECOVERY_S,
CRASH_LOOP_RESTARTS,
CameraWatchdog,
)
from frigate.video.ffmpeg import CameraWatchdog
def build_watchdog(
@@ -228,75 +224,3 @@ class TestCameraWatchdogStreamHealth(unittest.TestCase):
assert watchdog.record_stale_threshold[STREAM_TYPE_MAIN] == 120
assert watchdog.record_stale_threshold[STREAM_TYPE_SUB] == 150
class TestCameraWatchdogCrashLoop(unittest.TestCase):
"""The crash loop notice follows restarts in the trailing hour."""
def _build_watchdog(self) -> CameraWatchdog:
return build_watchdog()
def _notice_payloads(self, watchdog: CameraWatchdog) -> list[dict]:
return [
call.args[1]
for call in watchdog.requestor.send_data.call_args_list
if call.args[0] == "update_notice"
]
def test_below_threshold_raises_nothing(self):
watchdog = self._build_watchdog()
now = datetime.now().timestamp()
watchdog.reconnect_timestamps.extend([now] * (CRASH_LOOP_RESTARTS - 1))
watchdog.capture_thread = MagicMock()
watchdog.capture_thread.is_alive.return_value = False
watchdog._update_crash_loop_notice(now)
self.assertEqual(self._notice_payloads(watchdog), [])
def test_threshold_raises_once_per_restart_count(self):
watchdog = self._build_watchdog()
now = datetime.now().timestamp()
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
watchdog.capture_thread = MagicMock()
watchdog.capture_thread.is_alive.return_value = False
watchdog._update_crash_loop_notice(now)
watchdog._update_crash_loop_notice(now)
payloads = self._notice_payloads(watchdog)
self.assertEqual(len(payloads), 1)
self.assertEqual(payloads[0]["action"], "raise")
self.assertEqual(payloads[0]["kind"], "ffmpeg_crash_loop")
self.assertEqual(payloads[0]["scope"], "front_door")
self.assertEqual(payloads[0]["params"], {"restarts": CRASH_LOOP_RESTARTS})
def test_recovery_window_resolves(self):
watchdog = self._build_watchdog()
now = datetime.now().timestamp()
# the last restart is at `now`, so recovery is measured from there
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
watchdog.capture_thread = MagicMock()
watchdog.capture_thread.is_alive.return_value = False
watchdog._update_crash_loop_notice(now)
watchdog.capture_thread.is_alive.return_value = True
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S - 1)
self.assertEqual(len(self._notice_payloads(watchdog)), 1)
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S + 1)
payloads = self._notice_payloads(watchdog)
self.assertEqual(len(payloads), 2)
self.assertEqual(payloads[1]["action"], "resolve")
self.assertFalse(watchdog.crash_loop_raised)
def test_disable_resolves(self):
watchdog = self._build_watchdog()
watchdog.crash_loop_raised = True
watchdog._resolve_crash_loop()
payloads = self._notice_payloads(watchdog)
self.assertEqual(payloads[-1]["action"], "resolve")
self.assertFalse(watchdog.crash_loop_raised)
frigate/test/test_dispatcher_runtime_state.py
+15 -24
View File
@@ -518,7 +518,7 @@ class TestNoticeWiring(unittest.TestCase):
def setUp(self) -> None:
self.registry = MagicMock()
self.registry.active.return_value = [{"id": "retention_unmet"}]
self.registry.active.return_value = [{"id": "detector_stuck:ov"}]
config = MagicMock()
config.cameras = {}
@@ -540,37 +540,28 @@ class TestNoticeWiring(unittest.TestCase):
self.dispatcher._publish_notices
)
def test_raise_request_reaches_registry(self) -> None:
self.dispatcher._receive(
"update_notice",
{
"action": "raise",
"kind": "ffmpeg_crash_loop",
"scope": "front_door",
"params": {"restarts": 5},
},
)
def test_update_request_reaches_registry(self) -> None:
update = {
"action": "raise",
"kind": "model_download_failed",
"scope": "yolo/model.onnx",
"params": {"file": "model.onnx", "error": "timeout"},
}
self.registry.raise_notice.assert_called_once_with(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
)
self.dispatcher._receive("update_notice", update)
def test_resolve_request_reaches_registry(self) -> None:
self.dispatcher._receive(
"update_notice",
{"action": "resolve", "kind": "retention_unmet", "scope": None},
)
self.registry.resolve.assert_called_once_with("retention_unmet", None)
self.registry.apply.assert_called_once_with(update)
def test_malformed_request_does_not_raise(self) -> None:
self.registry.raise_notice.side_effect = RuntimeError("boom")
self.registry.apply.side_effect = RuntimeError("boom")
self.dispatcher._receive("update_notice", "not a dict")
self.dispatcher._receive(
"update_notice", {"action": "raise", "kind": "x", "params": {}}
)
self.registry.apply.assert_called_once()
def test_publish_local_skips_mqtt(self) -> None:
mqtt = MagicMock(spec=MqttClient)
other = MagicMock()
@@ -587,7 +578,7 @@ class TestNoticeWiring(unittest.TestCase):
self.dispatcher._publish_notices()
self.dispatcher.publish_local.assert_called_once_with(
"notices", '[{"id": "retention_unmet"}]'
"notices", '[{"id": "detector_stuck:ov"}]'
)
def test_snapshot_includes_notices(self) -> None:
@@ -599,4 +590,4 @@ class TestNoticeWiring(unittest.TestCase):
self.dispatcher.publish_runtime_snapshot(publisher)
publisher.assert_any_call("notices", '[{"id": "retention_unmet"}]', False)
publisher.assert_any_call("notices", '[{"id": "detector_stuck:ov"}]', False)
frigate/test/test_model_downloader.py
+26 -26
View File
@@ -13,13 +13,12 @@ class TestModelDownloadNotice(unittest.TestCase):
def setUp(self):
self.download_path = tempfile.mkdtemp()
downloader.last_download_error.clear()
def _notice_calls(self, requestor: MagicMock) -> list[dict]:
return [
call.args[1]
for call in requestor.send_data.call_args_list
if call.args[0] == "update_notice"
]
raise_patch = patch("frigate.util.downloader.raise_notice")
resolve_patch = patch("frigate.util.downloader.resolve_notice")
self.raise_notice = raise_patch.start()
self.resolve_notice = resolve_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
def _downloader(self, download_func) -> ModelDownloader:
with patch("frigate.util.downloader.InterProcessRequestor"):
@@ -36,15 +35,16 @@ class TestModelDownloadNotice(unittest.TestCase):
with self.assertRaises(RuntimeError):
model_downloader._download_models()
notices = self._notice_calls(model_downloader.requestor)
self.assertEqual(len(notices), 1)
self.assertEqual(notices[0]["action"], "raise")
self.assertEqual(notices[0]["kind"], "model_download_failed")
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
self.assertEqual(
notices[0]["params"],
{"file": "facedet.onnx", "error": "HTTP 503 from upstream"},
self.raise_notice.assert_called_once_with(
"model_download_failed",
scope="facedet/facedet.onnx",
params={
"file": "facedet.onnx",
"model": "facedet",
"error": "HTTP 503 from upstream",
},
)
self.resolve_notice.assert_not_called()
def test_swallowing_download_that_leaves_no_file_reports(self):
target = os.path.join(self.download_path, "facedet.onnx")
@@ -62,10 +62,11 @@ class TestModelDownloadNotice(unittest.TestCase):
model_downloader = self._downloader(swallowing)
model_downloader._download_models()
notices = self._notice_calls(model_downloader.requestor)
self.assertEqual(len(notices), 1)
self.assertEqual(notices[0]["action"], "raise")
self.assertEqual(notices[0]["params"], {"file": "facedet.onnx", "error": "dns"})
self.raise_notice.assert_called_once()
self.assertEqual(
self.raise_notice.call_args.kwargs["params"],
{"file": "facedet.onnx", "model": "facedet", "error": "dns"},
)
self.assertFalse(os.path.exists(target))
def test_success_resolves(self):
@@ -76,10 +77,10 @@ class TestModelDownloadNotice(unittest.TestCase):
model_downloader = self._downloader(succeeding)
model_downloader._download_models()
notices = self._notice_calls(model_downloader.requestor)
self.assertEqual(len(notices), 1)
self.assertEqual(notices[0]["action"], "resolve")
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
self.raise_notice.assert_not_called()
self.resolve_notice.assert_called_once_with(
"model_download_failed", "facedet/facedet.onnx"
)
def test_a_sibling_downloader_only_resolves_its_own_files(self):
"""PaddleOCR, facedet and jina each spread one model_name over several
@@ -96,9 +97,8 @@ class TestModelDownloadNotice(unittest.TestCase):
sibling._download_models()
self.assertEqual(
[n["scope"] for n in self._notice_calls(sibling.requestor)],
["paddleocr-onnx/det.onnx"],
self.resolve_notice.assert_called_once_with(
"model_download_failed", "paddleocr-onnx/det.onnx"
)
frigate/test/test_notice_facade.py
+85
View File
@@ -0,0 +1,85 @@
"""Tests for raising and resolving notices through frigate.notices."""
import unittest
from unittest.mock import MagicMock, patch
from frigate import notices
class TestNoticeFacade(unittest.TestCase):
def tearDown(self):
notices.install_registry(None)
def test_the_installed_registry_applies_each_update(self):
registry = MagicMock()
notices.install_registry(registry)
notices.raise_notice("detector_stuck", scope="ov", params={"detector": "ov"})
notices.resolve_notice("detector_stuck", "ov")
notices.resolve_kind("update_available")
self.assertEqual(
[call.args[0] for call in registry.apply.call_args_list],
[
{
"action": "raise",
"kind": "detector_stuck",
"scope": "ov",
"params": {"detector": "ov"},
},
{"action": "resolve", "kind": "detector_stuck", "scope": "ov"},
{"action": "resolve_kind", "kind": "update_available"},
],
)
def test_a_child_process_sends_over_ipc(self):
with (
patch.object(notices.mp, "parent_process", return_value=MagicMock()),
patch.object(notices, "InterProcessRequestor") as requestor,
patch.object(notices, "_requestor", None),
):
notices.raise_notice("detector_stuck", scope="ov")
notices.resolve_notice("detector_stuck", "ov")
# one requestor serves every call in the process
requestor.assert_called_once_with()
send_data = requestor.return_value.send_data
self.assertEqual(send_data.call_count, 2)
self.assertEqual(
send_data.call_args_list[0].args,
(
"update_notice",
{
"action": "raise",
"kind": "detector_stuck",
"scope": "ov",
"params": {},
},
),
)
def test_the_main_process_without_a_registry_drops_updates(self):
with (
patch.object(notices.mp, "parent_process", return_value=None),
patch.object(notices, "InterProcessRequestor") as requestor,
):
notices.raise_notice("detector_stuck", scope="ov")
requestor.assert_not_called()
def test_a_failing_registry_is_logged_not_raised(self):
registry = MagicMock()
registry.apply.side_effect = RuntimeError("db")
notices.install_registry(registry)
with self.assertLogs("frigate.notices", level="ERROR"):
notices.raise_notice("detector_stuck", scope="ov")
def test_flush_reaches_only_an_installed_registry(self):
notices.flush_notices()
registry = MagicMock()
notices.install_registry(registry)
notices.flush_notices()
registry.flush.assert_called_once_with()
frigate/test/test_notice_registry.py
+298 -111
View File
@@ -3,51 +3,53 @@
import logging
import os
import unittest
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from peewee_migrate import Router
from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqliteq import SqliteQueueDatabase
from frigate.models import Notice, NoticeStats
from frigate.notices.registry import DismissResult, NoticeRegistry
from frigate.notices.types import (
NOTICE_KINDS,
NoticeMode,
NoticeSeverity,
notice_id,
)
from frigate.notices.registry import NoticeRegistry
from frigate.notices.types import NOTICE_KINDS, NoticeKind, NoticeSeverity, notice_id
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
# kinds that switch on the lifecycle knobs, so the tests do not depend on the
# values the real catalog picks
BATCHED = NoticeKind(
"batched", NoticeSeverity.warning, "system", batch_repeats=True, reopen_at_count=3
)
PRUNED = NoticeKind("pruned", NoticeSeverity.info, "system", keep_latest=2)
TEST_KINDS = {kind.key: kind for kind in (BATCHED, PRUNED)}
class TestNoticeKinds(unittest.TestCase):
def test_every_kind_is_keyed_by_its_own_key(self):
for key, definition in NOTICE_KINDS.items():
self.assertEqual(key, definition.key)
def test_first_producers_are_defined(self):
self.assertEqual(NOTICE_KINDS["ffmpeg_crash_loop"].mode, NoticeMode.state)
self.assertEqual(
NOTICE_KINDS["ffmpeg_crash_loop"].severity, NoticeSeverity.error
)
self.assertEqual(NOTICE_KINDS["detector_stuck"].mode, NoticeMode.event)
self.assertEqual(
NOTICE_KINDS["detector_stuck"].severity, NoticeSeverity.warning
)
self.assertEqual(NOTICE_KINDS["model_download_failed"].mode, NoticeMode.event)
self.assertEqual(NOTICE_KINDS["retention_unmet"].mode, NoticeMode.state)
self.assertEqual(NOTICE_KINDS["update_available"].mode, NoticeMode.state)
self.assertEqual(NOTICE_KINDS["update_available"].severity, NoticeSeverity.info)
def test_notice_id_with_and_without_scope(self):
self.assertEqual(notice_id("retention_unmet", None), "retention_unmet")
self.assertEqual(notice_id("failed_login", None), "failed_login")
self.assertEqual(
notice_id("ffmpeg_crash_loop", "front_door"),
"ffmpeg_crash_loop:front_door",
notice_id("skipped_detections", "front_door"),
"skipped_detections:front_door",
)
def test_link_is_filled_in_from_params(self):
kind = NoticeKind(
"k", NoticeSeverity.info, "system", link="https://x/v{version}"
)
class TestNoticeRegistry(unittest.TestCase):
self.assertEqual(kind.link_for({"version": "0.19.1"}), "https://x/v0.19.1")
self.assertIsNone(kind.link_for({}))
def test_a_kind_without_a_link(self):
kind = NoticeKind("k", NoticeSeverity.info, "system")
self.assertIsNone(kind.link_for({"version": "0.19.1"}))
class RegistryTestCase(unittest.TestCase):
def setUp(self):
migrate_db = SqliteExtDatabase("test.db")
del logging.getLogger("peewee_migrate").handlers[:]
@@ -70,132 +72,128 @@ class TestNoticeRegistry(unittest.TestCase):
except OSError:
pass
def _raise_at(self, timestamp: float, kind: str, **kwargs) -> None:
with patch("frigate.notices.registry.datetime") as mock_datetime:
mock_datetime.now.return_value.timestamp.return_value = timestamp
self.registry.raise_notice(kind, **kwargs)
class TestNoticeRegistry(RegistryTestCase):
def test_raise_inserts_and_counts_one_occurrence(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
active = self.registry.active()
self.assertEqual(len(active), 1)
self.assertEqual(active[0]["id"], "ffmpeg_crash_loop:front_door")
self.assertEqual(active[0]["severity"], "error")
self.assertEqual(active[0]["mode"], "state")
self.assertEqual(active[0]["id"], "skipped_detections:front_door")
self.assertEqual(active[0]["severity"], "warning")
self.assertEqual(active[0]["category"], "camera")
self.assertEqual(active[0]["scope"], "front_door")
self.assertEqual(active[0]["params"], {"restarts": 5})
self.assertEqual(active[0]["params"], {"pct": 12.5})
self.assertEqual(active[0]["link"], "/system#cameras")
self.assertEqual(active[0]["count"], 1)
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
def test_state_re_raise_with_new_params_updates_without_counting(self):
def test_re_raise_counts_and_updates_params(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
self.listener.reset_mock()
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 6}
"skipped_detections", scope="front_door", params={"pct": 8.0}
)
active = self.registry.active()
self.assertEqual(active[0]["count"], 1)
self.assertEqual(active[0]["params"], {"restarts": 6})
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
self.assertEqual(active[0]["count"], 2)
self.assertEqual(active[0]["params"], {"pct": 8.0})
self.assertEqual(self.registry.stats()[0]["occurrences"], 2)
def test_state_re_raise_with_same_params_is_a_no_op(self):
def test_repeat_of_a_non_counting_kind_is_a_no_op(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
before = self.registry.active()[0]["last_seen"]
self.listener.reset_mock()
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
self.assertEqual(self.registry.active()[0]["last_seen"], before)
self.assertEqual(self.registry.active()[0]["count"], 1)
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_not_called()
def test_event_re_raise_counts_and_undismisses(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.assertEqual(self.registry.dismiss("detector_stuck"), DismissResult.ok)
self.assertEqual(self.registry.active(), [])
def test_dismissal_survives_a_re_raise(self):
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertTrue(self.registry.dismiss("skipped_detections:front_door"))
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
# no scope here on purpose: a global event id is still valid
active = self.registry.active()
self.assertEqual(len(active), 1)
self.assertEqual(active[0]["count"], 2)
self.assertIsNone(active[0]["dismissed_at"])
stats = self.registry.stats()[0]
self.assertEqual(stats["occurrences"], 2)
self.assertEqual(stats["dismissals"], 1)
def test_resolve_deletes_and_keeps_stats(self):
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
self.listener.reset_mock()
self.registry.resolve("retention_unmet")
self.registry.resolve("retention_unmet")
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertEqual(self.registry.active(), [])
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
history = self.registry.active(include_dismissed=True)
self.assertEqual(history[0]["count"], 2)
self.assertIsNotNone(history[0]["dismissed_at"])
def test_state_notice_cannot_be_dismissed(self):
self.registry.raise_notice("retention_unmet", params={})
def test_dismiss_counts_once(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.assertEqual(
self.registry.dismiss("retention_unmet"), DismissResult.not_dismissable
)
self.assertEqual(len(self.registry.active()), 1)
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
self.assertEqual(self.registry.stats()[0]["dismissals"], 1)
def test_dismiss_unknown_id(self):
self.assertEqual(self.registry.dismiss("nope"), DismissResult.not_found)
self.assertFalse(self.registry.dismiss("nope"))
def test_dismissed_hidden_unless_requested(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.dismiss("detector_stuck")
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.dismiss("detector_stuck:ov")
self.assertEqual(self.registry.active(), [])
self.assertEqual(len(self.registry.active(include_dismissed=True)), 1)
def test_startup_hides_state_notices_until_confirmed(self):
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
def test_resolve_deletes_and_keeps_stats(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
"model_download_failed", scope="yolo/model.onnx", params={}
)
self.listener.reset_mock()
self.registry.mark_state_notices_unconfirmed()
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["detector_stuck:ov"])
# the producer confirms the same episode: visible again, no new occurrence
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["retention_unmet", "detector_stuck:ov"])
self.assertEqual(
[s for s in self.registry.stats() if s["kind"] == "retention_unmet"][0][
"occurrences"
],
1,
)
def test_unconfirmed_across_two_startups_is_deleted(self):
self.registry.raise_notice("retention_unmet", params={})
self.registry.mark_state_notices_unconfirmed()
self.registry.mark_state_notices_unconfirmed()
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
def test_resolve_kind_drops_every_scope_of_that_kind(self):
self.registry.raise_notice("detector_stuck", params={})
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.resolve_kind("detector_stuck")
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["skipped_detections:garage"])
def test_active_fills_in_each_link(self):
self.registry.raise_notice(
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
links = {n["kind"]: n["link"] for n in self.registry.active()}
self.assertEqual(
links,
{
"update_available": "https://github.com/blakeblackshear/frigate/releases/tag/v0.19.1",
"model_download_failed": None,
},
)
def test_replay_camera_is_ignored(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="_replay_front_door", params={}
"skipped_detections", scope="_replay_front_door", params={}
)
self.assertEqual(self.registry.active(), [])
@@ -208,22 +206,26 @@ class TestNoticeRegistry(unittest.TestCase):
self.listener.assert_not_called()
def test_active_sorts_by_severity_then_recency(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.raise_notice("retention_unmet", params={})
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["retention_unmet", "detector_stuck"])
self.assertEqual(
ids, ["model_download_failed:yolo/model.onnx", "detector_stuck:ov"]
)
def test_listener_exception_does_not_propagate(self):
self.registry.subscribe(MagicMock(side_effect=RuntimeError("boom")))
self.registry.raise_notice("retention_unmet", params={})
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.assertEqual(len(self.registry.active()), 1)
def test_resolve_camera_drops_only_that_camera(self):
self.registry.raise_notice("ffmpeg_crash_loop", scope="front_door", params={})
self.registry.raise_notice("ffmpeg_crash_loop", scope="garage", params={})
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.raise_notice(
"model_download_failed", scope="front_door", params={}
)
@@ -232,5 +234,190 @@ class TestNoticeRegistry(unittest.TestCase):
ids = sorted(n["id"] for n in self.registry.active())
self.assertEqual(
ids, ["ffmpeg_crash_loop:garage", "model_download_failed:front_door"]
ids, ["model_download_failed:front_door", "skipped_detections:garage"]
)
def test_resolve_camera_drops_that_cameras_check_dismissals(self):
for check_id in (
"stream:front_door:0:probe",
"config:detect:fps-greater-than-five:camera.front_door",
"config:detect:fps-greater-than-five:global",
"stream:garage:0:probe",
):
self.assertTrue(self.registry.dismiss(check_id))
self.registry.resolve_camera("front_door")
ids = sorted(check["id"] for check in self.registry.dismissed_checks())
self.assertEqual(
ids,
["config:detect:fps-greater-than-five:global", "stream:garage:0:probe"],
)
def test_camera_named_global_keeps_global_check_dismissals(self):
self.registry.dismiss("config:detect:fps-greater-than-five:global")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.global")
self.registry.resolve_camera("global")
ids = [check["id"] for check in self.registry.dismissed_checks()]
self.assertEqual(ids, ["config:detect:fps-greater-than-five:global"])
def test_purge_dismissed_keeps_active_notices_and_counts(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.garage")
self.assertEqual(self.registry.purge_dismissed(), 2)
ids = [n["id"] for n in self.registry.active(include_dismissed=True)]
self.assertEqual(ids, ["skipped_detections:garage"])
self.assertEqual(self.registry.dismissed_checks(), [])
dismissals = {s["kind"]: s["dismissals"] for s in self.registry.stats()}
self.assertEqual(dismissals["detector_stuck"], 1)
class TestApply(RegistryTestCase):
def test_each_action_reaches_its_method(self):
self.registry.apply(
{
"action": "raise",
"kind": "detector_stuck",
"scope": "ov",
"params": {"detector": "ov"},
}
)
self.assertEqual(
[n["id"] for n in self.registry.active()], ["detector_stuck:ov"]
)
self.registry.apply(
{"action": "resolve", "kind": "detector_stuck", "scope": "ov"}
)
self.assertEqual(self.registry.active(), [])
for scope in ("a", "b"):
self.registry.apply(
{"action": "raise", "kind": "detector_stuck", "scope": scope}
)
self.registry.apply({"action": "resolve_kind", "kind": "detector_stuck"})
self.assertEqual(self.registry.active(), [])
def test_malformed_updates_are_logged_and_ignored(self):
with self.assertLogs("frigate.notices.registry", level="WARNING") as logs:
self.registry.apply({"action": "raise"})
self.registry.apply({"action": "explode", "kind": "detector_stuck"})
self.assertEqual(len(logs.output), 2)
self.assertEqual(self.registry.active(), [])
@patch.dict(NOTICE_KINDS, TEST_KINDS)
class TestLifecycleKnobs(RegistryTestCase):
def test_batched_repeats_wait_for_the_flush(self):
self.registry.raise_notice("batched", params={"n": 1})
self.listener.reset_mock()
for n in (2, 3, 4):
self.registry.raise_notice("batched", params={"n": n})
self.assertEqual(self.registry.active()[0]["count"], 1)
self.listener.assert_not_called()
self.registry.flush()
notice = self.registry.active()[0]
self.assertEqual(notice["count"], 4)
self.assertEqual(notice["params"], {"n": 4})
self.assertEqual(self.registry.stats()[0]["occurrences"], 4)
self.listener.assert_called_once()
def test_flush_with_nothing_held_is_a_no_op(self):
self.registry.raise_notice("batched")
self.listener.reset_mock()
self.registry.flush()
self.listener.assert_not_called()
def test_repeats_of_a_resolved_notice_are_dropped(self):
self.registry.raise_notice("batched")
self.registry.raise_notice("batched")
self.registry.resolve("batched")
self.listener.reset_mock()
self.registry.flush()
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.listener.assert_not_called()
def test_a_new_row_starts_without_stale_repeats(self):
self.registry.raise_notice("batched")
self.registry.raise_notice("batched")
self.registry.resolve("batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active()[0]["count"], 1)
def test_a_dismissed_notice_reopens_at_the_count(self):
self.registry.raise_notice("batched")
self.registry.dismiss("batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active(), [])
self.registry.raise_notice("batched")
self.registry.flush()
notice = self.registry.active()[0]
self.assertEqual(notice["count"], BATCHED.reopen_at_count)
self.assertIsNone(notice["dismissed_at"])
def test_a_notice_dismissed_past_the_count_stays_dismissed(self):
for _ in range(3):
self.registry.raise_notice("batched")
self.registry.flush()
self.registry.dismiss("batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active(), [])
self.assertEqual(self.registry.active(include_dismissed=True)[0]["count"], 4)
def test_keep_latest_drops_the_oldest_rows(self):
for index, scope in enumerate(("a", "b", "c")):
self._raise_at(1000.0 + index, "pruned", scope=scope)
ids = sorted(n["id"] for n in self.registry.active())
self.assertEqual(ids, ["pruned:b", "pruned:c"])
class TestCatalogLifecycles(RegistryTestCase):
def test_a_new_release_replaces_the_old_update_notice(self):
for index, version in enumerate(("0.19.0", "0.19.1")):
self._raise_at(
1000.0 + index,
"update_available",
scope=version,
params={"version": version},
)
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["update_available:0.19.1"])
def test_a_dismissed_failed_login_burst_reopens_at_five_attempts(self):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.dismiss("failed_login:1000")
for _ in range(4):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.flush()
burst = self.registry.active()[0]
self.assertEqual(burst["count"], 5)
self.assertIsNone(burst["dismissed_at"])
frigate/test/test_shm_notice.py
+64
View File
@@ -0,0 +1,64 @@
"""Tests for the notice raised while /dev/shm is too small."""
import unittest
from unittest.mock import patch
from frigate.stats import emitter
TOO_SMALL = {"total": 64.0, "min_shm": 180}
BIG_ENOUGH = {"total": 512.0, "min_shm": 180}
class TestShmNotice(unittest.TestCase):
def setUp(self):
raise_patch = patch.object(emitter, "raise_notice")
resolve_patch = patch.object(emitter, "resolve_notice")
self.raise_notice = raise_patch.start()
self.resolve_notice = resolve_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
self.emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
self.emitter._shm_checked = False
self.emitter._shm_params = None
def test_too_small_raises_once(self):
for _ in range(3):
self.emitter._update_shm_notice(TOO_SMALL)
self.raise_notice.assert_called_once_with(
"shm_too_low", params={"total": 64.0, "min": 180}
)
self.resolve_notice.assert_not_called()
def test_first_tick_clears_a_row_the_last_run_left(self):
self.emitter._update_shm_notice(BIG_ENOUGH)
self.emitter._update_shm_notice(BIG_ENOUGH)
self.resolve_notice.assert_called_once_with("shm_too_low")
self.raise_notice.assert_not_called()
def test_a_new_requirement_updates_the_notice(self):
self.emitter._update_shm_notice(TOO_SMALL)
self.emitter._update_shm_notice({"total": 64.0, "min_shm": 200})
self.assertEqual(self.raise_notice.call_count, 2)
self.raise_notice.assert_called_with(
"shm_too_low", params={"total": 64.0, "min": 200}
)
def test_growing_shm_resolves_the_notice(self):
self.emitter._update_shm_notice(TOO_SMALL)
self.emitter._update_shm_notice(BIG_ENOUGH)
self.resolve_notice.assert_called_once_with("shm_too_low")
def test_missing_shm_is_not_a_problem(self):
self.emitter._update_shm_notice({})
self.raise_notice.assert_not_called()
self.resolve_notice.assert_called_once_with("shm_too_low")
if __name__ == "__main__":
unittest.main()
frigate/test/test_skipped_detections.py
+129
View File
@@ -0,0 +1,129 @@
"""Tests for the skipped detection rate and the notice it can raise."""
import unittest
from unittest.mock import patch
from frigate.stats import emitter
from frigate.stats.emitter import SkippedDetectionsTracker
from frigate.stats.util import skipped_percent
class TestSkippedPercent(unittest.TestCase):
def test_share_of_all_frames(self):
self.assertEqual(skipped_percent(0.5, 5.0, True), 10.0)
def test_offline_camera_reports_zero(self):
self.assertEqual(skipped_percent(2.0, 0.0, True), 0.0)
def test_disabled_camera_reports_zero(self):
self.assertEqual(skipped_percent(2.0, 5.0, False), 0.0)
class TestSkippedDetectionsTracker(unittest.TestCase):
def _cameras(self, pct: float) -> dict:
return {"front_door": {"skipped_pct": pct}}
def test_below_threshold_never_qualifies(self):
tracker = SkippedDetectionsTracker()
for tick in range(10):
self.assertEqual(tracker.update(self._cameras(4.9), tick * 15.0), [])
def test_qualifies_once_after_the_hold(self):
tracker = SkippedDetectionsTracker()
results = [
tracker.update(self._cameras(10.0), tick * 15.0) for tick in range(8)
]
# ticks at 0, 15, 30, and 45 s are inside the hold; 60 s qualifies
self.assertEqual(results[:4], [[], [], [], []])
self.assertEqual(results[4], ["front_door"])
self.assertEqual(results[5:], [[], [], []])
def test_a_dip_restarts_the_hold(self):
tracker = SkippedDetectionsTracker()
for now, pct in ((0.0, 10.0), (15.0, 10.0), (30.0, 10.0), (45.0, 2.0)):
self.assertEqual(tracker.update(self._cameras(pct), now), [])
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
self.assertEqual(tracker.update(self._cameras(10.0), 120.0), ["front_door"])
def test_recovery_then_relapse_is_a_new_episode(self):
tracker = SkippedDetectionsTracker()
tracker.update(self._cameras(10.0), 0.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), ["front_door"])
tracker.update(self._cameras(0.0), 75.0)
tracker.update(self._cameras(10.0), 90.0)
self.assertEqual(tracker.update(self._cameras(10.0), 150.0), ["front_door"])
def test_cameras_are_tracked_separately(self):
tracker = SkippedDetectionsTracker()
tracker.update({"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 0.0}}, 0.0)
qualified = tracker.update(
{"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 10.0}}, 60.0
)
self.assertEqual(qualified, ["a"])
def test_a_removed_camera_starts_a_new_episode(self):
tracker = SkippedDetectionsTracker()
tracker.update(self._cameras(10.0), 0.0)
tracker.update({}, 15.0)
tracker.update(self._cameras(10.0), 30.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
self.assertEqual(tracker.update(self._cameras(10.0), 90.0), ["front_door"])
class TestEmitterNotices(unittest.TestCase):
def setUp(self):
raise_patch = patch.object(emitter, "raise_notice")
resolve_patch = patch.object(emitter, "resolve_notice")
flush_patch = patch.object(emitter, "flush_notices")
self.raise_notice = raise_patch.start()
resolve_patch.start()
self.flush_notices = flush_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
self.addCleanup(flush_patch.stop)
def _emitter(self) -> emitter.StatsEmitter:
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
stats_emitter.skipped_detections = SkippedDetectionsTracker()
stats_emitter._shm_checked = False
stats_emitter._shm_params = None
return stats_emitter
def _stats(self, uptime: int, pct: float) -> dict:
return {
"service": {"uptime": uptime, "storage": {"/dev/shm": {}}},
"cameras": {"front_door": {"skipped_pct": pct}},
}
def test_qualified_camera_raises_a_notice(self):
stats_emitter = self._emitter()
stats_emitter._update_notices(self._stats(300, 12.5), 0.0)
stats_emitter._update_notices(self._stats(360, 12.5), 60.0)
self.raise_notice.assert_called_once_with(
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
def test_startup_window_is_ignored(self):
stats_emitter = self._emitter()
stats_emitter._update_notices(self._stats(30, 50.0), 0.0)
stats_emitter._update_notices(self._stats(90, 50.0), 60.0)
self.raise_notice.assert_not_called()
def test_every_tick_flushes_held_repeats(self):
self._emitter()._update_notices(self._stats(30, 0.0), 0.0)
self.flush_notices.assert_called_once_with()
frigate/test/test_stats_emitter_version.py
+26 -26
View File
@@ -34,44 +34,46 @@ class TestIsNewerVersion(unittest.TestCase):
class TestUpdateNotice(unittest.TestCase):
def setUp(self):
raise_patch = patch.object(emitter, "raise_notice")
resolve_patch = patch.object(emitter, "resolve_kind")
self.raise_notice = raise_patch.start()
self.resolve_kind = resolve_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
def _emitter(self, latest: str) -> emitter.StatsEmitter:
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
stats_emitter.config = MagicMock()
stats_emitter.stats_tracking = {"latest_frigate_version": latest}
stats_emitter.notice_registry = MagicMock()
return stats_emitter
def test_newer_release_raises(self):
stats_emitter = self._emitter("0.19.0")
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
stats_emitter._check_update_notice()
self._emitter("0.19.0")._check_update_notice()
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
"update_available", params={"version": "0.19.0"}
self.raise_notice.assert_called_once_with(
"update_available", scope="0.19.0", params={"version": "0.19.0"}
)
stats_emitter.notice_registry.resolve.assert_not_called()
self.resolve_kind.assert_not_called()
def test_current_or_disabled_resolves(self):
for latest in ("0.18.1", "disabled"):
stats_emitter = self._emitter(latest)
with self.subTest(latest=latest):
self.resolve_kind.reset_mock()
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
stats_emitter._check_update_notice()
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
self._emitter(latest)._check_update_notice()
stats_emitter.notice_registry.raise_notice.assert_not_called()
stats_emitter.notice_registry.resolve.assert_called_once_with(
"update_available"
)
self.raise_notice.assert_not_called()
self.resolve_kind.assert_called_once_with("update_available")
def test_a_failed_lookup_leaves_the_notice_alone(self):
stats_emitter = self._emitter("unknown")
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
stats_emitter._check_update_notice()
self._emitter("unknown")._check_update_notice()
stats_emitter.notice_registry.raise_notice.assert_not_called()
stats_emitter.notice_registry.resolve.assert_not_called()
self.raise_notice.assert_not_called()
self.resolve_kind.assert_not_called()
def test_refresh_updates_tracking_on_a_thread_then_checks(self):
stats_emitter = self._emitter("0.18.0")
@@ -89,7 +91,7 @@ class TestUpdateNotice(unittest.TestCase):
self.assertEqual(
stats_emitter.stats_tracking["latest_frigate_version"], "0.19.0"
)
stats_emitter.notice_registry.raise_notice.assert_called_once()
self.raise_notice.assert_called_once()
def test_failed_refresh_keeps_the_last_known_version(self):
stats_emitter = self._emitter("0.20.0")
@@ -105,10 +107,10 @@ class TestUpdateNotice(unittest.TestCase):
self.assertEqual(
stats_emitter.stats_tracking["latest_frigate_version"], "0.20.0"
)
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
"update_available", params={"version": "0.20.0"}
self.raise_notice.assert_called_once_with(
"update_available", scope="0.20.0", params={"version": "0.20.0"}
)
stats_emitter.notice_registry.resolve.assert_not_called()
self.resolve_kind.assert_not_called()
def test_disabled_version_check_still_clears_the_notice(self):
stats_emitter = self._emitter("0.20.0")
@@ -124,6 +126,4 @@ class TestUpdateNotice(unittest.TestCase):
self.assertEqual(
stats_emitter.stats_tracking["latest_frigate_version"], "disabled"
)
stats_emitter.notice_registry.resolve.assert_called_once_with(
"update_available"
)
self.resolve_kind.assert_called_once_with("update_available")
frigate/test/test_storage.py
+7 -13
View File
@@ -556,10 +556,9 @@ class TestHttp(unittest.TestCase):
assert Recordings.get(Recordings.id == rec_k2_id)
assert Recordings.get(Recordings.id == rec_k3_id)
def test_unmet_retention_raises_notice(self):
registry = MagicMock()
def test_unmet_retention_sets_the_live_flag(self):
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock(), registry)
storage = StorageMaintainer(config, MagicMock())
storage.camera_storage_stats = {
"front_door": {"needs_refresh": False, "usage": 10, "bandwidth": 10}
}
@@ -567,22 +566,17 @@ class TestHttp(unittest.TestCase):
with patch.object(storage, "expected_hourly_bandwidth", return_value=100.0):
storage.reduce_storage_consumption()
registry.raise_notice.assert_called_once()
kind = registry.raise_notice.call_args.args[0]
params = registry.raise_notice.call_args.kwargs["params"]
self.assertEqual(kind, "retention_unmet")
self.assertEqual(params["needed_mb"], 100.0)
self.assertEqual(params["cleared_mb"], 0.0)
self.assertTrue(storage.retention_unmet)
def test_clean_run_resolves_notice(self):
registry = MagicMock()
def test_clean_run_clears_the_live_flag(self):
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock(), registry)
storage = StorageMaintainer(config, MagicMock())
storage.retention_unmet = True
with patch.object(storage, "check_storage_needs_cleanup", return_value=False):
storage._maintain_once()
registry.resolve.assert_called_once_with("retention_unmet")
self.assertFalse(storage.retention_unmet)
def _insert_mock_event(
frigate/test/test_watchdog_notices.py
+10 -8
View File
@@ -14,25 +14,27 @@ class TestDetectorStuckNotice(unittest.TestCase):
return detector
def test_stuck_restart_raises_notice(self):
registry = MagicMock()
detector = self._stuck_detector()
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
watchdog = FrigateWatchdog({"ov": detector}, MagicMock())
with patch("frigate.watchdog.datetime") as mock_datetime:
with (
patch("frigate.watchdog.datetime") as mock_datetime,
patch("frigate.watchdog.raise_notice") as raise_notice,
):
mock_datetime.datetime.now.return_value.timestamp.return_value = 100.0
watchdog._check_detectors()
detector.start_or_restart.assert_called_once()
registry.raise_notice.assert_called_once_with(
raise_notice.assert_called_once_with(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
def test_healthy_detector_raises_nothing(self):
registry = MagicMock()
detector = self._stuck_detector()
detector.detection_start.value = 0.0
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
watchdog = FrigateWatchdog({"ov": detector}, MagicMock())
watchdog._check_detectors()
with patch("frigate.watchdog.raise_notice") as raise_notice:
watchdog._check_detectors()
registry.raise_notice.assert_not_called()
raise_notice.assert_not_called()
frigate/types.py
+5 -1
View File
@@ -1,10 +1,13 @@
from enum import Enum
from typing import TypedDict
from typing import TYPE_CHECKING, TypedDict
from frigate.camera import CameraMetrics
from frigate.data_processing.types import DataProcessorMetrics
from frigate.object_detection.base import ObjectDetectProcess
if TYPE_CHECKING:
from frigate.storage import StorageMaintainer
class StatsTrackingTypes(TypedDict):
camera_metrics: dict[str, CameraMetrics]
@@ -14,6 +17,7 @@ class StatsTrackingTypes(TypedDict):
latest_frigate_version: str
last_updated: int
processes: dict[str, int]
storage_maintainer: "StorageMaintainer | None"
class ModelStatusTypesEnum(str, Enum):
frigate/util/downloader.py
+7 -18
View File
@@ -7,7 +7,8 @@ from pathlib import Path
import requests
from frigate.comms.inter_process import InterProcessRequestor
from frigate.const import UPDATE_MODEL_STATE, UPDATE_NOTICE
from frigate.const import UPDATE_MODEL_STATE
from frigate.notices import raise_notice, resolve_notice
from frigate.types import ModelStatusTypesEnum
from frigate.util.file import FileLock
@@ -64,26 +65,14 @@ class ModelDownloader:
return f"{self.model_name}/{file_name}"
def _report_failure(self, file_name: str, error: str) -> None:
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "raise",
"kind": "model_download_failed",
"scope": self._notice_scope(file_name),
"params": {"file": file_name, "error": error},
},
raise_notice(
"model_download_failed",
scope=self._notice_scope(file_name),
params={"file": file_name, "model": self.model_name, "error": error},
)
def _resolve_failure(self, file_name: str) -> None:
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "resolve",
"kind": "model_download_failed",
"scope": self._notice_scope(file_name),
"params": {},
},
)
resolve_notice("model_download_failed", self._notice_scope(file_name))
def _send_state(self, file_name: str, state: ModelStatusTypesEnum) -> None:
self.requestor.send_data(
frigate/video/ffmpeg.py
-60
View File
@@ -29,7 +29,6 @@ from frigate.const import (
STREAM_TYPE_MAIN,
STREAM_TYPE_SUB,
STREAM_TYPE_TO_ROLE,
UPDATE_NOTICE,
)
from frigate.log import LogPipe
from frigate.util.builtin import EventsPerSecond, get_record_segment_time
@@ -44,11 +43,6 @@ logger = logging.getLogger(__name__)
RECORD_GRACE_SECONDS = 90
# a camera whose ffmpeg restarts this many times in the trailing hour is in a
# crash loop; it recovers once the capture thread stays up this long
CRASH_LOOP_RESTARTS = 5
CRASH_LOOP_RECOVERY_S = 600
def capture_frames(
ffmpeg_process: sp.Popen[Any],
@@ -146,8 +140,6 @@ class CameraWatchdog(threading.Thread):
self.stop_event = stop_event
self.sleeptime = self.config.ffmpeg.retry_interval
self.reconnect_timestamps = deque()
self.crash_loop_raised = False
self._crash_loop_sent_restarts = 0
self.stalls = stalls
self.reconnects = reconnects
self.detection_frame = detection_frame
@@ -197,53 +189,6 @@ class CameraWatchdog(threading.Thread):
self._last_record_status: dict[str, str] = {}
self._last_status_update_time: float = 0.0
def _update_crash_loop_notice(self, now: float) -> None:
"""Raise the crash loop notice as restarts pile up, resolve once stable.
Called on every 1 second tick, so raises are sent only when the restart
count changes; the registry treats repeats of a state notice as updates.
"""
restarts = len(self.reconnect_timestamps)
alive = self.capture_thread is not None and self.capture_thread.is_alive()
if not alive and restarts >= CRASH_LOOP_RESTARTS:
if self.crash_loop_raised and restarts == self._crash_loop_sent_restarts:
return
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "raise",
"kind": "ffmpeg_crash_loop",
"scope": self.config.name,
"params": {"restarts": restarts},
},
)
self.crash_loop_raised = True
self._crash_loop_sent_restarts = restarts
elif (
self.crash_loop_raised
and alive
and (
restarts == 0
or now - self.reconnect_timestamps[-1] >= CRASH_LOOP_RECOVERY_S
)
):
self._resolve_crash_loop()
def _resolve_crash_loop(self) -> None:
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "resolve",
"kind": "ffmpeg_crash_loop",
"scope": self.config.name,
"params": {},
},
)
self.crash_loop_raised = False
self._crash_loop_sent_restarts = 0
def _send_detect_status(self, status: str, now: float) -> None:
"""Send detect status only if changed or retry_interval has elapsed."""
if (
@@ -434,9 +379,6 @@ class CameraWatchdog(threading.Thread):
# cameras without a sub stream never get a record_sub topic
if self.config.record.sub.enabled:
self._send_record_status(STREAM_TYPE_SUB, "disabled", now)
if self.crash_loop_raised:
self._resolve_crash_loop()
self.was_enabled = enabled
continue
@@ -514,7 +456,6 @@ class CameraWatchdog(threading.Thread):
self.logger.error(
f"Ffmpeg process crashed unexpectedly for {self.config.name}."
)
self._update_crash_loop_notice(now)
if can_restart:
self.reset_capture_thread(terminate=False)
last_restart_time = now
@@ -544,7 +485,6 @@ class CameraWatchdog(threading.Thread):
# process is running normally
self._send_detect_status("online", now)
self.fps_overflow_count = 0
self._update_crash_loop_notice(now)
for p in self.ffmpeg_other_processes:
poll = p["process"].poll()
frigate/watchdog.py
+2 -8
View File
@@ -7,7 +7,7 @@ from collections.abc import Callable
from dataclasses import dataclass, field
from multiprocessing.synchronize import Event as MpEvent
from frigate.notices.registry import NoticeRegistry
from frigate.notices import raise_notice
from frigate.object_detection.base import ObjectDetectProcess
from frigate.util.process import FrigateProcess
from frigate.util.services import restart_frigate
@@ -45,12 +45,10 @@ class FrigateWatchdog(threading.Thread):
self,
detectors: dict[str, ObjectDetectProcess],
stop_event: MpEvent,
notice_registry: NoticeRegistry | None = None,
):
super().__init__(name="frigate_watchdog")
self.detectors = detectors
self.stop_event = stop_event
self.notice_registry = notice_registry
self._monitored: list[MonitoredProcess] = []
def register(
@@ -126,11 +124,7 @@ class FrigateWatchdog(threading.Thread):
"Detection appears to be stuck. Restarting detection process..."
)
detector.start_or_restart()
if self.notice_registry is not None:
self.notice_registry.raise_notice(
"detector_stuck", scope=name, params={"detector": name}
)
raise_notice("detector_stuck", scope=name, params={"detector": name})
elif (
detector.detect_process is not None
and not detector.detect_process.is_alive()
generate_api_auth_spec.py
+2 -1
View File
@@ -331,7 +331,8 @@ def build_access_map(
for method in route.methods:
if method in ("HEAD", "OPTIONS"):
continue
access_map[(route.path, method.lower())] = {
# the OpenAPI paths drop convertors such as :path, like path_format
access_map[(route.path_format, method.lower())] = {
"level": level,
"roles": roles,
"flag": flag,
web/e2e/fixtures/mock-data/stats.ts
+2
View File
@@ -16,6 +16,7 @@ function cameraStats(_name: string) {
pid: 102,
process_fps: 5.0,
skipped_fps: 0,
skipped_pct: 0,
connection_quality: "excellent" as const,
expected_fps: 5,
reconnects_last_hour: 0,
@@ -75,6 +76,7 @@ export const BASE_STATS = {
uptime: 86400,
latest_version: "0.15.0",
version: "0.15.0-test",
retention_unmet: false,
},
camera_fps: 15.0,
process_fps: 15.0,
web/e2e/helpers/api-mocker.ts
+4 -5
View File
@@ -49,7 +49,7 @@ export interface ApiMockOverrides {
};
users?: { username: string; role: string }[];
notices?: unknown[];
noticeStats?: unknown[];
dismissedChecks?: unknown[];
/** camera name to the ffprobe entries returned for `paths=camera:<name>` */
ffprobe?: Record<string, unknown[]>;
}
@@ -234,13 +234,12 @@ export class ApiMocker {
return route.fulfill({ json: entries });
});
// Notices. The stats route is registered after the list route so it wins
// for /api/notices/stats; the list glob does not match a sub-path anyway.
// Notices
await this.page.route("**/api/notices", (route) =>
route.fulfill({ json: overrides?.notices ?? [] }),
);
await this.page.route("**/api/notices/stats", (route) =>
route.fulfill({ json: overrides?.noticeStats ?? [] }),
await this.page.route("**/api/notices/dismissed_checks", (route) =>
route.fulfill({ json: overrides?.dismissedChecks ?? [] }),
);
// Users. GET lists them; POST/PUT (create, password) just succeed, so
web/e2e/specs/system-health.spec.ts
+418 -81
View File
@@ -5,34 +5,35 @@
*/
import { test, expect } from "../fixtures/frigate-test";
import { viewerProfile } from "../fixtures/mock-data/profile";
const NOW = Math.floor(Date.now() / 1000);
// the fixture detector runs at 75.5 ms, above the live warning threshold
const QUIET_STATS = { detectors: { cpu: { inference_speed: 10 } } };
const STATE_NOTICE = {
id: "ffmpeg_crash_loop:front_door",
kind: "ffmpeg_crash_loop",
mode: "state",
const ERROR_NOTICE = {
id: "model_download_failed:yolo/model.onnx",
kind: "model_download_failed",
severity: "error",
category: "camera",
scope: "front_door",
params: { restarts: 6 },
category: "model",
scope: "yolo/model.onnx",
params: { file: "model.onnx", model: "yolo", error: "timeout" },
link: null,
first_seen: NOW - 600,
last_seen: NOW,
count: 1,
count: 2,
dismissed_at: null,
};
const EVENT_NOTICE = {
id: "detector_stuck",
kind: "detector_stuck",
mode: "event",
severity: "warning",
category: "detector",
scope: null,
params: { detector: "ov" },
link: "/system#general",
first_seen: NOW - 7200,
last_seen: NOW - 60,
count: 3,
@@ -45,16 +46,7 @@ test.describe("System — Health tab @medium", () => {
}) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [STATE_NOTICE, EVENT_NOTICE],
noticeStats: [
{
kind: "ffmpeg_crash_loop",
occurrences: 14,
dismissals: 0,
first_seen: NOW - 86400 * 20,
last_seen: NOW,
},
],
notices: [ERROR_NOTICE, EVENT_NOTICE],
});
await frigateApp.goto("/system");
@@ -67,11 +59,13 @@ test.describe("System — Health tab @medium", () => {
const rows = frigateApp.page.locator("[data-testid^='health-problem-']");
await expect(rows).toHaveCount(2);
await expect(rows.nth(0)).toHaveAttribute("data-severity", "error");
await expect(rows.nth(0)).toContainText("ffmpeg has crashed 6 times");
await expect(rows.nth(0)).toContainText("14 times since");
await expect(rows.nth(0)).toContainText(
"Downloading model.onnx for yolo failed: timeout",
);
await expect(rows.nth(0)).toContainText("2 times");
await expect(
rows.nth(0).getByRole("button", { name: "Dismiss" }),
).toHaveCount(0);
).toBeVisible();
await expect(rows.nth(1)).toContainText("Detector ov was restarted");
await expect(rows.nth(1)).toContainText("3 times");
await expect(
@@ -104,7 +98,10 @@ test.describe("System — Health tab @medium", () => {
req.url().includes("/api/notices/detector_stuck/dismiss") &&
req.method() === "POST",
);
await frigateApp.page.getByRole("button", { name: "Dismiss" }).click();
await frigateApp.page
.getByTestId("health-problem-notice:detector_stuck")
.getByRole("button", { name: "Dismiss" })
.click();
await request;
await expect(
@@ -133,13 +130,13 @@ test.describe("System — Health tab @medium", () => {
stats: QUIET_STATS,
notices: [
{
id: "update_available",
id: "update_available:0.19.0",
kind: "update_available",
mode: "state",
severity: "info",
category: "system",
scope: null,
scope: "0.19.0",
params: { version: "0.19.0" },
link: "https://github.com/blakeblackshear/frigate/releases/tag/v0.19.0",
first_seen: NOW - 3600,
last_seen: NOW,
count: 1,
@@ -150,7 +147,7 @@ test.describe("System — Health tab @medium", () => {
await frigateApp.goto("/system#health");
const row = frigateApp.page.getByTestId(
"health-problem-notice:update_available",
"health-problem-notice:update_available:0.19.0",
);
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(row).toHaveAttribute("data-severity", "info");
@@ -159,7 +156,229 @@ test.describe("System — Health tab @medium", () => {
"href",
"https://github.com/blakeblackshear/frigate/releases/tag/v0.19.0",
);
await expect(row.getByRole("button", { name: "Dismiss" })).toBeVisible();
});
test("the filter shows dismissed notices without a Dismiss button", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [EVENT_NOTICE],
});
await frigateApp.page.route(
(url) =>
url.pathname.endsWith("/api/notices") &&
url.searchParams.get("include_dismissed") === "true",
(route) =>
route.fulfill({
json: [EVENT_NOTICE, { ...ERROR_NOTICE, dismissed_at: NOW - 120 }],
}),
);
await frigateApp.goto("/system#health");
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
const showDismissed = frigateApp.page.getByRole("switch", {
name: "Show dismissed",
});
await expect(showDismissed).toHaveAttribute("aria-checked", "false");
await showDismissed.click();
const row = frigateApp.page.getByTestId(
"health-problem-notice:model_download_failed:yolo/model.onnx",
);
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(row).toContainText("Dismissed");
await expect(row.getByRole("button", { name: "Dismiss" })).toHaveCount(0);
await showDismissed.click();
await expect(row).toHaveCount(0);
});
test("clear dismissed deletes the dismissed rows after confirming", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [EVENT_NOTICE],
});
// the history loses its dismissed row once the DELETE lands
let cleared = false;
await frigateApp.page.route(
(url) =>
url.pathname.endsWith("/api/notices") &&
url.searchParams.get("include_dismissed") === "true",
(route) =>
route.fulfill({
json: cleared
? [EVENT_NOTICE]
: [EVENT_NOTICE, { ...ERROR_NOTICE, dismissed_at: NOW - 120 }],
}),
);
await frigateApp.page.route("**/api/notices/dismissed", (route) => {
cleared = true;
return route.fulfill({ json: { success: true } });
});
await frigateApp.goto("/system#health");
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
await frigateApp.page
.getByRole("switch", { name: "Show dismissed" })
.click();
const row = frigateApp.page.getByTestId(
"health-problem-notice:model_download_failed:yolo/model.onnx",
);
await expect(row).toBeVisible({ timeout: 15_000 });
await frigateApp.page.keyboard.press("Escape");
await frigateApp.page
.getByRole("button", { name: "Clear dismissed" })
.click();
const request = frigateApp.page.waitForRequest(
(req) =>
req.url().endsWith("/api/notices/dismissed") &&
req.method() === "DELETE",
);
await frigateApp.page
.getByRole("alertdialog")
.getByRole("button", { name: "Clear dismissed" })
.click();
await request;
await expect(row).toHaveCount(0);
await expect(
frigateApp.page.getByText("No dismissed notices"),
).toBeVisible();
});
test("severity switches hide notices of that severity", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [ERROR_NOTICE, EVENT_NOTICE],
});
await frigateApp.goto("/system#health");
const rows = frigateApp.page.locator("[data-testid^='health-problem-']");
await expect(rows).toHaveCount(2, { timeout: 15_000 });
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
await frigateApp.page.getByRole("switch", { name: "Warning" }).click();
await expect(rows).toHaveCount(1);
await expect(rows.first()).toHaveAttribute("data-severity", "error");
await frigateApp.page.getByRole("switch", { name: "Error" }).click();
await expect(rows).toHaveCount(0);
await expect(
frigateApp.page.getByText("No notices match the filter"),
).toBeVisible();
});
test("failed login bursts are per user and show their attempt count", async ({
frigateApp,
}) => {
const burst = (user: string, start: number, count: number) => ({
id: `failed_login:${user}:${start}`,
kind: "failed_login",
severity: "warning",
category: "system",
scope: `${user}:${start}`,
params: { user },
link: "/logs",
first_seen: start,
last_seen: start + 60,
count,
dismissed_at: null,
});
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [burst("admin", NOW - 60, 7), burst("ghost", NOW - 3600, 1)],
});
await frigateApp.goto("/system#health");
const attack = frigateApp.page.getByTestId(
`health-problem-notice:failed_login:admin:${NOW - 60}`,
);
await expect(attack).toBeVisible({ timeout: 15_000 });
await expect(attack).toContainText("Failed login attempts for admin");
await expect(attack).toContainText("7 times");
await expect(attack).not.toContainText(String(NOW - 60));
await expect(
attack.getByRole("link", { name: "Open settings" }),
).toHaveAttribute("href", "/logs");
await expect(
frigateApp.page.getByTestId(
`health-problem-notice:failed_login:ghost:${NOW - 3600}`,
),
).toContainText("Failed login attempt for ghost");
});
test("skipped detections notice links to camera stats", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [
{
id: "skipped_detections:front_door",
kind: "skipped_detections",
severity: "warning",
category: "camera",
scope: "front_door",
params: { pct: 12.5 },
link: "/system#cameras",
first_seen: NOW - 600,
last_seen: NOW - 600,
count: 1,
dismissed_at: null,
},
],
});
await frigateApp.goto("/system#health");
const row = frigateApp.page.getByTestId(
"health-problem-notice:skipped_detections:front_door",
);
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(row).toContainText("skipped 12.5% of frames");
await expect(
row.getByRole("link", { name: "Open settings" }),
).toHaveAttribute("href", "/system#cameras");
});
test("shm notice links to storage metrics", async ({ frigateApp }) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [
{
id: "shm_too_low",
kind: "shm_too_low",
severity: "warning",
category: "system",
scope: null,
params: { total: 64, min: 180 },
link: "/system#storage",
first_seen: NOW - 600,
last_seen: NOW - 600,
count: 1,
dismissed_at: null,
},
],
});
await frigateApp.goto("/system#health");
const row = frigateApp.page.getByTestId(
"health-problem-notice:shm_too_low",
);
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(row).toContainText(
"/dev/shm allocation (64 MB) should be increased to at least 180 MB",
);
await expect(
row.getByRole("link", { name: "Open settings" }),
).toHaveAttribute("href", "/system#storage");
});
});
@@ -169,13 +388,13 @@ test.describe("System — Health tab mobile @medium @mobile", () => {
test("notices render at mobile viewport", async ({ frigateApp }) => {
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [STATE_NOTICE],
notices: [ERROR_NOTICE],
});
await frigateApp.goto("/system#health");
await expect(
frigateApp.page.getByTestId(
"health-problem-notice:ffmpeg_crash_loop:front_door",
"health-problem-notice:model_download_failed:yolo/model.onnx",
),
).toBeVisible({ timeout: 15_000 });
});
@@ -298,14 +517,17 @@ test.describe("System — Health hardware pane @medium", () => {
).toHaveAttribute("data-state", "warning");
});
test("face recognition row reflects the runtime device", async ({
test("slow face recognition on the CPU warns when an accelerator exists", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
config: { face_recognition: { enabled: true } },
stats: {
...QUIET_STATS,
embeddings: { devices: { face_recognition: "CPU" } },
embeddings: {
face_recognition_speed: 800,
devices: { face_recognition: "CPU" },
},
},
});
await frigateApp.goto("/system#health");
@@ -321,6 +543,27 @@ test.describe("System — Health hardware pane @medium", () => {
);
});
test("fast face recognition on the CPU is ok", async ({ frigateApp }) => {
await frigateApp.installDefaults({
config: { face_recognition: { enabled: true } },
stats: {
...QUIET_STATS,
embeddings: {
face_recognition_speed: 40,
devices: { face_recognition: "CPU" },
},
},
});
await frigateApp.goto("/system#health");
const row = frigateApp.page.getByTestId(
"hardware-row-enrichment:face_recognition",
);
await expect(row).toHaveAttribute("data-state", "ok", { timeout: 15_000 });
await expect(row).toContainText("CPU");
await expect(row).not.toContainText("although an accelerator");
});
test("explicit GPU that loaded on CUDA is ok despite the probe", async ({
frigateApp,
}) => {
@@ -415,36 +658,24 @@ test.describe("System — Health hardware pane @medium", () => {
});
test.describe("System — Health notices sources @medium", () => {
test("live offline camera and config checks render with links", async ({
frigateApp,
}) => {
test("config checks render with links", async ({ frigateApp }) => {
await frigateApp.installDefaults({
config: {
cameras: {
garage: { detect: { width: 2560, height: 1440, fps: 10 } },
},
},
stats: { ...QUIET_STATS, cameras: { front_door: { camera_fps: 0 } } },
stats: QUIET_STATS,
});
await frigateApp.goto("/system#health");
const rows = frigateApp.page.locator("[data-testid^='health-problem-']");
await expect(rows.first()).toBeVisible({ timeout: 15_000 });
await expect(rows.filter({ hasText: "Front Door is offline" })).toHaveCount(
1,
);
await expect(
frigateApp.page.getByText(
"This detect resolution is higher than recommended",
),
).toBeVisible();
await expect(
rows
.filter({ hasText: "Front Door is offline" })
.getByRole("link", { name: "Open settings" }),
).toHaveAttribute("href", "/logs");
).toBeVisible({ timeout: 15_000 });
const fpsRow = frigateApp.page.getByTestId(
"health-problem-config:detect:fps-greater-than-five:garage",
"health-problem-config:detect:fps-greater-than-five:camera.garage",
);
await expect(fpsRow).toHaveAttribute("data-severity", "info");
await expect(
@@ -452,6 +683,28 @@ test.describe("System — Health notices sources @medium", () => {
).toHaveAttribute("href", "/settings?page=cameraDetect&camera=garage");
});
test("status bar problems stay out of the list", async ({ frigateApp }) => {
test.skip(frigateApp.isMobile, "Status bar is desktop-only");
await frigateApp.installDefaults({
stats: {
service: { retention_unmet: true },
cameras: { front_door: { camera_fps: 0 } },
},
});
await frigateApp.goto("/system#health");
// the status bar shows a problem, so stats have loaded
await expect(
frigateApp.page.getByText("Front Door is offline"),
).toBeVisible({ timeout: 15_000 });
await expect(
frigateApp.page.locator("[data-testid^='health-problem-']"),
).toHaveCount(0);
await expect(
frigateApp.page.getByText("Your Frigate installation is healthy"),
).toBeVisible();
});
// the fixture's cameras all have a record role with recording off, so
// dropping the role isolates the gate on record.enabled
const NO_RECORD_ROLE = {
@@ -553,43 +806,11 @@ test.describe("System — Health notices sources @medium", () => {
await expect(rows).toHaveCount(2);
await expect(
frigateApp.page.getByTestId(
"health-problem-config:detect:detect-resolution-high:garage",
"health-problem-config:detect:detect-resolution-high:camera.garage",
),
).toBeVisible();
});
test("registry rows sort ahead of live rows and keep Dismiss", async ({
frigateApp,
}) => {
await frigateApp.installDefaults({
notices: [
{
id: "detector_stuck:cpu",
kind: "detector_stuck",
mode: "event",
severity: "warning",
category: "detector",
scope: "cpu",
params: { detector: "cpu" },
first_seen: NOW - 60,
last_seen: NOW - 60,
count: 1,
dismissed_at: null,
},
],
// the default fixture's slow cpu detector is the live warning here
});
await frigateApp.goto("/system#health");
const rows = frigateApp.page.locator("[data-severity='warning']");
await expect(rows.first()).toBeVisible({ timeout: 15_000 });
await expect(rows.nth(0)).toContainText("Detector cpu was restarted");
await expect(
rows.nth(0).getByRole("button", { name: "Dismiss" }),
).toBeVisible();
await expect(rows.nth(1)).toContainText("Cpu is slow");
});
test("empty state when stats, config, and registry are clean", async ({
frigateApp,
}) => {
@@ -797,7 +1018,7 @@ test.describe("System — Health notices sources @medium", () => {
await frigateApp.goto("/system#health");
const row = frigateApp.page.getByTestId(
"health-problem-config:lpr:global-disabled:garage",
"health-problem-config:lpr:global-disabled:camera.garage",
);
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(
@@ -824,4 +1045,120 @@ test.describe("System — Health notices sources @medium", () => {
.click();
await expect(frigateApp.page).toHaveURL(/\/system#health/);
});
test("status bar counts undismissed notices next to the health text", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Status bar is desktop-only");
await frigateApp.installDefaults({
stats: QUIET_STATS,
notices: [EVENT_NOTICE],
});
await frigateApp.goto("/");
const notices = frigateApp.page.getByRole("link", {
name: "1 system notice",
});
await expect(notices).toBeVisible({ timeout: 15_000 });
await expect(notices).toHaveAttribute("href", "/system#health");
await expect(
frigateApp.page.getByRole("link", { name: "System is healthy" }),
).toBeVisible();
});
test("status bar shows viewers no problems or health text", async ({
frigateApp,
}) => {
test.skip(frigateApp.isMobile, "Status bar is desktop-only");
await frigateApp.installDefaults({
profile: viewerProfile(),
stats: {
cpu_usages: { "frigate.full_system": { cpu: "12.0" } },
cameras: { front_door: { camera_fps: 0 } },
},
});
await frigateApp.goto("/");
// the CPU reading comes from the same stats as the offline problem
await expect(frigateApp.page.getByText("CPU 12%")).toBeVisible({
timeout: 15_000,
});
await expect(
frigateApp.page.getByText("Front Door is offline"),
).toHaveCount(0);
await expect(frigateApp.page.getByText("System is healthy")).toHaveCount(0);
});
test("a config row can be dismissed", async ({ frigateApp }) => {
const id = "config:detect:fps-greater-than-five:camera.garage";
await frigateApp.installDefaults({
config: {
cameras: {
garage: { detect: { width: 2560, height: 1440, fps: 10 } },
},
},
stats: QUIET_STATS,
});
// the list gains the dismissal after the POST so the refetch hides the row
let dismissed = false;
await frigateApp.page.route(`**/api/notices/${id}/dismiss`, (route) => {
dismissed = true;
return route.fulfill({ json: { success: true } });
});
await frigateApp.page.route("**/api/notices/dismissed_checks", (route) =>
route.fulfill({ json: dismissed ? [{ id, dismissed_at: NOW }] : [] }),
);
await frigateApp.goto("/system#health");
const row = frigateApp.page.getByTestId(`health-problem-${id}`);
await expect(row).toBeVisible({ timeout: 15_000 });
const request = frigateApp.page.waitForRequest(
(req) =>
req.url().includes(`/api/notices/${id}/dismiss`) &&
req.method() === "POST",
);
await row.getByRole("button", { name: "Dismiss" }).click();
await request;
await expect(row).toHaveCount(0);
});
test("dismissed config rows move to the dismissed list", async ({
frigateApp,
}) => {
const id = "config:detect:fps-greater-than-five:camera.garage";
await frigateApp.installDefaults({
config: {
cameras: {
garage: { detect: { width: 2560, height: 1440, fps: 10 } },
},
},
stats: QUIET_STATS,
dismissedChecks: [{ id, dismissed_at: NOW - 120 }],
});
await frigateApp.page.route(
(url) =>
url.pathname.endsWith("/api/notices") &&
url.searchParams.get("include_dismissed") === "true",
(route) => route.fulfill({ json: [] }),
);
await frigateApp.goto("/system#health");
await expect(
frigateApp.page.getByText(
"This detect resolution is higher than recommended",
),
).toBeVisible({ timeout: 15_000 });
const row = frigateApp.page.getByTestId(`health-problem-${id}`);
await expect(row).toHaveCount(0);
await frigateApp.page.getByRole("button", { name: "Filter" }).click();
await frigateApp.page
.getByRole("switch", { name: "Show dismissed" })
.click();
await expect(row).toBeVisible();
await expect(row).toContainText("Dismissed");
await expect(row.getByRole("button", { name: "Dismiss" })).toHaveCount(0);
});
});
web/public/locales/en/views/system.json
+26 -10
View File
@@ -22,21 +22,35 @@
"dismiss": "Dismiss",
"openSettings": "Open settings",
"openLink": "Open link",
"since": "Since {{time}}",
"sinceWithCount": "Since {{time}} · {{times}} times since {{firstSeen}}",
"firstSeen": "First seen {{time}} · {{times}} times",
"noMatches": "No notices match the filter",
"dismissedTitle": "Dismissed",
"noneDismissed": "No dismissed notices",
"clearDismissed": "Clear dismissed",
"clearDismissedTitle": "Clear dismissed notices?",
"clearDismissedDesc": "Every dismissed notice is deleted. Config and stream checks show again right away, and other notices return the next time they happen.",
"firstSeen_one": "First seen {{time}}",
"firstSeen_other": "First seen {{time}} · {{count}} times",
"dismissedAt": "Dismissed {{time}}",
"filter": {
"showDismissed": "Show dismissed",
"severity": "Severity",
"error": "Error",
"warning": "Warning",
"info": "Info"
},
"kinds": {
"ffmpeg_crash_loop": "ffmpeg has crashed {{restarts}} times in the last hour",
"detector_stuck": "Detector {{detector}} was restarted after it stopped responding",
"model_download_failed": "Downloading {{file}} failed: {{error}}",
"retention_unmet": "Recordings were deleted before their retention period to free space ({{cleared_mb}} of {{needed_mb}} MB needed)",
"model_download_failed": "Downloading {{file}} for {{model}} failed: {{error}}",
"skipped_detections": "Detection could not keep up and skipped {{pct}}% of frames for over a minute",
"shm_too_low": "/dev/shm allocation ({{total}} MB) should be increased to at least {{min}} MB",
"failed_login_one": "Failed login attempt for {{user}}",
"failed_login_other": "Failed login attempts for {{user}}",
"update_available": "Frigate {{version}} is available"
},
"streamPrefix": "Stream {{index}}: {{message}}",
"streamPrefixRestream": "Stream {{index}} (via go2rtc): {{message}}",
"streamProbeFailed": "Stream {{index}} could not be probed: {{error}}",
"cameraProbeFailed": "Streams could not be probed: {{error}}",
"startupWindow": "Frigate started less than two minutes ago, live checks begin after startup"
"cameraProbeFailed": "Streams could not be probed: {{error}}"
},
"hardware": {
"title": "Hardware",
@@ -311,14 +325,16 @@
"lastRefreshed": "Last refreshed: ",
"stats": {
"ffmpegHighCpuUsage": "{{camera}} has high FFmpeg CPU usage ({{ffmpegAvg}}%)",
"cameraSkippedDetections": "{{camera}} has skipped detections ({{fps}}).",
"cameraSkippedDetections": "{{camera}} is skipping detection on {{pct}}% of frames",
"detectHighCpuUsage": "{{camera}} has high detect CPU usage ({{detectAvg}}%)",
"healthy": "System is healthy",
"systemNotices_one": "{{count}} system notice",
"systemNotices_other": "{{count}} system notices",
"reindexingEmbeddings": "Reindexing embeddings ({{processed}}% complete)",
"cameraIsOffline": "{{camera}} is offline",
"detectIsSlow": "{{detect}} is slow ({{speed}} ms)",
"detectIsVerySlow": "{{detect}} is very slow ({{speed}} ms)",
"shmTooLow": "/dev/shm allocation ({{total}} MB) should be increased to at least {{min}} MB.",
"retentionUnmet": "Recordings are being deleted before their retention period ends to free space",
"debugReplayActive": "Debug replay session is active"
},
"enrichments": {
web/src/components/Statusbar.tsx
+7 -15
View File
@@ -4,6 +4,7 @@ import {
StatusMessage,
} from "@/context/statusbar-provider";
import useStats, { useAutoFrigateStats } from "@/hooks/use-stats";
import StatusBarNotices from "@/components/health/StatusBarNotices";
import { cn } from "@/lib/utils";
import type { ProfilesApiResponse } from "@/types/profile";
import { getProfileColor } from "@/utils/profileColors";
@@ -186,21 +187,11 @@ export default function Statusbar() {
))}
</div>
<div className="no-scrollbar flex h-full max-w-[50%] items-center gap-2 overflow-x-auto">
{Object.entries(messages).length === 0 ? (
isAdmin ? (
<Link
to="/system#health"
className="flex items-center gap-2 text-sm"
>
<FaCheck className="size-3 text-green-500" />
{t("stats.healthy")}
</Link>
) : (
<div className="flex items-center gap-2 text-sm">
<FaCheck className="size-3 text-green-500" />
{t("stats.healthy")}
</div>
)
{!isAdmin ? null : Object.entries(messages).length === 0 ? (
<Link to="/system#health" className="flex items-center gap-2 text-sm">
<FaCheck className="size-3 text-green-500" />
{t("stats.healthy")}
</Link>
) : (
Object.entries(messages).map(([key, messageArray]) => (
<div key={key} className="flex h-full items-center gap-2">
@@ -230,6 +221,7 @@ export default function Statusbar() {
</div>
))
)}
{isAdmin && <StatusBarNotices />}
</div>
</div>
);
web/src/components/health/NoticeFilterButton.tsx
+101
View File
@@ -0,0 +1,101 @@
import { useState } from "react";
import { useTranslation } from "react-i18next";
import { FaFilter } from "react-icons/fa";
import FilterSwitch from "@/components/filter/FilterSwitch";
import PlatformAwareDialog from "@/components/overlay/dialog/PlatformAwareDialog";
import { Button } from "@/components/ui/button";
import { DropdownMenuSeparator } from "@/components/ui/dropdown-menu";
import { cn } from "@/lib/utils";
import {
DEFAULT_NOTICE_FILTER,
type HealthSeverity,
type NoticeFilter,
} from "@/types/health";
type NoticeFilterButtonProps = {
filter: NoticeFilter;
onFilterChange: (filter: NoticeFilter) => void;
};
export default function NoticeFilterButton({
filter,
onFilterChange,
}: NoticeFilterButtonProps) {
const { t } = useTranslation(["views/system", "components/filter"]);
const [open, setOpen] = useState(false);
const active =
filter.showDismissed ||
filter.severities.length < DEFAULT_NOTICE_FILTER.severities.length;
const severityLabels: Record<HealthSeverity, string> = {
error: t("health.notices.filter.error"),
warning: t("health.notices.filter.warning"),
info: t("health.notices.filter.info"),
};
const trigger = (
<Button
size="sm"
variant={active ? "select" : "default"}
className="flex items-center gap-2 smart-capitalize"
aria-label={t("filter", { ns: "components/filter" })}
>
<FaFilter
className={
active ? "text-selected-foreground" : "text-secondary-foreground"
}
/>
<div
className={cn(
"hidden md:block",
active ? "text-selected-foreground" : "text-primary",
)}
>
{t("filter", { ns: "components/filter" })}
</div>
</Button>
);
const content = (
<div className="space-y-3 p-4">
<FilterSwitch
label={t("health.notices.filter.showDismissed")}
isChecked={filter.showDismissed}
onCheckedChange={(showDismissed) =>
onFilterChange({ ...filter, showDismissed })
}
/>
<DropdownMenuSeparator />
<div className="space-y-2.5">
<div className="mx-2 text-sm text-muted-foreground">
{t("health.notices.filter.severity")}
</div>
{DEFAULT_NOTICE_FILTER.severities.map((severity) => (
<FilterSwitch
key={severity}
label={severityLabels[severity]}
isChecked={filter.severities.includes(severity)}
onCheckedChange={(checked) =>
onFilterChange({
...filter,
severities: checked
? [...filter.severities, severity]
: filter.severities.filter((s) => s !== severity),
})
}
/>
))}
</div>
</div>
);
return (
<PlatformAwareDialog
trigger={trigger}
content={content}
contentClassName="p-1"
open={open}
onOpenChange={setOpen}
/>
);
}
web/src/components/health/NoticesPane.tsx
+95 -150
View File
@@ -1,164 +1,49 @@
import { useMemo } from "react";
import { useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { FaCircleCheck } from "react-icons/fa6";
import useSWR from "swr";
import HealthProblemRow from "@/components/health/HealthProblemRow";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import { Button, buttonVariants } from "@/components/ui/button";
import { Skeleton } from "@/components/ui/skeleton";
import { useNotices } from "@/hooks/use-notices";
import { useDateLocale } from "@/hooks/use-date-locale";
import { useTimezone } from "@/hooks/use-date-utils";
import useStats, { useAutoFrigateStats } from "@/hooks/use-stats";
import { useHealthChecks } from "@/hooks/use-health-checks";
import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
import { releaseUrl } from "@/utils/versionUtil";
import { evaluateConfigHealth } from "@/utils/configHealth";
import { isStartupWindow } from "@/utils/health";
import { sortHealthProblems } from "@/utils/healthSort";
import { streamHealth } from "@/utils/streamHealth";
import type { FrigateConfig } from "@/types/frigateConfig";
import type { HealthProblem } from "@/types/health";
import type { Notice, NoticeKind, NoticeStats } from "@/types/notice";
import { useHealthProblems } from "@/hooks/use-health-problems";
import type { NoticeFilter } from "@/types/health";
const SETTINGS_LINK_BY_KIND: Partial<
Record<NoticeKind, (scope: string | null) => string>
> = {
ffmpeg_crash_loop: (scope) => `/settings?page=cameraFfmpeg&camera=${scope}`,
retention_unmet: () => "/system#storage",
detector_stuck: () => "/system#general",
type NoticesPaneProps = {
filter: NoticeFilter;
};
const EXTERNAL_LINK_BY_KIND: Partial<
Record<NoticeKind, (params: Notice["params"]) => string | undefined>
> = {
update_available: (params) =>
typeof params.version === "string" ? releaseUrl(params.version) : undefined,
};
function useNoticeProblems(
notices: Notice[] | undefined,
statsByKind: Partial<Record<NoticeKind, NoticeStats>>,
dismiss: (id: string) => Promise<void>,
): HealthProblem[] {
const { t } = useTranslation(["views/system"]);
const { data: config } = useSWR<FrigateConfig>("config");
const timezone = useTimezone(config);
const locale = useDateLocale();
return useMemo(() => {
if (!notices) {
return [];
}
const formatTime = (timestamp: number, date_format: string) =>
formatUnixTimestampToDateTime(timestamp, {
timezone,
date_format,
locale,
});
return notices.map((notice) => {
const stats = statsByKind[notice.kind];
const seen = formatTime(notice.first_seen, "MMM d, h:mm a");
let meta: string;
if (notice.mode === "event") {
meta = t("health.notices.firstSeen", {
time: seen,
times: notice.count,
});
} else if (stats && stats.occurrences > 1) {
meta = t("health.notices.sinceWithCount", {
time: seen,
times: stats.occurrences,
firstSeen: formatTime(stats.first_seen, "MMM d"),
});
} else {
meta = t("health.notices.since", { time: seen });
}
const link = SETTINGS_LINK_BY_KIND[notice.kind];
const external = EXTERNAL_LINK_BY_KIND[notice.kind];
return {
id: `notice:${notice.id}`,
source: "registry" as const,
severity: notice.severity,
scope: notice.scope ?? undefined,
scopeIsCamera: notice.category === "camera",
// replace keeps backend params out of i18next's own option names
text: t(`health.notices.kinds.${notice.kind}`, {
replace: notice.params,
}),
meta,
link: link ? link(notice.scope) : undefined,
externalLink: external ? external(notice.params) : undefined,
onDismiss:
notice.mode === "event" ? () => dismiss(notice.id) : undefined,
};
});
}, [notices, statsByKind, dismiss, t, timezone, locale]);
}
export default function NoticesPane() {
const { t } = useTranslation(["views/system", "views/settings"]);
const { data: config } = useSWR<FrigateConfig>("config", {
revalidateOnFocus: false,
});
const stats = useAutoFrigateStats();
const { notices, statsByKind, dismiss } = useNotices();
const registryProblems = useNoticeProblems(notices, statsByKind, dismiss);
const { potentialProblems } = useStats(stats);
const {
stream: { results },
} = useHealthChecks();
const liveProblems = useMemo<HealthProblem[]>(() => {
if (!stats) {
return [];
}
if (isStartupWindow(stats)) {
return [
{
id: "live:startup",
source: "live",
severity: "info",
text: t("health.notices.startupWindow"),
},
];
}
return potentialProblems.map((problem, index) => ({
id: `live:${index}:${problem.text}`,
source: "live",
severity: problem.severity,
text: problem.text,
link: problem.relevantLink?.replace(/^(?!\/)/, "/"),
}));
}, [stats, potentialProblems, t]);
const configProblems = useMemo<HealthProblem[]>(
() => (config ? evaluateConfigHealth(config, t) : []),
[config, t],
export default function NoticesPane({ filter }: NoticesPaneProps) {
const { t } = useTranslation(["views/system", "views/settings", "common"]);
const { problems, dismissed, loading, clearDismissed } = useHealthProblems(
t,
filter.showDismissed,
);
const [confirmClear, setConfirmClear] = useState(false);
const streamProblems = useMemo<HealthProblem[]>(
() => (config ? streamHealth(config, results, t).problems : []),
[config, results, t],
);
const problems = useMemo(
const shown = useMemo(
() =>
sortHealthProblems([
...registryProblems,
...liveProblems,
...configProblems,
...streamProblems,
]),
[registryProblems, liveProblems, configProblems, streamProblems],
problems.filter((problem) =>
filter.severities.includes(problem.severity),
),
[problems, filter.severities],
);
const loading = notices === undefined || !config;
const shownDismissed = useMemo(
() =>
dismissed?.filter((problem) =>
filter.severities.includes(problem.severity),
),
[dismissed, filter.severities],
);
return (
<div className="flex flex-col gap-4">
@@ -173,14 +58,74 @@ export default function NoticesPane() {
<FaCircleCheck className="size-4 text-success" />
<span>{t("health.notices.empty")}</span>
</div>
) : shown.length === 0 ? (
<div className="px-1 py-2 text-sm text-muted-foreground">
{t("health.notices.noMatches")}
</div>
) : (
<div className="flex flex-col">
{problems.map((problem) => (
{shown.map((problem) => (
<HealthProblemRow key={problem.id} problem={problem} />
))}
</div>
)}
</div>
{filter.showDismissed && (
<div className="flex flex-col gap-2">
<div className="flex items-center justify-between gap-2">
<div className="text-sm text-muted-foreground">
{t("health.notices.dismissedTitle")}
</div>
{dismissed && dismissed.length > 0 && (
<Button
variant="ghost"
size="sm"
onClick={() => setConfirmClear(true)}
>
{t("health.notices.clearDismissed")}
</Button>
)}
</div>
<div className="rounded-lg bg-background_alt p-2.5 md:rounded-2xl">
{shownDismissed === undefined ? (
<Skeleton className="h-10 w-full" />
) : shownDismissed.length === 0 ? (
<div className="px-1 py-2 text-sm text-muted-foreground">
{t("health.notices.noneDismissed")}
</div>
) : (
<div className="flex flex-col">
{shownDismissed.map((problem) => (
<HealthProblemRow key={problem.id} problem={problem} />
))}
</div>
)}
</div>
</div>
)}
<AlertDialog open={confirmClear} onOpenChange={setConfirmClear}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
{t("health.notices.clearDismissedTitle")}
</AlertDialogTitle>
<AlertDialogDescription>
{t("health.notices.clearDismissedDesc")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>
{t("button.cancel", { ns: "common" })}
</AlertDialogCancel>
<AlertDialogAction
className={buttonVariants({ variant: "destructive" })}
onClick={clearDismissed}
>
{t("health.notices.clearDismissed")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
);
}
web/src/components/health/StatusBarNotices.tsx
+25
View File
@@ -0,0 +1,25 @@
import { useTranslation } from "react-i18next";
import { Link } from "react-router-dom";
import { useHealthProblems } from "@/hooks/use-health-problems";
/** The count of undismissed Notices rows, shown before the status bar's health text. */
export default function StatusBarNotices() {
const { t } = useTranslation(["views/system"]);
const { problems, loading } = useHealthProblems(t);
if (loading || problems.length === 0) {
return null;
}
return (
<>
<span className="text-sm text-muted-foreground">•</span>
<Link
to="/system#health"
className="whitespace-nowrap text-sm hover:underline"
>
{t("stats.systemNotices", { count: problems.length })}
</Link>
</>
);
}
web/src/components/navigation/Bottombar.tsx
+5 -1
View File
@@ -13,6 +13,7 @@ import {
useState,
} from "react";
import useStats from "@/hooks/use-stats";
import { useIsAdmin } from "@/hooks/use-is-admin";
import GeneralSettings from "../menu/GeneralSettings";
import useNavigation from "@/hooks/use-navigation";
import {
@@ -151,7 +152,10 @@ function StatusAlertNav({ className, large }: StatusAlertNavProps) {
}
}, [reindexState, addMessage, clearMessages, t]);
if (!messages || Object.keys(messages).length === 0) {
const isAdmin = useIsAdmin();
// problems link to admin-only pages
if (!isAdmin || !messages || Object.keys(messages).length === 0) {
return;
}
web/src/hooks/use-health-problems.ts
+198
View File
@@ -0,0 +1,198 @@
import { useCallback, useMemo } from "react";
import axios from "axios";
import type { TFunction } from "i18next";
import useSWR from "swr";
import { useDateLocale } from "@/hooks/use-date-locale";
import { useTimezone } from "@/hooks/use-date-utils";
import { useHealthChecks } from "@/hooks/use-health-checks";
import { useNotices } from "@/hooks/use-notices";
import { evaluateConfigHealth } from "@/utils/configHealth";
import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
import { sortHealthProblems } from "@/utils/healthSort";
import { streamHealth } from "@/utils/streamHealth";
import type { FrigateConfig } from "@/types/frigateConfig";
import type { HealthProblem } from "@/types/health";
import type { DismissedCheck, Notice } from "@/types/notice";
const EXTERNAL_LINK = /^https?:\/\//;
type HealthProblems = {
/** undismissed rows, most severe first */
problems: HealthProblem[];
/** dismissed rows, most recently dismissed first; undefined until loaded */
dismissed?: HealthProblem[];
loading: boolean;
/** delete every dismissed row, so each can show again */
clearDismissed: () => Promise<void>;
};
/**
* Every row of the Health tab's Notices list: registry notices plus the config
* and stream checks the browser builds. The Notices pane and the status bar
* both read it, so they always agree. `t` only fills in the text; row ids and
* counts never depend on it.
*/
export function useHealthProblems(
t: TFunction,
showDismissed = false,
): HealthProblems {
const { data: config } = useSWR<FrigateConfig>("config", {
revalidateOnFocus: false,
});
const timezone = useTimezone(config);
const locale = useDateLocale();
const {
notices,
dismissed: dismissedNotices,
dismiss,
mutateDismissed,
} = useNotices(showDismissed);
const { data: dismissedChecks, mutate: mutateDismissedChecks } = useSWR<
DismissedCheck[]
>("notices/dismissed_checks");
const {
stream: { results },
} = useHealthChecks();
const dismissCheck = useCallback(
async (id: string) => {
await axios.post(`notices/${id}/dismiss`);
mutateDismissedChecks();
},
[mutateDismissedChecks],
);
const clearDismissed = useCallback(async () => {
await axios.delete("notices/dismissed");
mutateDismissed();
mutateDismissedChecks();
}, [mutateDismissed, mutateDismissedChecks]);
const formatTime = useCallback(
(timestamp: number) =>
formatUnixTimestampToDateTime(timestamp, {
timezone,
date_format: "MMM d, h:mm a",
locale,
}),
[timezone, locale],
);
const noticeRow = useCallback(
(notice: Notice): HealthProblem => {
const link = notice.link ?? undefined;
const external = link !== undefined && EXTERNAL_LINK.test(link);
const isCamera = notice.category === "camera";
return {
id: `notice:${notice.id}`,
source: "registry",
severity: notice.severity,
// only a camera scope is a name; other scopes are ids like a release
scope: isCamera ? (notice.scope ?? undefined) : undefined,
scopeIsCamera: isCamera,
// replace keeps backend params out of i18next's own option names
text: t(`health.notices.kinds.${notice.kind}`, {
ns: "views/system",
replace: notice.params,
count: notice.count,
}),
meta:
notice.dismissed_at === null
? t("health.notices.firstSeen", {
ns: "views/system",
time: formatTime(notice.first_seen),
count: notice.count,
})
: t("health.notices.dismissedAt", {
ns: "views/system",
time: formatTime(notice.dismissed_at),
}),
link: external ? undefined : link,
externalLink: external ? link : undefined,
onDismiss:
notice.dismissed_at === null ? () => dismiss(notice.id) : undefined,
};
},
[dismiss, formatTime, t],
);
const checks = useMemo<HealthProblem[]>(
() =>
config
? [
...evaluateConfigHealth(config, t),
...streamHealth(config, results, t).problems,
]
: [],
[config, results, t],
);
const dismissedAt = useMemo(
() =>
new Map(
(dismissedChecks ?? []).map((check) => [check.id, check.dismissed_at]),
),
[dismissedChecks],
);
const problems = useMemo(
() =>
sortHealthProblems([
...(notices ?? []).map(noticeRow),
...checks
.filter((check) => !dismissedAt.has(check.id))
.map((check) => ({
...check,
onDismiss: () => dismissCheck(check.id),
})),
]),
[notices, noticeRow, checks, dismissedAt, dismissCheck],
);
const dismissed = useMemo(() => {
if (!showDismissed || dismissedNotices === undefined) {
return undefined;
}
const rows = [
...dismissedNotices.map((notice) => ({
at: notice.dismissed_at ?? 0,
row: noticeRow(notice),
})),
...checks.flatMap((check) => {
const at = dismissedAt.get(check.id);
return at === undefined
? []
: [
{
at,
row: {
...check,
meta: t("health.notices.dismissedAt", {
ns: "views/system",
time: formatTime(at),
}),
},
},
];
}),
];
return rows.sort((a, b) => b.at - a.at).map(({ row }) => row);
}, [
showDismissed,
dismissedNotices,
noticeRow,
checks,
dismissedAt,
formatTime,
t,
]);
const loading =
notices === undefined || dismissedChecks === undefined || !config;
return { problems, dismissed, loading, clearDismissed };
}
web/src/hooks/use-notices.ts
+25 -19
View File
@@ -1,21 +1,22 @@
import { useCallback, useMemo } from "react";
import { useCallback, useEffect, useMemo } from "react";
import axios from "axios";
import useSWR from "swr";
import { useWs } from "@/api/ws";
import type { Notice, NoticeStats } from "@/types/notice";
import type { Notice } from "@/types/notice";
/**
* Active notices: a REST snapshot, replaced by every `notices` websocket
* payload after that. Stats are fetched separately and only change on
* dismiss or a new occurrence, so they revalidate after each dismiss.
* Active notices come from a REST snapshot, then from every `notices`
* websocket payload. Dismissed notices are fetched only while the history is
* shown, and again when the active list changes or the tab regains focus. A
* purge in another tab leaves the active list unchanged, so only focus
* catches it.
*/
export function useNotices() {
export function useNotices(showDismissed: boolean) {
const { data: initial, mutate } = useSWR<Notice[]>("notices", {
revalidateOnFocus: false,
});
const { data: stats, mutate: mutateStats } = useSWR<NoticeStats[]>(
"notices/stats",
{ revalidateOnFocus: false },
const { data: history, mutate: mutateHistory } = useSWR<Notice[]>(
showDismissed ? ["notices", { include_dismissed: true }] : null,
);
const {
value: { payload },
@@ -33,22 +34,27 @@ export function useNotices() {
// still shows up because the registry publishes a new frame after it
const notices = live ?? initial;
const statsByKind = useMemo(() => {
const byKind: Partial<Record<Notice["kind"], NoticeStats>> = {};
(stats ?? []).forEach((entry) => {
byKind[entry.kind] = entry;
});
return byKind;
}, [stats]);
// refetch the history whenever the active list changes; SWR ignores the
// call while the history is hidden
useEffect(() => {
mutateHistory();
}, [live, mutateHistory]);
const dismissed = useMemo(
() =>
history
?.filter((notice) => notice.dismissed_at !== null)
.sort((a, b) => (b.dismissed_at ?? 0) - (a.dismissed_at ?? 0)),
[history],
);
const dismiss = useCallback(
async (id: string) => {
await axios.post(`notices/${id}/dismiss`);
mutate();
mutateStats();
},
[mutate, mutateStats],
[mutate],
);
return { notices, statsByKind, dismiss };
return { notices, dismissed, dismiss, mutateDismissed: mutateHistory };
}
web/src/hooks/use-stats.ts
+10 -13
View File
@@ -31,6 +31,9 @@ function problem(
return { text, severity, color: SEVERITY_COLOR[severity], relevantLink };
}
// matches SKIPPED_DETECTIONS_PCT in frigate/stats/emitter.py
const SKIPPED_DETECTIONS_PCT = 5;
export default function useStats(stats: FrigateStats | undefined) {
const { t } = useTranslation(["views/system"]);
const { data: config } = useSWR<FrigateConfig>("config");
@@ -61,18 +64,9 @@ export default function useStats(stats: FrigateStats | undefined) {
return problems;
}
// check shm level
const shm = memoizedStats.service.storage["/dev/shm"];
if (shm?.total && shm?.min_shm && shm.total < shm.min_shm) {
if (memoizedStats.service.retention_unmet) {
problems.push(
problem(
"error",
t("stats.shmTooLow", {
total: shm.total,
min: shm.min_shm,
}),
"/system#storage",
),
problem("error", t("stats.retentionUnmet"), "/system#storage"),
);
}
@@ -144,13 +138,16 @@ export default function useStats(stats: FrigateStats | undefined) {
const cameraName = config?.cameras?.[name]?.friendly_name ?? name;
if (config?.cameras?.[name]?.enabled && cam["skipped_fps"] > 1) {
if (
config?.cameras?.[name]?.enabled &&
cam["skipped_pct"] >= SKIPPED_DETECTIONS_PCT
) {
problems.push(
problem(
"warning",
t("stats.cameraSkippedDetections", {
camera: capitalizeFirstLetter(capitalizeAll(cameraName)),
fps: cam["skipped_fps"],
pct: cam["skipped_pct"],
}),
"/system#cameras",
),
web/src/pages/System.tsx
+12 -1
View File
@@ -21,6 +21,8 @@ import { Toaster } from "@/components/ui/sonner";
import { FrigateConfig } from "@/types/frigateConfig";
import EnrichmentMetrics from "@/views/system/EnrichmentMetrics";
import HealthMetrics from "@/views/system/HealthMetrics";
import NoticeFilterButton from "@/components/health/NoticeFilterButton";
import { DEFAULT_NOTICE_FILTER, NoticeFilter } from "@/types/health";
import { useTranslation } from "react-i18next";
const allMetrics = [
@@ -65,6 +67,9 @@ function System() {
const [lastUpdated, setLastUpdated] = useState<number>(
Math.floor(Date.now() / 1000),
);
const [noticeFilter, setNoticeFilter] = useState<NoticeFilter>(
DEFAULT_NOTICE_FILTER,
);
// Track which tabs have been visited so we can keep them mounted after first visit.
// Using a ref updated during render avoids extra render cycles from state/effects.
@@ -122,6 +127,12 @@ function System() {
</ToggleGroup>
<div className="flex h-full items-center">
{pageToggle == "health" && (
<NoticeFilterButton
filter={noticeFilter}
onFilterChange={setNoticeFilter}
/>
)}
{lastUpdated && pageToggle != "health" && (
<div className="h-full content-center text-sm text-muted-foreground">
{t("lastRefreshed")}
@@ -140,7 +151,7 @@ function System() {
</div>
{visitedTabs.has("health") && (
<div className={pageToggle == "health" ? "contents" : "hidden"}>
<HealthMetrics />
<HealthMetrics noticeFilter={noticeFilter} />
</div>
)}
{visitedTabs.has("general") && (
web/src/types/health.ts
+15 -4
View File
@@ -3,14 +3,14 @@ import type { ReactNode } from "react";
export type HealthSeverity = "error" | "warning" | "info";
/**
* One row in the Health tab's Notices list. Every source (registry notices
* now, live stats, config checks, and stream checks in PR 2) maps into this
* shape so the row component never needs to know where a problem came from.
* One row in the Health tab's Notices list. Registry notices, config checks,
* and stream checks all map into this shape so the row component never needs
* to know where a problem came from.
*/
export type HealthProblem = {
id: string;
/** which source produced the row; part of the sort order */
source: "registry" | "live" | "config" | "stream";
source: "registry" | "config" | "stream";
severity: HealthSeverity;
/** camera name or other scope shown as a chip before the text */
scope?: string;
@@ -29,3 +29,14 @@ export type HealthProblem = {
pending?: boolean;
onDismiss?: () => void;
};
/** What the Health tab's filter shows. */
export type NoticeFilter = {
showDismissed: boolean;
severities: HealthSeverity[];
};
export const DEFAULT_NOTICE_FILTER: NoticeFilter = {
showDismissed: false,
severities: ["error", "warning", "info"],
};
web/src/types/notice.ts
+8 -21
View File
@@ -1,36 +1,23 @@
export type NoticeMode = "state" | "event";
export type NoticeSeverity = "error" | "warning" | "info";
export type NoticeCategory =
| "camera"
| "detector"
| "storage"
| "model"
| "system";
export type NoticeKind =
| "ffmpeg_crash_loop"
| "detector_stuck"
| "model_download_failed"
| "retention_unmet"
| "update_available";
export type NoticeCategory = "camera" | "detector" | "model" | "system";
export type Notice = {
id: string;
kind: NoticeKind;
mode: NoticeMode;
kind: string;
severity: NoticeSeverity;
category: NoticeCategory;
scope: string | null;
params: Record<string, string | number | boolean>;
/** app route or absolute URL */
link: string | null;
first_seen: number;
last_seen: number;
count: number;
dismissed_at: number | null;
};
export type NoticeStats = {
kind: NoticeKind;
occurrences: number;
dismissals: number;
first_seen: number;
last_seen: number;
/** a config or stream check row an admin dismissed */
export type DismissedCheck = {
id: string;
dismissed_at: number;
};
web/src/types/stats.ts
+4
View File
@@ -24,6 +24,7 @@ export type CameraStats = {
pid: number;
process_fps: number;
skipped_fps: number;
skipped_pct: number;
connection_quality: "excellent" | "fair" | "poor" | "unusable";
expected_fps: number;
reconnects_last_hour: number;
@@ -54,6 +55,8 @@ export type EmbeddingsStats = {
face_embedding_speed: number;
plate_recognition_speed: number;
text_embedding_speed: number;
face_recognition_speed?: number;
yolov9_plate_detection_speed?: number;
devices?: Record<string, string>;
};
@@ -91,6 +94,7 @@ export type ServiceStats = {
uptime: number;
latest_version: string;
version: string;
retention_unmet: boolean;
};
export type StorageStats = {
web/src/utils/configHealth.ts
+3 -1
View File
@@ -180,13 +180,15 @@ export function evaluateConfigHealth(
cameraMessages
.filter((message) => isActive(message, ctx))
.forEach((message) => {
// camera names have no dots, so the backend can tell this suffix
// from global and modelN when the camera is deleted
const problem = toProblem(
message,
section,
ctx,
camera.name,
true,
camera.name,
`camera.${camera.name}`,
t,
);
web/src/utils/health.ts
+28 -6
View File
@@ -8,8 +8,8 @@ import type {
DetectionModelConfig,
FrigateConfig,
} from "@/types/frigateConfig";
import type { FrigateStats, GpuVendor } from "@/types/stats";
import { InferenceThreshold } from "@/types/graph";
import type { EmbeddingsStats, FrigateStats, GpuVendor } from "@/types/stats";
import { EmbeddingThreshold, InferenceThreshold } from "@/types/graph";
import { summarizeDevices } from "@/utils/detectionHardware";
import { isReplayCamera } from "@/utils/cameraUtil";
import { resolveCameraName } from "@/hooks/use-camera-friendly-name";
@@ -527,6 +527,26 @@ type EnrichmentSpec = {
presenceOnly: boolean;
};
type SpeedKey = Exclude<keyof EmbeddingsStats, "devices">;
// the stats that time each enrichment's inference
const SPEED_KEYS: Record<EnrichmentSpec["id"], SpeedKey[]> = {
semantic_search: ["image_embedding_speed", "text_embedding_speed"],
face_recognition: ["face_recognition_speed"],
lpr: ["plate_recognition_speed", "yolov9_plate_detection_speed"],
audio_transcription: [],
};
/** Whether an enrichment's inference is past the warning line its chart draws. */
function inferenceIsSlow(
stats: FrigateStats | undefined,
id: EnrichmentSpec["id"],
): boolean {
return SPEED_KEYS[id].some(
(key) => (stats?.embeddings?.[key] ?? 0) > EmbeddingThreshold.warning,
);
}
function enrichmentSpecs(config: FrigateConfig): EnrichmentSpec[] {
const ss = config.semantic_search;
const anyCameraTranscribes = Object.values(config.cameras).some(
@@ -682,9 +702,10 @@ export function enrichmentRows({
id,
state: "unknown",
label,
message: t("health.hardware.modelNotRunYet", {
ns: "views/system",
}),
message:
startup || !stats
? t("health.hardware.justStarted", { ns: "views/system" })
: t("health.hardware.modelNotRunYet", { ns: "views/system" }),
};
}
@@ -706,7 +727,8 @@ export function enrichmentRows({
};
}
if (runtimeIsCpu && present) {
// a model that keeps up on the CPU needs no accelerator
if (runtimeIsCpu && present && inferenceIsSlow(stats, spec.id)) {
return {
id,
state: "warning",
web/src/utils/healthSort.ts
+1 -1
View File
@@ -1,7 +1,7 @@
import type { HealthProblem } from "@/types/health";
const SEVERITY_ORDER = { error: 0, warning: 1, info: 2 } as const;
const SOURCE_ORDER = { registry: 0, live: 1, config: 2, stream: 3 } as const;
const SOURCE_ORDER = { registry: 0, config: 1, stream: 2 } as const;
/** errors first, then warnings, then info; within a severity by source, then scope */
export function sortHealthProblems(problems: HealthProblem[]): HealthProblem[] {
web/src/utils/versionUtil.ts
-4
View File
@@ -1,4 +0,0 @@
/** GitHub release page for a Frigate version such as "0.19.0". */
export function releaseUrl(version: string): string {
return `https://github.com/blakeblackshear/frigate/releases/tag/v${version}`;
}
web/src/views/system/HealthMetrics.tsx
+7 -2
View File
@@ -1,10 +1,15 @@
import HardwarePane from "@/components/health/HardwarePane";
import NoticesPane from "@/components/health/NoticesPane";
import type { NoticeFilter } from "@/types/health";
export default function HealthMetrics() {
type HealthMetricsProps = {
noticeFilter: NoticeFilter;
};
export default function HealthMetrics({ noticeFilter }: HealthMetricsProps) {
return (
<div className="scrollbar-container mt-4 flex size-full flex-col gap-4 overflow-y-auto">
<NoticesPane />
<NoticesPane filter={noticeFilter} />
<HardwarePane />
</div>
);