administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-10-01 20:36:51 +03:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Refactor Notices and System Health pane (#24243)

Browse Source 
* refactor notices

* show startup message for enrichments in health pane

* tweaks
This commit is contained in:
Josh Hawkins
2026-09-12 07:30:04 -06:00
committed by Nicolas Mowen
parent 5cef6823a6
commit 7bc32fd4c9
50 changed files with 2471 additions and 898 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frigate/api/auth.py
+56
View File
@@ -8,6 +8,7 @@ import logging
import os
import re
import secrets
import threading
import time
from datetime import datetime
from pathlib import Path
@@ -34,6 +35,7 @@ from frigate.api.media_auth import (
from frigate.config import AuthConfig, ProxyConfig
from frigate.const import CONFIG_DIR, JWT_SECRET_ENV_VAR, PASSWORD_HASH_ALGORITHM
from frigate.models import User
from frigate.notices import raise_notice
logger = logging.getLogger(__name__)
@@ -253,6 +255,58 @@ class RateLimiter:
rateLimiter = RateLimiter()
# a failed login this long after the user's previous one opens a new burst
FAILED_LOGIN_BURST_GAP_S = 300
# the username comes from the request, so it is cut before it reaches a notice
MAX_NOTICE_USERNAME = 64
# unknown usernames are unbounded, so past this many open bursts a new one only
# reaches the log
MAX_OPEN_BURSTS = 100
class FailedLoginTracker:
"""Groups each user's failed logins into bursts, one notice per burst."""
def __init__(self) -> None:
self._lock = threading.Lock()
# user -> (burst start, last attempt), stalest attempt first
self._bursts: dict[str, tuple[int, float]] = {}
def record(self, user: str, now: float, *, known: bool = False) -> None:
"""Count a failed login toward the user's open burst, or open a new one.
Once MAX_OPEN_BURSTS are open, only a known user opens another.
"""
user = user[:MAX_NOTICE_USERNAME]
with self._lock:
# bursts that went quiet are over
while self._bursts:
stalest = next(iter(self._bursts))
if now - self._bursts[stalest][1] < FAILED_LOGIN_BURST_GAP_S:
break
del self._bursts[stalest]
if (
not known
and user not in self._bursts
and len(self._bursts) >= MAX_OPEN_BURSTS
):
return
start, _ = self._bursts.pop(user, (int(now), now))
self._bursts[user] = (start, now)
raise_notice("failed_login", scope=f"{user}:{start}", params={"user": user})
failed_logins = FailedLoginTracker()
def get_remote_addr(request: Request):
# fall back to the direct TCP peer when no proxy chain is present
@@ -880,6 +934,7 @@ def login(request: Request, body: AppPostLoginBody):
db_user: User = User.get_by_id(user)
except DoesNotExist:
logger.warning(f"Login failed for unknown user '{user}' from {remote_addr}")
failed_logins.record(user, time.time())
return JSONResponse(content={"message": "Login failed"}, status_code=401)
password_hash = db_user.password_hash
@@ -911,6 +966,7 @@ def login(request: Request, body: AppPostLoginBody):
logger.warning(
f"Login failed for user '{user}' (invalid password) from {remote_addr}"
)
failed_logins.record(user, time.time(), known=True)
return JSONResponse(content={"message": "Login failed"}, status_code=401)
frigate/api/notices.py
+27 -17
View File
@@ -7,7 +7,6 @@ from fastapi.responses import JSONResponse
from frigate.api.auth import require_role
from frigate.api.defs.tags import Tags
from frigate.notices.registry import DismissResult
logger = logging.getLogger(__name__)
@@ -16,13 +15,13 @@ router = APIRouter(tags=[Tags.notices])
@router.get("/notices", dependencies=[Depends(require_role(["admin"]))])
def get_notices(request: Request, include_dismissed: bool = False) -> JSONResponse:
"""Get the active notices, most severe first.
"""Get notices, most severe first.
Args:
include_dismissed: Also return event notices the user has dismissed
include_dismissed: Also return dismissed notices, for the history view
Returns:
The active notices
The notices
"""
return JSONResponse(
content=request.app.notice_registry.active(include_dismissed=include_dismissed)
@@ -35,26 +34,37 @@ def get_notice_stats(request: Request) -> JSONResponse:
return JSONResponse(content=request.app.notice_registry.stats())
@router.get(
"/notices/dismissed_checks", dependencies=[Depends(require_role(["admin"]))]
)
def get_dismissed_checks(request: Request) -> JSONResponse:
"""Get the dismissed config and stream check rows, newest first."""
return JSONResponse(content=request.app.notice_registry.dismissed_checks())
@router.delete("/notices/dismissed", dependencies=[Depends(require_role(["admin"]))])
def purge_dismissed(request: Request) -> JSONResponse:
"""Delete every dismissed notice and check row so each can show again."""
request.app.notice_registry.purge_dismissed()
return JSONResponse(
content={"success": True, "message": "Dismissed notices cleared"}
)
# model notice ids contain a slash, so the id is a path parameter
@router.post(
"/notices/{notice_id}/dismiss", dependencies=[Depends(require_role(["admin"]))]
"/notices/{notice_id:path}/dismiss",
dependencies=[Depends(require_role(["admin"]))],
)
def dismiss_notice(request: Request, notice_id: str) -> JSONResponse:
"""Hide an event notice until it is raised again."""
result = request.app.notice_registry.dismiss(notice_id)
"""Hide a notice or a config or stream check row.
if result == DismissResult.not_found:
It stays hidden if the same problem happens again.
"""
if not request.app.notice_registry.dismiss(notice_id):
return JSONResponse(
content={"success": False, "message": "Notice not found"},
status_code=404,
)
if result == DismissResult.not_dismissable:
return JSONResponse(
content={
"success": False,
"message": "This notice clears itself when the problem is fixed",
},
status_code=400,
)
return JSONResponse(content={"success": True, "message": "Notice dismissed"})
frigate/app.py
+9 -11
View File
@@ -73,6 +73,7 @@ from frigate.models import (
Trigger,
User,
)
from frigate.notices import install_registry
from frigate.notices.registry import NoticeRegistry
from frigate.object_detection.base import ObjectDetectProcess
from frigate.object_detection.util import detection_frame_size
@@ -310,10 +311,6 @@ class FrigateApp:
]
self.db.bind(models)
self.notice_registry = NoticeRegistry()
# producers confirm state notices again at startup if they still hold
self.notice_registry.mark_state_notices_unconfirmed()
def check_db_data_migrations(self) -> None:
# check if vacuum needs to be run
if not os.path.exists(f"{CONFIG_DIR}/.exports"):
@@ -325,6 +322,10 @@ class FrigateApp:
migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
def install_notice_registry(self) -> None:
self.notice_registry = NoticeRegistry()
install_registry(self.notice_registry)
def init_embeddings_client(self) -> None:
# Create a client for other processes to use
self.embeddings = EmbeddingsContext(self.db)
@@ -510,9 +511,7 @@ class FrigateApp:
self.record_cleanup.start()
def start_storage_maintainer(self) -> None:
self.storage_maintainer = StorageMaintainer(
self.config, self.stop_event, self.notice_registry
)
self.storage_maintainer = StorageMaintainer(self.config, self.stop_event)
self.storage_maintainer.start()
def start_stats_emitter(self) -> None:
@@ -524,16 +523,14 @@ class FrigateApp:
self.embeddings_metrics,
self.detectors,
self.processes,
self.storage_maintainer,
),
self.stop_event,
notice_registry=self.notice_registry,
)
self.stats_emitter.start()
def start_watchdog(self) -> None:
self.frigate_watchdog = FrigateWatchdog(
self.detectors, self.stop_event, self.notice_registry
)
self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event)
# (attribute on self, key in self.processes, factory)
specs: list[tuple[str, str, Callable[[], FrigateProcess]]] = [
@@ -644,6 +641,7 @@ class FrigateApp:
self.init_embeddings_manager()
self.bind_database()
self.check_db_data_migrations()
self.install_notice_registry()
# Clean up any stale replay camera artifacts (filesystem + DB)
cleanup_replay_cameras()
frigate/comms/dispatcher.py
+1 -14
View File
@@ -357,20 +357,7 @@ class Dispatcher:
return
try:
action = payload.get("action")
kind = payload.get("kind")
scope = payload.get("scope")
if not isinstance(kind, str):
logger.warning("Ignoring notice update without a kind")
elif action == "raise":
self.notice_registry.raise_notice(
kind, scope=scope, params=payload.get("params") or {}
)
elif action == "resolve":
self.notice_registry.resolve(kind, scope)
else:
logger.warning("Ignoring notice update with action %s", action)
self.notice_registry.apply(payload)
except Exception:
# a raise here would kill the REP thread for every process
logger.exception("Failed to apply notice update")
frigate/notices/__init__.py
+84
View File
@@ -0,0 +1,84 @@
"""Raise and resolve notices from any Frigate process or thread.
A notice records something worth seeing later, such as an event or a setup
problem only the backend can detect. A condition that comes and goes belongs in
stats and the status bar, and a config problem belongs in a settings message
flagged for health.
"""
import logging
import multiprocessing as mp
import threading
from typing import TYPE_CHECKING, Any
from frigate.comms.inter_process import InterProcessRequestor
from frigate.const import UPDATE_NOTICE
if TYPE_CHECKING:
from frigate.notices.registry import NoticeRegistry
logger = logging.getLogger(__name__)
_registry: "NoticeRegistry | None" = None
# zmq sockets are not thread safe, so every thread in a process shares one
# requestor behind a lock
_requestor: InterProcessRequestor | None = None
_requestor_lock = threading.Lock()
def install_registry(registry: "NoticeRegistry | None") -> None:
"""Send this process's notices straight to the registry.
Only the main process installs one. Children start from the forkserver,
so they never inherit it and send over IPC instead.
"""
global _registry
_registry = registry
def raise_notice(
kind: str, *, scope: str | None = None, params: dict[str, Any] | None = None
) -> None:
"""Record a notice, or count another occurrence of it. Never raises."""
_send({"action": "raise", "kind": kind, "scope": scope, "params": params or {}})
def resolve_notice(kind: str, scope: str | None = None) -> None:
"""Delete a notice once its problem is fixed. Never raises."""
_send({"action": "resolve", "kind": kind, "scope": scope})
def resolve_kind(kind: str) -> None:
"""Delete every notice of a kind. Never raises."""
_send({"action": "resolve_kind", "kind": kind})
def flush_notices() -> None:
"""Write repeats the registry held back. Only the main process has any."""
if _registry is None:
return
try:
_registry.flush()
except Exception:
logger.exception("Failed to flush notices")
def _send(update: dict[str, Any]) -> None:
global _requestor
try:
if _registry is not None:
_registry.apply(update)
elif mp.parent_process() is not None:
with _requestor_lock:
if _requestor is None:
_requestor = InterProcessRequestor()
_requestor.send_data(UPDATE_NOTICE, update)
else:
# the main process before startup installs the registry, or a test
logger.debug("No notice registry for %s", update["kind"])
except Exception:
logger.exception("Failed to update notice %s", update["kind"])
frigate/notices/registry.py
+187 -78
View File
@@ -1,49 +1,44 @@
"""Registry of active notices and per-kind occurrence counts."""
"""Registry of notices and per-kind occurrence counts."""
import logging
import threading
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Any
from frigate.const import REPLAY_CAMERA_PREFIX
from frigate.models import Notice, NoticeStats
from frigate.notices.types import (
NOTICE_KINDS,
SEVERITY_ORDER,
NoticeMode,
notice_id,
)
from frigate.notices.types import CHECK_KINDS, NOTICE_KINDS, SEVERITY_ORDER, notice_id
logger = logging.getLogger(__name__)
class DismissResult(str, Enum):
ok = "ok"
not_found = "not_found"
not_dismissable = "not_dismissable"
# last_seen value of a state notice that has not been raised since startup
UNCONFIRMED = 0
@dataclass
class _HeldRepeats:
kind: str
count: int
last_seen: float
params: dict[str, Any]
class NoticeRegistry:
"""Owns the notice tables. Lives in the main process only.
Producers in the main process call it directly. Other processes send an
update_notice request that the dispatcher forwards here. The FastAPI app
runs inside the main process too, so the API's dismiss shares this exact
object, its lock, and its listeners.
Producers call the functions in frigate.notices, which reach this object
directly in the main process and through the dispatcher everywhere else.
It holds no rules for particular kinds; those live in NOTICE_KINDS.
"""
def __init__(self) -> None:
self._lock = threading.Lock()
self._listeners: list[Callable[[], None]] = []
# row id -> repeats of a batch_repeats kind that flush() writes
self._held: dict[str, _HeldRepeats] = {}
def subscribe(self, listener: Callable[[], None]) -> None:
"""Call listener after every change to the active list."""
"""Call listener after every change to the notices."""
self._listeners.append(listener)
def _notify(self) -> None:
@@ -53,6 +48,26 @@ class NoticeRegistry:
except Exception:
logger.exception("Notice listener failed")
def apply(self, update: dict[str, Any]) -> None:
"""Apply one update message built by frigate.notices."""
kind = update.get("kind")
if not isinstance(kind, str):
logger.warning("Ignoring notice update without a kind")
return
match update.get("action"):
case "raise":
self.raise_notice(
kind, scope=update.get("scope"), params=update.get("params")
)
case "resolve":
self.resolve(kind, update.get("scope"))
case "resolve_kind":
self.resolve_kind(kind)
case action:
logger.warning("Ignoring notice update with action %s", action)
def raise_notice(
self,
kind: str,
@@ -60,7 +75,12 @@ class NoticeRegistry:
scope: str | None = None,
params: dict[str, Any] | None = None,
) -> None:
"""Insert or update a notice. See the counting rule in the spec."""
"""Insert a notice or count another occurrence of it.
A dismissed notice stays dismissed when it is raised again, unless its
kind sets reopen_at_count. A kind that should come back after a
dismissal gives each episode its own scope.
"""
definition = NOTICE_KINDS.get(kind)
if definition is None:
@@ -82,6 +102,8 @@ class NoticeRegistry:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
# repeats held for a row that has since been deleted are stale
self._held.pop(row_id, None)
Notice.create(
id=row_id,
kind=kind,
@@ -92,28 +114,48 @@ class NoticeRegistry:
count=1,
dismissed_at=None,
)
self._bump_occurrences(kind, now)
elif definition.mode == NoticeMode.event:
Notice.update(
count=Notice.count + 1,
last_seen=now,
params=params,
dismissed_at=None,
).where(Notice.id == row_id).execute()
self._bump_occurrences(kind, now)
self._bump_occurrences(kind, 1, now)
if definition.keep_latest is not None:
self._prune(kind, definition.keep_latest)
elif not definition.counts_repeats:
return
elif definition.batch_repeats:
held = self._held.setdefault(row_id, _HeldRepeats(kind, 0, now, params))
held.count += 1
held.last_seen = now
held.params = params
return
else:
# one episode counts once; a repeat with nothing new is free
confirmed = existing.last_seen != UNCONFIRMED
if confirmed and existing.params == params:
return
Notice.update(last_seen=now, params=params).where(
Notice.id == row_id
).execute()
self._write_repeats(existing, kind, 1, now, params)
self._notify()
def flush(self) -> None:
"""Write the repeats that batch_repeats kinds held back."""
with self._lock:
held, self._held = self._held, {}
written = False
for row_id, repeats in held.items():
row = Notice.get_or_none(Notice.id == row_id)
# resolved while its repeats waited
if row is None:
continue
self._write_repeats(
row,
repeats.kind,
repeats.count,
repeats.last_seen,
repeats.params,
)
written = True
if written:
self._notify()
def resolve(self, kind: str, scope: str | None = None) -> None:
"""Delete a notice if present. Safe to call when it is absent."""
with self._lock:
@@ -124,8 +166,16 @@ class NoticeRegistry:
if deleted:
self._notify()
def resolve_kind(self, kind: str) -> None:
"""Delete every notice of a kind."""
with self._lock:
deleted = Notice.delete().where(Notice.kind == kind).execute()
if deleted:
self._notify()
def resolve_camera(self, camera: str) -> None:
"""Drop every camera-category notice scoped to a camera being deleted."""
"""Drop the notices and check dismissals of a camera being deleted."""
camera_kinds = [
key
for key, definition in NOTICE_KINDS.items()
@@ -142,33 +192,82 @@ class NoticeRegistry:
.execute()
)
# a stream id names its camera first; a config id ends with camera.<name>
for check in self.dismissed_checks():
check_id = check["id"]
if check_id.startswith(f"stream:{camera}:") or (
check_id.startswith("config:")
and check_id.endswith(f":camera.{camera}")
):
Notice.delete_by_id(check_id)
if deleted:
self._notify()
def dismiss(self, row_id: str) -> DismissResult:
"""Hide an event notice until it is raised again."""
def purge_dismissed(self) -> int:
"""Delete every dismissed row so each can show again. Returns how many."""
with self._lock:
return int(
Notice.delete().where(Notice.dismissed_at.is_null(False)).execute()
)
def dismiss(self, row_id: str) -> bool:
"""Hide a notice or check row for good. Returns False for an unknown id."""
with self._lock:
existing = Notice.get_or_none(Notice.id == row_id)
if existing is None:
return DismissResult.not_found
# a check row gets a notice row only once it is dismissed
kind, _, scope = row_id.partition(":")
definition = NOTICE_KINDS.get(existing.kind)
if kind not in CHECK_KINDS or not scope:
return False
if definition is None or definition.mode == NoticeMode.state:
return DismissResult.not_dismissable
now = datetime.now().timestamp()
Notice.create(
id=row_id,
kind=kind,
scope=scope,
params={},
first_seen=now,
last_seen=now,
count=1,
dismissed_at=now,
)
return True
now = datetime.now().timestamp()
Notice.update(dismissed_at=now).where(Notice.id == row_id).execute()
if existing.dismissed_at is not None:
return True
if existing.kind not in NOTICE_KINDS:
return False
Notice.update(dismissed_at=datetime.now().timestamp()).where(
Notice.id == row_id
).execute()
NoticeStats.update(dismissals=NoticeStats.dismissals + 1).where(
NoticeStats.kind == existing.kind
).execute()
self._notify()
return DismissResult.ok
return True
def dismissed_checks(self) -> list[dict[str, Any]]:
"""Dismissed config and stream check rows, newest first."""
rows = (
Notice.select()
.where(Notice.kind.in_(list(CHECK_KINDS))) # type: ignore[call-arg, arg-type, misc]
.order_by(Notice.dismissed_at.desc())
)
return [{"id": row.id, "dismissed_at": row.dismissed_at} for row in rows]
def active(self, include_dismissed: bool = False) -> list[dict[str, Any]]:
"""The active notices, most severe first, then most recent first."""
"""Notices most severe first, then most recent first.
Args:
include_dismissed: Also return dismissed notices, for the history view
"""
rows = []
for row in Notice.select():
@@ -180,18 +279,15 @@ class NoticeRegistry:
if row.dismissed_at is not None and not include_dismissed:
continue
if definition.mode == NoticeMode.state and row.last_seen == UNCONFIRMED:
continue
rows.append(
{
"id": row.id,
"kind": row.kind,
"mode": definition.mode.value,
"severity": definition.severity.value,
"category": definition.category,
"scope": row.scope,
"params": row.params,
"link": definition.link_for(row.params),
"first_seen": row.first_seen,
"last_seen": row.last_seen,
"count": row.count,
@@ -208,7 +304,7 @@ class NoticeRegistry:
return rows
def stats(self) -> list[dict[str, Any]]:
"""Lifetime counts per kind, for the UI meta line and later analytics."""
"""Lifetime counts per kind, for later analytics."""
return [
{
"kind": row.kind,
@@ -223,41 +319,54 @@ class NoticeRegistry:
if row.kind in NOTICE_KINDS
]
def mark_state_notices_unconfirmed(self) -> None:
"""Hide state notices until their producer raises them again.
def _write_repeats(
self,
row: Notice,
kind: str,
count: int,
last_seen: float,
params: dict[str, Any],
) -> None:
# called with the lock held
fields: dict[str, Any] = {
"count": row.count + count,
"last_seen": last_seen,
"params": params,
}
reopen_at = NOTICE_KINDS[kind].reopen_at_count
Rows already unconfirmed from the previous run are deleted. Hiding
rather than deleting keeps one episode that spans a restart at one
occurrence.
"""
state_kinds = [
key
for key, definition in NOTICE_KINDS.items()
if definition.mode == NoticeMode.state
]
if reopen_at is not None and row.count < reopen_at <= row.count + count:
fields["dismissed_at"] = None
with self._lock:
Notice.delete().where(
Notice.kind.in_(state_kinds), # type: ignore[call-arg, arg-type, misc]
Notice.last_seen == UNCONFIRMED,
).execute()
Notice.update(last_seen=UNCONFIRMED).where(
Notice.kind.in_(state_kinds) # type: ignore[call-arg, arg-type, misc]
).execute()
Notice.update(**fields).where(Notice.id == row.id).execute()
self._bump_occurrences(kind, count, last_seen)
def _bump_occurrences(self, kind: str, now: float) -> None:
def _prune(self, kind: str, keep: int) -> None:
# called with the lock held
newest = (
Notice.select(Notice.id)
.where(Notice.kind == kind)
.order_by(Notice.first_seen.desc())
.limit(keep)
)
Notice.delete().where(
Notice.kind == kind,
Notice.id.not_in(newest), # type: ignore[call-arg, misc]
).execute()
def _bump_occurrences(self, kind: str, count: int, now: float) -> None:
# called with the lock held
stats = NoticeStats.get_or_none(NoticeStats.kind == kind)
if stats is None:
NoticeStats.create(
kind=kind,
occurrences=1,
occurrences=count,
dismissals=0,
first_seen=now,
last_seen=now,
)
else:
NoticeStats.update(
occurrences=NoticeStats.occurrences + 1, last_seen=now
occurrences=NoticeStats.occurrences + count, last_seen=now
).where(NoticeStats.kind == kind).execute()
frigate/notices/types.py
+55 -17
View File
@@ -1,14 +1,8 @@
"""Notice kinds and the definitions that fix their mode, severity, and category."""
"""Notice kinds and the definitions that fix their severity and category."""
from dataclasses import dataclass
from enum import Enum
class NoticeMode(str, Enum):
"""Whether a producer resolves the notice itself or the user dismisses it."""
state = "state"
event = "event"
from typing import Any
class NoticeSeverity(str, Enum):
@@ -30,33 +24,77 @@ class NoticeKind:
Attributes:
key: Stable machine-readable name, also the translation key suffix
mode: state notices are resolved by their producer, event notices by the user
severity: error, warning, or info
category: what the scope names: camera, detector, and model kinds carry a scope
category: camera, detector, model, or system; a camera scope is a
camera name, and the UI shows it
link: app route or absolute URL for the row, filled in from params
counts_repeats: whether raising an existing notice counts another occurrence
batch_repeats: whether repeats wait in memory for the next flush
reopen_at_count: count at which a dismissed notice shows again
keep_latest: rows of this kind to keep; a new row drops the oldest
reportable: whether a future analytics reporter may send this kind's counts
"""
key: str
mode: NoticeMode
severity: NoticeSeverity
category: str
link: str | None = None
counts_repeats: bool = True
batch_repeats: bool = False
reopen_at_count: int | None = None
keep_latest: int | None = None
reportable: bool = True
def link_for(self, params: dict[str, Any]) -> str | None:
"""The link with params filled in, or None if a param is missing."""
if self.link is None:
return None
try:
return self.link.format(**params)
except (KeyError, IndexError, TypeError, ValueError):
return None
SCOPED_CATEGORIES = frozenset({"camera", "detector", "model"})
_KINDS = (
NoticeKind("ffmpeg_crash_loop", NoticeMode.state, NoticeSeverity.error, "camera"),
NoticeKind("detector_stuck", NoticeMode.event, NoticeSeverity.warning, "detector"),
NoticeKind(
"model_download_failed", NoticeMode.event, NoticeSeverity.error, "model"
"detector_stuck", NoticeSeverity.warning, "detector", link="/system#general"
),
NoticeKind("model_download_failed", NoticeSeverity.error, "model"),
NoticeKind(
"skipped_detections",
NoticeSeverity.warning,
"camera",
link="/system#cameras",
),
NoticeKind("shm_too_low", NoticeSeverity.warning, "system", link="/system#storage"),
# one row per user per burst; the login log lines carry the address
NoticeKind(
"failed_login",
NoticeSeverity.warning,
"system",
link="/logs",
batch_repeats=True,
reopen_at_count=5,
keep_latest=100,
),
# one row per release, so a dismissal lasts until the next release
NoticeKind(
"update_available",
NoticeSeverity.info,
"system",
link="https://github.com/blakeblackshear/frigate/releases/tag/v{version}",
counts_repeats=False,
keep_latest=1,
),
NoticeKind("retention_unmet", NoticeMode.state, NoticeSeverity.error, "storage"),
NoticeKind("update_available", NoticeMode.state, NoticeSeverity.info, "system"),
)
NOTICE_KINDS: dict[str, NoticeKind] = {kind.key: kind for kind in _KINDS}
# the Health tab builds config and stream check rows in the browser, so a notice
# row of these kinds only records a dismissal; its other fields are placeholders
CHECK_KINDS = frozenset({"config", "stream"})
def notice_id(kind: str, scope: str | None) -> str:
"""The row id for a kind and optional scope."""
frigate/stats/emitter.py
+91 -10
View File
@@ -11,7 +11,7 @@ from typing import Any
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.const import FREQUENCY_STATS_POINTS
from frigate.notices.registry import NoticeRegistry
from frigate.notices import flush_notices, raise_notice, resolve_kind, resolve_notice
from frigate.stats.hardware import HardwareStats
from frigate.stats.prometheus import update_metrics
from frigate.stats.util import get_latest_version, is_newer_version, stats_snapshot
@@ -26,6 +26,46 @@ MAX_STATS_POINTS = 80
# how often to ask GitHub for the latest release
VERSION_REFRESH_S = 24 * 60 * 60
# a camera skipping at least this percent of its frames is falling behind
SKIPPED_DETECTIONS_PCT = 5
# for at least this long before it becomes a notice
SKIPPED_DETECTIONS_HOLD_S = 60
# detectors warm up after a start; the status bar waits this long too
STARTUP_GRACE_S = 120
class SkippedDetectionsTracker:
"""Finds cameras whose skipped share stays high long enough for a notice."""
def __init__(self) -> None:
self._since: dict[str, float] = {}
self._raised: set[str] = set()
def update(self, cameras: dict[str, dict[str, Any]], now: float) -> list[str]:
"""Return the cameras whose episode qualified on this sample."""
qualified: list[str] = []
# a removed camera that comes back starts a new episode
for camera in self._since.keys() - cameras.keys():
self._since.pop(camera)
self._raised.discard(camera)
for camera, camera_stats in cameras.items():
if camera_stats["skipped_pct"] < SKIPPED_DETECTIONS_PCT:
self._since.pop(camera, None)
self._raised.discard(camera)
continue
since = self._since.setdefault(camera, now)
if camera not in self._raised and now - since >= SKIPPED_DETECTIONS_HOLD_S:
self._raised.add(camera)
qualified.append(camera)
return qualified
class StatsEmitter(threading.Thread):
def __init__(
@@ -33,15 +73,18 @@ class StatsEmitter(threading.Thread):
config: FrigateConfig,
stats_tracking: StatsTrackingTypes,
stop_event: MpEvent,
notice_registry: NoticeRegistry | None = None,
):
super().__init__(name="frigate_stats_emitter")
self.config = config
self.stats_tracking = stats_tracking
self.stop_event = stop_event
self.notice_registry = notice_registry
self.hardware_stats = HardwareStats(config)
self.stats_history: list[dict[str, Any]] = []
self.skipped_detections = SkippedDetectionsTracker()
# the shm notice's params as last sent, so only a change is written
self._shm_checked = False
self._shm_params: dict[str, Any] | None = None
# create communication for stats
self.requestor = InterProcessRequestor()
@@ -134,9 +177,6 @@ class StatsEmitter(threading.Thread):
def _check_update_notice(self) -> None:
"""Raise or resolve the update notice from the tracked latest version."""
if self.notice_registry is None:
return
latest = self.stats_tracking["latest_frigate_version"]
# a failed lookup says nothing about whether an update exists
@@ -144,11 +184,9 @@ class StatsEmitter(threading.Thread):
return
if is_newer_version(VERSION, latest):
self.notice_registry.raise_notice(
"update_available", params={"version": latest}
)
raise_notice("update_available", scope=latest, params={"version": latest})
else:
self.notice_registry.resolve("update_available")
resolve_kind("update_available")
def _refresh_latest_version(self) -> None:
"""Refresh the latest release on a daemon thread so the request never stalls stats."""
@@ -166,6 +204,46 @@ class StatsEmitter(threading.Thread):
target=refresh, name="frigate_version_check", daemon=True
).start()
def _update_shm_notice(self, shm: dict[str, Any]) -> None:
"""Raise the shm notice while /dev/shm is smaller than the cameras need."""
params = (
{"total": shm["total"], "min": shm["min_shm"]}
if shm and shm["total"] < shm["min_shm"]
else None
)
# the first tick always sends, so a row the last run left is raised
# again or cleared
if self._shm_checked and params == self._shm_params:
return
self._shm_checked = True
self._shm_params = params
if params is None:
resolve_notice("shm_too_low")
else:
raise_notice("shm_too_low", params=params)
def _update_notices(self, stats: dict[str, Any], now: float) -> None:
"""Update notices based on current stats or time."""
# skipped detections
if stats["service"]["uptime"] >= STARTUP_GRACE_S:
for camera in self.skipped_detections.update(stats["cameras"], now):
raise_notice(
"skipped_detections",
scope=camera,
params={"pct": stats["cameras"][camera]["skipped_pct"]},
)
# shm too small for the cameras
self._update_shm_notice(stats["service"]["storage"]["/dev/shm"])
# repeats of batched kinds, such as failed logins
flush_notices()
# add any additional notice types here
def run(self) -> None:
time.sleep(10)
self._check_update_notice()
@@ -186,11 +264,14 @@ class StatsEmitter(threading.Thread):
)
self.stats_history.append(stats)
self.stats_history = self.stats_history[-MAX_STATS_POINTS:]
self._update_notices(stats, time.time())
if counter == 0:
self.requestor.send_data("stats", json.dumps(stats))
logger.debug("Finished stats collection")
# write the repeats held back since the last tick
flush_notices()
self.hardware_stats.stop()
logger.info("Exiting stats emitter...")
frigate/stats/util.py
+28 -1
View File
@@ -6,7 +6,7 @@ import shutil
import time
from json import JSONDecodeError
from multiprocessing.managers import DictProxy
from typing import Any
from typing import TYPE_CHECKING, Any
import requests
from requests.exceptions import RequestException
@@ -24,6 +24,9 @@ from frigate.util.services import (
)
from frigate.version import VERSION
if TYPE_CHECKING:
from frigate.storage import StorageMaintainer
logger = logging.getLogger(__name__)
@@ -88,6 +91,7 @@ def stats_init(
embeddings_metrics: DataProcessorMetrics,
detectors: dict[str, ObjectDetectProcess],
processes: dict[str, int],
storage_maintainer: "StorageMaintainer | None" = None,
) -> StatsTrackingTypes:
stats_tracking: StatsTrackingTypes = {
"camera_metrics": camera_metrics,
@@ -97,6 +101,7 @@ def stats_init(
"latest_frigate_version": get_latest_version(config),
"last_updated": int(time.time()),
"processes": processes,
"storage_maintainer": storage_maintainer,
}
return stats_tracking
@@ -216,6 +221,18 @@ def embeddings_stats(
return stats
def skipped_percent(skipped_fps: float, camera_fps: float, enabled: bool) -> float:
"""Percent of a camera's frames dropped before detection.
camera_fps counts the dropped frames too. A disabled camera keeps its last
readings, so it reports zero.
"""
if not enabled or camera_fps <= 0:
return 0.0
return round(skipped_fps / camera_fps * 100, 1)
def stats_snapshot(
config: FrigateConfig,
stats_tracking: StatsTrackingTypes,
@@ -273,6 +290,11 @@ def stats_snapshot(
"camera_fps": round(camera_stats.camera_fps.value, 2),
"process_fps": round(camera_stats.process_fps.value, 2),
"skipped_fps": round(camera_stats.skipped_fps.value, 2),
"skipped_pct": skipped_percent(
camera_stats.skipped_fps.value,
current_fps,
config.cameras[name].enabled,
),
"detection_fps": round(camera_stats.detection_fps.value, 2),
"detection_enabled": config.cameras[name].detect.enabled,
"pid": pid,
@@ -301,12 +323,17 @@ def stats_snapshot(
if bandwidth_stats:
stats["bandwidth_usages"] = bandwidth_stats
storage_maintainer = stats_tracking.get("storage_maintainer")
stats["service"] = {
"uptime": (int(time.time()) - stats_tracking["started"]),
"version": VERSION,
"latest_version": stats_tracking["latest_frigate_version"],
"storage": {},
"last_updated": int(time.time()),
"retention_unmet": bool(
storage_maintainer and storage_maintainer.retention_unmet
),
}
for path in [RECORD_DIR, CLIPS_DIR, CACHE_DIR]:
frigate/storage.py
+8 -21
View File
@@ -20,7 +20,6 @@ from frigate.const import (
STREAM_TYPE_SUB,
)
from frigate.models import Event, Recordings
from frigate.notices.registry import NoticeRegistry
from frigate.util.builtin import clear_and_unlink
logger = logging.getLogger(__name__)
@@ -35,17 +34,14 @@ BANDWIDTH_SAMPLE_TARGET = 50
class StorageMaintainer(threading.Thread):
"""Maintain frigates recording storage."""
def __init__(
self,
config: FrigateConfig,
stop_event: MpEvent,
notice_registry: NoticeRegistry | None = None,
) -> None:
def __init__(self, config: FrigateConfig, stop_event: MpEvent) -> None:
super().__init__(name="storage_maintainer")
self.config = config
self.stop_event = stop_event
self.notice_registry = notice_registry
self.camera_storage_stats: dict[str, dict] = {}
# read by stats; true while maintenance has to delete retained recordings
self.retention_unmet = False
self.config_subscriber = CameraConfigUpdateSubscriber(
self.config,
self.config.cameras,
@@ -350,19 +346,10 @@ class StorageMaintainer(threading.Thread):
# this file was not found so we must assume no space was cleaned up
pass
if self.notice_registry is not None:
self.notice_registry.raise_notice(
"retention_unmet",
params={
"needed_mb": round(float(hourly_bandwidth), 1),
"cleared_mb": round(float(deleted_segments_size), 1),
},
)
self.retention_unmet = True
else:
logger.info(f"Cleaned up {deleted_segments_size:.2f} MB of recordings")
if self.notice_registry is not None:
self.notice_registry.resolve("retention_unmet")
self.retention_unmet = False
logger.debug(f"Expiring {len(deleted_recordings)} recordings")
# delete up to 100,000 at a time
@@ -448,5 +435,5 @@ class StorageMaintainer(threading.Thread):
"Less than 1 hour of recording space left, running storage maintenance..."
)
self.reduce_storage_consumption()
elif self.notice_registry is not None:
self.notice_registry.resolve("retention_unmet")
else:
self.retention_unmet = False
frigate/test/http_api/test_http_auth.py
+122 -2
View File
@@ -1,9 +1,16 @@
"""Tests for authentication endpoints."""
import os
from unittest.mock import patch
import unittest
from unittest.mock import ANY, patch
from frigate.api.auth import hash_password
from frigate.api.auth import (
FAILED_LOGIN_BURST_GAP_S,
MAX_NOTICE_USERNAME,
MAX_OPEN_BURSTS,
FailedLoginTracker,
hash_password,
)
from frigate.const import JWT_SECRET_ENV_VAR
from frigate.models import User
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
@@ -43,3 +50,116 @@ class TestHttpAuth(BaseTestHttp):
)
assert response.status_code == 401
assert any("Login failed" in m and "admin" in m for m in logs.output)
@patch.dict(os.environ, {JWT_SECRET_ENV_VAR: "test-secret"})
class TestFailedLoginNotices(BaseTestHttp):
def setUp(self):
super().setUp([User])
self.app = super().create_app()
User.insert(
username="admin",
password_hash=hash_password("correct-horse-battery", iterations=1000),
role="admin",
notification_tokens=[],
).execute()
tracker_patch = patch("frigate.api.auth.failed_logins")
self.failed_logins = tracker_patch.start()
self.addCleanup(tracker_patch.stop)
def tearDown(self):
User.delete().execute()
super().tearDown()
def _login(self, user: str, password: str) -> int:
with AuthTestClient(self.app) as client:
response = client.post("/login", json={"user": user, "password": password})
return response.status_code
def test_unknown_user_is_recorded(self):
self.assertEqual(self._login("ghost", "irrelevant"), 401)
self.failed_logins.record.assert_called_once_with("ghost", ANY)
def test_bad_password_is_recorded_as_known(self):
self.assertEqual(self._login("admin", "wrong"), 401)
self.failed_logins.record.assert_called_once_with("admin", ANY, known=True)
def test_success_is_not_recorded(self):
self.assertEqual(self._login("admin", "correct-horse-battery"), 200)
self.failed_logins.record.assert_not_called()
class TestFailedLoginTracker(unittest.TestCase):
def setUp(self):
self.tracker = FailedLoginTracker()
raise_patch = patch("frigate.api.auth.raise_notice")
self.raise_notice = raise_patch.start()
self.addCleanup(raise_patch.stop)
def _scopes(self) -> list[str]:
return [call.kwargs["scope"] for call in self.raise_notice.call_args_list]
def _fill(self, now: float) -> None:
for index in range(MAX_OPEN_BURSTS):
self.tracker.record(f"ghost{index}", now)
def test_attempts_inside_the_gap_share_a_burst(self):
for now in (1000.0, 1010.0, 1299.0):
self.tracker.record("admin", now)
self.raise_notice.assert_called_with(
"failed_login", scope="admin:1000", params={"user": "admin"}
)
self.assertEqual(self._scopes(), ["admin:1000"] * 3)
def test_each_user_gets_their_own_burst(self):
self.tracker.record("admin", 1000.0)
self.tracker.record("ghost", 1010.0)
self.tracker.record("admin", 1020.0)
self.assertEqual(self._scopes(), ["admin:1000", "ghost:1010", "admin:1000"])
def test_a_quiet_gap_opens_a_new_burst(self):
self.tracker.record("admin", 1000.0)
self.tracker.record("admin", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
self.assertEqual(self._scopes(), ["admin:1000", "admin:1300"])
def test_the_gap_runs_from_the_latest_attempt(self):
for now in (1000.0, 1200.0, 1450.0):
self.tracker.record("admin", now)
self.assertEqual(self._scopes(), ["admin:1000"] * 3)
def test_quiet_users_are_forgotten(self):
self.tracker.record("ghost", 1000.0)
self.tracker.record("admin", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
self.assertEqual(list(self.tracker._bursts), ["admin"])
def test_a_long_username_is_cut(self):
self.tracker.record("x" * 500, 1000.0)
user = self.raise_notice.call_args.kwargs["params"]["user"]
self.assertEqual(len(user), MAX_NOTICE_USERNAME)
def test_unknown_users_past_the_cap_get_no_notice(self):
self._fill(1000.0)
self.tracker.record("one-too-many", 1001.0)
self.tracker.record("ghost0", 1002.0)
self.assertEqual(self.raise_notice.call_count, MAX_OPEN_BURSTS + 1)
self.assertEqual(self._scopes()[-1], "ghost0:1000")
def test_known_users_skip_the_cap(self):
self._fill(1000.0)
self.tracker.record("admin", 1001.0, known=True)
self.assertEqual(self._scopes()[-1], "admin:1001")
def test_quiet_bursts_free_their_slots(self):
self._fill(1000.0)
self.tracker.record("late", 1000.0 + FAILED_LOGIN_BURST_GAP_S)
self.assertEqual(self._scopes()[-1], "late:1300")
frigate/test/http_api/test_http_notices.py
+87 -15
View File
@@ -6,6 +6,8 @@ from frigate.models import Notice, NoticeStats
from frigate.notices.registry import NoticeRegistry
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
CONFIG_CHECK = "config:detect:fps-greater-than-five:camera.garage"
class TestHttpNotices(BaseTestHttp):
def setUp(self):
@@ -17,20 +19,32 @@ class TestHttpNotices(BaseTestHttp):
return AuthTestClient(self.app)
def test_get_notices_orders_by_severity(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.raise_notice(
"model_download_failed",
scope="yolo/model.onnx",
params={"file": "model.onnx", "error": "timeout"},
)
with self.client() as client:
response = client.get("/notices")
self.assertEqual(response.status_code, 200)
self.assertEqual(
[n["id"] for n in response.json()], ["retention_unmet", "detector_stuck"]
[n["id"] for n in response.json()],
["model_download_failed:yolo/model.onnx", "detector_stuck:ov"],
)
self.assertEqual(response.json()[0]["severity"], "error")
self.assertEqual(
[n["link"] for n in response.json()], [None, "/system#general"]
)
def test_get_stats(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
with self.client() as client:
response = client.get("/notices/stats")
@@ -39,15 +53,33 @@ class TestHttpNotices(BaseTestHttp):
self.assertEqual(response.json()[0]["kind"], "detector_stuck")
self.assertEqual(response.json()[0]["occurrences"], 1)
def test_dismiss_event(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
def test_dismiss_hides_and_history_keeps_it(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
with self.client() as client:
response = client.post("/notices/detector_stuck/dismiss")
response = client.post("/notices/detector_stuck:ov/dismiss")
listed = client.get("/notices").json()
history = client.get("/notices", params={"include_dismissed": True}).json()
self.assertEqual(response.status_code, 200)
self.assertEqual(listed, [])
self.assertEqual([n["id"] for n in history], ["detector_stuck:ov"])
self.assertIsNotNone(history[0]["dismissed_at"])
def test_dismiss_an_id_with_a_slash(self):
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
with self.client() as client:
response = client.post(
"/notices/model_download_failed:yolo/model.onnx/dismiss"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(self.registry.active(), [])
def test_dismiss_unknown_is_404(self):
with self.client() as client:
@@ -55,14 +87,6 @@ class TestHttpNotices(BaseTestHttp):
self.assertEqual(response.status_code, 404)
def test_dismiss_state_is_400(self):
self.registry.raise_notice("retention_unmet", params={})
with self.client() as client:
response = client.post("/notices/retention_unmet/dismiss")
self.assertEqual(response.status_code, 400)
def test_requires_admin(self):
with TestClient(self.app) as client:
response = client.get(
@@ -70,3 +94,51 @@ class TestHttpNotices(BaseTestHttp):
)
self.assertEqual(response.status_code, 403)
def test_dismissed_checks_are_listed_once(self):
with self.client() as client:
first = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
again = client.post(f"/notices/{CONFIG_CHECK}/dismiss")
listed = client.get("/notices/dismissed_checks").json()
history = client.get("/notices", params={"include_dismissed": True}).json()
self.assertEqual(first.status_code, 200)
self.assertEqual(again.status_code, 200)
self.assertEqual([check["id"] for check in listed], [CONFIG_CHECK])
self.assertIsNotNone(listed[0]["dismissed_at"])
# the Health tab builds the row itself, so it never comes back as a notice
self.assertEqual(history, [])
def test_dismissed_checks_require_admin(self):
with TestClient(self.app) as client:
response = client.get(
"/notices/dismissed_checks",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
self.assertEqual(response.status_code, 403)
def test_purge_dismissed_clears_notices_and_checks(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss(CONFIG_CHECK)
with self.client() as client:
response = client.delete("/notices/dismissed")
history = client.get("/notices", params={"include_dismissed": True}).json()
checks = client.get("/notices/dismissed_checks").json()
self.assertEqual(response.status_code, 200)
self.assertEqual(history, [])
self.assertEqual(checks, [])
def test_purge_dismissed_requires_admin(self):
with TestClient(self.app) as client:
response = client.delete(
"/notices/dismissed",
headers={"remote-user": "viewer", "remote-role": "viewer"},
)
self.assertEqual(response.status_code, 403)
frigate/test/test_camera_watchdog.py
+1 -77
View File
@@ -6,11 +6,7 @@ from unittest.mock import MagicMock, patch
from frigate.config import FrigateConfig
from frigate.const import STREAM_TYPE_MAIN, STREAM_TYPE_SUB
from frigate.video.ffmpeg import (
CRASH_LOOP_RECOVERY_S,
CRASH_LOOP_RESTARTS,
CameraWatchdog,
)
from frigate.video.ffmpeg import CameraWatchdog
def build_watchdog(
@@ -228,75 +224,3 @@ class TestCameraWatchdogStreamHealth(unittest.TestCase):
assert watchdog.record_stale_threshold[STREAM_TYPE_MAIN] == 120
assert watchdog.record_stale_threshold[STREAM_TYPE_SUB] == 150
class TestCameraWatchdogCrashLoop(unittest.TestCase):
"""The crash loop notice follows restarts in the trailing hour."""
def _build_watchdog(self) -> CameraWatchdog:
return build_watchdog()
def _notice_payloads(self, watchdog: CameraWatchdog) -> list[dict]:
return [
call.args[1]
for call in watchdog.requestor.send_data.call_args_list
if call.args[0] == "update_notice"
]
def test_below_threshold_raises_nothing(self):
watchdog = self._build_watchdog()
now = datetime.now().timestamp()
watchdog.reconnect_timestamps.extend([now] * (CRASH_LOOP_RESTARTS - 1))
watchdog.capture_thread = MagicMock()
watchdog.capture_thread.is_alive.return_value = False
watchdog._update_crash_loop_notice(now)
self.assertEqual(self._notice_payloads(watchdog), [])
def test_threshold_raises_once_per_restart_count(self):
watchdog = self._build_watchdog()
now = datetime.now().timestamp()
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
watchdog.capture_thread = MagicMock()
watchdog.capture_thread.is_alive.return_value = False
watchdog._update_crash_loop_notice(now)
watchdog._update_crash_loop_notice(now)
payloads = self._notice_payloads(watchdog)
self.assertEqual(len(payloads), 1)
self.assertEqual(payloads[0]["action"], "raise")
self.assertEqual(payloads[0]["kind"], "ffmpeg_crash_loop")
self.assertEqual(payloads[0]["scope"], "front_door")
self.assertEqual(payloads[0]["params"], {"restarts": CRASH_LOOP_RESTARTS})
def test_recovery_window_resolves(self):
watchdog = self._build_watchdog()
now = datetime.now().timestamp()
# the last restart is at `now`, so recovery is measured from there
watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS)
watchdog.capture_thread = MagicMock()
watchdog.capture_thread.is_alive.return_value = False
watchdog._update_crash_loop_notice(now)
watchdog.capture_thread.is_alive.return_value = True
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S - 1)
self.assertEqual(len(self._notice_payloads(watchdog)), 1)
watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S + 1)
payloads = self._notice_payloads(watchdog)
self.assertEqual(len(payloads), 2)
self.assertEqual(payloads[1]["action"], "resolve")
self.assertFalse(watchdog.crash_loop_raised)
def test_disable_resolves(self):
watchdog = self._build_watchdog()
watchdog.crash_loop_raised = True
watchdog._resolve_crash_loop()
payloads = self._notice_payloads(watchdog)
self.assertEqual(payloads[-1]["action"], "resolve")
self.assertFalse(watchdog.crash_loop_raised)
frigate/test/test_dispatcher_runtime_state.py
+15 -24
View File
@@ -518,7 +518,7 @@ class TestNoticeWiring(unittest.TestCase):
def setUp(self) -> None:
self.registry = MagicMock()
self.registry.active.return_value = [{"id": "retention_unmet"}]
self.registry.active.return_value = [{"id": "detector_stuck:ov"}]
config = MagicMock()
config.cameras = {}
@@ -540,37 +540,28 @@ class TestNoticeWiring(unittest.TestCase):
self.dispatcher._publish_notices
)
def test_raise_request_reaches_registry(self) -> None:
self.dispatcher._receive(
"update_notice",
{
"action": "raise",
"kind": "ffmpeg_crash_loop",
"scope": "front_door",
"params": {"restarts": 5},
},
)
def test_update_request_reaches_registry(self) -> None:
update = {
"action": "raise",
"kind": "model_download_failed",
"scope": "yolo/model.onnx",
"params": {"file": "model.onnx", "error": "timeout"},
}
self.registry.raise_notice.assert_called_once_with(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
)
self.dispatcher._receive("update_notice", update)
def test_resolve_request_reaches_registry(self) -> None:
self.dispatcher._receive(
"update_notice",
{"action": "resolve", "kind": "retention_unmet", "scope": None},
)
self.registry.resolve.assert_called_once_with("retention_unmet", None)
self.registry.apply.assert_called_once_with(update)
def test_malformed_request_does_not_raise(self) -> None:
self.registry.raise_notice.side_effect = RuntimeError("boom")
self.registry.apply.side_effect = RuntimeError("boom")
self.dispatcher._receive("update_notice", "not a dict")
self.dispatcher._receive(
"update_notice", {"action": "raise", "kind": "x", "params": {}}
)
self.registry.apply.assert_called_once()
def test_publish_local_skips_mqtt(self) -> None:
mqtt = MagicMock(spec=MqttClient)
other = MagicMock()
@@ -587,7 +578,7 @@ class TestNoticeWiring(unittest.TestCase):
self.dispatcher._publish_notices()
self.dispatcher.publish_local.assert_called_once_with(
"notices", '[{"id": "retention_unmet"}]'
"notices", '[{"id": "detector_stuck:ov"}]'
)
def test_snapshot_includes_notices(self) -> None:
@@ -599,4 +590,4 @@ class TestNoticeWiring(unittest.TestCase):
self.dispatcher.publish_runtime_snapshot(publisher)
publisher.assert_any_call("notices", '[{"id": "retention_unmet"}]', False)
publisher.assert_any_call("notices", '[{"id": "detector_stuck:ov"}]', False)
frigate/test/test_model_downloader.py
+26 -26
View File
@@ -13,13 +13,12 @@ class TestModelDownloadNotice(unittest.TestCase):
def setUp(self):
self.download_path = tempfile.mkdtemp()
downloader.last_download_error.clear()
def _notice_calls(self, requestor: MagicMock) -> list[dict]:
return [
call.args[1]
for call in requestor.send_data.call_args_list
if call.args[0] == "update_notice"
]
raise_patch = patch("frigate.util.downloader.raise_notice")
resolve_patch = patch("frigate.util.downloader.resolve_notice")
self.raise_notice = raise_patch.start()
self.resolve_notice = resolve_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
def _downloader(self, download_func) -> ModelDownloader:
with patch("frigate.util.downloader.InterProcessRequestor"):
@@ -36,15 +35,16 @@ class TestModelDownloadNotice(unittest.TestCase):
with self.assertRaises(RuntimeError):
model_downloader._download_models()
notices = self._notice_calls(model_downloader.requestor)
self.assertEqual(len(notices), 1)
self.assertEqual(notices[0]["action"], "raise")
self.assertEqual(notices[0]["kind"], "model_download_failed")
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
self.assertEqual(
notices[0]["params"],
{"file": "facedet.onnx", "error": "HTTP 503 from upstream"},
self.raise_notice.assert_called_once_with(
"model_download_failed",
scope="facedet/facedet.onnx",
params={
"file": "facedet.onnx",
"model": "facedet",
"error": "HTTP 503 from upstream",
},
)
self.resolve_notice.assert_not_called()
def test_swallowing_download_that_leaves_no_file_reports(self):
target = os.path.join(self.download_path, "facedet.onnx")
@@ -62,10 +62,11 @@ class TestModelDownloadNotice(unittest.TestCase):
model_downloader = self._downloader(swallowing)
model_downloader._download_models()
notices = self._notice_calls(model_downloader.requestor)
self.assertEqual(len(notices), 1)
self.assertEqual(notices[0]["action"], "raise")
self.assertEqual(notices[0]["params"], {"file": "facedet.onnx", "error": "dns"})
self.raise_notice.assert_called_once()
self.assertEqual(
self.raise_notice.call_args.kwargs["params"],
{"file": "facedet.onnx", "model": "facedet", "error": "dns"},
)
self.assertFalse(os.path.exists(target))
def test_success_resolves(self):
@@ -76,10 +77,10 @@ class TestModelDownloadNotice(unittest.TestCase):
model_downloader = self._downloader(succeeding)
model_downloader._download_models()
notices = self._notice_calls(model_downloader.requestor)
self.assertEqual(len(notices), 1)
self.assertEqual(notices[0]["action"], "resolve")
self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx")
self.raise_notice.assert_not_called()
self.resolve_notice.assert_called_once_with(
"model_download_failed", "facedet/facedet.onnx"
)
def test_a_sibling_downloader_only_resolves_its_own_files(self):
"""PaddleOCR, facedet and jina each spread one model_name over several
@@ -96,9 +97,8 @@ class TestModelDownloadNotice(unittest.TestCase):
sibling._download_models()
self.assertEqual(
[n["scope"] for n in self._notice_calls(sibling.requestor)],
["paddleocr-onnx/det.onnx"],
self.resolve_notice.assert_called_once_with(
"model_download_failed", "paddleocr-onnx/det.onnx"
)
frigate/test/test_notice_facade.py
+85
View File
@@ -0,0 +1,85 @@
"""Tests for raising and resolving notices through frigate.notices."""
import unittest
from unittest.mock import MagicMock, patch
from frigate import notices
class TestNoticeFacade(unittest.TestCase):
def tearDown(self):
notices.install_registry(None)
def test_the_installed_registry_applies_each_update(self):
registry = MagicMock()
notices.install_registry(registry)
notices.raise_notice("detector_stuck", scope="ov", params={"detector": "ov"})
notices.resolve_notice("detector_stuck", "ov")
notices.resolve_kind("update_available")
self.assertEqual(
[call.args[0] for call in registry.apply.call_args_list],
[
{
"action": "raise",
"kind": "detector_stuck",
"scope": "ov",
"params": {"detector": "ov"},
},
{"action": "resolve", "kind": "detector_stuck", "scope": "ov"},
{"action": "resolve_kind", "kind": "update_available"},
],
)
def test_a_child_process_sends_over_ipc(self):
with (
patch.object(notices.mp, "parent_process", return_value=MagicMock()),
patch.object(notices, "InterProcessRequestor") as requestor,
patch.object(notices, "_requestor", None),
):
notices.raise_notice("detector_stuck", scope="ov")
notices.resolve_notice("detector_stuck", "ov")
# one requestor serves every call in the process
requestor.assert_called_once_with()
send_data = requestor.return_value.send_data
self.assertEqual(send_data.call_count, 2)
self.assertEqual(
send_data.call_args_list[0].args,
(
"update_notice",
{
"action": "raise",
"kind": "detector_stuck",
"scope": "ov",
"params": {},
},
),
)
def test_the_main_process_without_a_registry_drops_updates(self):
with (
patch.object(notices.mp, "parent_process", return_value=None),
patch.object(notices, "InterProcessRequestor") as requestor,
):
notices.raise_notice("detector_stuck", scope="ov")
requestor.assert_not_called()
def test_a_failing_registry_is_logged_not_raised(self):
registry = MagicMock()
registry.apply.side_effect = RuntimeError("db")
notices.install_registry(registry)
with self.assertLogs("frigate.notices", level="ERROR"):
notices.raise_notice("detector_stuck", scope="ov")
def test_flush_reaches_only_an_installed_registry(self):
notices.flush_notices()
registry = MagicMock()
notices.install_registry(registry)
notices.flush_notices()
registry.flush.assert_called_once_with()
frigate/test/test_notice_registry.py
+298 -111
View File
@@ -3,51 +3,53 @@
import logging
import os
import unittest
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from peewee_migrate import Router
from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqliteq import SqliteQueueDatabase
from frigate.models import Notice, NoticeStats
from frigate.notices.registry import DismissResult, NoticeRegistry
from frigate.notices.types import (
NOTICE_KINDS,
NoticeMode,
NoticeSeverity,
notice_id,
)
from frigate.notices.registry import NoticeRegistry
from frigate.notices.types import NOTICE_KINDS, NoticeKind, NoticeSeverity, notice_id
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
# kinds that switch on the lifecycle knobs, so the tests do not depend on the
# values the real catalog picks
BATCHED = NoticeKind(
"batched", NoticeSeverity.warning, "system", batch_repeats=True, reopen_at_count=3
)
PRUNED = NoticeKind("pruned", NoticeSeverity.info, "system", keep_latest=2)
TEST_KINDS = {kind.key: kind for kind in (BATCHED, PRUNED)}
class TestNoticeKinds(unittest.TestCase):
def test_every_kind_is_keyed_by_its_own_key(self):
for key, definition in NOTICE_KINDS.items():
self.assertEqual(key, definition.key)
def test_first_producers_are_defined(self):
self.assertEqual(NOTICE_KINDS["ffmpeg_crash_loop"].mode, NoticeMode.state)
self.assertEqual(
NOTICE_KINDS["ffmpeg_crash_loop"].severity, NoticeSeverity.error
)
self.assertEqual(NOTICE_KINDS["detector_stuck"].mode, NoticeMode.event)
self.assertEqual(
NOTICE_KINDS["detector_stuck"].severity, NoticeSeverity.warning
)
self.assertEqual(NOTICE_KINDS["model_download_failed"].mode, NoticeMode.event)
self.assertEqual(NOTICE_KINDS["retention_unmet"].mode, NoticeMode.state)
self.assertEqual(NOTICE_KINDS["update_available"].mode, NoticeMode.state)
self.assertEqual(NOTICE_KINDS["update_available"].severity, NoticeSeverity.info)
def test_notice_id_with_and_without_scope(self):
self.assertEqual(notice_id("retention_unmet", None), "retention_unmet")
self.assertEqual(notice_id("failed_login", None), "failed_login")
self.assertEqual(
notice_id("ffmpeg_crash_loop", "front_door"),
"ffmpeg_crash_loop:front_door",
notice_id("skipped_detections", "front_door"),
"skipped_detections:front_door",
)
def test_link_is_filled_in_from_params(self):
kind = NoticeKind(
"k", NoticeSeverity.info, "system", link="https://x/v{version}"
)
class TestNoticeRegistry(unittest.TestCase):
self.assertEqual(kind.link_for({"version": "0.19.1"}), "https://x/v0.19.1")
self.assertIsNone(kind.link_for({}))
def test_a_kind_without_a_link(self):
kind = NoticeKind("k", NoticeSeverity.info, "system")
self.assertIsNone(kind.link_for({"version": "0.19.1"}))
class RegistryTestCase(unittest.TestCase):
def setUp(self):
migrate_db = SqliteExtDatabase("test.db")
del logging.getLogger("peewee_migrate").handlers[:]
@@ -70,132 +72,128 @@ class TestNoticeRegistry(unittest.TestCase):
except OSError:
pass
def _raise_at(self, timestamp: float, kind: str, **kwargs) -> None:
with patch("frigate.notices.registry.datetime") as mock_datetime:
mock_datetime.now.return_value.timestamp.return_value = timestamp
self.registry.raise_notice(kind, **kwargs)
class TestNoticeRegistry(RegistryTestCase):
def test_raise_inserts_and_counts_one_occurrence(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
active = self.registry.active()
self.assertEqual(len(active), 1)
self.assertEqual(active[0]["id"], "ffmpeg_crash_loop:front_door")
self.assertEqual(active[0]["severity"], "error")
self.assertEqual(active[0]["mode"], "state")
self.assertEqual(active[0]["id"], "skipped_detections:front_door")
self.assertEqual(active[0]["severity"], "warning")
self.assertEqual(active[0]["category"], "camera")
self.assertEqual(active[0]["scope"], "front_door")
self.assertEqual(active[0]["params"], {"restarts": 5})
self.assertEqual(active[0]["params"], {"pct": 12.5})
self.assertEqual(active[0]["link"], "/system#cameras")
self.assertEqual(active[0]["count"], 1)
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
def test_state_re_raise_with_new_params_updates_without_counting(self):
def test_re_raise_counts_and_updates_params(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
self.listener.reset_mock()
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 6}
"skipped_detections", scope="front_door", params={"pct": 8.0}
)
active = self.registry.active()
self.assertEqual(active[0]["count"], 1)
self.assertEqual(active[0]["params"], {"restarts": 6})
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
self.assertEqual(active[0]["count"], 2)
self.assertEqual(active[0]["params"], {"pct": 8.0})
self.assertEqual(self.registry.stats()[0]["occurrences"], 2)
def test_state_re_raise_with_same_params_is_a_no_op(self):
def test_repeat_of_a_non_counting_kind_is_a_no_op(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
before = self.registry.active()[0]["last_seen"]
self.listener.reset_mock()
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="front_door", params={"restarts": 5}
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
self.assertEqual(self.registry.active()[0]["last_seen"], before)
self.assertEqual(self.registry.active()[0]["count"], 1)
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_not_called()
def test_event_re_raise_counts_and_undismisses(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.assertEqual(self.registry.dismiss("detector_stuck"), DismissResult.ok)
self.assertEqual(self.registry.active(), [])
def test_dismissal_survives_a_re_raise(self):
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertTrue(self.registry.dismiss("skipped_detections:front_door"))
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
# no scope here on purpose: a global event id is still valid
active = self.registry.active()
self.assertEqual(len(active), 1)
self.assertEqual(active[0]["count"], 2)
self.assertIsNone(active[0]["dismissed_at"])
stats = self.registry.stats()[0]
self.assertEqual(stats["occurrences"], 2)
self.assertEqual(stats["dismissals"], 1)
def test_resolve_deletes_and_keeps_stats(self):
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
self.listener.reset_mock()
self.registry.resolve("retention_unmet")
self.registry.resolve("retention_unmet")
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.assertEqual(self.registry.active(), [])
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
history = self.registry.active(include_dismissed=True)
self.assertEqual(history[0]["count"], 2)
self.assertIsNotNone(history[0]["dismissed_at"])
def test_state_notice_cannot_be_dismissed(self):
self.registry.raise_notice("retention_unmet", params={})
def test_dismiss_counts_once(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.assertEqual(
self.registry.dismiss("retention_unmet"), DismissResult.not_dismissable
)
self.assertEqual(len(self.registry.active()), 1)
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
self.assertTrue(self.registry.dismiss("detector_stuck:ov"))
self.assertEqual(self.registry.stats()[0]["dismissals"], 1)
def test_dismiss_unknown_id(self):
self.assertEqual(self.registry.dismiss("nope"), DismissResult.not_found)
self.assertFalse(self.registry.dismiss("nope"))
def test_dismissed_hidden_unless_requested(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.dismiss("detector_stuck")
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.dismiss("detector_stuck:ov")
self.assertEqual(self.registry.active(), [])
self.assertEqual(len(self.registry.active(include_dismissed=True)), 1)
def test_startup_hides_state_notices_until_confirmed(self):
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
def test_resolve_deletes_and_keeps_stats(self):
self.registry.raise_notice(
"detector_stuck", scope="ov", params={"detector": "ov"}
"model_download_failed", scope="yolo/model.onnx", params={}
)
self.listener.reset_mock()
self.registry.mark_state_notices_unconfirmed()
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["detector_stuck:ov"])
# the producer confirms the same episode: visible again, no new occurrence
self.registry.raise_notice("retention_unmet", params={"needed_mb": 1})
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["retention_unmet", "detector_stuck:ov"])
self.assertEqual(
[s for s in self.registry.stats() if s["kind"] == "retention_unmet"][0][
"occurrences"
],
1,
)
def test_unconfirmed_across_two_startups_is_deleted(self):
self.registry.raise_notice("retention_unmet", params={})
self.registry.mark_state_notices_unconfirmed()
self.registry.mark_state_notices_unconfirmed()
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.registry.resolve("model_download_failed", "yolo/model.onnx")
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.assertEqual(self.registry.stats()[0]["occurrences"], 1)
self.listener.assert_called_once()
def test_resolve_kind_drops_every_scope_of_that_kind(self):
self.registry.raise_notice("detector_stuck", params={})
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.resolve_kind("detector_stuck")
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["skipped_detections:garage"])
def test_active_fills_in_each_link(self):
self.registry.raise_notice(
"update_available", scope="0.19.1", params={"version": "0.19.1"}
)
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
links = {n["kind"]: n["link"] for n in self.registry.active()}
self.assertEqual(
links,
{
"update_available": "https://github.com/blakeblackshear/frigate/releases/tag/v0.19.1",
"model_download_failed": None,
},
)
def test_replay_camera_is_ignored(self):
self.registry.raise_notice(
"ffmpeg_crash_loop", scope="_replay_front_door", params={}
"skipped_detections", scope="_replay_front_door", params={}
)
self.assertEqual(self.registry.active(), [])
@@ -208,22 +206,26 @@ class TestNoticeRegistry(unittest.TestCase):
self.listener.assert_not_called()
def test_active_sorts_by_severity_then_recency(self):
self.registry.raise_notice("detector_stuck", params={"detector": "ov"})
self.registry.raise_notice("retention_unmet", params={})
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice(
"model_download_failed", scope="yolo/model.onnx", params={}
)
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["retention_unmet", "detector_stuck"])
self.assertEqual(
ids, ["model_download_failed:yolo/model.onnx", "detector_stuck:ov"]
)
def test_listener_exception_does_not_propagate(self):
self.registry.subscribe(MagicMock(side_effect=RuntimeError("boom")))
self.registry.raise_notice("retention_unmet", params={})
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.assertEqual(len(self.registry.active()), 1)
def test_resolve_camera_drops_only_that_camera(self):
self.registry.raise_notice("ffmpeg_crash_loop", scope="front_door", params={})
self.registry.raise_notice("ffmpeg_crash_loop", scope="garage", params={})
self.registry.raise_notice("skipped_detections", scope="front_door", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.raise_notice(
"model_download_failed", scope="front_door", params={}
)
@@ -232,5 +234,190 @@ class TestNoticeRegistry(unittest.TestCase):
ids = sorted(n["id"] for n in self.registry.active())
self.assertEqual(
ids, ["ffmpeg_crash_loop:garage", "model_download_failed:front_door"]
ids, ["model_download_failed:front_door", "skipped_detections:garage"]
)
def test_resolve_camera_drops_that_cameras_check_dismissals(self):
for check_id in (
"stream:front_door:0:probe",
"config:detect:fps-greater-than-five:camera.front_door",
"config:detect:fps-greater-than-five:global",
"stream:garage:0:probe",
):
self.assertTrue(self.registry.dismiss(check_id))
self.registry.resolve_camera("front_door")
ids = sorted(check["id"] for check in self.registry.dismissed_checks())
self.assertEqual(
ids,
["config:detect:fps-greater-than-five:global", "stream:garage:0:probe"],
)
def test_camera_named_global_keeps_global_check_dismissals(self):
self.registry.dismiss("config:detect:fps-greater-than-five:global")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.global")
self.registry.resolve_camera("global")
ids = [check["id"] for check in self.registry.dismissed_checks()]
self.assertEqual(ids, ["config:detect:fps-greater-than-five:global"])
def test_purge_dismissed_keeps_active_notices_and_counts(self):
self.registry.raise_notice("detector_stuck", scope="ov", params={})
self.registry.raise_notice("skipped_detections", scope="garage", params={})
self.registry.dismiss("detector_stuck:ov")
self.registry.dismiss("config:detect:fps-greater-than-five:camera.garage")
self.assertEqual(self.registry.purge_dismissed(), 2)
ids = [n["id"] for n in self.registry.active(include_dismissed=True)]
self.assertEqual(ids, ["skipped_detections:garage"])
self.assertEqual(self.registry.dismissed_checks(), [])
dismissals = {s["kind"]: s["dismissals"] for s in self.registry.stats()}
self.assertEqual(dismissals["detector_stuck"], 1)
class TestApply(RegistryTestCase):
def test_each_action_reaches_its_method(self):
self.registry.apply(
{
"action": "raise",
"kind": "detector_stuck",
"scope": "ov",
"params": {"detector": "ov"},
}
)
self.assertEqual(
[n["id"] for n in self.registry.active()], ["detector_stuck:ov"]
)
self.registry.apply(
{"action": "resolve", "kind": "detector_stuck", "scope": "ov"}
)
self.assertEqual(self.registry.active(), [])
for scope in ("a", "b"):
self.registry.apply(
{"action": "raise", "kind": "detector_stuck", "scope": scope}
)
self.registry.apply({"action": "resolve_kind", "kind": "detector_stuck"})
self.assertEqual(self.registry.active(), [])
def test_malformed_updates_are_logged_and_ignored(self):
with self.assertLogs("frigate.notices.registry", level="WARNING") as logs:
self.registry.apply({"action": "raise"})
self.registry.apply({"action": "explode", "kind": "detector_stuck"})
self.assertEqual(len(logs.output), 2)
self.assertEqual(self.registry.active(), [])
@patch.dict(NOTICE_KINDS, TEST_KINDS)
class TestLifecycleKnobs(RegistryTestCase):
def test_batched_repeats_wait_for_the_flush(self):
self.registry.raise_notice("batched", params={"n": 1})
self.listener.reset_mock()
for n in (2, 3, 4):
self.registry.raise_notice("batched", params={"n": n})
self.assertEqual(self.registry.active()[0]["count"], 1)
self.listener.assert_not_called()
self.registry.flush()
notice = self.registry.active()[0]
self.assertEqual(notice["count"], 4)
self.assertEqual(notice["params"], {"n": 4})
self.assertEqual(self.registry.stats()[0]["occurrences"], 4)
self.listener.assert_called_once()
def test_flush_with_nothing_held_is_a_no_op(self):
self.registry.raise_notice("batched")
self.listener.reset_mock()
self.registry.flush()
self.listener.assert_not_called()
def test_repeats_of_a_resolved_notice_are_dropped(self):
self.registry.raise_notice("batched")
self.registry.raise_notice("batched")
self.registry.resolve("batched")
self.listener.reset_mock()
self.registry.flush()
self.assertEqual(self.registry.active(include_dismissed=True), [])
self.listener.assert_not_called()
def test_a_new_row_starts_without_stale_repeats(self):
self.registry.raise_notice("batched")
self.registry.raise_notice("batched")
self.registry.resolve("batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active()[0]["count"], 1)
def test_a_dismissed_notice_reopens_at_the_count(self):
self.registry.raise_notice("batched")
self.registry.dismiss("batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active(), [])
self.registry.raise_notice("batched")
self.registry.flush()
notice = self.registry.active()[0]
self.assertEqual(notice["count"], BATCHED.reopen_at_count)
self.assertIsNone(notice["dismissed_at"])
def test_a_notice_dismissed_past_the_count_stays_dismissed(self):
for _ in range(3):
self.registry.raise_notice("batched")
self.registry.flush()
self.registry.dismiss("batched")
self.registry.raise_notice("batched")
self.registry.flush()
self.assertEqual(self.registry.active(), [])
self.assertEqual(self.registry.active(include_dismissed=True)[0]["count"], 4)
def test_keep_latest_drops_the_oldest_rows(self):
for index, scope in enumerate(("a", "b", "c")):
self._raise_at(1000.0 + index, "pruned", scope=scope)
ids = sorted(n["id"] for n in self.registry.active())
self.assertEqual(ids, ["pruned:b", "pruned:c"])
class TestCatalogLifecycles(RegistryTestCase):
def test_a_new_release_replaces_the_old_update_notice(self):
for index, version in enumerate(("0.19.0", "0.19.1")):
self._raise_at(
1000.0 + index,
"update_available",
scope=version,
params={"version": version},
)
ids = [n["id"] for n in self.registry.active()]
self.assertEqual(ids, ["update_available:0.19.1"])
def test_a_dismissed_failed_login_burst_reopens_at_five_attempts(self):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.dismiss("failed_login:1000")
for _ in range(4):
self.registry.raise_notice("failed_login", scope="1000")
self.registry.flush()
burst = self.registry.active()[0]
self.assertEqual(burst["count"], 5)
self.assertIsNone(burst["dismissed_at"])
frigate/test/test_shm_notice.py
+64
View File
@@ -0,0 +1,64 @@
"""Tests for the notice raised while /dev/shm is too small."""
import unittest
from unittest.mock import patch
from frigate.stats import emitter
TOO_SMALL = {"total": 64.0, "min_shm": 180}
BIG_ENOUGH = {"total": 512.0, "min_shm": 180}
class TestShmNotice(unittest.TestCase):
def setUp(self):
raise_patch = patch.object(emitter, "raise_notice")
resolve_patch = patch.object(emitter, "resolve_notice")
self.raise_notice = raise_patch.start()
self.resolve_notice = resolve_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
self.emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
self.emitter._shm_checked = False
self.emitter._shm_params = None
def test_too_small_raises_once(self):
for _ in range(3):
self.emitter._update_shm_notice(TOO_SMALL)
self.raise_notice.assert_called_once_with(
"shm_too_low", params={"total": 64.0, "min": 180}
)
self.resolve_notice.assert_not_called()
def test_first_tick_clears_a_row_the_last_run_left(self):
self.emitter._update_shm_notice(BIG_ENOUGH)
self.emitter._update_shm_notice(BIG_ENOUGH)
self.resolve_notice.assert_called_once_with("shm_too_low")
self.raise_notice.assert_not_called()
def test_a_new_requirement_updates_the_notice(self):
self.emitter._update_shm_notice(TOO_SMALL)
self.emitter._update_shm_notice({"total": 64.0, "min_shm": 200})
self.assertEqual(self.raise_notice.call_count, 2)
self.raise_notice.assert_called_with(
"shm_too_low", params={"total": 64.0, "min": 200}
)
def test_growing_shm_resolves_the_notice(self):
self.emitter._update_shm_notice(TOO_SMALL)
self.emitter._update_shm_notice(BIG_ENOUGH)
self.resolve_notice.assert_called_once_with("shm_too_low")
def test_missing_shm_is_not_a_problem(self):
self.emitter._update_shm_notice({})
self.raise_notice.assert_not_called()
self.resolve_notice.assert_called_once_with("shm_too_low")
if __name__ == "__main__":
unittest.main()
frigate/test/test_skipped_detections.py
+129
View File
@@ -0,0 +1,129 @@
"""Tests for the skipped detection rate and the notice it can raise."""
import unittest
from unittest.mock import patch
from frigate.stats import emitter
from frigate.stats.emitter import SkippedDetectionsTracker
from frigate.stats.util import skipped_percent
class TestSkippedPercent(unittest.TestCase):
def test_share_of_all_frames(self):
self.assertEqual(skipped_percent(0.5, 5.0, True), 10.0)
def test_offline_camera_reports_zero(self):
self.assertEqual(skipped_percent(2.0, 0.0, True), 0.0)
def test_disabled_camera_reports_zero(self):
self.assertEqual(skipped_percent(2.0, 5.0, False), 0.0)
class TestSkippedDetectionsTracker(unittest.TestCase):
def _cameras(self, pct: float) -> dict:
return {"front_door": {"skipped_pct": pct}}
def test_below_threshold_never_qualifies(self):
tracker = SkippedDetectionsTracker()
for tick in range(10):
self.assertEqual(tracker.update(self._cameras(4.9), tick * 15.0), [])
def test_qualifies_once_after_the_hold(self):
tracker = SkippedDetectionsTracker()
results = [
tracker.update(self._cameras(10.0), tick * 15.0) for tick in range(8)
]
# ticks at 0, 15, 30, and 45 s are inside the hold; 60 s qualifies
self.assertEqual(results[:4], [[], [], [], []])
self.assertEqual(results[4], ["front_door"])
self.assertEqual(results[5:], [[], [], []])
def test_a_dip_restarts_the_hold(self):
tracker = SkippedDetectionsTracker()
for now, pct in ((0.0, 10.0), (15.0, 10.0), (30.0, 10.0), (45.0, 2.0)):
self.assertEqual(tracker.update(self._cameras(pct), now), [])
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
self.assertEqual(tracker.update(self._cameras(10.0), 120.0), ["front_door"])
def test_recovery_then_relapse_is_a_new_episode(self):
tracker = SkippedDetectionsTracker()
tracker.update(self._cameras(10.0), 0.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), ["front_door"])
tracker.update(self._cameras(0.0), 75.0)
tracker.update(self._cameras(10.0), 90.0)
self.assertEqual(tracker.update(self._cameras(10.0), 150.0), ["front_door"])
def test_cameras_are_tracked_separately(self):
tracker = SkippedDetectionsTracker()
tracker.update({"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 0.0}}, 0.0)
qualified = tracker.update(
{"a": {"skipped_pct": 10.0}, "b": {"skipped_pct": 10.0}}, 60.0
)
self.assertEqual(qualified, ["a"])
def test_a_removed_camera_starts_a_new_episode(self):
tracker = SkippedDetectionsTracker()
tracker.update(self._cameras(10.0), 0.0)
tracker.update({}, 15.0)
tracker.update(self._cameras(10.0), 30.0)
self.assertEqual(tracker.update(self._cameras(10.0), 60.0), [])
self.assertEqual(tracker.update(self._cameras(10.0), 90.0), ["front_door"])
class TestEmitterNotices(unittest.TestCase):
def setUp(self):
raise_patch = patch.object(emitter, "raise_notice")
resolve_patch = patch.object(emitter, "resolve_notice")
flush_patch = patch.object(emitter, "flush_notices")
self.raise_notice = raise_patch.start()
resolve_patch.start()
self.flush_notices = flush_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
self.addCleanup(flush_patch.stop)
def _emitter(self) -> emitter.StatsEmitter:
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
stats_emitter.skipped_detections = SkippedDetectionsTracker()
stats_emitter._shm_checked = False
stats_emitter._shm_params = None
return stats_emitter
def _stats(self, uptime: int, pct: float) -> dict:
return {
"service": {"uptime": uptime, "storage": {"/dev/shm": {}}},
"cameras": {"front_door": {"skipped_pct": pct}},
}
def test_qualified_camera_raises_a_notice(self):
stats_emitter = self._emitter()
stats_emitter._update_notices(self._stats(300, 12.5), 0.0)
stats_emitter._update_notices(self._stats(360, 12.5), 60.0)
self.raise_notice.assert_called_once_with(
"skipped_detections", scope="front_door", params={"pct": 12.5}
)
def test_startup_window_is_ignored(self):
stats_emitter = self._emitter()
stats_emitter._update_notices(self._stats(30, 50.0), 0.0)
stats_emitter._update_notices(self._stats(90, 50.0), 60.0)
self.raise_notice.assert_not_called()
def test_every_tick_flushes_held_repeats(self):
self._emitter()._update_notices(self._stats(30, 0.0), 0.0)
self.flush_notices.assert_called_once_with()
frigate/test/test_stats_emitter_version.py
+26 -26
View File
@@ -34,44 +34,46 @@ class TestIsNewerVersion(unittest.TestCase):
class TestUpdateNotice(unittest.TestCase):
def setUp(self):
raise_patch = patch.object(emitter, "raise_notice")
resolve_patch = patch.object(emitter, "resolve_kind")
self.raise_notice = raise_patch.start()
self.resolve_kind = resolve_patch.start()
self.addCleanup(raise_patch.stop)
self.addCleanup(resolve_patch.stop)
def _emitter(self, latest: str) -> emitter.StatsEmitter:
stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter)
stats_emitter.config = MagicMock()
stats_emitter.stats_tracking = {"latest_frigate_version": latest}
stats_emitter.notice_registry = MagicMock()
return stats_emitter
def test_newer_release_raises(self):
stats_emitter = self._emitter("0.19.0")
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
stats_emitter._check_update_notice()
self._emitter("0.19.0")._check_update_notice()
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
"update_available", params={"version": "0.19.0"}
self.raise_notice.assert_called_once_with(
"update_available", scope="0.19.0", params={"version": "0.19.0"}
)
stats_emitter.notice_registry.resolve.assert_not_called()
self.resolve_kind.assert_not_called()
def test_current_or_disabled_resolves(self):
for latest in ("0.18.1", "disabled"):
stats_emitter = self._emitter(latest)
with self.subTest(latest=latest):
self.resolve_kind.reset_mock()
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
stats_emitter._check_update_notice()
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
self._emitter(latest)._check_update_notice()
stats_emitter.notice_registry.raise_notice.assert_not_called()
stats_emitter.notice_registry.resolve.assert_called_once_with(
"update_available"
)
self.raise_notice.assert_not_called()
self.resolve_kind.assert_called_once_with("update_available")
def test_a_failed_lookup_leaves_the_notice_alone(self):
stats_emitter = self._emitter("unknown")
with patch.object(emitter, "VERSION", "0.18.1-abcdef"):
stats_emitter._check_update_notice()
self._emitter("unknown")._check_update_notice()
stats_emitter.notice_registry.raise_notice.assert_not_called()
stats_emitter.notice_registry.resolve.assert_not_called()
self.raise_notice.assert_not_called()
self.resolve_kind.assert_not_called()
def test_refresh_updates_tracking_on_a_thread_then_checks(self):
stats_emitter = self._emitter("0.18.0")
@@ -89,7 +91,7 @@ class TestUpdateNotice(unittest.TestCase):
self.assertEqual(
stats_emitter.stats_tracking["latest_frigate_version"], "0.19.0"
)
stats_emitter.notice_registry.raise_notice.assert_called_once()
self.raise_notice.assert_called_once()
def test_failed_refresh_keeps_the_last_known_version(self):
stats_emitter = self._emitter("0.20.0")
@@ -105,10 +107,10 @@ class TestUpdateNotice(unittest.TestCase):
self.assertEqual(
stats_emitter.stats_tracking["latest_frigate_version"], "0.20.0"
)
stats_emitter.notice_registry.raise_notice.assert_called_once_with(
"update_available", params={"version": "0.20.0"}
self.raise_notice.assert_called_once_with(
"update_available", scope="0.20.0", params={"version": "0.20.0"}
)
stats_emitter.notice_registry.resolve.assert_not_called()
self.resolve_kind.assert_not_called()
def test_disabled_version_check_still_clears_the_notice(self):
stats_emitter = self._emitter("0.20.0")
@@ -124,6 +126,4 @@ class TestUpdateNotice(unittest.TestCase):
self.assertEqual(
stats_emitter.stats_tracking["latest_frigate_version"], "disabled"
)
stats_emitter.notice_registry.resolve.assert_called_once_with(
"update_available"
)
self.resolve_kind.assert_called_once_with("update_available")
frigate/test/test_storage.py
+7 -13
View File
@@ -556,10 +556,9 @@ class TestHttp(unittest.TestCase):
assert Recordings.get(Recordings.id == rec_k2_id)
assert Recordings.get(Recordings.id == rec_k3_id)
def test_unmet_retention_raises_notice(self):
registry = MagicMock()
def test_unmet_retention_sets_the_live_flag(self):
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock(), registry)
storage = StorageMaintainer(config, MagicMock())
storage.camera_storage_stats = {
"front_door": {"needs_refresh": False, "usage": 10, "bandwidth": 10}
}
@@ -567,22 +566,17 @@ class TestHttp(unittest.TestCase):
with patch.object(storage, "expected_hourly_bandwidth", return_value=100.0):
storage.reduce_storage_consumption()
registry.raise_notice.assert_called_once()
kind = registry.raise_notice.call_args.args[0]
params = registry.raise_notice.call_args.kwargs["params"]
self.assertEqual(kind, "retention_unmet")
self.assertEqual(params["needed_mb"], 100.0)
self.assertEqual(params["cleared_mb"], 0.0)
self.assertTrue(storage.retention_unmet)
def test_clean_run_resolves_notice(self):
registry = MagicMock()
def test_clean_run_clears_the_live_flag(self):
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock(), registry)
storage = StorageMaintainer(config, MagicMock())
storage.retention_unmet = True
with patch.object(storage, "check_storage_needs_cleanup", return_value=False):
storage._maintain_once()
registry.resolve.assert_called_once_with("retention_unmet")
self.assertFalse(storage.retention_unmet)
def _insert_mock_event(
frigate/test/test_watchdog_notices.py
+10 -8
View File
@@ -14,25 +14,27 @@ class TestDetectorStuckNotice(unittest.TestCase):
return detector
def test_stuck_restart_raises_notice(self):
registry = MagicMock()
detector = self._stuck_detector()
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
watchdog = FrigateWatchdog({"ov": detector}, MagicMock())
with patch("frigate.watchdog.datetime") as mock_datetime:
with (
patch("frigate.watchdog.datetime") as mock_datetime,
patch("frigate.watchdog.raise_notice") as raise_notice,
):
mock_datetime.datetime.now.return_value.timestamp.return_value = 100.0
watchdog._check_detectors()
detector.start_or_restart.assert_called_once()
registry.raise_notice.assert_called_once_with(
raise_notice.assert_called_once_with(
"detector_stuck", scope="ov", params={"detector": "ov"}
)
def test_healthy_detector_raises_nothing(self):
registry = MagicMock()
detector = self._stuck_detector()
detector.detection_start.value = 0.0
watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry)
watchdog = FrigateWatchdog({"ov": detector}, MagicMock())
watchdog._check_detectors()
with patch("frigate.watchdog.raise_notice") as raise_notice:
watchdog._check_detectors()
registry.raise_notice.assert_not_called()
raise_notice.assert_not_called()
frigate/types.py
+5 -1
View File
@@ -1,10 +1,13 @@
from enum import Enum
from typing import TypedDict
from typing import TYPE_CHECKING, TypedDict
from frigate.camera import CameraMetrics
from frigate.data_processing.types import DataProcessorMetrics
from frigate.object_detection.base import ObjectDetectProcess
if TYPE_CHECKING:
from frigate.storage import StorageMaintainer
class StatsTrackingTypes(TypedDict):
camera_metrics: dict[str, CameraMetrics]
@@ -14,6 +17,7 @@ class StatsTrackingTypes(TypedDict):
latest_frigate_version: str
last_updated: int
processes: dict[str, int]
storage_maintainer: "StorageMaintainer | None"
class ModelStatusTypesEnum(str, Enum):
frigate/util/downloader.py
+7 -18
View File
@@ -7,7 +7,8 @@ from pathlib import Path
import requests
from frigate.comms.inter_process import InterProcessRequestor
from frigate.const import UPDATE_MODEL_STATE, UPDATE_NOTICE
from frigate.const import UPDATE_MODEL_STATE
from frigate.notices import raise_notice, resolve_notice
from frigate.types import ModelStatusTypesEnum
from frigate.util.file import FileLock
@@ -64,26 +65,14 @@ class ModelDownloader:
return f"{self.model_name}/{file_name}"
def _report_failure(self, file_name: str, error: str) -> None:
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "raise",
"kind": "model_download_failed",
"scope": self._notice_scope(file_name),
"params": {"file": file_name, "error": error},
},
raise_notice(
"model_download_failed",
scope=self._notice_scope(file_name),
params={"file": file_name, "model": self.model_name, "error": error},
)
def _resolve_failure(self, file_name: str) -> None:
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "resolve",
"kind": "model_download_failed",
"scope": self._notice_scope(file_name),
"params": {},
},
)
resolve_notice("model_download_failed", self._notice_scope(file_name))
def _send_state(self, file_name: str, state: ModelStatusTypesEnum) -> None:
self.requestor.send_data(
frigate/video/ffmpeg.py
-60
View File
@@ -29,7 +29,6 @@ from frigate.const import (
STREAM_TYPE_MAIN,
STREAM_TYPE_SUB,
STREAM_TYPE_TO_ROLE,
UPDATE_NOTICE,
)
from frigate.log import LogPipe
from frigate.util.builtin import EventsPerSecond, get_record_segment_time
@@ -44,11 +43,6 @@ logger = logging.getLogger(__name__)
RECORD_GRACE_SECONDS = 90
# a camera whose ffmpeg restarts this many times in the trailing hour is in a
# crash loop; it recovers once the capture thread stays up this long
CRASH_LOOP_RESTARTS = 5
CRASH_LOOP_RECOVERY_S = 600
def capture_frames(
ffmpeg_process: sp.Popen[Any],
@@ -146,8 +140,6 @@ class CameraWatchdog(threading.Thread):
self.stop_event = stop_event
self.sleeptime = self.config.ffmpeg.retry_interval
self.reconnect_timestamps = deque()
self.crash_loop_raised = False
self._crash_loop_sent_restarts = 0
self.stalls = stalls
self.reconnects = reconnects
self.detection_frame = detection_frame
@@ -197,53 +189,6 @@ class CameraWatchdog(threading.Thread):
self._last_record_status: dict[str, str] = {}
self._last_status_update_time: float = 0.0
def _update_crash_loop_notice(self, now: float) -> None:
"""Raise the crash loop notice as restarts pile up, resolve once stable.
Called on every 1 second tick, so raises are sent only when the restart
count changes; the registry treats repeats of a state notice as updates.
"""
restarts = len(self.reconnect_timestamps)
alive = self.capture_thread is not None and self.capture_thread.is_alive()
if not alive and restarts >= CRASH_LOOP_RESTARTS:
if self.crash_loop_raised and restarts == self._crash_loop_sent_restarts:
return
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "raise",
"kind": "ffmpeg_crash_loop",
"scope": self.config.name,
"params": {"restarts": restarts},
},
)
self.crash_loop_raised = True
self._crash_loop_sent_restarts = restarts
elif (
self.crash_loop_raised
and alive
and (
restarts == 0
or now - self.reconnect_timestamps[-1] >= CRASH_LOOP_RECOVERY_S
)
):
self._resolve_crash_loop()
def _resolve_crash_loop(self) -> None:
self.requestor.send_data(
UPDATE_NOTICE,
{
"action": "resolve",
"kind": "ffmpeg_crash_loop",
"scope": self.config.name,
"params": {},
},
)
self.crash_loop_raised = False
self._crash_loop_sent_restarts = 0
def _send_detect_status(self, status: str, now: float) -> None:
"""Send detect status only if changed or retry_interval has elapsed."""
if (
@@ -434,9 +379,6 @@ class CameraWatchdog(threading.Thread):
# cameras without a sub stream never get a record_sub topic
if self.config.record.sub.enabled:
self._send_record_status(STREAM_TYPE_SUB, "disabled", now)
if self.crash_loop_raised:
self._resolve_crash_loop()
self.was_enabled = enabled
continue
@@ -514,7 +456,6 @@ class CameraWatchdog(threading.Thread):
self.logger.error(
f"Ffmpeg process crashed unexpectedly for {self.config.name}."
)
self._update_crash_loop_notice(now)
if can_restart:
self.reset_capture_thread(terminate=False)
last_restart_time = now
@@ -544,7 +485,6 @@ class CameraWatchdog(threading.Thread):
# process is running normally
self._send_detect_status("online", now)
self.fps_overflow_count = 0
self._update_crash_loop_notice(now)
for p in self.ffmpeg_other_processes:
poll = p["process"].poll()
frigate/watchdog.py
+2 -8
View File
@@ -7,7 +7,7 @@ from collections.abc import Callable
from dataclasses import dataclass, field
from multiprocessing.synchronize import Event as MpEvent
from frigate.notices.registry import NoticeRegistry
from frigate.notices import raise_notice
from frigate.object_detection.base import ObjectDetectProcess
from frigate.util.process import FrigateProcess
from frigate.util.services import restart_frigate
@@ -45,12 +45,10 @@ class FrigateWatchdog(threading.Thread):
self,
detectors: dict[str, ObjectDetectProcess],
stop_event: MpEvent,
notice_registry: NoticeRegistry | None = None,
):
super().__init__(name="frigate_watchdog")
self.detectors = detectors
self.stop_event = stop_event
self.notice_registry = notice_registry
self._monitored: list[MonitoredProcess] = []
def register(
@@ -126,11 +124,7 @@ class FrigateWatchdog(threading.Thread):
"Detection appears to be stuck. Restarting detection process..."
)
detector.start_or_restart()
if self.notice_registry is not None:
self.notice_registry.raise_notice(
"detector_stuck", scope=name, params={"detector": name}
)
raise_notice("detector_stuck", scope=name, params={"detector": name})
elif (
detector.detect_process is not None
and not detector.detect_process.is_alive()