From f3a31e2fb4c0f17d2bb4f8956dfc3145346def1c Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:58:13 -0500 Subject: [PATCH] Add a notice registry and System Health tab (#24178) * add a notice registry and System Health tab Problems Frigate detects on its own (ffmpeg crash loops, stuck detectors, failed model downloads, recordings deleted before their retention period) only ever existed as log lines. This adds a `NoticeRegistry` in the main process backed by two tables, an `update_notice` IPC topic so producers in other processes can reach it through the dispatcher, an admin-only API and websocket topic, and a Health tab that lists them. Kinds declare their own mode, severity, and category in one place: state notices are resolved by their producer, event notices are dismissed by the user. * treat a prerelease as behind its final release * fix notices clearing early * rename menu items and update docs * don't resolve the update notice on a failed version lookup --- docs/docs/configuration/genai/config.md | 2 +- .../license_plate_recognition.md | 2 +- docs/docs/configuration/object_detectors.md | 2 +- docs/docs/configuration/record.md | 2 +- docs/docs/frigate/updating.md | 2 +- docs/docs/troubleshooting/faqs.md | 6 +- docs/docs/troubleshooting/go2rtc.md | 2 +- docs/docs/troubleshooting/recordings.md | 2 +- docs/static/frigate-api.yaml | 90 ++++++ frigate/api/camera.py | 3 + frigate/api/defs/tags.py | 1 + frigate/api/fastapi_app.py | 5 + frigate/api/notices.py | 60 ++++ frigate/app.py | 20 +- frigate/comms/dispatcher.py | 54 ++++ frigate/comms/ws.py | 18 ++ frigate/const.py | 1 + frigate/models.py | 22 ++ frigate/notices/__init__.py | 0 frigate/notices/registry.py | 263 ++++++++++++++++++ frigate/notices/types.py | 63 +++++ frigate/stats/emitter.py | 49 +++- frigate/stats/util.py | 37 +++ frigate/storage.py | 60 ++-- frigate/test/http_api/base_http_test.py | 5 +- frigate/test/http_api/test_http_notices.py | 72 +++++ frigate/test/test_camera_watchdog.py | 182 ++++++++---- frigate/test/test_dispatcher_runtime_state.py | 90 ++++++ frigate/test/test_model_downloader.py | 102 +++++++ frigate/test/test_notice_registry.py | 236 ++++++++++++++++ frigate/test/test_stats_emitter_version.py | 129 +++++++++ frigate/test/test_storage.py | 30 +- frigate/test/test_watchdog_notices.py | 38 +++ frigate/test/test_ws_outbound_filter.py | 20 ++ frigate/util/downloader.py | 54 +++- frigate/video/ffmpeg.py | 60 ++++ frigate/watchdog.py | 46 +-- generate_api_auth_spec.py | 2 + migrations/038_create_notices.py | 38 +++ web/e2e/fixtures/mock-data/config.ts | 2 +- web/e2e/fixtures/mock-data/stats.ts | 4 +- web/e2e/helpers/api-mocker.ts | 11 + web/e2e/specs/navigation.spec.ts | 6 +- web/e2e/specs/system-health.spec.ts | 169 +++++++++++ web/public/locales/en/common.json | 4 +- web/public/locales/en/views/system.json | 23 +- .../components/health/HealthProblemRow.tsx | 88 ++++++ web/src/components/health/NoticesPane.tsx | 125 +++++++++ web/src/components/menu/GeneralSettings.tsx | 2 +- web/src/hooks/use-notices.ts | 54 ++++ web/src/pages/System.tsx | 29 +- web/src/types/health.ts | 27 ++ web/src/types/notice.ts | 36 +++ web/src/utils/versionUtil.ts | 4 + web/src/views/system/HealthMetrics.tsx | 9 + 55 files changed, 2346 insertions(+), 117 deletions(-) create mode 100644 frigate/api/notices.py create mode 100644 frigate/notices/__init__.py create mode 100644 frigate/notices/registry.py create mode 100644 frigate/notices/types.py create mode 100644 frigate/test/http_api/test_http_notices.py create mode 100644 frigate/test/test_model_downloader.py create mode 100644 frigate/test/test_notice_registry.py create mode 100644 frigate/test/test_stats_emitter_version.py create mode 100644 frigate/test/test_watchdog_notices.py create mode 100644 migrations/038_create_notices.py create mode 100644 web/e2e/specs/system-health.spec.ts create mode 100644 web/src/components/health/HealthProblemRow.tsx create mode 100644 web/src/components/health/NoticesPane.tsx create mode 100644 web/src/hooks/use-notices.ts create mode 100644 web/src/types/health.ts create mode 100644 web/src/types/notice.ts create mode 100644 web/src/utils/versionUtil.ts create mode 100644 web/src/views/system/HealthMetrics.tsx diff --git a/docs/docs/configuration/genai/config.md b/docs/docs/configuration/genai/config.md index a3cabcbace..43494e2388 100644 --- a/docs/docs/configuration/genai/config.md +++ b/docs/docs/configuration/genai/config.md @@ -517,6 +517,6 @@ objects: 7. If descriptions are generated but the results are poor or inconsistent, look at the model and the context window. - Empty fields, missing `shortSummary` values, or `Failed to parse review description` errors usually mean the model is not following the requested JSON schema. Smaller models struggle with structured output. Try a larger parameter size or one of the [recommended models](#recommended-local-models). - Frigate calculates how many frames to send from the context size the provider reports. If your server reports a different value than it is actually running with, frames will be truncated or the request will fail. Pin the value by adding `context_size` under (`genai..provider_options`), and for Ollama also confirm `options.num_ctx` there matches the context you have configured. - - Check **Review Description Speed** and **Object Description Speed** in . If inference takes tens of seconds, requests will queue behind each other and descriptions will appear to stop. For Ollama, review `OLLAMA_NUM_PARALLEL`, `OLLAMA_MAX_QUEUE`, and `OLLAMA_MAX_LOADED_MODELS` so that concurrent requests from Frigate are handled the way you expect. + - Check **Review Description Speed** and **Object Description Speed** in . If inference takes tens of seconds, requests will queue behind each other and descriptions will appear to stop. For Ollama, review `OLLAMA_NUM_PARALLEL`, `OLLAMA_MAX_QUEUE`, and `OLLAMA_MAX_LOADED_MODELS` so that concurrent requests from Frigate are handled the way you expect. diff --git a/docs/docs/configuration/license_plate_recognition.md b/docs/docs/configuration/license_plate_recognition.md index 2c7c8e4cbb..2f0645fb1a 100644 --- a/docs/docs/configuration/license_plate_recognition.md +++ b/docs/docs/configuration/license_plate_recognition.md @@ -697,7 +697,7 @@ lpr: - You may need to adjust your `detection_threshold` if your plates are not being detected. 4. Ensure the characters on detected plates are being _recognized_. - - Check the **Plate recognition** inference time in Enrichment metrics (). High inference times (> 100ms) could lead to poor recognition results, especially for dedicated LPR cameras where the plate crosses the frame quickly. + - Check the **Plate recognition** inference time in Enrichment metrics (). High inference times (> 100ms) could lead to poor recognition results, especially for dedicated LPR cameras where the plate crosses the frame quickly. - Enable `debug_save_plates` to save images of detected text on plates to the clips directory (`/media/frigate/clips/lpr`). Ensure these images are readable and the text is clear. - Watch the debug view to see plates recognized in real-time. For non-dedicated LPR cameras, the vehicle's label will change to the recognized plate when LPR is enabled and working. - Adjust `recognition_threshold` settings per the suggestions [above](#advanced-configuration). diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 755d4e24ff..5dd163299b 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -134,7 +134,7 @@ Along with picking a detector for your hardware, you will choose a model's **inp **Resolution (320x320 vs 640x640):** Frigate is optimized for `320x320` models, and `320x320` is the best choice for the vast majority of setups. Frigate is specifically designed to compensate for the smaller model by cropping a region of motion from the full frame and zooming into it before running detection, so a `320x320` model is actually _better_ at small and distant objects, not worse. A `640x640` model is slower and uses more resources, and its main benefit is fitting more objects into a single inference when many objects are spread across a large area. Recent versions of Frigate have improved support for `640x640` models, but `320x320` remains the recommended starting point for nearly all setups. -**Variant size (tiny/small/medium):** Larger variants are gradually more accurate but slower. Whether the difference is noticeable depends on your specific cameras and scenes. A good rule of thumb is to use the largest model your hardware can run without skipping detections, which you can monitor on the page in the UI. Better accuracy only helps if your detector keeps up with the detection load across all cameras. +**Variant size (tiny/small/medium):** Larger variants are gradually more accurate but slower. Whether the difference is noticeable depends on your specific cameras and scenes. A good rule of thumb is to use the largest model your hardware can run without skipping detections, which you can monitor on the page in the UI. Better accuracy only helps if your detector keeps up with the detection load across all cameras. **Acceptable inference time depends on your hardware.** Inference time alone does not tell the whole story, because different hardware has different capacity. A GPU can run multiple instances of the same model concurrently, so an inference time around 30ms can still keep up with several cameras. A Google Coral runs only a single instance of the model, so it needs a much lower inference time (around 10ms) to keep up. diff --git a/docs/docs/configuration/record.md b/docs/docs/configuration/record.md index aeba4bff8e..f573777ab9 100644 --- a/docs/docs/configuration/record.md +++ b/docs/docs/configuration/record.md @@ -515,7 +515,7 @@ The storage usage Frigate reports will not exactly match what the operating syst ### How Frigate measures recording usage -The **Recordings** value on the Storage Metrics page (), and the per-camera **Camera Storage** breakdown, is the sum of the recording segment sizes Frigate has written, taken from Frigate's database. It is **not** computed by a scan of the disk. Frigate tracks usage this way by design: repeatedly walking the entire drive to total its size would keep hard drives spun up and add unnecessary I/O. +The **Recordings** value on the Storage Metrics page (), and the per-camera **Camera Storage** breakdown, is the sum of the recording segment sizes Frigate has written, taken from Frigate's database. It is **not** computed by a scan of the disk. Frigate tracks usage this way by design: repeatedly walking the entire drive to total its size would keep hard drives spun up and add unnecessary I/O. The disk **total** shown beside it, and the free-space figure Frigate uses to decide when to delete recordings, instead come from the operating system's report for the whole filesystem mounted at `/media/frigate`. As a result, the **Unused** value on the page is _total disk capacity minus Frigate's recordings_, not the drive's real free space, which will be lower whenever anything else is stored on the disk. diff --git a/docs/docs/frigate/updating.md b/docs/docs/frigate/updating.md index a4dfb7f0a4..d1b2e14350 100644 --- a/docs/docs/frigate/updating.md +++ b/docs/docs/frigate/updating.md @@ -60,7 +60,7 @@ If you’re running Frigate via Docker (recommended method), follow these steps: ```bash docker logs frigate ``` - - Visit the Frigate Web UI (default: `http://:5000`) to confirm the new version is running. The version number is displayed at the top of the System Metrics page. + - Visit the Frigate Web UI (default: `http://:5000`) to confirm the new version is running. The version number is displayed at the top of the Health and Metrics page. ### Notes diff --git a/docs/docs/troubleshooting/faqs.md b/docs/docs/troubleshooting/faqs.md index ae29f6c9eb..f70cda27fc 100644 --- a/docs/docs/troubleshooting/faqs.md +++ b/docs/docs/troubleshooting/faqs.md @@ -135,7 +135,7 @@ You can still configure Frigate to use UDP by using ffmpeg input args or the pre ### Frigate is slow to start up with a "probing detect stream" message in the logs -When `detect.width` and `detect.height` are not set, Frigate probes each camera's detect stream on startup (and when saving the config) to auto-detect its resolution. For RTSP streams Frigate probes with ffprobe and automatically retries over TCP if UDP doesn't respond, with a 5 second timeout per attempt. A camera that cannot be reached over either transport will add up to ~10 seconds to startup before Frigate falls through with default dimensions, which may show up as width `0` and height `0` in Camera Probe Info under System Metrics. +When `detect.width` and `detect.height` are not set, Frigate probes each camera's detect stream on startup (and when saving the config) to auto-detect its resolution. For RTSP streams Frigate probes with ffprobe and automatically retries over TCP if UDP doesn't respond, with a 5 second timeout per attempt. A camera that cannot be reached over either transport will add up to ~10 seconds to startup before Frigate falls through with default dimensions, which may show up as width `0` and height `0` in Camera Probe Info under Health and Metrics. To skip the probe entirely and make startup instant, set `detect.width` and `detect.height` explicitly in your camera config: @@ -181,3 +181,7 @@ Frigate's object detection relies on a machine learning [model](../frigate/gloss - If the false positive is always in the same fixed spot (like a statue or mailbox that reads as a person), add an [object filter mask](../configuration/masks.md#object-filter-masks) over that location. Filters and masks only hide the incorrect result - they don't teach Frigate what the object actually is. For that, fine-tune your own model or use Frigate+. + +### Where do I see problems Frigate has detected? + +Open System > Health. The Notices list shows problems the backend has noticed on its own, such as a camera whose ffmpeg keeps crashing, a detector that had to be restarted, a model download that failed, or recordings being deleted before their retention period. Entries that describe a one-time event can be dismissed; entries that describe an ongoing condition clear themselves once it is fixed. diff --git a/docs/docs/troubleshooting/go2rtc.md b/docs/docs/troubleshooting/go2rtc.md index 00ecd6f46e..2ee8bd756e 100644 --- a/docs/docs/troubleshooting/go2rtc.md +++ b/docs/docs/troubleshooting/go2rtc.md @@ -15,7 +15,7 @@ When a stream won't play or behaves oddly, the most important first step is to f ### 1. Read the go2rtc logs -Access the go2rtc logs in the Frigate UI under in the sidebar (select the **go2rtc** tab). If go2rtc cannot connect to your camera you will usually see a clear error here: `401 Unauthorized` (bad or incorrectly encoded credentials), `Connection refused` / `timeout` (wrong IP, port, or the camera is at its connection limit), or `404 Not Found` (wrong RTSP path, or the referenced stream name does not exist). +Access the go2rtc logs in the Frigate UI under in the sidebar (select the **go2rtc** tab). If go2rtc cannot connect to your camera you will usually see a clear error here: `401 Unauthorized` (bad or incorrectly encoded credentials), `Connection refused` / `timeout` (wrong IP, port, or the camera is at its connection limit), or `404 Not Found` (wrong RTSP path, or the referenced stream name does not exist). ### 2. Test the stream in the go2rtc web interface diff --git a/docs/docs/troubleshooting/recordings.md b/docs/docs/troubleshooting/recordings.md index 04b0240557..5516d9b8db 100644 --- a/docs/docs/troubleshooting/recordings.md +++ b/docs/docs/troubleshooting/recordings.md @@ -374,7 +374,7 @@ If segments are only ~1 second instead of ~10 seconds, the camera is sending cor :::tip -You don't have to run `ffprobe` by hand to catch this. Open a camera's **Camera Probe Info** dialog (the info icon on the System → Metrics → Cameras page) and check the **Keyframe analysis** section. It probes the record stream and flags sparse or variable keyframes, which is what smart/"+" codecs (H.264+/H.265+) and long keyframe intervals produce. +You don't have to run `ffprobe` by hand to catch this. Open a camera's **Camera Probe Info** dialog (the info icon on the Health and Metrics → Cameras page) and check the **Keyframe analysis** section. It probes the record stream and flags sparse or variable keyframes, which is what smart/"+" codecs (H.264+/H.265+) and long keyframe intervals produce. ::: diff --git a/docs/static/frigate-api.yaml b/docs/static/frigate-api.yaml index 3ed7a2c1f5..ec0153a581 100644 --- a/docs/static/frigate-api.yaml +++ b/docs/static/frigate-api.yaml @@ -4108,6 +4108,96 @@ paths: security: - frigateAdminAuth: [] x-required-role: admin + /notices: + get: + tags: + - Notices + summary: Get Notices + description: |- + **Access:** Admin role required. + + Get the active notices, most severe first. + + Args: + include_dismissed: Also return event notices the user has dismissed + + Returns: + The active notices + operationId: get_notices_notices_get + parameters: + - name: include_dismissed + in: query + required: false + schema: + type: boolean + default: false + title: Include Dismissed + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + security: + - frigateAdminAuth: [] + x-required-role: admin + /notices/stats: + get: + tags: + - Notices + summary: Get Notice Stats + description: |- + **Access:** Admin role required. + + Get lifetime occurrence counts per notice kind. + operationId: get_notice_stats_notices_stats_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} + security: + - frigateAdminAuth: [] + x-required-role: admin + /notices/{notice_id}/dismiss: + post: + tags: + - Notices + summary: Dismiss Notice + description: |- + **Access:** Admin role required. + + Hide an event notice until it is raised again. + operationId: dismiss_notice_notices__notice_id__dismiss_post + parameters: + - name: notice_id + in: path + required: true + schema: + type: string + title: Notice Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + security: + - frigateAdminAuth: [] + x-required-role: admin /events: get: tags: diff --git a/frigate/api/camera.py b/frigate/api/camera.py index 6425d1a992..0e9f8c2984 100644 --- a/frigate/api/camera.py +++ b/frigate/api/camera.py @@ -1301,6 +1301,9 @@ async def delete_camera( if request.app.dispatcher is not None: request.app.dispatcher.clear_runtime_state_for_camera(camera_name) + if request.app.notice_registry is not None: + request.app.notice_registry.resolve_camera(camera_name) + # Publish removal to stop ffmpeg processes and clean up runtime state request.app.config_publisher.publish_update( CameraConfigUpdateTopic(CameraConfigUpdateEnum.remove, camera_name), diff --git a/frigate/api/defs/tags.py b/frigate/api/defs/tags.py index 2b46be0ea4..9ce733329c 100644 --- a/frigate/api/defs/tags.py +++ b/frigate/api/defs/tags.py @@ -13,6 +13,7 @@ class Tags(Enum): logs = "Logs" media = "Media" motion_search = "Motion Search" + notices = "Notices" notifications = "Notifications" preview = "Preview" recordings = "Recordings" diff --git a/frigate/api/fastapi_app.py b/frigate/api/fastapi_app.py index a252d4c7c7..3b712dae6d 100644 --- a/frigate/api/fastapi_app.py +++ b/frigate/api/fastapi_app.py @@ -24,6 +24,7 @@ from frigate.api import ( hardware, media, motion_search, + notices, notification, preview, record, @@ -41,6 +42,7 @@ from frigate.config.profile_manager import ProfileManager from frigate.debug_replay import DebugReplayManager, debug_replay_auto_stop_watchdog from frigate.embeddings import EmbeddingsContext from frigate.genai import GenAIClientManager +from frigate.notices.registry import NoticeRegistry from frigate.ptz.onvif import OnvifController from frigate.stats.emitter import StatsEmitter from frigate.storage import StorageMaintainer @@ -77,6 +79,7 @@ def create_fastapi_app( profile_manager: ProfileManager | None = None, enforce_default_admin: bool = True, config_holder: ConfigHolder | None = None, + notice_registry: NoticeRegistry | None = None, ): logger.info("Starting FastAPI app") app = FastAPI( @@ -147,6 +150,7 @@ def create_fastapi_app( app.include_router(notification.router) app.include_router(export.router) app.include_router(hardware.router) + app.include_router(notices.router) app.include_router(event.router) app.include_router(media.router) app.include_router(motion_search.router) @@ -163,6 +167,7 @@ def create_fastapi_app( app.camera_error_image = None app.onvif = onvif app.stats_emitter = stats_emitter + app.notice_registry = notice_registry app.event_metadata_updater = event_metadata_updater app.config_publisher = config_publisher app.replay_manager = replay_manager diff --git a/frigate/api/notices.py b/frigate/api/notices.py new file mode 100644 index 0000000000..57a622081f --- /dev/null +++ b/frigate/api/notices.py @@ -0,0 +1,60 @@ +"""Notice APIs.""" + +import logging + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from frigate.api.auth import require_role +from frigate.api.defs.tags import Tags +from frigate.notices.registry import DismissResult + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=[Tags.notices]) + + +@router.get("/notices", dependencies=[Depends(require_role(["admin"]))]) +def get_notices(request: Request, include_dismissed: bool = False) -> JSONResponse: + """Get the active notices, most severe first. + + Args: + include_dismissed: Also return event notices the user has dismissed + + Returns: + The active notices + """ + return JSONResponse( + content=request.app.notice_registry.active(include_dismissed=include_dismissed) + ) + + +@router.get("/notices/stats", dependencies=[Depends(require_role(["admin"]))]) +def get_notice_stats(request: Request) -> JSONResponse: + """Get lifetime occurrence counts per notice kind.""" + return JSONResponse(content=request.app.notice_registry.stats()) + + +@router.post( + "/notices/{notice_id}/dismiss", dependencies=[Depends(require_role(["admin"]))] +) +def dismiss_notice(request: Request, notice_id: str) -> JSONResponse: + """Hide an event notice until it is raised again.""" + result = request.app.notice_registry.dismiss(notice_id) + + if result == DismissResult.not_found: + return JSONResponse( + content={"success": False, "message": "Notice not found"}, + status_code=404, + ) + + if result == DismissResult.not_dismissable: + return JSONResponse( + content={ + "success": False, + "message": "This notice clears itself when the problem is fixed", + }, + status_code=400, + ) + + return JSONResponse(content={"success": True, "message": "Notice dismissed"}) diff --git a/frigate/app.py b/frigate/app.py index e5e952bd8d..fb4be714b2 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -62,6 +62,8 @@ from frigate.log import _stop_logging from frigate.models import ( Event, Export, + Notice, + NoticeStats, Previews, Recordings, RecordingsToDelete, @@ -71,6 +73,7 @@ from frigate.models import ( Trigger, User, ) +from frigate.notices.registry import NoticeRegistry from frigate.object_detection.base import ObjectDetectProcess from frigate.object_detection.util import detection_frame_size from frigate.output.output import OutputProcess @@ -294,6 +297,8 @@ class FrigateApp: models = [ Event, Export, + Notice, + NoticeStats, Previews, Recordings, RecordingsToDelete, @@ -305,6 +310,10 @@ class FrigateApp: ] self.db.bind(models) + self.notice_registry = NoticeRegistry() + # producers confirm state notices again at startup if they still hold + self.notice_registry.mark_state_notices_unconfirmed() + def check_db_data_migrations(self) -> None: # check if vacuum needs to be run if not os.path.exists(f"{CONFIG_DIR}/.exports"): @@ -354,6 +363,7 @@ class FrigateApp: self.onvif_controller, self.ptz_metrics, comms, + notice_registry=self.notice_registry, ) self.dispatcher.start_communicators() @@ -500,7 +510,9 @@ class FrigateApp: self.record_cleanup.start() def start_storage_maintainer(self) -> None: - self.storage_maintainer = StorageMaintainer(self.config, self.stop_event) + self.storage_maintainer = StorageMaintainer( + self.config, self.stop_event, self.notice_registry + ) self.storage_maintainer.start() def start_stats_emitter(self) -> None: @@ -514,11 +526,14 @@ class FrigateApp: self.processes, ), self.stop_event, + notice_registry=self.notice_registry, ) self.stats_emitter.start() def start_watchdog(self) -> None: - self.frigate_watchdog = FrigateWatchdog(self.detectors, self.stop_event) + self.frigate_watchdog = FrigateWatchdog( + self.detectors, self.stop_event, self.notice_registry + ) # (attribute on self, key in self.processes, factory) specs: list[tuple[str, str, Callable[[], FrigateProcess]]] = [ @@ -686,6 +701,7 @@ class FrigateApp: self.dispatcher, self.profile_manager, config_holder=self.config_holder, + notice_registry=self.notice_registry, ), host="127.0.0.1", port=5001, diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index ef260aa821..359137e3b1 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -11,6 +11,7 @@ from peewee import IntegrityError from frigate.camera import PTZMetrics from frigate.camera.activity_manager import AudioActivityManager, CameraActivityManager from frigate.comms.base_communicator import Communicator +from frigate.comms.mqtt import MqttClient from frigate.comms.runtime_state import RuntimeStatePersistence from frigate.comms.webpush import WebPushClient from frigate.config import ( @@ -40,10 +41,12 @@ from frigate.const import ( UPDATE_EVENT_DESCRIPTION, UPDATE_JOB_STATE, UPDATE_MODEL_STATE, + UPDATE_NOTICE, UPDATE_REVIEW_DESCRIPTION, UPSERT_REVIEW_SEGMENT, ) from frigate.models import Event, Previews, Recordings, ReviewSegment +from frigate.notices.registry import NoticeRegistry from frigate.ptz.onvif import OnvifCommandEnum, OnvifController from frigate.types import ModelStatusTypesEnum, TrackedObjectUpdateTypesEnum from frigate.util.object import get_camera_regions_grid @@ -67,12 +70,18 @@ class Dispatcher: onvif: OnvifController, ptz_metrics: dict[str, PTZMetrics], communicators: list[Communicator], + notice_registry: NoticeRegistry | None = None, ) -> None: self.config = config self.config_updater = config_updater self.onvif = onvif self.ptz_metrics = ptz_metrics self.comms = communicators + self.notice_registry = notice_registry + + if notice_registry is not None: + notice_registry.subscribe(self._publish_notices) + self.camera_activity = CameraActivityManager(config, self.publish) self.audio_activity = AudioActivityManager(config, self.publish) self.model_state: dict[str, ModelStatusTypesEnum] = {} @@ -212,6 +221,10 @@ class Dispatcher: False, ) publish("birdseye_layout", json.dumps(self.birdseye_layout.copy()), False) + + if self.notice_registry is not None: + publish("notices", json.dumps(self.notice_registry.active()), False) + publish("audio_detections", json.dumps(audio_detections), False) publish( "profile/state", @@ -339,6 +352,29 @@ class Dispatcher: self.model_state[model] = ModelStatusTypesEnum[state] self.publish("model_state", json.dumps(self.model_state)) + def handle_update_notice() -> None: + if self.notice_registry is None or not isinstance(payload, dict): + return + + try: + action = payload.get("action") + kind = payload.get("kind") + scope = payload.get("scope") + + if not isinstance(kind, str): + logger.warning("Ignoring notice update without a kind") + elif action == "raise": + self.notice_registry.raise_notice( + kind, scope=scope, params=payload.get("params") or {} + ) + elif action == "resolve": + self.notice_registry.resolve(kind, scope) + else: + logger.warning("Ignoring notice update with action %s", action) + except Exception: + # a raise here would kill the REP thread for every process + logger.exception("Failed to apply notice update") + def handle_model_state() -> None: self.publish("model_state", json.dumps(self.model_state.copy())) @@ -406,6 +442,7 @@ class Dispatcher: UPDATE_REVIEW_DESCRIPTION: handle_update_review_description, UPDATE_MODEL_STATE: handle_update_model_state, UPDATE_JOB_STATE: handle_update_job_state, + UPDATE_NOTICE: handle_update_notice, UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress, UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout, UPDATE_AUDIO_TRANSCRIPTION_STATE: handle_update_audio_transcription_state, @@ -464,6 +501,23 @@ class Dispatcher: for comm in self.comms: comm.publish(topic, payload, retain) + def publish_local(self, topic: str, payload: Any) -> None: + """Publish to every communicator except MQTT. + + Used for topics whose external schema is not settled yet. + """ + for comm in self.comms: + if isinstance(comm, MqttClient): + continue + + comm.publish(topic, payload, False) + + def _publish_notices(self) -> None: + if self.notice_registry is None: + return + + self.publish_local("notices", json.dumps(self.notice_registry.active())) + def stop(self) -> None: self.camera_activity.stop() diff --git a/frigate/comms/ws.py b/frigate/comms/ws.py index 6e4eb212a5..348640e2d6 100644 --- a/frigate/comms/ws.py +++ b/frigate/comms/ws.py @@ -31,6 +31,7 @@ from frigate.const import ( UPDATE_EMBEDDINGS_REINDEX_PROGRESS, UPDATE_EVENT_DESCRIPTION, UPDATE_MODEL_STATE, + UPDATE_NOTICE, UPDATE_REVIEW_DESCRIPTION, UPSERT_REVIEW_SEGMENT, ) @@ -56,6 +57,7 @@ _WS_BLOCKED_TOPICS = frozenset( UPDATE_EMBEDDINGS_REINDEX_PROGRESS, UPDATE_BIRDSEYE_LAYOUT, UPDATE_AUDIO_TRANSCRIPTION_STATE, + UPDATE_NOTICE, } ) @@ -158,6 +160,16 @@ _WS_UNRESTRICTED_ONLY_TOPICS = frozenset( } ) +# Topics only an admin connection may receive. unrestricted_only is not +# enough: the built-in viewer role has an empty camera allow-list, which the +# camera policy treats as full access, while the REST side of these topics +# is admin-only. +_WS_ADMIN_ONLY_TOPICS = frozenset( + { + "notices", + } +) + # Topics whose payload (parsed as JSON) names a single owning camera at the # given key path. Used to scope events, reviews, triggers, etc. _WS_PAYLOAD_CAMERA_TOPICS: dict[str, tuple[str, ...]] = { @@ -280,6 +292,7 @@ def _classify_outbound( - "global" : send to every authenticated client - "drop" : send to nobody (fail-closed for unknowns) - "unrestricted_only" : send only to admin/full-access roles + - "admin_only" : send only to connections with the admin role - "camera" : extra is the owning camera name - "payload_camera" : extra is the JSON key path to the camera name - "reshape_by_camera_key" @@ -288,6 +301,8 @@ def _classify_outbound( """ if topic in _WS_GLOBAL_OUTBOUND_TOPICS: return ("global", None) + if topic in _WS_ADMIN_ONLY_TOPICS: + return ("admin_only", None) if topic in _WS_UNRESTRICTED_ONLY_TOPICS: return ("unrestricted_only", None) if topic in _WS_RESHAPE_BY_CAMERA_KEY_TOPICS: @@ -391,6 +406,9 @@ def _materialize_for_ws( if kind == "unrestricted_only": return full_message if _ws_is_unrestricted(ws, config) else None + if kind == "admin_only": + return full_message if "admin" in _ws_valid_roles(ws, config) else None + if kind == "camera": return full_message if ws_has_camera_access(ws, extra, config) else None diff --git a/frigate/const.py b/frigate/const.py index d313c64dd4..029d46989f 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -155,6 +155,7 @@ UPDATE_MODEL_STATE = "update_model_state" UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress" UPDATE_BIRDSEYE_LAYOUT = "update_birdseye_layout" UPDATE_JOB_STATE = "update_job_state" +UPDATE_NOTICE = "update_notice" NOTIFICATION_TEST = "notification_test" # IO Nice Values diff --git a/frigate/models.py b/frigate/models.py index eba076ed9a..e9d98046c9 100644 --- a/frigate/models.py +++ b/frigate/models.py @@ -183,3 +183,25 @@ class Trigger(Model): class Meta: primary_key = CompositeKey("camera", "name") + + +class Notice(Model): + id = CharField(null=False, primary_key=True, max_length=150) + kind = CharField(index=True, max_length=50) + scope = CharField(max_length=100, null=True) + params = JSONField() + first_seen = DateTimeField() + last_seen = DateTimeField() + count = IntegerField(default=1) + dismissed_at = DateTimeField(null=True) + + +class NoticeStats(Model): + kind = CharField(null=False, primary_key=True, max_length=50) + occurrences = IntegerField(default=0) + dismissals = IntegerField(default=0) + first_seen = DateTimeField() + last_seen = DateTimeField() + # watermarks for a future analytics reporter; unused until then + reported_occurrences = IntegerField(default=0) + reported_dismissals = IntegerField(default=0) diff --git a/frigate/notices/__init__.py b/frigate/notices/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frigate/notices/registry.py b/frigate/notices/registry.py new file mode 100644 index 0000000000..c8297d3b85 --- /dev/null +++ b/frigate/notices/registry.py @@ -0,0 +1,263 @@ +"""Registry of active notices and per-kind occurrence counts.""" + +import logging +import threading +from collections.abc import Callable +from datetime import datetime +from enum import Enum +from typing import Any + +from frigate.const import REPLAY_CAMERA_PREFIX +from frigate.models import Notice, NoticeStats +from frigate.notices.types import ( + NOTICE_KINDS, + SEVERITY_ORDER, + NoticeMode, + notice_id, +) + +logger = logging.getLogger(__name__) + + +class DismissResult(str, Enum): + ok = "ok" + not_found = "not_found" + not_dismissable = "not_dismissable" + + +# last_seen value of a state notice that has not been raised since startup +UNCONFIRMED = 0 + + +class NoticeRegistry: + """Owns the notice tables. Lives in the main process only. + + Producers in the main process call it directly. Other processes send an + update_notice request that the dispatcher forwards here. The FastAPI app + runs inside the main process too, so the API's dismiss shares this exact + object, its lock, and its listeners. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._listeners: list[Callable[[], None]] = [] + + def subscribe(self, listener: Callable[[], None]) -> None: + """Call listener after every change to the active list.""" + self._listeners.append(listener) + + def _notify(self) -> None: + for listener in self._listeners: + try: + listener() + except Exception: + logger.exception("Notice listener failed") + + def raise_notice( + self, + kind: str, + *, + scope: str | None = None, + params: dict[str, Any] | None = None, + ) -> None: + """Insert or update a notice. See the counting rule in the spec.""" + definition = NOTICE_KINDS.get(kind) + + if definition is None: + logger.warning("Ignoring notice of unknown kind %s", kind) + return + + if ( + definition.category == "camera" + and scope + and scope.startswith(REPLAY_CAMERA_PREFIX) + ): + return + + now = datetime.now().timestamp() + row_id = notice_id(kind, scope) + params = params or {} + + with self._lock: + existing = Notice.get_or_none(Notice.id == row_id) + + if existing is None: + Notice.create( + id=row_id, + kind=kind, + scope=scope, + params=params, + first_seen=now, + last_seen=now, + count=1, + dismissed_at=None, + ) + self._bump_occurrences(kind, now) + elif definition.mode == NoticeMode.event: + Notice.update( + count=Notice.count + 1, + last_seen=now, + params=params, + dismissed_at=None, + ).where(Notice.id == row_id).execute() + self._bump_occurrences(kind, now) + else: + # one episode counts once; a repeat with nothing new is free + confirmed = existing.last_seen != UNCONFIRMED + + if confirmed and existing.params == params: + return + + Notice.update(last_seen=now, params=params).where( + Notice.id == row_id + ).execute() + + self._notify() + + def resolve(self, kind: str, scope: str | None = None) -> None: + """Delete a notice if present. Safe to call when it is absent.""" + with self._lock: + deleted = ( + Notice.delete().where(Notice.id == notice_id(kind, scope)).execute() + ) + + if deleted: + self._notify() + + def resolve_camera(self, camera: str) -> None: + """Drop every camera-category notice scoped to a camera being deleted.""" + camera_kinds = [ + key + for key, definition in NOTICE_KINDS.items() + if definition.category == "camera" + ] + + with self._lock: + deleted = ( + Notice.delete() + .where( + Notice.kind.in_(camera_kinds), # type: ignore[call-arg, arg-type, misc] + Notice.scope == camera, + ) + .execute() + ) + + if deleted: + self._notify() + + def dismiss(self, row_id: str) -> DismissResult: + """Hide an event notice until it is raised again.""" + with self._lock: + existing = Notice.get_or_none(Notice.id == row_id) + + if existing is None: + return DismissResult.not_found + + definition = NOTICE_KINDS.get(existing.kind) + + if definition is None or definition.mode == NoticeMode.state: + return DismissResult.not_dismissable + + now = datetime.now().timestamp() + Notice.update(dismissed_at=now).where(Notice.id == row_id).execute() + NoticeStats.update(dismissals=NoticeStats.dismissals + 1).where( + NoticeStats.kind == existing.kind + ).execute() + + self._notify() + return DismissResult.ok + + def active(self, include_dismissed: bool = False) -> list[dict[str, Any]]: + """The active notices, most severe first, then most recent first.""" + rows = [] + + for row in Notice.select(): + definition = NOTICE_KINDS.get(row.kind) + + if definition is None: + continue + + if row.dismissed_at is not None and not include_dismissed: + continue + + if definition.mode == NoticeMode.state and row.last_seen == UNCONFIRMED: + continue + + rows.append( + { + "id": row.id, + "kind": row.kind, + "mode": definition.mode.value, + "severity": definition.severity.value, + "category": definition.category, + "scope": row.scope, + "params": row.params, + "first_seen": row.first_seen, + "last_seen": row.last_seen, + "count": row.count, + "dismissed_at": row.dismissed_at, + } + ) + + rows.sort( + key=lambda n: ( + SEVERITY_ORDER[NOTICE_KINDS[n["kind"]].severity], + -n["last_seen"], + ) + ) + return rows + + def stats(self) -> list[dict[str, Any]]: + """Lifetime counts per kind, for the UI meta line and later analytics.""" + return [ + { + "kind": row.kind, + "occurrences": row.occurrences, + "dismissals": row.dismissals, + "first_seen": row.first_seen, + "last_seen": row.last_seen, + "reported_occurrences": row.reported_occurrences, + "reported_dismissals": row.reported_dismissals, + } + for row in NoticeStats.select() + if row.kind in NOTICE_KINDS + ] + + def mark_state_notices_unconfirmed(self) -> None: + """Hide state notices until their producer raises them again. + + Rows already unconfirmed from the previous run are deleted. Hiding + rather than deleting keeps one episode that spans a restart at one + occurrence. + """ + state_kinds = [ + key + for key, definition in NOTICE_KINDS.items() + if definition.mode == NoticeMode.state + ] + + with self._lock: + Notice.delete().where( + Notice.kind.in_(state_kinds), # type: ignore[call-arg, arg-type, misc] + Notice.last_seen == UNCONFIRMED, + ).execute() + Notice.update(last_seen=UNCONFIRMED).where( + Notice.kind.in_(state_kinds) # type: ignore[call-arg, arg-type, misc] + ).execute() + + def _bump_occurrences(self, kind: str, now: float) -> None: + # called with the lock held + stats = NoticeStats.get_or_none(NoticeStats.kind == kind) + + if stats is None: + NoticeStats.create( + kind=kind, + occurrences=1, + dismissals=0, + first_seen=now, + last_seen=now, + ) + else: + NoticeStats.update( + occurrences=NoticeStats.occurrences + 1, last_seen=now + ).where(NoticeStats.kind == kind).execute() diff --git a/frigate/notices/types.py b/frigate/notices/types.py new file mode 100644 index 0000000000..e3fa7c581f --- /dev/null +++ b/frigate/notices/types.py @@ -0,0 +1,63 @@ +"""Notice kinds and the definitions that fix their mode, severity, and category.""" + +from dataclasses import dataclass +from enum import Enum + + +class NoticeMode(str, Enum): + """Whether a producer resolves the notice itself or the user dismisses it.""" + + state = "state" + event = "event" + + +class NoticeSeverity(str, Enum): + error = "error" + warning = "warning" + info = "info" + + +SEVERITY_ORDER: dict[NoticeSeverity, int] = { + NoticeSeverity.error: 0, + NoticeSeverity.warning: 1, + NoticeSeverity.info: 2, +} + + +@dataclass(frozen=True) +class NoticeKind: + """One kind of notice. Producers pass the key; everything else comes from here. + + Attributes: + key: Stable machine-readable name, also the translation key suffix + mode: state notices are resolved by their producer, event notices by the user + severity: error, warning, or info + category: what the scope names: camera, detector, and model kinds carry a scope + reportable: whether a future analytics reporter may send this kind's counts + """ + + key: str + mode: NoticeMode + severity: NoticeSeverity + category: str + reportable: bool = True + + +SCOPED_CATEGORIES = frozenset({"camera", "detector", "model"}) + +_KINDS = ( + NoticeKind("ffmpeg_crash_loop", NoticeMode.state, NoticeSeverity.error, "camera"), + NoticeKind("detector_stuck", NoticeMode.event, NoticeSeverity.warning, "detector"), + NoticeKind( + "model_download_failed", NoticeMode.event, NoticeSeverity.error, "model" + ), + NoticeKind("retention_unmet", NoticeMode.state, NoticeSeverity.error, "storage"), + NoticeKind("update_available", NoticeMode.state, NoticeSeverity.info, "system"), +) + +NOTICE_KINDS: dict[str, NoticeKind] = {kind.key: kind for kind in _KINDS} + + +def notice_id(kind: str, scope: str | None) -> str: + """The row id for a kind and optional scope.""" + return f"{kind}:{scope}" if scope else kind diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index 6072314118..01c3a6b7d5 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -11,16 +11,21 @@ from typing import Any from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig from frigate.const import FREQUENCY_STATS_POINTS +from frigate.notices.registry import NoticeRegistry from frigate.stats.hardware import HardwareStats from frigate.stats.prometheus import update_metrics -from frigate.stats.util import stats_snapshot +from frigate.stats.util import get_latest_version, is_newer_version, stats_snapshot from frigate.types import StatsTrackingTypes +from frigate.version import VERSION logger = logging.getLogger(__name__) MAX_STATS_POINTS = 80 +# how often to ask GitHub for the latest release +VERSION_REFRESH_S = 24 * 60 * 60 + class StatsEmitter(threading.Thread): def __init__( @@ -28,11 +33,13 @@ class StatsEmitter(threading.Thread): config: FrigateConfig, stats_tracking: StatsTrackingTypes, stop_event: MpEvent, + notice_registry: NoticeRegistry | None = None, ): super().__init__(name="frigate_stats_emitter") self.config = config self.stats_tracking = stats_tracking self.stop_event = stop_event + self.notice_registry = notice_registry self.hardware_stats = HardwareStats(config) self.stats_history: list[dict[str, Any]] = [] @@ -125,14 +132,54 @@ class StatsEmitter(threading.Thread): update_metrics(stats) return stats + def _check_update_notice(self) -> None: + """Raise or resolve the update notice from the tracked latest version.""" + if self.notice_registry is None: + return + + latest = self.stats_tracking["latest_frigate_version"] + + # a failed lookup says nothing about whether an update exists + if latest == "unknown": + return + + if is_newer_version(VERSION, latest): + self.notice_registry.raise_notice( + "update_available", params={"version": latest} + ) + else: + self.notice_registry.resolve("update_available") + + def _refresh_latest_version(self) -> None: + """Refresh the latest release on a daemon thread so the request never stalls stats.""" + + def refresh() -> None: + latest = get_latest_version(self.config) + + # keep the last good value; stats surface it as service.latest_version + if latest != "unknown": + self.stats_tracking["latest_frigate_version"] = latest + + self._check_update_notice() + + threading.Thread( + target=refresh, name="frigate_version_check", daemon=True + ).start() + def run(self) -> None: time.sleep(10) + self._check_update_notice() + last_version_check = time.time() for counter in itertools.cycle( range(int(self.config.mqtt.stats_interval / FREQUENCY_STATS_POINTS)) ): if self.stop_event.wait(FREQUENCY_STATS_POINTS): break + if time.time() - last_version_check >= VERSION_REFRESH_S: + self._refresh_latest_version() + last_version_check = time.time() + logger.debug("Starting stats collection") stats = stats_snapshot( self.config, self.stats_tracking, self.hardware_stats diff --git a/frigate/stats/util.py b/frigate/stats/util.py index 575903e8f7..1252db1f4e 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -1,6 +1,7 @@ """Utilities for stats.""" import logging +import re import shutil import time from json import JSONDecodeError @@ -45,6 +46,42 @@ def get_latest_version(config: FrigateConfig) -> str: return "unknown" +def _version_tuple(value: str) -> tuple[int, int, int] | None: + match = re.match(r"^(\d+)\.(\d+)\.(\d+)", value or "") + + if match is None: + return None + + return int(match.group(1)), int(match.group(2)), int(match.group(3)) + + +# a build's git hash suffix is not a prerelease marker +PRERELEASE_PATTERN = re.compile(r"^\d+\.\d+\.\d+-(?:beta|rc)", re.IGNORECASE) + + +def _is_prerelease(value: str) -> bool: + return PRERELEASE_PATTERN.match(value or "") is not None + + +def is_newer_version(current: str, latest: str) -> bool: + """Whether latest is a release newer than the running version. + + Build suffixes after the third number are ignored, and a value that does + not parse (disabled, unknown) is never newer. A prerelease is behind the + final release of the same number, so 0.19.0-beta2 is behind 0.19.0. + """ + current_tuple = _version_tuple(current) + latest_tuple = _version_tuple(latest) + + if current_tuple is None or latest_tuple is None: + return False + + if current_tuple == latest_tuple: + return _is_prerelease(current) and not _is_prerelease(latest) + + return latest_tuple > current_tuple + + def stats_init( config: FrigateConfig, camera_metrics: DictProxy, diff --git a/frigate/storage.py b/frigate/storage.py index aa959b61ec..dbccb83b29 100644 --- a/frigate/storage.py +++ b/frigate/storage.py @@ -20,6 +20,7 @@ from frigate.const import ( STREAM_TYPE_SUB, ) from frigate.models import Event, Recordings +from frigate.notices.registry import NoticeRegistry from frigate.util.builtin import clear_and_unlink logger = logging.getLogger(__name__) @@ -34,10 +35,16 @@ BANDWIDTH_SAMPLE_TARGET = 50 class StorageMaintainer(threading.Thread): """Maintain frigates recording storage.""" - def __init__(self, config: FrigateConfig, stop_event: MpEvent) -> None: + def __init__( + self, + config: FrigateConfig, + stop_event: MpEvent, + notice_registry: NoticeRegistry | None = None, + ) -> None: super().__init__(name="storage_maintainer") self.config = config self.stop_event = stop_event + self.notice_registry = notice_registry self.camera_storage_stats: dict[str, dict] = {} self.config_subscriber = CameraConfigUpdateSubscriber( self.config, @@ -342,9 +349,21 @@ class StorageMaintainer(threading.Thread): except FileNotFoundError: # this file was not found so we must assume no space was cleaned up pass + + if self.notice_registry is not None: + self.notice_registry.raise_notice( + "retention_unmet", + params={ + "needed_mb": round(float(hourly_bandwidth), 1), + "cleared_mb": round(float(deleted_segments_size), 1), + }, + ) else: logger.info(f"Cleaned up {deleted_segments_size:.2f} MB of recordings") + if self.notice_registry is not None: + self.notice_registry.resolve("retention_unmet") + logger.debug(f"Expiring {len(deleted_recordings)} recordings") # delete up to 100,000 at a time max_deletes = 100000 @@ -406,23 +425,28 @@ class StorageMaintainer(threading.Thread): self.calculate_camera_bandwidth() while not self.stop_event.wait(300): - updated_topics = self.config_subscriber.check_for_updates() - - for camera in updated_topics.get(CameraConfigUpdateEnum.record.name, []): - if camera in self.camera_storage_stats: - self.camera_storage_stats[camera]["needs_refresh"] = True - - if not self.camera_storage_stats or True in [ - r["needs_refresh"] for r in self.camera_storage_stats.values() - ]: - self.calculate_camera_bandwidth() - logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.") - - if self.check_storage_needs_cleanup(): - logger.info( - "Less than 1 hour of recording space left, running storage maintenance..." - ) - self.reduce_storage_consumption() + self._maintain_once() self.config_subscriber.stop() logger.info("Exiting storage maintainer...") + + def _maintain_once(self) -> None: + updated_topics = self.config_subscriber.check_for_updates() + + for camera in updated_topics.get(CameraConfigUpdateEnum.record.name, []): + if camera in self.camera_storage_stats: + self.camera_storage_stats[camera]["needs_refresh"] = True + + if not self.camera_storage_stats or True in [ + r["needs_refresh"] for r in self.camera_storage_stats.values() + ]: + self.calculate_camera_bandwidth() + logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.") + + if self.check_storage_needs_cleanup(): + logger.info( + "Less than 1 hour of recording space left, running storage maintenance..." + ) + self.reduce_storage_consumption() + elif self.notice_registry is not None: + self.notice_registry.resolve("retention_unmet") diff --git a/frigate/test/http_api/base_http_test.py b/frigate/test/http_api/base_http_test.py index 32d110962e..1cee0ae3ef 100644 --- a/frigate/test/http_api/base_http_test.py +++ b/frigate/test/http_api/base_http_test.py @@ -144,7 +144,9 @@ class BaseTestHttp(unittest.TestCase): except OSError: pass - def create_app(self, stats=None, event_metadata_publisher=None): + def create_app( + self, stats=None, event_metadata_publisher=None, notice_registry=None + ): from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user app = create_fastapi_app( @@ -159,6 +161,7 @@ class BaseTestHttp(unittest.TestCase): None, DebugReplayManager(), enforce_default_admin=False, + notice_registry=notice_registry, ) # Default test mocks for authentication diff --git a/frigate/test/http_api/test_http_notices.py b/frigate/test/http_api/test_http_notices.py new file mode 100644 index 0000000000..424a53a92a --- /dev/null +++ b/frigate/test/http_api/test_http_notices.py @@ -0,0 +1,72 @@ +"""Tests for the notices API.""" + +from fastapi.testclient import TestClient + +from frigate.models import Notice, NoticeStats +from frigate.notices.registry import NoticeRegistry +from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp + + +class TestHttpNotices(BaseTestHttp): + def setUp(self): + super().setUp([Notice, NoticeStats]) + self.registry = NoticeRegistry() + self.app = self.create_app(notice_registry=self.registry) + + def client(self) -> TestClient: + return AuthTestClient(self.app) + + def test_get_notices_orders_by_severity(self): + self.registry.raise_notice("detector_stuck", params={"detector": "ov"}) + self.registry.raise_notice("retention_unmet", params={"needed_mb": 1}) + + with self.client() as client: + response = client.get("/notices") + + self.assertEqual(response.status_code, 200) + self.assertEqual( + [n["id"] for n in response.json()], ["retention_unmet", "detector_stuck"] + ) + self.assertEqual(response.json()[0]["severity"], "error") + + def test_get_stats(self): + self.registry.raise_notice("detector_stuck", params={"detector": "ov"}) + + with self.client() as client: + response = client.get("/notices/stats") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()[0]["kind"], "detector_stuck") + self.assertEqual(response.json()[0]["occurrences"], 1) + + def test_dismiss_event(self): + self.registry.raise_notice("detector_stuck", params={"detector": "ov"}) + + with self.client() as client: + response = client.post("/notices/detector_stuck/dismiss") + listed = client.get("/notices").json() + + self.assertEqual(response.status_code, 200) + self.assertEqual(listed, []) + + def test_dismiss_unknown_is_404(self): + with self.client() as client: + response = client.post("/notices/nope/dismiss") + + self.assertEqual(response.status_code, 404) + + def test_dismiss_state_is_400(self): + self.registry.raise_notice("retention_unmet", params={}) + + with self.client() as client: + response = client.post("/notices/retention_unmet/dismiss") + + self.assertEqual(response.status_code, 400) + + def test_requires_admin(self): + with TestClient(self.app) as client: + response = client.get( + "/notices", headers={"remote-user": "viewer", "remote-role": "viewer"} + ) + + self.assertEqual(response.status_code, 403) diff --git a/frigate/test/test_camera_watchdog.py b/frigate/test/test_camera_watchdog.py index 0167c1aa45..ba31621587 100644 --- a/frigate/test/test_camera_watchdog.py +++ b/frigate/test/test_camera_watchdog.py @@ -6,62 +6,72 @@ from unittest.mock import MagicMock, patch from frigate.config import FrigateConfig from frigate.const import STREAM_TYPE_MAIN, STREAM_TYPE_SUB -from frigate.video.ffmpeg import CameraWatchdog +from frigate.video.ffmpeg import ( + CRASH_LOOP_RECOVERY_S, + CRASH_LOOP_RESTARTS, + CameraWatchdog, +) + + +def build_watchdog( + sub_enabled: bool = True, output_args: dict | None = None +) -> CameraWatchdog: + config = FrigateConfig( + **{ + "mqtt": {"host": "mqtt"}, + "cameras": { + "front_door": { + "ffmpeg": { + "output_args": output_args or {}, + "inputs": [ + { + "path": "rtsp://10.0.0.1:554/video", + "roles": ["record"], + }, + { + "path": "rtsp://10.0.0.1:554/video2", + "roles": ["detect", "record_sub"], + }, + ], + }, + "record": { + "enabled": True, + "sub": {"enabled": sub_enabled}, + }, + } + }, + } + ) + camera_config = config.cameras["front_door"] + + with ( + patch("frigate.video.ffmpeg.LogPipe"), + patch("frigate.video.ffmpeg.InterProcessRequestor"), + patch("frigate.video.ffmpeg.RecordingsDataSubscriber"), + patch("frigate.video.ffmpeg.CameraConfigUpdateSubscriber"), + ): + watchdog = CameraWatchdog( + camera_config, + 1, + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(), + MagicMock(), + ) + + watchdog.requestor = MagicMock() + return watchdog class TestCameraWatchdogStreamHealth(unittest.TestCase): def _build_watchdog( self, sub_enabled: bool = True, output_args: dict | None = None ) -> CameraWatchdog: - config = FrigateConfig( - **{ - "mqtt": {"host": "mqtt"}, - "cameras": { - "front_door": { - "ffmpeg": { - "output_args": output_args or {}, - "inputs": [ - { - "path": "rtsp://10.0.0.1:554/video", - "roles": ["record"], - }, - { - "path": "rtsp://10.0.0.1:554/video2", - "roles": ["detect", "record_sub"], - }, - ], - }, - "record": { - "enabled": True, - "sub": {"enabled": sub_enabled}, - }, - } - }, - } - ) - camera_config = config.cameras["front_door"] - - with ( - patch("frigate.video.ffmpeg.LogPipe"), - patch("frigate.video.ffmpeg.InterProcessRequestor"), - patch("frigate.video.ffmpeg.RecordingsDataSubscriber"), - patch("frigate.video.ffmpeg.CameraConfigUpdateSubscriber"), - ): - watchdog = CameraWatchdog( - camera_config, - 1, - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - MagicMock(), - ) - - watchdog.requestor = MagicMock() - return watchdog + return build_watchdog(sub_enabled, output_args) def test_stale_sub_does_not_mark_main_stale(self): watchdog = self._build_watchdog() @@ -218,3 +228,75 @@ class TestCameraWatchdogStreamHealth(unittest.TestCase): assert watchdog.record_stale_threshold[STREAM_TYPE_MAIN] == 120 assert watchdog.record_stale_threshold[STREAM_TYPE_SUB] == 150 + + +class TestCameraWatchdogCrashLoop(unittest.TestCase): + """The crash loop notice follows restarts in the trailing hour.""" + + def _build_watchdog(self) -> CameraWatchdog: + return build_watchdog() + + def _notice_payloads(self, watchdog: CameraWatchdog) -> list[dict]: + return [ + call.args[1] + for call in watchdog.requestor.send_data.call_args_list + if call.args[0] == "update_notice" + ] + + def test_below_threshold_raises_nothing(self): + watchdog = self._build_watchdog() + now = datetime.now().timestamp() + watchdog.reconnect_timestamps.extend([now] * (CRASH_LOOP_RESTARTS - 1)) + watchdog.capture_thread = MagicMock() + watchdog.capture_thread.is_alive.return_value = False + + watchdog._update_crash_loop_notice(now) + + self.assertEqual(self._notice_payloads(watchdog), []) + + def test_threshold_raises_once_per_restart_count(self): + watchdog = self._build_watchdog() + now = datetime.now().timestamp() + watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS) + watchdog.capture_thread = MagicMock() + watchdog.capture_thread.is_alive.return_value = False + + watchdog._update_crash_loop_notice(now) + watchdog._update_crash_loop_notice(now) + + payloads = self._notice_payloads(watchdog) + self.assertEqual(len(payloads), 1) + self.assertEqual(payloads[0]["action"], "raise") + self.assertEqual(payloads[0]["kind"], "ffmpeg_crash_loop") + self.assertEqual(payloads[0]["scope"], "front_door") + self.assertEqual(payloads[0]["params"], {"restarts": CRASH_LOOP_RESTARTS}) + + def test_recovery_window_resolves(self): + watchdog = self._build_watchdog() + now = datetime.now().timestamp() + # the last restart is at `now`, so recovery is measured from there + watchdog.reconnect_timestamps.extend([now] * CRASH_LOOP_RESTARTS) + watchdog.capture_thread = MagicMock() + watchdog.capture_thread.is_alive.return_value = False + watchdog._update_crash_loop_notice(now) + + watchdog.capture_thread.is_alive.return_value = True + watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S - 1) + self.assertEqual(len(self._notice_payloads(watchdog)), 1) + + watchdog._update_crash_loop_notice(now + CRASH_LOOP_RECOVERY_S + 1) + + payloads = self._notice_payloads(watchdog) + self.assertEqual(len(payloads), 2) + self.assertEqual(payloads[1]["action"], "resolve") + self.assertFalse(watchdog.crash_loop_raised) + + def test_disable_resolves(self): + watchdog = self._build_watchdog() + watchdog.crash_loop_raised = True + + watchdog._resolve_crash_loop() + + payloads = self._notice_payloads(watchdog) + self.assertEqual(payloads[-1]["action"], "resolve") + self.assertFalse(watchdog.crash_loop_raised) diff --git a/frigate/test/test_dispatcher_runtime_state.py b/frigate/test/test_dispatcher_runtime_state.py index e8a1382d73..162496f29d 100644 --- a/frigate/test/test_dispatcher_runtime_state.py +++ b/frigate/test/test_dispatcher_runtime_state.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch from frigate.app import FrigateApp from frigate.comms.dispatcher import Dispatcher +from frigate.comms.mqtt import MqttClient from frigate.comms.runtime_state import RuntimeStatePersistence from frigate.config import BirdseyeModeEnum @@ -510,3 +511,92 @@ class TestStartupAppliesConfigLayersBeforeWorkersStart(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class TestNoticeWiring(unittest.TestCase): + """The dispatcher forwards update_notice requests and publishes the list.""" + + def setUp(self) -> None: + self.registry = MagicMock() + self.registry.active.return_value = [{"id": "retention_unmet"}] + config = MagicMock() + config.cameras = {} + + with ( + patch("frigate.comms.dispatcher.CameraActivityManager"), + patch("frigate.comms.dispatcher.AudioActivityManager"), + ): + self.dispatcher = Dispatcher( + config, + MagicMock(), + MagicMock(), + {}, + [], + notice_registry=self.registry, + ) + + def test_registry_listener_is_subscribed(self) -> None: + self.registry.subscribe.assert_called_once_with( + self.dispatcher._publish_notices + ) + + def test_raise_request_reaches_registry(self) -> None: + self.dispatcher._receive( + "update_notice", + { + "action": "raise", + "kind": "ffmpeg_crash_loop", + "scope": "front_door", + "params": {"restarts": 5}, + }, + ) + + self.registry.raise_notice.assert_called_once_with( + "ffmpeg_crash_loop", scope="front_door", params={"restarts": 5} + ) + + def test_resolve_request_reaches_registry(self) -> None: + self.dispatcher._receive( + "update_notice", + {"action": "resolve", "kind": "retention_unmet", "scope": None}, + ) + + self.registry.resolve.assert_called_once_with("retention_unmet", None) + + def test_malformed_request_does_not_raise(self) -> None: + self.registry.raise_notice.side_effect = RuntimeError("boom") + + self.dispatcher._receive("update_notice", "not a dict") + self.dispatcher._receive( + "update_notice", {"action": "raise", "kind": "x", "params": {}} + ) + + def test_publish_local_skips_mqtt(self) -> None: + mqtt = MagicMock(spec=MqttClient) + other = MagicMock() + self.dispatcher.comms = [mqtt, other] + + self.dispatcher.publish_local("notices", "[]") + + mqtt.publish.assert_not_called() + other.publish.assert_called_once_with("notices", "[]", False) + + def test_listener_publishes_active_list(self) -> None: + self.dispatcher.publish_local = MagicMock() + + self.dispatcher._publish_notices() + + self.dispatcher.publish_local.assert_called_once_with( + "notices", '[{"id": "retention_unmet"}]' + ) + + def test_snapshot_includes_notices(self) -> None: + publisher = MagicMock() + self.dispatcher._build_camera_activity_snapshot = MagicMock( + return_value=({}, {}) + ) + self.dispatcher.web_push_client = None + + self.dispatcher.publish_runtime_snapshot(publisher) + + publisher.assert_any_call("notices", '[{"id": "retention_unmet"}]', False) diff --git a/frigate/test/test_model_downloader.py b/frigate/test/test_model_downloader.py new file mode 100644 index 0000000000..e4857bcbd1 --- /dev/null +++ b/frigate/test/test_model_downloader.py @@ -0,0 +1,102 @@ +"""Tests for the model download failure notice.""" + +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from frigate.util import downloader +from frigate.util.downloader import ModelDownloader + + +class TestModelDownloadNotice(unittest.TestCase): + def setUp(self): + self.download_path = tempfile.mkdtemp() + downloader.last_download_error.clear() + + def _notice_calls(self, requestor: MagicMock) -> list[dict]: + return [ + call.args[1] + for call in requestor.send_data.call_args_list + if call.args[0] == "update_notice" + ] + + def _downloader(self, download_func) -> ModelDownloader: + with patch("frigate.util.downloader.InterProcessRequestor"): + return ModelDownloader( + "facedet", self.download_path, ["facedet.onnx"], download_func + ) + + def test_raising_download_reports_and_reraises(self): + def failing(path: str) -> None: + raise RuntimeError("HTTP 503 from upstream\nmore detail") + + model_downloader = self._downloader(failing) + + with self.assertRaises(RuntimeError): + model_downloader._download_models() + + notices = self._notice_calls(model_downloader.requestor) + self.assertEqual(len(notices), 1) + self.assertEqual(notices[0]["action"], "raise") + self.assertEqual(notices[0]["kind"], "model_download_failed") + self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx") + self.assertEqual( + notices[0]["params"], + {"file": "facedet.onnx", "error": "HTTP 503 from upstream"}, + ) + + def test_swallowing_download_that_leaves_no_file_reports(self): + target = os.path.join(self.download_path, "facedet.onnx") + + def swallowing(path: str) -> None: + # mirrors the processors: download_from_url fails, they only log + with patch( + "frigate.util.downloader.requests.get", side_effect=OSError("dns") + ): + try: + ModelDownloader.download_from_url("http://x/facedet.onnx", path) + except Exception: + pass + + model_downloader = self._downloader(swallowing) + model_downloader._download_models() + + notices = self._notice_calls(model_downloader.requestor) + self.assertEqual(len(notices), 1) + self.assertEqual(notices[0]["action"], "raise") + self.assertEqual(notices[0]["params"], {"file": "facedet.onnx", "error": "dns"}) + self.assertFalse(os.path.exists(target)) + + def test_success_resolves(self): + def succeeding(path: str) -> None: + with open(path, "w") as f: + f.write("model") + + model_downloader = self._downloader(succeeding) + model_downloader._download_models() + + notices = self._notice_calls(model_downloader.requestor) + self.assertEqual(len(notices), 1) + self.assertEqual(notices[0]["action"], "resolve") + self.assertEqual(notices[0]["scope"], "facedet/facedet.onnx") + + def test_a_sibling_downloader_only_resolves_its_own_files(self): + """PaddleOCR, facedet and jina each spread one model_name over several + downloaders, so a resolve must never reach a sibling's file.""" + + def succeeding(path: str) -> None: + with open(path, "w") as f: + f.write("model") + + with patch("frigate.util.downloader.InterProcessRequestor"): + sibling = ModelDownloader( + "paddleocr-onnx", self.download_path, ["det.onnx"], succeeding + ) + + sibling._download_models() + + self.assertEqual( + [n["scope"] for n in self._notice_calls(sibling.requestor)], + ["paddleocr-onnx/det.onnx"], + ) diff --git a/frigate/test/test_notice_registry.py b/frigate/test/test_notice_registry.py new file mode 100644 index 0000000000..8c05804190 --- /dev/null +++ b/frigate/test/test_notice_registry.py @@ -0,0 +1,236 @@ +"""Tests for the notice registry and its kind definitions.""" + +import logging +import os +import unittest +from unittest.mock import MagicMock + +from peewee_migrate import Router +from playhouse.sqlite_ext import SqliteExtDatabase +from playhouse.sqliteq import SqliteQueueDatabase + +from frigate.models import Notice, NoticeStats +from frigate.notices.registry import DismissResult, NoticeRegistry +from frigate.notices.types import ( + NOTICE_KINDS, + NoticeMode, + NoticeSeverity, + notice_id, +) +from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS + + +class TestNoticeKinds(unittest.TestCase): + def test_every_kind_is_keyed_by_its_own_key(self): + for key, definition in NOTICE_KINDS.items(): + self.assertEqual(key, definition.key) + + def test_first_producers_are_defined(self): + self.assertEqual(NOTICE_KINDS["ffmpeg_crash_loop"].mode, NoticeMode.state) + self.assertEqual( + NOTICE_KINDS["ffmpeg_crash_loop"].severity, NoticeSeverity.error + ) + self.assertEqual(NOTICE_KINDS["detector_stuck"].mode, NoticeMode.event) + self.assertEqual( + NOTICE_KINDS["detector_stuck"].severity, NoticeSeverity.warning + ) + self.assertEqual(NOTICE_KINDS["model_download_failed"].mode, NoticeMode.event) + self.assertEqual(NOTICE_KINDS["retention_unmet"].mode, NoticeMode.state) + self.assertEqual(NOTICE_KINDS["update_available"].mode, NoticeMode.state) + self.assertEqual(NOTICE_KINDS["update_available"].severity, NoticeSeverity.info) + + def test_notice_id_with_and_without_scope(self): + self.assertEqual(notice_id("retention_unmet", None), "retention_unmet") + self.assertEqual( + notice_id("ffmpeg_crash_loop", "front_door"), + "ffmpeg_crash_loop:front_door", + ) + + +class TestNoticeRegistry(unittest.TestCase): + def setUp(self): + migrate_db = SqliteExtDatabase("test.db") + del logging.getLogger("peewee_migrate").handlers[:] + router = Router(migrate_db) + router.run() + migrate_db.close() + self.db = SqliteQueueDatabase(TEST_DB) + self.db.bind([Notice, NoticeStats]) + self.registry = NoticeRegistry() + self.listener = MagicMock() + self.registry.subscribe(self.listener) + + def tearDown(self): + if not self.db.is_closed(): + self.db.close() + + try: + for file in TEST_DB_CLEANUPS: + os.remove(file) + except OSError: + pass + + def test_raise_inserts_and_counts_one_occurrence(self): + self.registry.raise_notice( + "ffmpeg_crash_loop", scope="front_door", params={"restarts": 5} + ) + + active = self.registry.active() + self.assertEqual(len(active), 1) + self.assertEqual(active[0]["id"], "ffmpeg_crash_loop:front_door") + self.assertEqual(active[0]["severity"], "error") + self.assertEqual(active[0]["mode"], "state") + self.assertEqual(active[0]["category"], "camera") + self.assertEqual(active[0]["scope"], "front_door") + self.assertEqual(active[0]["params"], {"restarts": 5}) + self.assertEqual(active[0]["count"], 1) + self.assertEqual(self.registry.stats()[0]["occurrences"], 1) + self.listener.assert_called_once() + + def test_state_re_raise_with_new_params_updates_without_counting(self): + self.registry.raise_notice( + "ffmpeg_crash_loop", scope="front_door", params={"restarts": 5} + ) + self.listener.reset_mock() + + self.registry.raise_notice( + "ffmpeg_crash_loop", scope="front_door", params={"restarts": 6} + ) + + active = self.registry.active() + self.assertEqual(active[0]["count"], 1) + self.assertEqual(active[0]["params"], {"restarts": 6}) + self.assertEqual(self.registry.stats()[0]["occurrences"], 1) + self.listener.assert_called_once() + + def test_state_re_raise_with_same_params_is_a_no_op(self): + self.registry.raise_notice( + "ffmpeg_crash_loop", scope="front_door", params={"restarts": 5} + ) + before = self.registry.active()[0]["last_seen"] + self.listener.reset_mock() + + self.registry.raise_notice( + "ffmpeg_crash_loop", scope="front_door", params={"restarts": 5} + ) + + self.assertEqual(self.registry.active()[0]["last_seen"], before) + self.listener.assert_not_called() + + def test_event_re_raise_counts_and_undismisses(self): + self.registry.raise_notice("detector_stuck", params={"detector": "ov"}) + self.assertEqual(self.registry.dismiss("detector_stuck"), DismissResult.ok) + self.assertEqual(self.registry.active(), []) + + self.registry.raise_notice("detector_stuck", params={"detector": "ov"}) + # no scope here on purpose: a global event id is still valid + + active = self.registry.active() + self.assertEqual(len(active), 1) + self.assertEqual(active[0]["count"], 2) + self.assertIsNone(active[0]["dismissed_at"]) + stats = self.registry.stats()[0] + self.assertEqual(stats["occurrences"], 2) + self.assertEqual(stats["dismissals"], 1) + + def test_resolve_deletes_and_keeps_stats(self): + self.registry.raise_notice("retention_unmet", params={"needed_mb": 1}) + self.listener.reset_mock() + + self.registry.resolve("retention_unmet") + self.registry.resolve("retention_unmet") + + self.assertEqual(self.registry.active(), []) + self.assertEqual(self.registry.stats()[0]["occurrences"], 1) + self.listener.assert_called_once() + + def test_state_notice_cannot_be_dismissed(self): + self.registry.raise_notice("retention_unmet", params={}) + + self.assertEqual( + self.registry.dismiss("retention_unmet"), DismissResult.not_dismissable + ) + self.assertEqual(len(self.registry.active()), 1) + + def test_dismiss_unknown_id(self): + self.assertEqual(self.registry.dismiss("nope"), DismissResult.not_found) + + def test_dismissed_hidden_unless_requested(self): + self.registry.raise_notice("detector_stuck", params={"detector": "ov"}) + self.registry.dismiss("detector_stuck") + + self.assertEqual(self.registry.active(), []) + self.assertEqual(len(self.registry.active(include_dismissed=True)), 1) + + def test_startup_hides_state_notices_until_confirmed(self): + self.registry.raise_notice("retention_unmet", params={"needed_mb": 1}) + self.registry.raise_notice( + "detector_stuck", scope="ov", params={"detector": "ov"} + ) + + self.registry.mark_state_notices_unconfirmed() + + ids = [n["id"] for n in self.registry.active()] + self.assertEqual(ids, ["detector_stuck:ov"]) + + # the producer confirms the same episode: visible again, no new occurrence + self.registry.raise_notice("retention_unmet", params={"needed_mb": 1}) + + ids = [n["id"] for n in self.registry.active()] + self.assertEqual(ids, ["retention_unmet", "detector_stuck:ov"]) + self.assertEqual( + [s for s in self.registry.stats() if s["kind"] == "retention_unmet"][0][ + "occurrences" + ], + 1, + ) + + def test_unconfirmed_across_two_startups_is_deleted(self): + self.registry.raise_notice("retention_unmet", params={}) + + self.registry.mark_state_notices_unconfirmed() + self.registry.mark_state_notices_unconfirmed() + + self.assertEqual(self.registry.active(include_dismissed=True), []) + + def test_replay_camera_is_ignored(self): + self.registry.raise_notice( + "ffmpeg_crash_loop", scope="_replay_front_door", params={} + ) + + self.assertEqual(self.registry.active(), []) + self.assertEqual(self.registry.stats(), []) + + def test_unknown_kind_is_dropped(self): + self.registry.raise_notice("not_a_kind", params={}) + + self.assertEqual(self.registry.active(), []) + self.listener.assert_not_called() + + def test_active_sorts_by_severity_then_recency(self): + self.registry.raise_notice("detector_stuck", params={"detector": "ov"}) + self.registry.raise_notice("retention_unmet", params={}) + + ids = [n["id"] for n in self.registry.active()] + self.assertEqual(ids, ["retention_unmet", "detector_stuck"]) + + def test_listener_exception_does_not_propagate(self): + self.registry.subscribe(MagicMock(side_effect=RuntimeError("boom"))) + + self.registry.raise_notice("retention_unmet", params={}) + + self.assertEqual(len(self.registry.active()), 1) + + def test_resolve_camera_drops_only_that_camera(self): + self.registry.raise_notice("ffmpeg_crash_loop", scope="front_door", params={}) + self.registry.raise_notice("ffmpeg_crash_loop", scope="garage", params={}) + self.registry.raise_notice( + "model_download_failed", scope="front_door", params={} + ) + + self.registry.resolve_camera("front_door") + + ids = sorted(n["id"] for n in self.registry.active()) + self.assertEqual( + ids, ["ffmpeg_crash_loop:garage", "model_download_failed:front_door"] + ) diff --git a/frigate/test/test_stats_emitter_version.py b/frigate/test/test_stats_emitter_version.py new file mode 100644 index 0000000000..5296ca96e2 --- /dev/null +++ b/frigate/test/test_stats_emitter_version.py @@ -0,0 +1,129 @@ +"""Tests for the daily latest version refresh and the update notice.""" + +import unittest +from unittest.mock import MagicMock, patch + +from frigate.stats import emitter +from frigate.stats.util import is_newer_version + + +class TestIsNewerVersion(unittest.TestCase): + def test_newer(self): + self.assertTrue(is_newer_version("0.18.1-abcdef", "0.19.0")) + self.assertTrue(is_newer_version("0.19.0-abcdef", "0.19.1")) + self.assertTrue(is_newer_version("0.19.0-abcdef", "1.0.0")) + + def test_not_newer(self): + self.assertFalse(is_newer_version("0.19.0-abcdef", "0.19.0")) + self.assertFalse(is_newer_version("0.19.1-abcdef", "0.19.0")) + self.assertFalse(is_newer_version("0.19.0-abcdef", "0.19.0-beta2")) + + def test_prerelease_is_behind_its_final_release(self): + self.assertTrue(is_newer_version("0.19.0-beta2", "0.19.0")) + self.assertTrue(is_newer_version("0.19.0-rc1", "0.19.0")) + self.assertTrue(is_newer_version("0.19.0-RC1", "0.19.0")) + + def test_prerelease_of_a_later_line_is_not_behind(self): + self.assertFalse(is_newer_version("0.20.0-beta1", "0.19.0")) + self.assertFalse(is_newer_version("0.19.0-beta2", "0.19.0-beta2")) + + def test_unparseable_is_never_newer(self): + self.assertFalse(is_newer_version("0.19.0-abcdef", "disabled")) + self.assertFalse(is_newer_version("0.19.0-abcdef", "unknown")) + self.assertFalse(is_newer_version("dev", "0.19.0")) + + +class TestUpdateNotice(unittest.TestCase): + def _emitter(self, latest: str) -> emitter.StatsEmitter: + stats_emitter = emitter.StatsEmitter.__new__(emitter.StatsEmitter) + stats_emitter.config = MagicMock() + stats_emitter.stats_tracking = {"latest_frigate_version": latest} + stats_emitter.notice_registry = MagicMock() + return stats_emitter + + def test_newer_release_raises(self): + stats_emitter = self._emitter("0.19.0") + + with patch.object(emitter, "VERSION", "0.18.1-abcdef"): + stats_emitter._check_update_notice() + + stats_emitter.notice_registry.raise_notice.assert_called_once_with( + "update_available", params={"version": "0.19.0"} + ) + stats_emitter.notice_registry.resolve.assert_not_called() + + def test_current_or_disabled_resolves(self): + for latest in ("0.18.1", "disabled"): + stats_emitter = self._emitter(latest) + + with patch.object(emitter, "VERSION", "0.18.1-abcdef"): + stats_emitter._check_update_notice() + + stats_emitter.notice_registry.raise_notice.assert_not_called() + stats_emitter.notice_registry.resolve.assert_called_once_with( + "update_available" + ) + + def test_a_failed_lookup_leaves_the_notice_alone(self): + stats_emitter = self._emitter("unknown") + + with patch.object(emitter, "VERSION", "0.18.1-abcdef"): + stats_emitter._check_update_notice() + + stats_emitter.notice_registry.raise_notice.assert_not_called() + stats_emitter.notice_registry.resolve.assert_not_called() + + def test_refresh_updates_tracking_on_a_thread_then_checks(self): + stats_emitter = self._emitter("0.18.0") + + with ( + patch.object(emitter, "get_latest_version", return_value="0.19.0"), + patch.object(emitter, "VERSION", "0.18.1-abcdef"), + patch.object(emitter.threading, "Thread") as thread, + ): + stats_emitter._refresh_latest_version() + thread.return_value.start.assert_called_once() + # run the thread body inline + thread.call_args.kwargs["target"]() + + self.assertEqual( + stats_emitter.stats_tracking["latest_frigate_version"], "0.19.0" + ) + stats_emitter.notice_registry.raise_notice.assert_called_once() + + def test_failed_refresh_keeps_the_last_known_version(self): + stats_emitter = self._emitter("0.20.0") + + with ( + patch.object(emitter, "get_latest_version", return_value="unknown"), + patch.object(emitter, "VERSION", "0.19.0-abcdef"), + patch.object(emitter.threading, "Thread") as thread, + ): + stats_emitter._refresh_latest_version() + thread.call_args.kwargs["target"]() + + self.assertEqual( + stats_emitter.stats_tracking["latest_frigate_version"], "0.20.0" + ) + stats_emitter.notice_registry.raise_notice.assert_called_once_with( + "update_available", params={"version": "0.20.0"} + ) + stats_emitter.notice_registry.resolve.assert_not_called() + + def test_disabled_version_check_still_clears_the_notice(self): + stats_emitter = self._emitter("0.20.0") + + with ( + patch.object(emitter, "get_latest_version", return_value="disabled"), + patch.object(emitter, "VERSION", "0.19.0-abcdef"), + patch.object(emitter.threading, "Thread") as thread, + ): + stats_emitter._refresh_latest_version() + thread.call_args.kwargs["target"]() + + self.assertEqual( + stats_emitter.stats_tracking["latest_frigate_version"], "disabled" + ) + stats_emitter.notice_registry.resolve.assert_called_once_with( + "update_available" + ) diff --git a/frigate/test/test_storage.py b/frigate/test/test_storage.py index 67b2db15bb..070a8881f2 100644 --- a/frigate/test/test_storage.py +++ b/frigate/test/test_storage.py @@ -3,7 +3,7 @@ import logging import os import tempfile import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from peewee import DoesNotExist from peewee_migrate import Router @@ -556,6 +556,34 @@ class TestHttp(unittest.TestCase): assert Recordings.get(Recordings.id == rec_k2_id) assert Recordings.get(Recordings.id == rec_k3_id) + def test_unmet_retention_raises_notice(self): + registry = MagicMock() + config = FrigateConfig(**self.minimal_config) + storage = StorageMaintainer(config, MagicMock(), registry) + storage.camera_storage_stats = { + "front_door": {"needs_refresh": False, "usage": 10, "bandwidth": 10} + } + + with patch.object(storage, "expected_hourly_bandwidth", return_value=100.0): + storage.reduce_storage_consumption() + + registry.raise_notice.assert_called_once() + kind = registry.raise_notice.call_args.args[0] + params = registry.raise_notice.call_args.kwargs["params"] + self.assertEqual(kind, "retention_unmet") + self.assertEqual(params["needed_mb"], 100.0) + self.assertEqual(params["cleared_mb"], 0.0) + + def test_clean_run_resolves_notice(self): + registry = MagicMock() + config = FrigateConfig(**self.minimal_config) + storage = StorageMaintainer(config, MagicMock(), registry) + + with patch.object(storage, "check_storage_needs_cleanup", return_value=False): + storage._maintain_once() + + registry.resolve.assert_called_once_with("retention_unmet") + def _insert_mock_event( id: str, diff --git a/frigate/test/test_watchdog_notices.py b/frigate/test/test_watchdog_notices.py new file mode 100644 index 0000000000..ff8fdede1a --- /dev/null +++ b/frigate/test/test_watchdog_notices.py @@ -0,0 +1,38 @@ +"""Tests for the detector stuck notice raised by the Frigate watchdog.""" + +import unittest +from unittest.mock import MagicMock, patch + +from frigate.watchdog import FrigateWatchdog + + +class TestDetectorStuckNotice(unittest.TestCase): + def _stuck_detector(self) -> MagicMock: + detector = MagicMock() + detector.detection_start.value = 1.0 + detector.detect_process.is_alive.return_value = True + return detector + + def test_stuck_restart_raises_notice(self): + registry = MagicMock() + detector = self._stuck_detector() + watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry) + + with patch("frigate.watchdog.datetime") as mock_datetime: + mock_datetime.datetime.now.return_value.timestamp.return_value = 100.0 + watchdog._check_detectors() + + detector.start_or_restart.assert_called_once() + registry.raise_notice.assert_called_once_with( + "detector_stuck", scope="ov", params={"detector": "ov"} + ) + + def test_healthy_detector_raises_nothing(self): + registry = MagicMock() + detector = self._stuck_detector() + detector.detection_start.value = 0.0 + watchdog = FrigateWatchdog({"ov": detector}, MagicMock(), registry) + + watchdog._check_detectors() + + registry.raise_notice.assert_not_called() diff --git a/frigate/test/test_ws_outbound_filter.py b/frigate/test/test_ws_outbound_filter.py index ab1489da54..b401024a70 100644 --- a/frigate/test/test_ws_outbound_filter.py +++ b/frigate/test/test_ws_outbound_filter.py @@ -7,6 +7,7 @@ from types import SimpleNamespace from typing import Any from frigate.comms.ws import ( + _WS_BLOCKED_TOPICS, WebSocketClient, _classify_outbound, _collect_zone_names, @@ -106,6 +107,12 @@ class TestClassifyOutbound(unittest.TestCase): def test_birdseye_layout_is_unrestricted_only(self): self.assertEqual(self._classify("birdseye_layout"), ("unrestricted_only", None)) + def test_notices_is_admin_only(self): + self.assertEqual(self._classify("notices"), ("admin_only", None)) + + def test_update_notice_is_blocked_inbound(self): + self.assertIn("update_notice", _WS_BLOCKED_TOPICS) + # --- Camera-prefixed --- def test_camera_state_topic_resolves_to_camera(self): @@ -295,6 +302,19 @@ class TestMaterializeForWs(unittest.TestCase): full = json.dumps({"topic": topic, "payload": payload}) return _materialize_for_ws(ws, topic, full, scope, parsed, self.config) + def test_admin_only_topic_reaches_admin_not_viewer(self): + admin = SimpleNamespace(environ={"HTTP_REMOTE_ROLE": "admin"}) + viewer = SimpleNamespace(environ={"HTTP_REMOTE_ROLE": "viewer"}) + scope = ("admin_only", None) + + self.assertEqual( + _materialize_for_ws(admin, "notices", "msg", scope, None, self.config), + "msg", + ) + self.assertIsNone( + _materialize_for_ws(viewer, "notices", "msg", scope, None, self.config) + ) + # --- Globals: every authenticated client sees them --- def test_globals_reach_admin(self): diff --git a/frigate/util/downloader.py b/frigate/util/downloader.py index a8b593f159..9249286663 100644 --- a/frigate/util/downloader.py +++ b/frigate/util/downloader.py @@ -7,12 +7,22 @@ from pathlib import Path import requests from frigate.comms.inter_process import InterProcessRequestor -from frigate.const import UPDATE_MODEL_STATE +from frigate.const import UPDATE_MODEL_STATE, UPDATE_NOTICE from frigate.types import ModelStatusTypesEnum from frigate.util.file import FileLock logger = logging.getLogger(__name__) +# target path -> first line of the last download error for it; every existing +# download function swallows its exceptions, so this is how the downloader +# thread learns why a file is still missing +last_download_error: dict[str, str] = {} + + +def _first_line(error: BaseException) -> str: + text = str(error).strip().splitlines() + return (text[0] if text else type(error).__name__)[:200] + class ModelDownloader: def __init__( @@ -48,6 +58,33 @@ class ModelDownloader: ) self.download_thread.start() + def _notice_scope(self, file_name: str) -> str: + # per file: several loaders share one model_name with disjoint files, + # so a model wide scope lets one of them clear another's notice + return f"{self.model_name}/{file_name}" + + def _report_failure(self, file_name: str, error: str) -> None: + self.requestor.send_data( + UPDATE_NOTICE, + { + "action": "raise", + "kind": "model_download_failed", + "scope": self._notice_scope(file_name), + "params": {"file": file_name, "error": error}, + }, + ) + + def _resolve_failure(self, file_name: str) -> None: + self.requestor.send_data( + UPDATE_NOTICE, + { + "action": "resolve", + "kind": "model_download_failed", + "scope": self._notice_scope(file_name), + "params": {}, + }, + ) + def _download_models(self): for file_name in self.file_names: path = os.path.join(self.download_path, file_name) @@ -57,8 +94,20 @@ class ModelDownloader: if not os.path.exists(path): with lock: if not os.path.exists(path): - self.download_func(path) + try: + self.download_func(path) + except Exception as e: + self._report_failure(file_name, _first_line(e)) + raise + if not os.path.exists(path): + self._report_failure( + file_name, + last_download_error.pop(path, "download failed"), + ) + continue + + self._resolve_failure(file_name) self.requestor.send_data( UPDATE_MODEL_STATE, { @@ -93,6 +142,7 @@ class ModelDownloader: temporary_filename.rename(save_path) except Exception as e: logger.error(f"Error downloading model: {str(e)}") + last_download_error[save_path] = _first_line(e) raise if not silent: diff --git a/frigate/video/ffmpeg.py b/frigate/video/ffmpeg.py index c55c127498..b765a9de08 100644 --- a/frigate/video/ffmpeg.py +++ b/frigate/video/ffmpeg.py @@ -29,6 +29,7 @@ from frigate.const import ( STREAM_TYPE_MAIN, STREAM_TYPE_SUB, STREAM_TYPE_TO_ROLE, + UPDATE_NOTICE, ) from frigate.log import LogPipe from frigate.util.builtin import EventsPerSecond, get_record_segment_time @@ -43,6 +44,11 @@ logger = logging.getLogger(__name__) RECORD_GRACE_SECONDS = 90 +# a camera whose ffmpeg restarts this many times in the trailing hour is in a +# crash loop; it recovers once the capture thread stays up this long +CRASH_LOOP_RESTARTS = 5 +CRASH_LOOP_RECOVERY_S = 600 + def capture_frames( ffmpeg_process: sp.Popen[Any], @@ -140,6 +146,8 @@ class CameraWatchdog(threading.Thread): self.stop_event = stop_event self.sleeptime = self.config.ffmpeg.retry_interval self.reconnect_timestamps = deque() + self.crash_loop_raised = False + self._crash_loop_sent_restarts = 0 self.stalls = stalls self.reconnects = reconnects self.detection_frame = detection_frame @@ -189,6 +197,53 @@ class CameraWatchdog(threading.Thread): self._last_record_status: dict[str, str] = {} self._last_status_update_time: float = 0.0 + def _update_crash_loop_notice(self, now: float) -> None: + """Raise the crash loop notice as restarts pile up, resolve once stable. + + Called on every 1 second tick, so raises are sent only when the restart + count changes; the registry treats repeats of a state notice as updates. + """ + restarts = len(self.reconnect_timestamps) + alive = self.capture_thread is not None and self.capture_thread.is_alive() + + if not alive and restarts >= CRASH_LOOP_RESTARTS: + if self.crash_loop_raised and restarts == self._crash_loop_sent_restarts: + return + + self.requestor.send_data( + UPDATE_NOTICE, + { + "action": "raise", + "kind": "ffmpeg_crash_loop", + "scope": self.config.name, + "params": {"restarts": restarts}, + }, + ) + self.crash_loop_raised = True + self._crash_loop_sent_restarts = restarts + elif ( + self.crash_loop_raised + and alive + and ( + restarts == 0 + or now - self.reconnect_timestamps[-1] >= CRASH_LOOP_RECOVERY_S + ) + ): + self._resolve_crash_loop() + + def _resolve_crash_loop(self) -> None: + self.requestor.send_data( + UPDATE_NOTICE, + { + "action": "resolve", + "kind": "ffmpeg_crash_loop", + "scope": self.config.name, + "params": {}, + }, + ) + self.crash_loop_raised = False + self._crash_loop_sent_restarts = 0 + def _send_detect_status(self, status: str, now: float) -> None: """Send detect status only if changed or retry_interval has elapsed.""" if ( @@ -379,6 +434,9 @@ class CameraWatchdog(threading.Thread): # cameras without a sub stream never get a record_sub topic if self.config.record.sub.enabled: self._send_record_status(STREAM_TYPE_SUB, "disabled", now) + + if self.crash_loop_raised: + self._resolve_crash_loop() self.was_enabled = enabled continue @@ -456,6 +514,7 @@ class CameraWatchdog(threading.Thread): self.logger.error( f"Ffmpeg process crashed unexpectedly for {self.config.name}." ) + self._update_crash_loop_notice(now) if can_restart: self.reset_capture_thread(terminate=False) last_restart_time = now @@ -485,6 +544,7 @@ class CameraWatchdog(threading.Thread): # process is running normally self._send_detect_status("online", now) self.fps_overflow_count = 0 + self._update_crash_loop_notice(now) for p in self.ffmpeg_other_processes: poll = p["process"].poll() diff --git a/frigate/watchdog.py b/frigate/watchdog.py index 88023d5122..489c23f76c 100644 --- a/frigate/watchdog.py +++ b/frigate/watchdog.py @@ -7,6 +7,7 @@ from collections.abc import Callable from dataclasses import dataclass, field from multiprocessing.synchronize import Event as MpEvent +from frigate.notices.registry import NoticeRegistry from frigate.object_detection.base import ObjectDetectProcess from frigate.util.process import FrigateProcess from frigate.util.services import restart_frigate @@ -44,10 +45,12 @@ class FrigateWatchdog(threading.Thread): self, detectors: dict[str, ObjectDetectProcess], stop_event: MpEvent, + notice_registry: NoticeRegistry | None = None, ): super().__init__(name="frigate_watchdog") self.detectors = detectors self.stop_event = stop_event + self.notice_registry = notice_registry self._monitored: list[MonitoredProcess] = [] def register( @@ -111,27 +114,34 @@ class FrigateWatchdog(threading.Thread): except Exception: logger.exception("Failed to restart %s", entry.name) + def _check_detectors(self) -> None: + now = datetime.datetime.now().timestamp() + + for name, detector in self.detectors.items(): + detection_start = detector.detection_start.value # type: ignore[attr-defined] + # issue https://github.com/python/typeshed/issues/8799 + # from mypy 0.981 onwards + if detection_start > 0.0 and now - detection_start > 10: + logger.info( + "Detection appears to be stuck. Restarting detection process..." + ) + detector.start_or_restart() + + if self.notice_registry is not None: + self.notice_registry.raise_notice( + "detector_stuck", scope=name, params={"detector": name} + ) + elif ( + detector.detect_process is not None + and not detector.detect_process.is_alive() + ): + logger.info("Detection appears to have stopped. Exiting Frigate...") + restart_frigate() + def run(self) -> None: time.sleep(10) while not self.stop_event.wait(10): - now = datetime.datetime.now().timestamp() - - # check the detection processes - for detector in self.detectors.values(): - detection_start = detector.detection_start.value # type: ignore[attr-defined] - # issue https://github.com/python/typeshed/issues/8799 - # from mypy 0.981 onwards - if detection_start > 0.0 and now - detection_start > 10: - logger.info( - "Detection appears to be stuck. Restarting detection process..." - ) - detector.start_or_restart() - elif ( - detector.detect_process is not None - and not detector.detect_process.is_alive() - ): - logger.info("Detection appears to have stopped. Exiting Frigate...") - restart_frigate() + self._check_detectors() for entry in self._monitored: self._check_process(entry) diff --git a/generate_api_auth_spec.py b/generate_api_auth_spec.py index c1d82ab155..74193496b9 100644 --- a/generate_api_auth_spec.py +++ b/generate_api_auth_spec.py @@ -59,6 +59,7 @@ from frigate.api import ( hardware, media, motion_search, + notices, notification, preview, record, @@ -154,6 +155,7 @@ def build_app() -> FastAPI: notification.router, export.router, hardware.router, + notices.router, event.router, media.router, motion_search.router, diff --git a/migrations/038_create_notices.py b/migrations/038_create_notices.py new file mode 100644 index 0000000000..5ab92c5846 --- /dev/null +++ b/migrations/038_create_notices.py @@ -0,0 +1,38 @@ +"""Peewee migrations -- 038_create_notices.py. + +Creates the two notice tables: active notices, and per-kind lifetime counts. +""" + +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + migrator.sql( + 'CREATE TABLE IF NOT EXISTS "notice" (' + '"id" VARCHAR(150) NOT NULL PRIMARY KEY, ' + '"kind" VARCHAR(50) NOT NULL, ' + '"scope" VARCHAR(100), ' + '"params" TEXT NOT NULL, ' + '"first_seen" DATETIME NOT NULL, ' + '"last_seen" DATETIME NOT NULL, ' + '"count" INTEGER NOT NULL, ' + '"dismissed_at" DATETIME)' + ) + migrator.sql('CREATE INDEX IF NOT EXISTS "notice_kind" ON "notice" ("kind")') + migrator.sql( + 'CREATE TABLE IF NOT EXISTS "noticestats" (' + '"kind" VARCHAR(50) NOT NULL PRIMARY KEY, ' + '"occurrences" INTEGER NOT NULL, ' + '"dismissals" INTEGER NOT NULL, ' + '"first_seen" DATETIME NOT NULL, ' + '"last_seen" DATETIME NOT NULL, ' + '"reported_occurrences" INTEGER NOT NULL DEFAULT 0, ' + '"reported_dismissals" INTEGER NOT NULL DEFAULT 0)' + ) + + +def rollback(migrator, database, fake=False, **kwargs): + migrator.sql('DROP TABLE IF EXISTS "notice"') + migrator.sql('DROP TABLE IF EXISTS "noticestats"') diff --git a/web/e2e/fixtures/mock-data/config.ts b/web/e2e/fixtures/mock-data/config.ts index e337218e5a..6267aa908a 100644 --- a/web/e2e/fixtures/mock-data/config.ts +++ b/web/e2e/fixtures/mock-data/config.ts @@ -19,7 +19,7 @@ export type DeepPartial = { [P in keyof T]?: T[P] extends object ? DeepPartial : T[P]; }; -function deepMerge>( +export function deepMerge>( base: T, overrides?: DeepPartial, ): T { diff --git a/web/e2e/fixtures/mock-data/stats.ts b/web/e2e/fixtures/mock-data/stats.ts index d34ea25fc0..c1b5d1a78c 100644 --- a/web/e2e/fixtures/mock-data/stats.ts +++ b/web/e2e/fixtures/mock-data/stats.ts @@ -2,7 +2,7 @@ * FrigateStats factory for E2E tests. */ -import type { DeepPartial } from "./config"; +import { deepMerge, type DeepPartial } from "./config"; function cameraStats(_name: string) { return { @@ -72,5 +72,5 @@ export function statsFactory( overrides?: DeepPartial, ): typeof BASE_STATS { if (!overrides) return BASE_STATS; - return { ...BASE_STATS, ...overrides } as typeof BASE_STATS; + return deepMerge(BASE_STATS, overrides) as typeof BASE_STATS; } diff --git a/web/e2e/helpers/api-mocker.ts b/web/e2e/helpers/api-mocker.ts index 22dce5338e..3209d6fe6d 100644 --- a/web/e2e/helpers/api-mocker.ts +++ b/web/e2e/helpers/api-mocker.ts @@ -48,6 +48,8 @@ export interface ApiMockOverrides { available?: { key: string; presets: Record }[]; }; users?: { username: string; role: string }[]; + notices?: unknown[]; + noticeStats?: unknown[]; } export class ApiMocker { @@ -201,6 +203,15 @@ export class ApiMocker { }), ); + // Notices. The stats route is registered after the list route so it wins + // for /api/notices/stats; the list glob does not match a sub-path anyway. + await this.page.route("**/api/notices", (route) => + route.fulfill({ json: overrides?.notices ?? [] }), + ); + await this.page.route("**/api/notices/stats", (route) => + route.fulfill({ json: overrides?.noticeStats ?? [] }), + ); + // Users. GET lists them; POST/PUT (create, password) just succeed, so // tests assert on the intercepted request body instead of a response. await this.page.route("**/api/users**", (route) => diff --git a/web/e2e/specs/navigation.spec.ts b/web/e2e/specs/navigation.spec.ts index e14887c157..1bcaf10b2d 100644 --- a/web/e2e/specs/navigation.spec.ts +++ b/web/e2e/specs/navigation.spec.ts @@ -130,8 +130,8 @@ test.describe("Navigation — settings menu (desktop) @critical", () => { const TARGETS = [ { label: "Settings", url: /\/settings/ }, - { label: "System metrics", url: /\/system/ }, - { label: "System logs", url: /\/logs/ }, + { label: "Health and Metrics", url: /\/system/ }, + { label: "Logs", url: /\/logs/ }, { label: "Configuration Editor", url: /\/config/ }, ]; @@ -142,7 +142,7 @@ test.describe("Navigation — settings menu (desktop) @critical", () => { .locator("aside .mb-8 div[class*='cursor-pointer']") .first(); await gear.click(); - await frigateApp.page.getByLabel(target.label).click(); + await frigateApp.page.getByLabel(target.label, { exact: true }).click(); await expect(frigateApp.page).toHaveURL(target.url); }); } diff --git a/web/e2e/specs/system-health.spec.ts b/web/e2e/specs/system-health.spec.ts new file mode 100644 index 0000000000..06d1dfa0c6 --- /dev/null +++ b/web/e2e/specs/system-health.spec.ts @@ -0,0 +1,169 @@ +/** + * Health tab tests -- MEDIUM tier. + * + * Default tab, notice list rendering, dismiss, empty state, update notice. + */ + +import { test, expect } from "../fixtures/frigate-test"; + +const NOW = Math.floor(Date.now() / 1000); + +const STATE_NOTICE = { + id: "ffmpeg_crash_loop:front_door", + kind: "ffmpeg_crash_loop", + mode: "state", + severity: "error", + category: "camera", + scope: "front_door", + params: { restarts: 6 }, + first_seen: NOW - 600, + last_seen: NOW, + count: 1, + dismissed_at: null, +}; + +const EVENT_NOTICE = { + id: "detector_stuck", + kind: "detector_stuck", + mode: "event", + severity: "warning", + category: "detector", + scope: null, + params: { detector: "ov" }, + first_seen: NOW - 7200, + last_seen: NOW - 60, + count: 3, + dismissed_at: null, +}; + +test.describe("System — Health tab @medium", () => { + test("Health is the default tab and lists notices by severity", async ({ + frigateApp, + }) => { + await frigateApp.installDefaults({ + notices: [STATE_NOTICE, EVENT_NOTICE], + noticeStats: [ + { + kind: "ffmpeg_crash_loop", + occurrences: 14, + dismissals: 0, + first_seen: NOW - 86400 * 20, + last_seen: NOW, + }, + ], + }); + await frigateApp.goto("/system"); + + await expect(frigateApp.page.getByLabel("Select health")).toHaveAttribute( + "data-state", + "on", + { timeout: 15_000 }, + ); + + const rows = frigateApp.page.locator("[data-testid^='health-problem-']"); + await expect(rows).toHaveCount(2); + await expect(rows.nth(0)).toHaveAttribute("data-severity", "error"); + await expect(rows.nth(0)).toContainText("ffmpeg has crashed 6 times"); + await expect(rows.nth(0)).toContainText("14 times since"); + // the default fixture's cpu detector is slow (75.5 ms); PR 2 merges live + // problems into this list, PR 1 does not, so no extra row here + await expect( + rows.nth(0).getByRole("button", { name: "Dismiss" }), + ).toHaveCount(0); + await expect(rows.nth(1)).toContainText("Detector ov was restarted"); + await expect(rows.nth(1)).toContainText("3 times"); + await expect( + rows.nth(1).getByRole("button", { name: "Dismiss" }), + ).toBeVisible(); + }); + + test("dismiss posts and removes the row", async ({ frigateApp }) => { + await frigateApp.installDefaults({ notices: [EVENT_NOTICE] }); + + // the list shrinks after the dismiss so the refetch shows the row gone + let dismissed = false; + await frigateApp.page.route("**/api/notices", (route) => + route.fulfill({ json: dismissed ? [] : [EVENT_NOTICE] }), + ); + await frigateApp.page.route( + "**/api/notices/detector_stuck/dismiss", + (route) => { + dismissed = true; + return route.fulfill({ json: { success: true } }); + }, + ); + + await frigateApp.goto("/system#health"); + const request = frigateApp.page.waitForRequest( + (req) => + req.url().includes("/api/notices/detector_stuck/dismiss") && + req.method() === "POST", + ); + await frigateApp.page.getByRole("button", { name: "Dismiss" }).click(); + await request; + + await expect( + frigateApp.page.locator("[data-testid^='health-problem-']"), + ).toHaveCount(0, { timeout: 5_000 }); + await expect(frigateApp.page.getByText("No notices")).toBeVisible(); + }); + + test("empty state with no notices", async ({ frigateApp }) => { + await frigateApp.installDefaults(); + await frigateApp.goto("/system#health"); + + await expect(frigateApp.page.getByText("No notices")).toBeVisible({ + timeout: 15_000, + }); + }); + + test("update notice renders as info with a release link", async ({ + frigateApp, + }) => { + await frigateApp.installDefaults({ + notices: [ + { + id: "update_available", + kind: "update_available", + mode: "state", + severity: "info", + category: "system", + scope: null, + params: { version: "0.19.0" }, + first_seen: NOW - 3600, + last_seen: NOW, + count: 1, + dismissed_at: null, + }, + ], + }); + await frigateApp.goto("/system#health"); + + const row = frigateApp.page.getByTestId( + "health-problem-notice:update_available", + ); + await expect(row).toBeVisible({ timeout: 15_000 }); + await expect(row).toHaveAttribute("data-severity", "info"); + await expect(row).toContainText("Frigate 0.19.0 is available"); + await expect(row.getByRole("link", { name: "Open link" })).toHaveAttribute( + "href", + "https://github.com/blakeblackshear/frigate/releases/tag/v0.19.0", + ); + await expect(row.getByRole("button", { name: "Dismiss" })).toHaveCount(0); + }); +}); + +test.describe("System — Health tab mobile @medium @mobile", () => { + test.skip(({ frigateApp }) => !frigateApp.isMobile, "Mobile-only"); + + test("notices render at mobile viewport", async ({ frigateApp }) => { + await frigateApp.installDefaults({ notices: [STATE_NOTICE] }); + await frigateApp.goto("/system#health"); + + await expect( + frigateApp.page.getByTestId( + "health-problem-notice:ffmpeg_crash_loop:front_door", + ), + ).toBeVisible({ timeout: 15_000 }); + }); +}); diff --git a/web/public/locales/en/common.json b/web/public/locales/en/common.json index d2b803b2db..202e5cddc2 100644 --- a/web/public/locales/en/common.json +++ b/web/public/locales/en/common.json @@ -168,9 +168,9 @@ }, "menu": { "system": "System", - "systemMetrics": "System metrics", + "systemMetrics": "Health and Metrics", "configuration": "Configuration", - "systemLogs": "System logs", + "systemLogs": "Logs", "profiles": "Profiles", "settings": "Settings", "configurationEditor": "Configuration Editor", diff --git a/web/public/locales/en/views/system.json b/web/public/locales/en/views/system.json index 5f261eba4f..2b0d0e629f 100644 --- a/web/public/locales/en/views/system.json +++ b/web/public/locales/en/views/system.json @@ -9,10 +9,31 @@ "go2rtc": "Go2RTC Logs - Frigate", "nginx": "Nginx Logs - Frigate", "websocket": "Messages Logs - Frigate" - } + }, + "health": "Health - Frigate" }, "title": "System", "metrics": "System metrics", + "health": { + "title": "Health", + "notices": { + "title": "Notices", + "empty": "No notices", + "dismiss": "Dismiss", + "openSettings": "Open settings", + "openLink": "Open link", + "since": "Since {{time}}", + "sinceWithCount": "Since {{time}} · {{times}} times since {{firstSeen}}", + "firstSeen": "First seen {{time}} · {{times}} times", + "kinds": { + "ffmpeg_crash_loop": "ffmpeg has crashed {{restarts}} times in the last hour", + "detector_stuck": "Detector {{detector}} was restarted after it stopped responding", + "model_download_failed": "Downloading {{file}} failed: {{error}}", + "retention_unmet": "Recordings were deleted before their retention period to free space ({{cleared_mb}} of {{needed_mb}} MB needed)", + "update_available": "Frigate {{version}} is available" + } + } + }, "logs": { "websocket": { "label": "Messages", diff --git a/web/src/components/health/HealthProblemRow.tsx b/web/src/components/health/HealthProblemRow.tsx new file mode 100644 index 0000000000..11163fe1fe --- /dev/null +++ b/web/src/components/health/HealthProblemRow.tsx @@ -0,0 +1,88 @@ +import { Link } from "react-router-dom"; +import { useTranslation } from "react-i18next"; +import { FaTriangleExclamation } from "react-icons/fa6"; +import { + LuExternalLink, + LuInfo, + LuSlidersHorizontal, + LuX, +} from "react-icons/lu"; +import { Button } from "@/components/ui/button"; +import { CameraNameLabel } from "@/components/camera/FriendlyNameLabel"; +import type { HealthProblem } from "@/types/health"; + +type HealthProblemRowProps = { + problem: HealthProblem; +}; + +export default function HealthProblemRow({ problem }: HealthProblemRowProps) { + const { t } = useTranslation(["views/system"]); + + return ( +
+
+ {problem.severity === "error" && } + {problem.severity === "warning" && ( + + )} + {problem.severity === "info" && ( + + )} +
+ {problem.scope && ( + + {problem.scopeIsCamera ? ( + + ) : ( + problem.scope + )} + + )} +
+
{problem.text}
+ {problem.meta && ( +
+ {problem.meta} +
+ )} +
+
+ {problem.onDismiss && ( + + )} + {problem.link && ( + + + + )} + {problem.externalLink && ( + + + + )} +
+
+ ); +} diff --git a/web/src/components/health/NoticesPane.tsx b/web/src/components/health/NoticesPane.tsx new file mode 100644 index 0000000000..3a655c46d5 --- /dev/null +++ b/web/src/components/health/NoticesPane.tsx @@ -0,0 +1,125 @@ +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import { FaCircleCheck } from "react-icons/fa6"; +import useSWR from "swr"; +import HealthProblemRow from "@/components/health/HealthProblemRow"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useNotices } from "@/hooks/use-notices"; +import { useDateLocale } from "@/hooks/use-date-locale"; +import { useTimezone } from "@/hooks/use-date-utils"; +import { formatUnixTimestampToDateTime } from "@/utils/dateUtil"; +import { releaseUrl } from "@/utils/versionUtil"; +import type { FrigateConfig } from "@/types/frigateConfig"; +import type { HealthProblem } from "@/types/health"; +import type { Notice, NoticeKind, NoticeStats } from "@/types/notice"; + +const SETTINGS_LINK_BY_KIND: Partial< + Record string> +> = { + ffmpeg_crash_loop: (scope) => `/settings?page=cameraFfmpeg&camera=${scope}`, + retention_unmet: () => "/system#storage", + detector_stuck: () => "/system#general", +}; + +const EXTERNAL_LINK_BY_KIND: Partial< + Record string | undefined> +> = { + update_available: (params) => + typeof params.version === "string" ? releaseUrl(params.version) : undefined, +}; + +function useNoticeProblems( + notices: Notice[] | undefined, + statsByKind: Partial>, + dismiss: (id: string) => Promise, +): HealthProblem[] { + const { t } = useTranslation(["views/system"]); + const { data: config } = useSWR("config"); + const timezone = useTimezone(config); + const locale = useDateLocale(); + + return useMemo(() => { + if (!notices) { + return []; + } + + const formatTime = (timestamp: number, date_format: string) => + formatUnixTimestampToDateTime(timestamp, { + timezone, + date_format, + locale, + }); + + return notices.map((notice) => { + const stats = statsByKind[notice.kind]; + const seen = formatTime(notice.first_seen, "MMM d, h:mm a"); + let meta: string; + + if (notice.mode === "event") { + meta = t("health.notices.firstSeen", { + time: seen, + times: notice.count, + }); + } else if (stats && stats.occurrences > 1) { + meta = t("health.notices.sinceWithCount", { + time: seen, + times: stats.occurrences, + firstSeen: formatTime(stats.first_seen, "MMM d"), + }); + } else { + meta = t("health.notices.since", { time: seen }); + } + + const link = SETTINGS_LINK_BY_KIND[notice.kind]; + const external = EXTERNAL_LINK_BY_KIND[notice.kind]; + + return { + id: `notice:${notice.id}`, + severity: notice.severity, + scope: notice.scope ?? undefined, + scopeIsCamera: notice.category === "camera", + // replace keeps backend params out of i18next's own option names + text: t(`health.notices.kinds.${notice.kind}`, { + replace: notice.params, + }), + meta, + link: link ? link(notice.scope) : undefined, + externalLink: external ? external(notice.params) : undefined, + onDismiss: + notice.mode === "event" ? () => dismiss(notice.id) : undefined, + }; + }); + }, [notices, statsByKind, dismiss, t, timezone, locale]); +} + +export default function NoticesPane() { + const { t } = useTranslation(["views/system"]); + const { notices, statsByKind, dismiss } = useNotices(); + const problems = useNoticeProblems(notices, statsByKind, dismiss); + + return ( +
+
+
+ {t("health.notices.title")} +
+
+
+ {notices === undefined ? ( + + ) : problems.length === 0 ? ( +
+ + {t("health.notices.empty")} +
+ ) : ( +
+ {problems.map((problem) => ( + + ))} +
+ )} +
+
+ ); +} diff --git a/web/src/components/menu/GeneralSettings.tsx b/web/src/components/menu/GeneralSettings.tsx index 5dcac82530..6ae862f092 100644 --- a/web/src/components/menu/GeneralSettings.tsx +++ b/web/src/components/menu/GeneralSettings.tsx @@ -326,7 +326,7 @@ export default function GeneralSettings({ {t("menu.system")} - + ("notices", { + revalidateOnFocus: false, + }); + const { data: stats, mutate: mutateStats } = useSWR( + "notices/stats", + { revalidateOnFocus: false }, + ); + const { + value: { payload }, + } = useWs("notices", ""); + + const live = useMemo( + () => + payload && typeof payload === "string" + ? (JSON.parse(payload) as Notice[]) + : undefined, + [payload], + ); + + // once a websocket frame has arrived it is the source of truth; a dismiss + // still shows up because the registry publishes a new frame after it + const notices = live ?? initial; + + const statsByKind = useMemo(() => { + const byKind: Partial> = {}; + (stats ?? []).forEach((entry) => { + byKind[entry.kind] = entry; + }); + return byKind; + }, [stats]); + + const dismiss = useCallback( + async (id: string) => { + await axios.post(`notices/${id}/dismiss`); + mutate(); + mutateStats(); + }, + [mutate, mutateStats], + ); + + return { notices, statsByKind, dismiss }; +} diff --git a/web/src/pages/System.tsx b/web/src/pages/System.tsx index c7c543e7fd..1acb1a5006 100644 --- a/web/src/pages/System.tsx +++ b/web/src/pages/System.tsx @@ -6,7 +6,12 @@ import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; import { isDesktop, isMobile } from "react-device-detect"; import GeneralMetrics from "@/views/system/GeneralMetrics"; import StorageMetrics from "@/views/system/StorageMetrics"; -import { LuActivity, LuHardDrive, LuSearchCode } from "react-icons/lu"; +import { + LuActivity, + LuHardDrive, + LuHeartPulse, + LuSearchCode, +} from "react-icons/lu"; import { FaVideo } from "react-icons/fa"; import Logo from "@/components/Logo"; import useOptimisticState from "@/hooks/use-optimistic-state"; @@ -15,9 +20,16 @@ import { useHashState } from "@/hooks/use-overlay-state"; import { Toaster } from "@/components/ui/sonner"; import { FrigateConfig } from "@/types/frigateConfig"; import EnrichmentMetrics from "@/views/system/EnrichmentMetrics"; +import HealthMetrics from "@/views/system/HealthMetrics"; import { useTranslation } from "react-i18next"; -const allMetrics = ["general", "enrichments", "storage", "cameras"] as const; +const allMetrics = [ + "health", + "general", + "enrichments", + "storage", + "cameras", +] as const; type SystemMetric = (typeof allMetrics)[number]; function System() { @@ -44,8 +56,9 @@ function System() { // stats page const [page, setPage] = useHashState(); + // useHashState yields "" with no hash, which ?? would not catch const [pageToggle, setPageToggle] = useOptimisticState( - page ?? "general", + page || "health", setPage, 100, ); @@ -56,9 +69,7 @@ function System() { // Track which tabs have been visited so we can keep them mounted after first visit. // Using a ref updated during render avoids extra render cycles from state/effects. const visitedTabsRef = useRef(new Set()); - if (page) { - visitedTabsRef.current.add(page); - } + visitedTabsRef.current.add(pageToggle); const visitedTabs = visitedTabsRef.current; useEffect(() => { @@ -98,6 +109,7 @@ function System() { value={item} aria-label={`Select ${item}`} > + {item == "health" && } {item == "general" && } {item == "enrichments" && } {item == "storage" && } @@ -126,6 +138,11 @@ function System() { )} + {visitedTabs.has("health") && ( +
+ +
+ )} {visitedTabs.has("general") && (
void; +}; diff --git a/web/src/types/notice.ts b/web/src/types/notice.ts new file mode 100644 index 0000000000..47e3a9a654 --- /dev/null +++ b/web/src/types/notice.ts @@ -0,0 +1,36 @@ +export type NoticeMode = "state" | "event"; +export type NoticeSeverity = "error" | "warning" | "info"; +export type NoticeCategory = + | "camera" + | "detector" + | "storage" + | "model" + | "system"; +export type NoticeKind = + | "ffmpeg_crash_loop" + | "detector_stuck" + | "model_download_failed" + | "retention_unmet" + | "update_available"; + +export type Notice = { + id: string; + kind: NoticeKind; + mode: NoticeMode; + severity: NoticeSeverity; + category: NoticeCategory; + scope: string | null; + params: Record; + first_seen: number; + last_seen: number; + count: number; + dismissed_at: number | null; +}; + +export type NoticeStats = { + kind: NoticeKind; + occurrences: number; + dismissals: number; + first_seen: number; + last_seen: number; +}; diff --git a/web/src/utils/versionUtil.ts b/web/src/utils/versionUtil.ts new file mode 100644 index 0000000000..b97bd63968 --- /dev/null +++ b/web/src/utils/versionUtil.ts @@ -0,0 +1,4 @@ +/** GitHub release page for a Frigate version such as "0.19.0". */ +export function releaseUrl(version: string): string { + return `https://github.com/blakeblackshear/frigate/releases/tag/v${version}`; +} diff --git a/web/src/views/system/HealthMetrics.tsx b/web/src/views/system/HealthMetrics.tsx new file mode 100644 index 0000000000..70e6c8b67c --- /dev/null +++ b/web/src/views/system/HealthMetrics.tsx @@ -0,0 +1,9 @@ +import NoticesPane from "@/components/health/NoticesPane"; + +export default function HealthMetrics() { + return ( +
+ +
+ ); +}