From 36c61a3607d7cb52d940e0eaa09469fa4c7e074d Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 12 Jun 2026 16:41:33 +0100 Subject: [PATCH] =?UTF-8?q?=EF=BB=BFCorroborate=20record=20watchdog=20stal?= =?UTF-8?q?eness=20against=20the=20cache=20dir=20before=20restarting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The record watchdog treats a stale maintainer heartbeat as a dead recorder. But the heartbeat is published by the recording maintainer, so whenever the maintainer lags (e.g. "Unable to keep up with recording segments in cache", #9661) every camera looks stale at once and all record processes restart together - while recording was actually healthy. The restart churn then produces more, shorter segments, making the maintainer fall further behind. Before restarting on staleness, check the camera's newest cache segment on disk: if a segment is fresher than the staleness threshold, the recorder is demonstrably writing - log a warning, adopt the disk mtime as the heartbeat, and skip the restart. The invalid-segment path is untouched. Validated on a 26-camera production deployment (0.17.1 backport of this change): synchronized mass restarts went from 52/hour to zero, with heartbeat-stale events still occurring ~2/hour but now correctly identified as maintainer lag instead of recording failure. Co-Authored-By: Claude Fable 5 --- frigate/video/ffmpeg.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/frigate/video/ffmpeg.py b/frigate/video/ffmpeg.py index 40b18b7da3..53f819cc98 100644 --- a/frigate/video/ffmpeg.py +++ b/frigate/video/ffmpeg.py @@ -9,6 +9,7 @@ from collections import deque from datetime import datetime, timedelta, timezone from multiprocessing import Queue, Value from multiprocessing.synchronize import Event as MpEvent +from pathlib import Path from typing import Any from frigate.camera import CameraMetrics @@ -22,7 +23,7 @@ from frigate.config.camera.updater import ( CameraConfigUpdateEnum, CameraConfigUpdateSubscriber, ) -from frigate.const import PROCESS_PRIORITY_HIGH +from frigate.const import CACHE_DIR, PROCESS_PRIORITY_HIGH from frigate.log import LogPipe from frigate.util.builtin import EventsPerSecond, get_record_segment_time from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg @@ -453,6 +454,38 @@ class CameraWatchdog(threading.Thread): invalid_stale = invalid_stale_condition if cache_stale or valid_stale or invalid_stale: + # The staleness above is measured from the recording + # maintainer's IPC heartbeat, which lags whenever the + # maintainer falls behind (e.g. "Unable to keep up with + # recording segments in cache"). A late message is not + # a dead recorder: corroborate against the cache dir + # before restarting, otherwise every camera restarts + # together on maintainer lag and the resulting segment + # churn makes the overload worse. + if not invalid_stale: + newest_on_disk = max( + ( + f.stat().st_mtime + for f in Path(CACHE_DIR).glob( + f"{self.config.name}@*" + ) + ), + default=0.0, + ) + if ( + newest_on_disk > 0 + and now_utc.timestamp() - newest_on_disk + < self.record_stale_threshold + ): + self.logger.warning( + f"Recording heartbeat for {self.config.name} is stale but a cache " + f"segment is only {now_utc.timestamp() - newest_on_disk:.0f}s old — " + "skipping the record process restart (maintainer heartbeat lag, " + "not a recording failure)." + ) + self.latest_cache_segment_time = newest_on_disk + continue + if cache_stale: reason = "No new recording segments were created" elif valid_stale: