From ac9ac50df55f66346baa011a630932e8d0cc5ae8 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:44:47 -0500 Subject: [PATCH] back off restarts when a recording stream goes stale (#24420) The watchdog loop runs every second and the record staleness check restarted ffmpeg on every pass, so once a camera's segments went stale it got one restart per second and never had time to finish a 10 second segment. The restart is now gated on `can_restart` like the detect paths and grants 90 seconds of grace afterward. Backport of https://github.com/blakeblackshear/frigate/pull/24072, already in 0.19. --- frigate/video/ffmpeg.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/frigate/video/ffmpeg.py b/frigate/video/ffmpeg.py index 24b7805333..d2f60c0aea 100644 --- a/frigate/video/ffmpeg.py +++ b/frigate/video/ffmpeg.py @@ -34,6 +34,8 @@ from frigate.util.process import FrigateProcess logger = logging.getLogger(__name__) +RECORD_GRACE_SECONDS = 90 + def capture_frames( ffmpeg_process: sp.Popen[Any], @@ -164,6 +166,7 @@ class CameraWatchdog(threading.Thread): self.latest_invalid_segment_time: float = 0 self.latest_cache_segment_time: float = 0 self.record_enable_time: datetime | None = None + self.record_grace_until: datetime | None = None # `valid` segments are published with the segment's start time, so the # gap between consecutive publishes can reach 2 * segment_time. Pad the @@ -280,6 +283,7 @@ class CameraWatchdog(threading.Thread): self.latest_valid_segment_time = 0 self.latest_invalid_segment_time = 0 self.latest_cache_segment_time = 0 + self.record_grace_until = None self.record_enable_time = datetime.now().astimezone(UTC) last_restart_time = datetime.now().timestamp() continue @@ -294,6 +298,7 @@ class CameraWatchdog(threading.Thread): self.latest_valid_segment_time = 0 self.latest_invalid_segment_time = 0 self.latest_cache_segment_time = 0 + self.record_grace_until = None self.record_enable_time = datetime.now().astimezone(UTC) else: self.logger.debug(f"Disabling camera {self.config.name}") @@ -318,6 +323,7 @@ class CameraWatchdog(threading.Thread): self.latest_valid_segment_time = 0 self.latest_invalid_segment_time = 0 self.latest_cache_segment_time = 0 + self.record_grace_until = None self.record_enable_time = datetime.now().astimezone(UTC) last_restart_time = datetime.now().timestamp() self.was_record_enabled_in_config = record_enabled_in_config @@ -404,11 +410,16 @@ class CameraWatchdog(threading.Thread): if self.config.record.enabled and "record" in p["roles"]: now_utc = datetime.now().astimezone(UTC) - # Check if we're within the grace period after enabling recording - # Grace period: 90 seconds allows time for ffmpeg to start and create first segment - in_grace_period = self.record_enable_time is not None and ( - now_utc - self.record_enable_time - ) < timedelta(seconds=90) + # ffmpeg needs time to create a first segment after + # recording is enabled and after a restart + in_grace_period = ( + self.record_enable_time is not None + and (now_utc - self.record_enable_time) + < timedelta(seconds=RECORD_GRACE_SECONDS) + ) or ( + self.record_grace_until is not None + and now_utc < self.record_grace_until + ) latest_cache_dt = ( datetime.fromtimestamp(self.latest_cache_segment_time, tz=UTC) @@ -445,8 +456,9 @@ class CameraWatchdog(threading.Thread): <= self.latest_invalid_segment_time ) invalid_stale = invalid_stale_condition + stale = cache_stale or valid_stale or invalid_stale - if cache_stale or valid_stale or invalid_stale: + if stale and can_restart: if cache_stale: reason = "No new recording segments were created" elif valid_stale: @@ -471,8 +483,13 @@ class CameraWatchdog(threading.Thread): f"{self.config.name}/status/{role.value}", "offline" ) + self.record_grace_until = now_utc + timedelta( + seconds=RECORD_GRACE_SECONDS + ) + last_restart_time = now + continue - else: + elif not stale: self._send_record_status("online", now) p["latest_segment_time"] = self.latest_cache_segment_time