2026-03-25 08:44:12 -06:00
"""Manages ffmpeg processes for camera frame capture."""
2020-11-04 06:31:25 -06:00
import logging
import queue
2021-10-31 11:12:44 -05:00
import subprocess as sp
2020-11-04 06:31:25 -06:00
import threading
import time
2026-08-25 12:51:58 -05:00
from collections import defaultdict , deque
2026-07-06 09:28:02 -08:00
from datetime import UTC , datetime , timedelta
2025-04-23 17:06:06 -06:00
from multiprocessing import Queue , Value
from multiprocessing.synchronize import Event as MpEvent
from typing import Any
2020-11-04 06:31:25 -06:00
2026-03-25 08:44:12 -06:00
from frigate.camera import CameraMetrics
2024-02-14 17:24:36 -07:00
from frigate.comms.inter_process import InterProcessRequestor
2025-09-28 11:52:14 -05:00
from frigate.comms.recordings_updater import (
RecordingsDataSubscriber ,
RecordingsDataTypeEnum ,
)
2026-03-25 08:44:12 -06:00
from frigate.config import CameraConfig , LoggerConfig
2025-05-22 12:16:51 -06:00
from frigate.config.camera.updater import (
CameraConfigUpdateEnum ,
CameraConfigUpdateSubscriber ,
)
2026-08-25 12:51:58 -05:00
from frigate.const import (
PROCESS_PRIORITY_HIGH ,
RECORD_STREAM_TYPES ,
ROLE_TO_STREAM_TYPE ,
STREAM_TYPE_MAIN ,
STREAM_TYPE_SUB ,
STREAM_TYPE_TO_ROLE ,
)
2020-12-04 06:59:03 -06:00
from frigate.log import LogPipe
2026-06-11 15:16:41 -05:00
from frigate.util.builtin import EventsPerSecond , get_record_segment_time
2026-03-25 08:44:12 -06:00
from frigate.util.ffmpeg import start_or_restart_ffmpeg , stop_ffmpeg
2023-07-06 08:28:50 -06:00
from frigate.util.image import (
2021-02-17 07:23:32 -06:00
FrameManager ,
SharedMemoryFrameManager ,
)
2025-06-13 11:09:51 -06:00
from frigate.util.process import FrigateProcess
2019-02-25 20:27:02 -06:00
2020-11-03 21:26:39 -06:00
logger = logging . getLogger ( __name__ )
2026-08-25 12:51:58 -05:00
RECORD_GRACE_SECONDS = 90
2021-02-17 07:23:32 -06:00
def capture_frames (
2025-08-22 06:42:36 -06:00
ffmpeg_process : sp . Popen [ Any ],
2024-09-03 10:22:30 -06:00
config : CameraConfig ,
shm_frame_count : int ,
2024-11-17 10:25:49 -07:00
frame_index : int ,
frame_shape : tuple [ int , int ],
2021-02-17 07:23:32 -06:00
frame_manager : FrameManager ,
frame_queue ,
2025-04-23 17:06:06 -06:00
fps : Value ,
skipped_fps : Value ,
current_frame : Value ,
stop_event : MpEvent ,
2025-08-22 06:42:36 -06:00
) -> None :
2020-11-03 08:15:58 -06:00
frame_size = frame_shape [ 0 ] * frame_shape [ 1 ]
2020-10-25 10:05:21 -05:00
frame_rate = EventsPerSecond ()
2020-10-26 07:59:22 -05:00
frame_rate . start ()
2020-10-25 10:05:21 -05:00
skipped_eps = EventsPerSecond ()
skipped_eps . start ()
2025-12-15 15:02:03 -06:00
2026-08-15 15:24:00 -05:00
while not stop_event . is_set ():
# CameraWatchdog applies enabled updates onto this same CameraConfig
# before it stops ffmpeg. Do not subscribe here: it would be rebuilt per
# ffmpeg restart and strand a pipe in the idle main process config PUB.
if not config . enabled :
logger . debug ( f "Stopping capture thread for disabled { config . name } " )
break
2025-03-03 09:30:52 -06:00
2026-08-15 15:24:00 -05:00
fps . value = frame_rate . eps ()
skipped_fps . value = skipped_eps . eps ()
current_frame . value = datetime . now () . timestamp ()
frame_name = f " { config . name } _frame { frame_index } "
frame_buffer = frame_manager . write ( frame_name )
try :
frame_buffer [:] = ffmpeg_process . stdout . read ( frame_size )
except Exception :
# shutdown has been initiated
if stop_event . is_set ():
2023-02-04 08:58:45 -06:00
break
2024-09-03 10:22:30 -06:00
2026-08-15 15:24:00 -05:00
logger . error ( f " { config . name } : Unable to read frames from ffmpeg process." )
2020-08-22 07:05:20 -05:00
2026-08-15 15:24:00 -05:00
if ffmpeg_process . poll () is not None :
2022-02-06 08:46:41 -06:00
logger . error (
2026-08-15 15:24:00 -05:00
f " { config . name } : ffmpeg process is not running. exiting capture thread..."
2021-02-17 07:23:32 -06:00
)
2026-08-15 15:24:00 -05:00
break
2024-09-03 10:22:30 -06:00
2026-08-15 15:24:00 -05:00
continue
2020-08-22 07:05:20 -05:00
2026-08-15 15:24:00 -05:00
frame_rate . update ()
2020-08-22 07:05:20 -05:00
2026-08-15 15:24:00 -05:00
# don't lock the queue to check, just try since it should rarely be full
try :
# add to the queue
frame_queue . put (( frame_name , current_frame . value ), False )
frame_manager . close ( frame_name )
except queue . Full :
# if the queue is full, skip this frame
skipped_eps . update ()
2024-09-03 10:22:30 -06:00
2026-08-15 15:24:00 -05:00
frame_index = 0 if frame_index == shm_frame_count - 1 else frame_index + 1
2024-11-16 16:00:19 -07:00
2021-02-17 07:23:32 -06:00
2020-10-25 10:05:21 -05:00
class CameraWatchdog ( threading . Thread ):
2021-02-17 07:23:32 -06:00
def __init__ (
2022-12-08 20:03:54 -07:00
self ,
config : CameraConfig ,
2024-09-03 10:22:30 -06:00
shm_frame_count : int ,
2025-04-23 17:06:06 -06:00
frame_queue : Queue ,
2022-12-08 20:03:54 -07:00
camera_fps ,
2023-06-28 04:53:28 -06:00
skipped_fps ,
2022-12-08 20:03:54 -07:00
ffmpeg_pid ,
2025-12-15 15:02:03 -06:00
stalls ,
reconnects ,
detection_frame ,
2022-12-08 20:03:54 -07:00
stop_event ,
2021-02-17 07:23:32 -06:00
):
2020-10-25 10:05:21 -05:00
threading . Thread . __init__ ( self )
2025-05-22 12:16:51 -06:00
self . logger = logging . getLogger ( f "watchdog. { config . name } " )
2020-10-25 10:05:21 -05:00
self . config = config
2024-09-03 10:22:30 -06:00
self . shm_frame_count = shm_frame_count
2020-10-25 10:05:21 -05:00
self . capture_thread = None
2020-11-29 15:55:53 -06:00
self . ffmpeg_detect_process = None
2025-05-22 12:16:51 -06:00
self . logpipe = LogPipe ( f "ffmpeg. { self . config . name } .detect" )
2025-05-13 16:27:20 +02:00
self . ffmpeg_other_processes : list [ dict [ str , Any ]] = []
2020-10-25 10:05:21 -05:00
self . camera_fps = camera_fps
2023-06-28 04:53:28 -06:00
self . skipped_fps = skipped_fps
2020-10-26 07:59:05 -05:00
self . ffmpeg_pid = ffmpeg_pid
2020-10-25 10:05:21 -05:00
self . frame_queue = frame_queue
2020-11-03 08:15:58 -06:00
self . frame_shape = self . config . frame_shape_yuv
self . frame_size = self . frame_shape [ 0 ] * self . frame_shape [ 1 ]
2024-09-10 07:39:37 -06:00
self . fps_overflow_count = 0
2024-11-17 10:25:49 -07:00
self . frame_index = 0
2020-11-29 16:19:59 -06:00
self . stop_event = stop_event
2023-06-30 08:14:39 -04:00
self . sleeptime = self . config . ffmpeg . retry_interval
2025-12-15 15:02:03 -06:00
self . reconnect_timestamps = deque ()
self . stalls = stalls
self . reconnects = reconnects
self . detection_frame = detection_frame
2020-10-25 10:05:21 -05:00
2025-05-22 12:16:51 -06:00
self . config_subscriber = CameraConfigUpdateSubscriber (
2025-11-06 07:21:07 -07:00
None ,
{ config . name : config },
2026-03-06 14:45:39 -06:00
[
CameraConfigUpdateEnum . enabled ,
CameraConfigUpdateEnum . ffmpeg ,
CameraConfigUpdateEnum . record ,
],
2025-05-22 12:16:51 -06:00
)
2025-08-22 06:42:36 -06:00
self . requestor = InterProcessRequestor ()
2025-03-03 09:30:52 -06:00
self . was_enabled = self . config . enabled
2026-05-07 08:53:07 -05:00
self . was_record_enabled_in_config = self . config . record . enabled_in_config
2026-08-16 13:59:03 -05:00
self . was_record_sub_enabled = self . config . record . sub . enabled
2020-11-29 15:55:53 -06:00
2025-09-28 11:52:14 -05:00
self . segment_subscriber = RecordingsDataSubscriber ( RecordingsDataTypeEnum . all )
2026-08-25 12:51:58 -05:00
self . latest_valid_segment_time : dict [ str , float ] = defaultdict ( float )
self . latest_invalid_segment_time : dict [ str , float ] = defaultdict ( float )
self . latest_cache_segment_time : dict [ str , float ] = defaultdict ( float )
2026-02-12 14:42:08 -06:00
self . record_enable_time : datetime | None = None
2026-08-25 12:51:58 -05:00
self . stream_grace_until : dict [ str , datetime ] = {}
2025-09-28 11:52:14 -05:00
2026-04-29 17:20:19 -05:00
# `valid` segments are published with the segment's start time, so the
# gap between consecutive publishes can reach 2 * segment_time. Pad the
# staleness threshold so it's never tighter than that worst case.
2026-08-25 12:51:58 -05:00
self . record_stale_threshold : dict [ str , int ] = {
stream_type : max (
120 , 2 * get_record_segment_time ( self . config , stream_type ) + 30
)
for stream_type in RECORD_STREAM_TYPES
}
# the sub stream usually shares its input, and therefore its ffmpeg
# process, with detect, so it isn't in ffmpeg_other_processes and needs
# its own staleness check
self . detect_process_records_sub = False
2026-04-29 17:20:19 -05:00
2025-12-15 15:02:03 -06:00
# Stall tracking (based on last processed frame)
self . _stall_timestamps : deque [ float ] = deque ()
self . _stall_active : bool = False
# Status caching to reduce message volume
self . _last_detect_status : str | None = None
2026-08-25 12:51:58 -05:00
self . _last_record_status : dict [ str , str ] = {}
2025-12-15 15:02:03 -06:00
self . _last_status_update_time : float = 0.0
def _send_detect_status ( self , status : str , now : float ) -> None :
"""Send detect status only if changed or retry_interval has elapsed."""
if (
status != self . _last_detect_status
or ( now - self . _last_status_update_time ) >= self . sleeptime
):
self . requestor . send_data ( f " { self . config . name } /status/detect" , status )
self . _last_detect_status = status
self . _last_status_update_time = now
2026-08-25 12:51:58 -05:00
def _send_record_status ( self , stream_type : str , status : str , now : float ) -> None :
"""Send a record stream's status only if changed or retry_interval has elapsed."""
2025-12-15 15:02:03 -06:00
if (
2026-08-25 12:51:58 -05:00
status != self . _last_record_status . get ( stream_type )
2025-12-15 15:02:03 -06:00
or ( now - self . _last_status_update_time ) >= self . sleeptime
):
2026-08-25 12:51:58 -05:00
self . requestor . send_data (
f " { self . config . name } /status/ { STREAM_TYPE_TO_ROLE [ stream_type ] } " , status
)
self . _last_record_status [ stream_type ] = status
2025-12-15 15:02:03 -06:00
self . _last_status_update_time = now
2026-08-25 12:51:58 -05:00
def _reset_segment_times ( self ) -> None :
self . latest_valid_segment_time . clear ()
self . latest_invalid_segment_time . clear ()
self . latest_cache_segment_time . clear ()
self . stream_grace_until . clear ()
def _grant_restart_grace ( self , stream_types : list [ str ], now_utc : datetime ) -> None :
for stream_type in stream_types :
self . stream_grace_until [ stream_type ] = now_utc + timedelta (
seconds = RECORD_GRACE_SECONDS
)
def _stream_staleness ( self , stream_type : str , now_utc : datetime ) -> str | None :
"""Return why the stream's segments are stale, or None if they're healthy."""
# ffmpeg needs time to create a first segment after recording is
# enabled and after a restart, per stream
in_grace_period = (
self . record_enable_time is not None
and ( now_utc - self . record_enable_time )
< timedelta ( seconds = RECORD_GRACE_SECONDS )
) or now_utc < self . stream_grace_until . get ( stream_type , now_utc )
if in_grace_period :
return None
latest_cache = self . latest_cache_segment_time [ stream_type ]
latest_valid = self . latest_valid_segment_time [ stream_type ]
latest_invalid = self . latest_invalid_segment_time [ stream_type ]
def as_dt ( timestamp : float ) -> datetime :
if timestamp > 0 :
return datetime . fromtimestamp ( timestamp , tz = UTC )
return now_utc - timedelta ( seconds = 1 )
stale_window = timedelta ( seconds = self . record_stale_threshold [ stream_type ])
if now_utc > ( as_dt ( latest_cache ) + stale_window ):
return "No new recording segments were created"
if now_utc > ( as_dt ( latest_valid ) + stale_window ):
return "No new valid recording segments were created"
if (
latest_invalid > 0
and now_utc > ( as_dt ( latest_invalid ) + stale_window )
and latest_valid <= latest_invalid
):
return "No valid segments created since last invalid segment"
return None
def _recorded_streams ( self , roles : list [ Any ]) -> list [ str ]:
"""Record stream types the given roles cover that are currently recording."""
return [
stream_type
for role , stream_type in ROLE_TO_STREAM_TYPE . items ()
if role in roles and self . config . record . stream_enabled ( stream_type )
]
2026-03-06 14:45:39 -06:00
def _check_config_updates ( self ) -> dict [ str , list [ str ]]:
"""Check for config updates and return the update dict."""
return self . config_subscriber . check_for_updates ()
2025-03-03 09:30:52 -06:00
def _update_enabled_state ( self ) -> bool :
"""Fetch the latest config and update enabled state."""
2026-03-06 14:45:39 -06:00
self . _check_config_updates ()
2025-03-05 07:30:23 -06:00
return self . config . enabled
2025-03-03 09:30:52 -06:00
2025-07-11 13:19:43 -06:00
def reset_capture_thread (
self , terminate : bool = True , drain_output : bool = True
) -> None :
if terminate :
self . ffmpeg_detect_process . terminate ()
try :
self . logger . info ( "Waiting for ffmpeg to exit gracefully..." )
if drain_output :
self . ffmpeg_detect_process . communicate ( timeout = 30 )
else :
self . ffmpeg_detect_process . wait ( timeout = 30 )
except sp . TimeoutExpired :
self . logger . info ( "FFmpeg did not exit. Force killing..." )
self . ffmpeg_detect_process . kill ()
if drain_output :
self . ffmpeg_detect_process . communicate ()
else :
self . ffmpeg_detect_process . wait ()
2025-12-15 15:02:03 -06:00
# Update reconnects
now = datetime . now () . timestamp ()
self . reconnect_timestamps . append ( now )
while self . reconnect_timestamps and self . reconnect_timestamps [ 0 ] < now - 3600 :
self . reconnect_timestamps . popleft ()
if self . reconnects :
self . reconnects . value = len ( self . reconnect_timestamps )
2025-11-30 06:54:42 -06:00
# Wait for old capture thread to fully exit before starting a new one
if self . capture_thread is not None and self . capture_thread . is_alive ():
self . logger . info ( "Waiting for capture thread to exit..." )
self . capture_thread . join ( timeout = 5 )
if self . capture_thread . is_alive ():
self . logger . warning (
f "Capture thread for { self . config . name } did not exit in time"
)
2025-07-11 13:19:43 -06:00
self . logger . error (
"The following ffmpeg logs include the last 100 lines prior to exit."
)
self . logpipe . dump ()
self . logger . info ( "Restarting ffmpeg..." )
self . start_ffmpeg_detect ()
2026-08-25 12:51:58 -05:00
# this process produces the sub stream's segments too, so it gets the
# same startup grace however the reset was triggered
if self . detect_process_records_sub :
self . _grant_restart_grace ([ STREAM_TYPE_SUB ], datetime . now () . astimezone ( UTC ))
2025-07-11 13:19:43 -06:00
def run ( self ) -> None :
2025-03-03 09:30:52 -06:00
if self . _update_enabled_state ():
self . start_all_ffmpeg ()
2026-02-12 14:42:08 -06:00
# If recording is enabled at startup, set the grace period timer
if self . config . record . enabled :
2026-07-06 09:28:02 -08:00
self . record_enable_time = datetime . now () . astimezone ( UTC )
2021-02-17 07:23:32 -06:00
2023-06-30 08:14:39 -04:00
time . sleep ( self . sleeptime )
2025-12-15 15:02:03 -06:00
last_restart_time = datetime . now () . timestamp ()
# 1 second watchdog loop
while not self . stop_event . wait ( 1 ):
2026-03-06 14:45:39 -06:00
updates = self . _check_config_updates ()
# Handle ffmpeg config changes by restarting all ffmpeg processes
if "ffmpeg" in updates and self . config . enabled :
self . logger . debug (
"FFmpeg config updated for %s , restarting ffmpeg processes" ,
self . config . name ,
)
self . stop_all_ffmpeg ()
self . start_all_ffmpeg ()
2026-08-25 12:51:58 -05:00
self . _reset_segment_times ()
2026-07-06 09:28:02 -08:00
self . record_enable_time = datetime . now () . astimezone ( UTC )
2026-03-06 14:45:39 -06:00
last_restart_time = datetime . now () . timestamp ()
continue
enabled = self . config . enabled
2025-03-03 09:30:52 -06:00
if enabled != self . was_enabled :
if enabled :
2025-05-22 12:16:51 -06:00
self . logger . debug ( f "Enabling camera { self . config . name } " )
2025-03-03 09:30:52 -06:00
self . start_all_ffmpeg ()
2025-09-28 11:52:14 -05:00
2026-02-12 14:42:08 -06:00
# reset all timestamps and record the enable time for grace period
2026-08-25 12:51:58 -05:00
self . _reset_segment_times ()
2026-07-06 09:28:02 -08:00
self . record_enable_time = datetime . now () . astimezone ( UTC )
2025-03-03 09:30:52 -06:00
else :
2025-05-22 12:16:51 -06:00
self . logger . debug ( f "Disabling camera { self . config . name } " )
2025-03-03 09:30:52 -06:00
self . stop_all_ffmpeg ()
2026-02-12 14:42:08 -06:00
self . record_enable_time = None
2025-08-22 06:42:36 -06:00
# update camera status
2025-12-15 15:02:03 -06:00
now = datetime . now () . timestamp ()
self . _send_detect_status ( "disabled" , now )
2026-08-25 12:51:58 -05:00
self . _send_record_status ( STREAM_TYPE_MAIN , "disabled" , now )
# cameras without a sub stream never get a record_sub topic
if self . config . record . sub . enabled :
self . _send_record_status ( STREAM_TYPE_SUB , "disabled" , now )
2025-03-03 09:30:52 -06:00
self . was_enabled = enabled
continue
2026-05-07 08:53:07 -05:00
record_enabled_in_config = self . config . record . enabled_in_config
if record_enabled_in_config != self . was_record_enabled_in_config :
if record_enabled_in_config and enabled :
self . logger . debug (
f "Record enabled in config for { self . config . name } , restarting ffmpeg"
)
self . stop_all_ffmpeg ()
self . start_all_ffmpeg ()
2026-08-25 12:51:58 -05:00
self . _reset_segment_times ()
2026-07-06 09:28:02 -08:00
self . record_enable_time = datetime . now () . astimezone ( UTC )
2026-05-07 08:53:07 -05:00
last_restart_time = datetime . now () . timestamp ()
self . was_record_enabled_in_config = record_enabled_in_config
continue
2026-08-16 13:59:03 -05:00
record_sub_enabled = self . config . record . sub . enabled
if record_sub_enabled != self . was_record_sub_enabled :
# adding and removing the record_sub output both require a
# restart, unlike the main record toggle
if record_enabled_in_config and enabled :
self . logger . debug (
f "Sub stream recording toggled in config for { self . config . name } , restarting ffmpeg"
)
self . stop_all_ffmpeg ()
self . start_all_ffmpeg ()
2026-08-25 12:51:58 -05:00
self . _reset_segment_times ()
2026-08-16 13:59:03 -05:00
self . record_enable_time = datetime . now () . astimezone ( UTC )
last_restart_time = datetime . now () . timestamp ()
self . was_record_sub_enabled = record_sub_enabled
continue
2025-03-03 09:30:52 -06:00
if not enabled :
continue
2025-09-28 11:52:14 -05:00
while True :
update = self . segment_subscriber . check_for_update ( timeout = 0 )
if update == ( None , None ):
break
raw_topic , payload = update
if raw_topic and payload :
topic = str ( raw_topic )
2026-08-25 12:51:58 -05:00
camera , stream_type , segment_time , _ = payload
2025-09-28 11:52:14 -05:00
if camera != self . config . name :
continue
2026-04-28 09:54:09 -05:00
if topic . endswith ( RecordingsDataTypeEnum . invalid . value ):
2025-09-28 11:52:14 -05:00
self . logger . warning (
2026-08-25 12:51:58 -05:00
f "Invalid recording segment detected for { camera } ( { stream_type } ) at { segment_time } "
2025-09-28 11:52:14 -05:00
)
2026-08-25 12:51:58 -05:00
self . latest_invalid_segment_time [ stream_type ] = segment_time
2026-04-28 09:54:09 -05:00
elif topic . endswith ( RecordingsDataTypeEnum . valid . value ):
self . logger . debug (
2026-08-25 12:51:58 -05:00
f "Latest valid recording segment time on { camera } ( { stream_type } ): { segment_time } "
2026-04-28 09:54:09 -05:00
)
2026-08-25 12:51:58 -05:00
self . latest_valid_segment_time [ stream_type ] = segment_time
2025-09-28 11:52:14 -05:00
elif topic . endswith ( RecordingsDataTypeEnum . latest . value ):
2026-08-25 12:51:58 -05:00
self . latest_cache_segment_time [ stream_type ] = (
segment_time if segment_time is not None else 0
)
2025-09-28 11:52:14 -05:00
now = datetime . now () . timestamp ()
2020-10-25 10:05:21 -05:00
2025-12-15 15:02:03 -06:00
# Check if enough time has passed to allow ffmpeg restart (backoff pacing)
time_since_last_restart = now - last_restart_time
can_restart = time_since_last_restart >= self . sleeptime
2020-10-25 10:05:21 -05:00
if not self . capture_thread . is_alive ():
2025-12-15 15:02:03 -06:00
self . _send_detect_status ( "offline" , now )
2022-11-28 20:47:20 -07:00
self . camera_fps . value = 0
2021-08-14 14:04:00 -05:00
self . logger . error (
2025-05-22 12:16:51 -06:00
f "Ffmpeg process crashed unexpectedly for { self . config . name } ."
2021-08-14 14:04:00 -05:00
)
2025-12-15 15:02:03 -06:00
if can_restart :
self . reset_capture_thread ( terminate = False )
last_restart_time = now
2023-01-29 16:20:42 -07:00
elif self . camera_fps . value >= ( self . config . detect . fps + 10 ):
2024-09-10 07:39:37 -06:00
self . fps_overflow_count += 1
if self . fps_overflow_count == 3 :
2025-12-15 15:02:03 -06:00
self . _send_detect_status ( "offline" , now )
2024-09-10 07:39:37 -06:00
self . fps_overflow_count = 0
self . camera_fps . value = 0
self . logger . info (
2025-05-22 12:16:51 -06:00
f " { self . config . name } exceeded fps limit. Exiting ffmpeg..."
2024-09-10 07:39:37 -06:00
)
2025-12-15 15:02:03 -06:00
if can_restart :
self . reset_capture_thread ( drain_output = False )
last_restart_time = now
2025-07-11 13:19:43 -06:00
elif now - self . capture_thread . current_frame . value > 20 :
2025-12-15 15:02:03 -06:00
self . _send_detect_status ( "offline" , now )
2025-07-11 13:19:43 -06:00
self . camera_fps . value = 0
self . logger . info (
2025-05-22 12:16:51 -06:00
f "No frames received from { self . config . name } in 20 seconds. Exiting ffmpeg..."
2025-07-11 13:19:43 -06:00
)
2025-12-15 15:02:03 -06:00
if can_restart :
self . reset_capture_thread ()
last_restart_time = now
2024-09-10 07:39:37 -06:00
else :
# process is running normally
2025-12-15 15:02:03 -06:00
self . _send_detect_status ( "online" , now )
2024-09-10 07:39:37 -06:00
self . fps_overflow_count = 0
2021-02-17 07:23:32 -06:00
2020-11-29 15:55:53 -06:00
for p in self . ffmpeg_other_processes :
2021-02-17 07:23:32 -06:00
poll = p [ "process" ] . poll ()
2022-12-08 20:03:54 -07:00
2026-08-25 12:51:58 -05:00
recorded_streams = self . _recorded_streams ( p [ "roles" ])
if recorded_streams :
2026-07-06 09:28:02 -08:00
now_utc = datetime . now () . astimezone ( UTC )
2025-09-28 11:52:14 -05:00
2026-08-25 12:51:58 -05:00
# ensure segments are still being created and that they have
# valid video data. each stream is tracked separately so a
# healthy one can't mask a stalled one.
stale_stream = None
stale_reason = None
for stream_type in recorded_streams :
stale_reason = self . _stream_staleness ( stream_type , now_utc )
2026-02-12 14:42:08 -06:00
2026-08-25 12:51:58 -05:00
if stale_reason is not None :
stale_stream = stream_type
break
2025-09-28 11:52:14 -05:00
2026-08-25 12:51:58 -05:00
if stale_stream is not None and can_restart :
2022-12-08 20:03:54 -07:00
self . logger . error (
2026-08-25 12:51:58 -05:00
f " { stale_reason } for { self . config . name } ( { stale_stream } ) in the last { self . record_stale_threshold [ stale_stream ] } s. Restarting the ffmpeg record process..."
2022-12-08 20:03:54 -07:00
)
p [ "process" ] = start_or_restart_ffmpeg (
p [ "cmd" ],
self . logger ,
p [ "logpipe" ],
ffmpeg_process = p [ "process" ],
)
2025-08-22 06:42:36 -06:00
for role in p [ "roles" ]:
self . requestor . send_data (
2026-03-10 13:26:45 -06:00
f " { self . config . name } /status/ { role . value } " , "offline"
2025-08-22 06:42:36 -06:00
)
2026-08-25 12:51:58 -05:00
self . _grant_restart_grace ( recorded_streams , now_utc )
last_restart_time = now
2022-12-08 20:03:54 -07:00
continue
2026-08-25 12:51:58 -05:00
elif stale_stream is None :
for stream_type in recorded_streams :
self . _send_record_status ( stream_type , "online" , now )
p [ "latest_segment_time" ] = max (
self . latest_cache_segment_time [ stream_type ]
for stream_type in recorded_streams
)
2022-12-08 20:03:54 -07:00
2021-06-25 10:37:21 -06:00
if poll is None :
2020-11-29 15:55:53 -06:00
continue
2022-12-08 20:03:54 -07:00
2025-08-22 06:42:36 -06:00
for role in p [ "roles" ]:
self . requestor . send_data (
2026-03-10 13:26:45 -06:00
f " { self . config . name } /status/ { role . value } " , "offline"
2025-08-22 06:42:36 -06:00
)
2021-02-17 07:23:32 -06:00
p [ "logpipe" ] . dump ()
p [ "process" ] = start_or_restart_ffmpeg (
p [ "cmd" ], self . logger , p [ "logpipe" ], ffmpeg_process = p [ "process" ]
)
2026-08-25 12:51:58 -05:00
if (
self . detect_process_records_sub
and self . config . record . stream_enabled ( STREAM_TYPE_SUB )
and self . capture_thread is not None
and self . capture_thread . is_alive ()
):
now_utc = datetime . now () . astimezone ( UTC )
stale_reason = self . _stream_staleness ( STREAM_TYPE_SUB , now_utc )
if stale_reason is None :
self . _send_record_status ( STREAM_TYPE_SUB , "online" , now )
elif can_restart :
self . logger . error (
f " { stale_reason } for { self . config . name } (sub, shared with detect) in the last { self . record_stale_threshold [ STREAM_TYPE_SUB ] } s. Restarting ffmpeg..."
)
self . _send_record_status ( STREAM_TYPE_SUB , "offline" , now )
self . reset_capture_thread ()
last_restart_time = now
2026-04-06 08:53:23 -05:00
# Prune expired reconnect timestamps
2025-12-15 15:02:03 -06:00
now = datetime . now () . timestamp ()
2026-04-06 08:53:23 -05:00
while (
self . reconnect_timestamps and self . reconnect_timestamps [ 0 ] < now - 3600
):
self . reconnect_timestamps . popleft ()
if self . reconnects :
self . reconnects . value = len ( self . reconnect_timestamps )
# Update stall metrics based on last processed frame timestamp
2025-12-15 15:02:03 -06:00
processed_ts = (
float ( self . detection_frame . value ) if self . detection_frame else 0.0
)
if processed_ts > 0 :
delta = now - processed_ts
observed_fps = (
self . camera_fps . value
if self . camera_fps . value > 0
else self . config . detect . fps
)
interval = 1.0 / max ( observed_fps , 0.1 )
stall_threshold = max ( 2.0 * interval , 2.0 )
if delta > stall_threshold :
if not self . _stall_active :
self . _stall_timestamps . append ( now )
self . _stall_active = True
else :
self . _stall_active = False
while self . _stall_timestamps and self . _stall_timestamps [ 0 ] < now - 3600 :
self . _stall_timestamps . popleft ()
if self . stalls :
self . stalls . value = len ( self . _stall_timestamps )
2025-03-03 09:30:52 -06:00
self . stop_all_ffmpeg ()
2021-05-21 11:39:14 -04:00
self . logpipe . close ()
2025-03-03 09:30:52 -06:00
self . config_subscriber . stop ()
2025-09-28 11:52:14 -05:00
self . segment_subscriber . stop ()
2021-02-17 07:23:32 -06:00
2020-11-29 15:55:53 -06:00
def start_ffmpeg_detect ( self ):
2026-08-25 12:51:58 -05:00
detect_cmd = [ c for c in self . config . ffmpeg_cmds if "detect" in c [ "roles" ]][ 0 ]
ffmpeg_cmd = detect_cmd [ "cmd" ]
self . detect_process_records_sub = "record_sub" in detect_cmd [ "roles" ]
2021-02-17 07:23:32 -06:00
self . ffmpeg_detect_process = start_or_restart_ffmpeg (
ffmpeg_cmd , self . logger , self . logpipe , self . frame_size
)
2020-11-29 15:55:53 -06:00
self . ffmpeg_pid . value = self . ffmpeg_detect_process . pid
2025-06-12 12:12:34 -06:00
self . capture_thread = CameraCaptureRunner (
2024-09-03 10:22:30 -06:00
self . config ,
self . shm_frame_count ,
2024-11-17 10:25:49 -07:00
self . frame_index ,
2021-02-17 07:23:32 -06:00
self . ffmpeg_detect_process ,
self . frame_shape ,
self . frame_queue ,
self . camera_fps ,
2023-06-28 04:53:28 -06:00
self . skipped_fps ,
2023-02-04 08:58:45 -06:00
self . stop_event ,
2021-02-17 07:23:32 -06:00
)
2020-11-01 10:55:11 -06:00
self . capture_thread . start ()
2020-10-25 10:05:21 -05:00
2025-03-03 09:30:52 -06:00
def start_all_ffmpeg ( self ):
"""Start all ffmpeg processes (detection and others)."""
2025-05-22 12:16:51 -06:00
logger . debug ( f "Starting all ffmpeg processes for { self . config . name } " )
2025-03-03 09:30:52 -06:00
self . start_ffmpeg_detect ()
for c in self . config . ffmpeg_cmds :
if "detect" in c [ "roles" ]:
continue
logpipe = LogPipe (
2025-05-22 12:16:51 -06:00
f "ffmpeg. { self . config . name } . { '_' . join ( sorted ( c [ 'roles' ])) } "
2025-03-03 09:30:52 -06:00
)
self . ffmpeg_other_processes . append (
{
"cmd" : c [ "cmd" ],
"roles" : c [ "roles" ],
"logpipe" : logpipe ,
"process" : start_or_restart_ffmpeg ( c [ "cmd" ], self . logger , logpipe ),
}
)
def stop_all_ffmpeg ( self ):
"""Stop all ffmpeg processes (detection and others)."""
2025-05-22 12:16:51 -06:00
logger . debug ( f "Stopping all ffmpeg processes for { self . config . name } " )
2025-03-03 09:30:52 -06:00
if self . capture_thread is not None and self . capture_thread . is_alive ():
self . capture_thread . join ( timeout = 5 )
if self . capture_thread . is_alive ():
self . logger . warning (
2025-05-22 12:16:51 -06:00
f "Capture thread for { self . config . name } did not stop gracefully."
2025-03-03 09:30:52 -06:00
)
if self . ffmpeg_detect_process is not None :
stop_ffmpeg ( self . ffmpeg_detect_process , self . logger )
self . ffmpeg_detect_process = None
for p in self . ffmpeg_other_processes [:]:
if p [ "process" ] is not None :
stop_ffmpeg ( p [ "process" ], self . logger )
p [ "logpipe" ] . close ()
self . ffmpeg_other_processes . clear ()
2021-02-17 07:23:32 -06:00
2025-06-12 12:12:34 -06:00
class CameraCaptureRunner ( threading . Thread ):
2023-02-04 08:58:45 -06:00
def __init__ (
2023-06-28 04:53:28 -06:00
self ,
2024-09-03 10:22:30 -06:00
config : CameraConfig ,
shm_frame_count : int ,
2024-11-17 10:25:49 -07:00
frame_index : int ,
2023-06-28 04:53:28 -06:00
ffmpeg_process ,
2024-11-17 10:25:49 -07:00
frame_shape : tuple [ int , int ],
2025-04-23 17:06:06 -06:00
frame_queue : Queue ,
fps : Value ,
skipped_fps : Value ,
stop_event : MpEvent ,
2023-02-04 08:58:45 -06:00
):
2020-03-14 15:32:51 -05:00
threading . Thread . __init__ ( self )
2024-09-03 10:22:30 -06:00
self . name = f "capture: { config . name } "
self . config = config
self . shm_frame_count = shm_frame_count
2024-11-17 10:25:49 -07:00
self . frame_index = frame_index
2020-03-14 15:32:51 -05:00
self . frame_shape = frame_shape
self . frame_queue = frame_queue
self . fps = fps
2023-02-04 08:58:45 -06:00
self . stop_event = stop_event
2023-06-28 04:53:28 -06:00
self . skipped_fps = skipped_fps
2020-09-21 21:02:00 -05:00
self . frame_manager = SharedMemoryFrameManager ()
2020-03-14 15:32:51 -05:00
self . ffmpeg_process = ffmpeg_process
2025-04-23 17:06:06 -06:00
self . current_frame = Value ( "d" , 0.0 )
2020-04-19 10:07:27 -05:00
self . last_frame = 0
2020-03-14 15:32:51 -05:00
def run ( self ):
2021-02-17 07:23:32 -06:00
capture_frames (
self . ffmpeg_process ,
2024-09-03 10:22:30 -06:00
self . config ,
self . shm_frame_count ,
2024-11-17 10:25:49 -07:00
self . frame_index ,
2021-02-17 07:23:32 -06:00
self . frame_shape ,
self . frame_manager ,
self . frame_queue ,
self . fps ,
self . skipped_fps ,
self . current_frame ,
2023-02-04 08:58:45 -06:00
self . stop_event ,
2021-02-17 07:23:32 -06:00
)
2020-08-02 08:46:36 -05:00
2025-06-13 11:09:51 -06:00
class CameraCapture ( FrigateProcess ):
2025-06-12 12:12:34 -06:00
def __init__ (
2025-06-24 11:41:11 -06:00
self ,
config : CameraConfig ,
shm_frame_count : int ,
camera_metrics : CameraMetrics ,
stop_event : MpEvent ,
2025-11-17 08:12:05 -06:00
log_config : LoggerConfig | None = None ,
2025-06-12 12:12:34 -06:00
) -> None :
2025-07-18 11:23:06 -06:00
super () . __init__ (
stop_event ,
PROCESS_PRIORITY_HIGH ,
name = f "frigate.capture: { config . name } " ,
daemon = True ,
)
2025-06-12 12:12:34 -06:00
self . config = config
self . shm_frame_count = shm_frame_count
self . camera_metrics = camera_metrics
2025-11-17 08:12:05 -06:00
self . log_config = log_config
2021-02-17 07:23:32 -06:00
2025-06-12 12:12:34 -06:00
def run ( self ) -> None :
2025-11-17 08:12:05 -06:00
self . pre_run_setup ( self . log_config )
2025-06-12 12:12:34 -06:00
camera_watchdog = CameraWatchdog (
self . config ,
self . shm_frame_count ,
self . camera_metrics . frame_queue ,
self . camera_metrics . camera_fps ,
self . camera_metrics . skipped_fps ,
self . camera_metrics . ffmpeg_pid ,
2025-12-15 15:02:03 -06:00
self . camera_metrics . stalls_last_hour ,
self . camera_metrics . reconnects_last_hour ,
self . camera_metrics . detection_frame ,
2025-06-12 12:12:34 -06:00
self . stop_event ,
)
camera_watchdog . start ()
camera_watchdog . join ()