Files
frigate/frigate/video/ffmpeg.py
T

750 lines
29 KiB
Python
Raw Normal View History

2026-03-25 08:44:12 -06:00
"""Manages ffmpeg processes for camera frame capture."""
2020-11-04 06:31:25 -06:00
import logging
import queue
2021-10-31 11:12:44 -05:00
import subprocess as sp
2020-11-04 06:31:25 -06:00
import threading
import time
2026-08-25 12:51:58 -05:00
from collections import defaultdict, deque
2026-07-06 09:28:02 -08:00
from datetime import UTC, datetime, timedelta
2025-04-23 17:06:06 -06:00
from multiprocessing import Queue, Value
from multiprocessing.synchronize import Event as MpEvent
from typing import Any
2020-11-04 06:31:25 -06:00
2026-03-25 08:44:12 -06:00
from frigate.camera import CameraMetrics
from frigate.comms.inter_process import InterProcessRequestor
2025-09-28 11:52:14 -05:00
from frigate.comms.recordings_updater import (
RecordingsDataSubscriber,
RecordingsDataTypeEnum,
)
2026-03-25 08:44:12 -06:00
from frigate.config import CameraConfig, LoggerConfig
2025-05-22 12:16:51 -06:00
from frigate.config.camera.updater import (
CameraConfigUpdateEnum,
CameraConfigUpdateSubscriber,
)
2026-08-25 12:51:58 -05:00
from frigate.const import (
PROCESS_PRIORITY_HIGH,
RECORD_STREAM_TYPES,
ROLE_TO_STREAM_TYPE,
STREAM_TYPE_MAIN,
STREAM_TYPE_SUB,
STREAM_TYPE_TO_ROLE,
)
2020-12-04 06:59:03 -06:00
from frigate.log import LogPipe
from frigate.util.builtin import EventsPerSecond, get_record_segment_time
2026-03-25 08:44:12 -06:00
from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg
2023-07-06 08:28:50 -06:00
from frigate.util.image import (
2021-02-17 07:23:32 -06:00
FrameManager,
SharedMemoryFrameManager,
)
2025-06-13 11:09:51 -06:00
from frigate.util.process import FrigateProcess
2019-02-25 20:27:02 -06:00
2020-11-03 21:26:39 -06:00
logger = logging.getLogger(__name__)
2026-08-25 12:51:58 -05:00
RECORD_GRACE_SECONDS = 90
2021-02-17 07:23:32 -06:00
def capture_frames(
2025-08-22 06:42:36 -06:00
ffmpeg_process: sp.Popen[Any],
2024-09-03 10:22:30 -06:00
config: CameraConfig,
shm_frame_count: int,
2024-11-17 10:25:49 -07:00
frame_index: int,
frame_shape: tuple[int, int],
2021-02-17 07:23:32 -06:00
frame_manager: FrameManager,
frame_queue,
2025-04-23 17:06:06 -06:00
fps: Value,
skipped_fps: Value,
current_frame: Value,
stop_event: MpEvent,
2025-08-22 06:42:36 -06:00
) -> None:
2020-11-03 08:15:58 -06:00
frame_size = frame_shape[0] * frame_shape[1]
2020-10-25 10:05:21 -05:00
frame_rate = EventsPerSecond()
2020-10-26 07:59:22 -05:00
frame_rate.start()
2020-10-25 10:05:21 -05:00
skipped_eps = EventsPerSecond()
skipped_eps.start()
while not stop_event.is_set():
# CameraWatchdog applies enabled updates onto this same CameraConfig
# before it stops ffmpeg. Do not subscribe here: it would be rebuilt per
# ffmpeg restart and strand a pipe in the idle main process config PUB.
if not config.enabled:
logger.debug(f"Stopping capture thread for disabled {config.name}")
break
2025-03-03 09:30:52 -06:00
fps.value = frame_rate.eps()
skipped_fps.value = skipped_eps.eps()
current_frame.value = datetime.now().timestamp()
frame_name = f"{config.name}_frame{frame_index}"
frame_buffer = frame_manager.write(frame_name)
try:
frame_buffer[:] = ffmpeg_process.stdout.read(frame_size)
except Exception:
# shutdown has been initiated
if stop_event.is_set():
2023-02-04 08:58:45 -06:00
break
2024-09-03 10:22:30 -06:00
logger.error(f"{config.name}: Unable to read frames from ffmpeg process.")
2020-08-22 07:05:20 -05:00
if ffmpeg_process.poll() is not None:
2022-02-06 08:46:41 -06:00
logger.error(
f"{config.name}: ffmpeg process is not running. exiting capture thread..."
2021-02-17 07:23:32 -06:00
)
break
2024-09-03 10:22:30 -06:00
continue
2020-08-22 07:05:20 -05:00
frame_rate.update()
2020-08-22 07:05:20 -05:00
# don't lock the queue to check, just try since it should rarely be full
try:
# add to the queue
frame_queue.put((frame_name, current_frame.value), False)
frame_manager.close(frame_name)
except queue.Full:
# if the queue is full, skip this frame
skipped_eps.update()
2024-09-03 10:22:30 -06:00
frame_index = 0 if frame_index == shm_frame_count - 1 else frame_index + 1
2024-11-16 16:00:19 -07:00
2021-02-17 07:23:32 -06:00
2020-10-25 10:05:21 -05:00
class CameraWatchdog(threading.Thread):
2021-02-17 07:23:32 -06:00
def __init__(
self,
config: CameraConfig,
2024-09-03 10:22:30 -06:00
shm_frame_count: int,
2025-04-23 17:06:06 -06:00
frame_queue: Queue,
camera_fps,
skipped_fps,
ffmpeg_pid,
stalls,
reconnects,
detection_frame,
stop_event,
2021-02-17 07:23:32 -06:00
):
2020-10-25 10:05:21 -05:00
threading.Thread.__init__(self)
2025-05-22 12:16:51 -06:00
self.logger = logging.getLogger(f"watchdog.{config.name}")
2020-10-25 10:05:21 -05:00
self.config = config
2024-09-03 10:22:30 -06:00
self.shm_frame_count = shm_frame_count
2020-10-25 10:05:21 -05:00
self.capture_thread = None
2020-11-29 15:55:53 -06:00
self.ffmpeg_detect_process = None
2025-05-22 12:16:51 -06:00
self.logpipe = LogPipe(f"ffmpeg.{self.config.name}.detect")
2025-05-13 16:27:20 +02:00
self.ffmpeg_other_processes: list[dict[str, Any]] = []
2020-10-25 10:05:21 -05:00
self.camera_fps = camera_fps
self.skipped_fps = skipped_fps
self.ffmpeg_pid = ffmpeg_pid
2020-10-25 10:05:21 -05:00
self.frame_queue = frame_queue
2020-11-03 08:15:58 -06:00
self.frame_shape = self.config.frame_shape_yuv
self.frame_size = self.frame_shape[0] * self.frame_shape[1]
2024-09-10 07:39:37 -06:00
self.fps_overflow_count = 0
2024-11-17 10:25:49 -07:00
self.frame_index = 0
2020-11-29 16:19:59 -06:00
self.stop_event = stop_event
2023-06-30 08:14:39 -04:00
self.sleeptime = self.config.ffmpeg.retry_interval
self.reconnect_timestamps = deque()
self.stalls = stalls
self.reconnects = reconnects
self.detection_frame = detection_frame
2020-10-25 10:05:21 -05:00
2025-05-22 12:16:51 -06:00
self.config_subscriber = CameraConfigUpdateSubscriber(
2025-11-06 07:21:07 -07:00
None,
{config.name: config},
[
CameraConfigUpdateEnum.enabled,
CameraConfigUpdateEnum.ffmpeg,
CameraConfigUpdateEnum.record,
],
2025-05-22 12:16:51 -06:00
)
2025-08-22 06:42:36 -06:00
self.requestor = InterProcessRequestor()
2025-03-03 09:30:52 -06:00
self.was_enabled = self.config.enabled
2026-05-07 08:53:07 -05:00
self.was_record_enabled_in_config = self.config.record.enabled_in_config
self.was_record_sub_enabled = self.config.record.sub.enabled
2020-11-29 15:55:53 -06:00
2025-09-28 11:52:14 -05:00
self.segment_subscriber = RecordingsDataSubscriber(RecordingsDataTypeEnum.all)
2026-08-25 12:51:58 -05:00
self.latest_valid_segment_time: dict[str, float] = defaultdict(float)
self.latest_invalid_segment_time: dict[str, float] = defaultdict(float)
self.latest_cache_segment_time: dict[str, float] = defaultdict(float)
2026-02-12 14:42:08 -06:00
self.record_enable_time: datetime | None = None
2026-08-25 12:51:58 -05:00
self.stream_grace_until: dict[str, datetime] = {}
2025-09-28 11:52:14 -05:00
2026-04-29 17:20:19 -05:00
# `valid` segments are published with the segment's start time, so the
# gap between consecutive publishes can reach 2 * segment_time. Pad the
# staleness threshold so it's never tighter than that worst case.
2026-08-25 12:51:58 -05:00
self.record_stale_threshold: dict[str, int] = {
stream_type: max(
120, 2 * get_record_segment_time(self.config, stream_type) + 30
)
for stream_type in RECORD_STREAM_TYPES
}
# the sub stream usually shares its input, and therefore its ffmpeg
# process, with detect, so it isn't in ffmpeg_other_processes and needs
# its own staleness check
self.detect_process_records_sub = False
2026-04-29 17:20:19 -05:00
# Stall tracking (based on last processed frame)
self._stall_timestamps: deque[float] = deque()
self._stall_active: bool = False
# Status caching to reduce message volume
self._last_detect_status: str | None = None
2026-08-25 12:51:58 -05:00
self._last_record_status: dict[str, str] = {}
self._last_status_update_time: float = 0.0
def _send_detect_status(self, status: str, now: float) -> None:
"""Send detect status only if changed or retry_interval has elapsed."""
if (
status != self._last_detect_status
or (now - self._last_status_update_time) >= self.sleeptime
):
self.requestor.send_data(f"{self.config.name}/status/detect", status)
self._last_detect_status = status
self._last_status_update_time = now
2026-08-25 12:51:58 -05:00
def _send_record_status(self, stream_type: str, status: str, now: float) -> None:
"""Send a record stream's status only if changed or retry_interval has elapsed."""
if (
2026-08-25 12:51:58 -05:00
status != self._last_record_status.get(stream_type)
or (now - self._last_status_update_time) >= self.sleeptime
):
2026-08-25 12:51:58 -05:00
self.requestor.send_data(
f"{self.config.name}/status/{STREAM_TYPE_TO_ROLE[stream_type]}", status
)
self._last_record_status[stream_type] = status
self._last_status_update_time = now
2026-08-25 12:51:58 -05:00
def _reset_segment_times(self) -> None:
self.latest_valid_segment_time.clear()
self.latest_invalid_segment_time.clear()
self.latest_cache_segment_time.clear()
self.stream_grace_until.clear()
def _grant_restart_grace(self, stream_types: list[str], now_utc: datetime) -> None:
for stream_type in stream_types:
self.stream_grace_until[stream_type] = now_utc + timedelta(
seconds=RECORD_GRACE_SECONDS
)
def _stream_staleness(self, stream_type: str, now_utc: datetime) -> str | None:
"""Return why the stream's segments are stale, or None if they're healthy."""
# ffmpeg needs time to create a first segment after recording is
# enabled and after a restart, per stream
in_grace_period = (
self.record_enable_time is not None
and (now_utc - self.record_enable_time)
< timedelta(seconds=RECORD_GRACE_SECONDS)
) or now_utc < self.stream_grace_until.get(stream_type, now_utc)
if in_grace_period:
return None
latest_cache = self.latest_cache_segment_time[stream_type]
latest_valid = self.latest_valid_segment_time[stream_type]
latest_invalid = self.latest_invalid_segment_time[stream_type]
def as_dt(timestamp: float) -> datetime:
if timestamp > 0:
return datetime.fromtimestamp(timestamp, tz=UTC)
return now_utc - timedelta(seconds=1)
stale_window = timedelta(seconds=self.record_stale_threshold[stream_type])
if now_utc > (as_dt(latest_cache) + stale_window):
return "No new recording segments were created"
if now_utc > (as_dt(latest_valid) + stale_window):
return "No new valid recording segments were created"
if (
latest_invalid > 0
and now_utc > (as_dt(latest_invalid) + stale_window)
and latest_valid <= latest_invalid
):
return "No valid segments created since last invalid segment"
return None
def _recorded_streams(self, roles: list[Any]) -> list[str]:
"""Record stream types the given roles cover that are currently recording."""
return [
stream_type
for role, stream_type in ROLE_TO_STREAM_TYPE.items()
if role in roles and self.config.record.stream_enabled(stream_type)
]
def _check_config_updates(self) -> dict[str, list[str]]:
"""Check for config updates and return the update dict."""
return self.config_subscriber.check_for_updates()
2025-03-03 09:30:52 -06:00
def _update_enabled_state(self) -> bool:
"""Fetch the latest config and update enabled state."""
self._check_config_updates()
return self.config.enabled
2025-03-03 09:30:52 -06:00
2025-07-11 13:19:43 -06:00
def reset_capture_thread(
self, terminate: bool = True, drain_output: bool = True
) -> None:
if terminate:
self.ffmpeg_detect_process.terminate()
try:
self.logger.info("Waiting for ffmpeg to exit gracefully...")
if drain_output:
self.ffmpeg_detect_process.communicate(timeout=30)
else:
self.ffmpeg_detect_process.wait(timeout=30)
except sp.TimeoutExpired:
self.logger.info("FFmpeg did not exit. Force killing...")
self.ffmpeg_detect_process.kill()
if drain_output:
self.ffmpeg_detect_process.communicate()
else:
self.ffmpeg_detect_process.wait()
# Update reconnects
now = datetime.now().timestamp()
self.reconnect_timestamps.append(now)
while self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600:
self.reconnect_timestamps.popleft()
if self.reconnects:
self.reconnects.value = len(self.reconnect_timestamps)
2025-11-30 06:54:42 -06:00
# Wait for old capture thread to fully exit before starting a new one
if self.capture_thread is not None and self.capture_thread.is_alive():
self.logger.info("Waiting for capture thread to exit...")
self.capture_thread.join(timeout=5)
if self.capture_thread.is_alive():
self.logger.warning(
f"Capture thread for {self.config.name} did not exit in time"
)
2025-07-11 13:19:43 -06:00
self.logger.error(
"The following ffmpeg logs include the last 100 lines prior to exit."
)
self.logpipe.dump()
self.logger.info("Restarting ffmpeg...")
self.start_ffmpeg_detect()
2026-08-25 12:51:58 -05:00
# this process produces the sub stream's segments too, so it gets the
# same startup grace however the reset was triggered
if self.detect_process_records_sub:
self._grant_restart_grace([STREAM_TYPE_SUB], datetime.now().astimezone(UTC))
2025-07-11 13:19:43 -06:00
def run(self) -> None:
2025-03-03 09:30:52 -06:00
if self._update_enabled_state():
self.start_all_ffmpeg()
2026-02-12 14:42:08 -06:00
# If recording is enabled at startup, set the grace period timer
if self.config.record.enabled:
2026-07-06 09:28:02 -08:00
self.record_enable_time = datetime.now().astimezone(UTC)
2021-02-17 07:23:32 -06:00
2023-06-30 08:14:39 -04:00
time.sleep(self.sleeptime)
last_restart_time = datetime.now().timestamp()
# 1 second watchdog loop
while not self.stop_event.wait(1):
updates = self._check_config_updates()
# Handle ffmpeg config changes by restarting all ffmpeg processes
if "ffmpeg" in updates and self.config.enabled:
self.logger.debug(
"FFmpeg config updated for %s, restarting ffmpeg processes",
self.config.name,
)
self.stop_all_ffmpeg()
self.start_all_ffmpeg()
2026-08-25 12:51:58 -05:00
self._reset_segment_times()
2026-07-06 09:28:02 -08:00
self.record_enable_time = datetime.now().astimezone(UTC)
last_restart_time = datetime.now().timestamp()
continue
enabled = self.config.enabled
2025-03-03 09:30:52 -06:00
if enabled != self.was_enabled:
if enabled:
2025-05-22 12:16:51 -06:00
self.logger.debug(f"Enabling camera {self.config.name}")
2025-03-03 09:30:52 -06:00
self.start_all_ffmpeg()
2025-09-28 11:52:14 -05:00
2026-02-12 14:42:08 -06:00
# reset all timestamps and record the enable time for grace period
2026-08-25 12:51:58 -05:00
self._reset_segment_times()
2026-07-06 09:28:02 -08:00
self.record_enable_time = datetime.now().astimezone(UTC)
2025-03-03 09:30:52 -06:00
else:
2025-05-22 12:16:51 -06:00
self.logger.debug(f"Disabling camera {self.config.name}")
2025-03-03 09:30:52 -06:00
self.stop_all_ffmpeg()
2026-02-12 14:42:08 -06:00
self.record_enable_time = None
2025-08-22 06:42:36 -06:00
# update camera status
now = datetime.now().timestamp()
self._send_detect_status("disabled", now)
2026-08-25 12:51:58 -05:00
self._send_record_status(STREAM_TYPE_MAIN, "disabled", now)
# cameras without a sub stream never get a record_sub topic
if self.config.record.sub.enabled:
self._send_record_status(STREAM_TYPE_SUB, "disabled", now)
2025-03-03 09:30:52 -06:00
self.was_enabled = enabled
continue
2026-05-07 08:53:07 -05:00
record_enabled_in_config = self.config.record.enabled_in_config
if record_enabled_in_config != self.was_record_enabled_in_config:
if record_enabled_in_config and enabled:
self.logger.debug(
f"Record enabled in config for {self.config.name}, restarting ffmpeg"
)
self.stop_all_ffmpeg()
self.start_all_ffmpeg()
2026-08-25 12:51:58 -05:00
self._reset_segment_times()
2026-07-06 09:28:02 -08:00
self.record_enable_time = datetime.now().astimezone(UTC)
2026-05-07 08:53:07 -05:00
last_restart_time = datetime.now().timestamp()
self.was_record_enabled_in_config = record_enabled_in_config
continue
record_sub_enabled = self.config.record.sub.enabled
if record_sub_enabled != self.was_record_sub_enabled:
# adding and removing the record_sub output both require a
# restart, unlike the main record toggle
if record_enabled_in_config and enabled:
self.logger.debug(
f"Sub stream recording toggled in config for {self.config.name}, restarting ffmpeg"
)
self.stop_all_ffmpeg()
self.start_all_ffmpeg()
2026-08-25 12:51:58 -05:00
self._reset_segment_times()
self.record_enable_time = datetime.now().astimezone(UTC)
last_restart_time = datetime.now().timestamp()
self.was_record_sub_enabled = record_sub_enabled
continue
2025-03-03 09:30:52 -06:00
if not enabled:
continue
2025-09-28 11:52:14 -05:00
while True:
update = self.segment_subscriber.check_for_update(timeout=0)
if update == (None, None):
break
raw_topic, payload = update
if raw_topic and payload:
topic = str(raw_topic)
2026-08-25 12:51:58 -05:00
camera, stream_type, segment_time, _ = payload
2025-09-28 11:52:14 -05:00
if camera != self.config.name:
continue
2026-04-28 09:54:09 -05:00
if topic.endswith(RecordingsDataTypeEnum.invalid.value):
2025-09-28 11:52:14 -05:00
self.logger.warning(
2026-08-25 12:51:58 -05:00
f"Invalid recording segment detected for {camera} ({stream_type}) at {segment_time}"
2025-09-28 11:52:14 -05:00
)
2026-08-25 12:51:58 -05:00
self.latest_invalid_segment_time[stream_type] = segment_time
2026-04-28 09:54:09 -05:00
elif topic.endswith(RecordingsDataTypeEnum.valid.value):
self.logger.debug(
2026-08-25 12:51:58 -05:00
f"Latest valid recording segment time on {camera} ({stream_type}): {segment_time}"
2026-04-28 09:54:09 -05:00
)
2026-08-25 12:51:58 -05:00
self.latest_valid_segment_time[stream_type] = segment_time
2025-09-28 11:52:14 -05:00
elif topic.endswith(RecordingsDataTypeEnum.latest.value):
2026-08-25 12:51:58 -05:00
self.latest_cache_segment_time[stream_type] = (
segment_time if segment_time is not None else 0
)
2025-09-28 11:52:14 -05:00
now = datetime.now().timestamp()
2020-10-25 10:05:21 -05:00
# Check if enough time has passed to allow ffmpeg restart (backoff pacing)
time_since_last_restart = now - last_restart_time
can_restart = time_since_last_restart >= self.sleeptime
2020-10-25 10:05:21 -05:00
if not self.capture_thread.is_alive():
self._send_detect_status("offline", now)
self.camera_fps.value = 0
2021-08-14 14:04:00 -05:00
self.logger.error(
2025-05-22 12:16:51 -06:00
f"Ffmpeg process crashed unexpectedly for {self.config.name}."
2021-08-14 14:04:00 -05:00
)
if can_restart:
self.reset_capture_thread(terminate=False)
last_restart_time = now
elif self.camera_fps.value >= (self.config.detect.fps + 10):
2024-09-10 07:39:37 -06:00
self.fps_overflow_count += 1
if self.fps_overflow_count == 3:
self._send_detect_status("offline", now)
2024-09-10 07:39:37 -06:00
self.fps_overflow_count = 0
self.camera_fps.value = 0
self.logger.info(
2025-05-22 12:16:51 -06:00
f"{self.config.name} exceeded fps limit. Exiting ffmpeg..."
2024-09-10 07:39:37 -06:00
)
if can_restart:
self.reset_capture_thread(drain_output=False)
last_restart_time = now
2025-07-11 13:19:43 -06:00
elif now - self.capture_thread.current_frame.value > 20:
self._send_detect_status("offline", now)
2025-07-11 13:19:43 -06:00
self.camera_fps.value = 0
self.logger.info(
2025-05-22 12:16:51 -06:00
f"No frames received from {self.config.name} in 20 seconds. Exiting ffmpeg..."
2025-07-11 13:19:43 -06:00
)
if can_restart:
self.reset_capture_thread()
last_restart_time = now
2024-09-10 07:39:37 -06:00
else:
# process is running normally
self._send_detect_status("online", now)
2024-09-10 07:39:37 -06:00
self.fps_overflow_count = 0
2021-02-17 07:23:32 -06:00
2020-11-29 15:55:53 -06:00
for p in self.ffmpeg_other_processes:
2021-02-17 07:23:32 -06:00
poll = p["process"].poll()
2026-08-25 12:51:58 -05:00
recorded_streams = self._recorded_streams(p["roles"])
if recorded_streams:
2026-07-06 09:28:02 -08:00
now_utc = datetime.now().astimezone(UTC)
2025-09-28 11:52:14 -05:00
2026-08-25 12:51:58 -05:00
# ensure segments are still being created and that they have
# valid video data. each stream is tracked separately so a
# healthy one can't mask a stalled one.
stale_stream = None
stale_reason = None
for stream_type in recorded_streams:
stale_reason = self._stream_staleness(stream_type, now_utc)
2026-02-12 14:42:08 -06:00
2026-08-25 12:51:58 -05:00
if stale_reason is not None:
stale_stream = stream_type
break
2025-09-28 11:52:14 -05:00
2026-08-25 12:51:58 -05:00
if stale_stream is not None and can_restart:
self.logger.error(
2026-08-25 12:51:58 -05:00
f"{stale_reason} for {self.config.name} ({stale_stream}) in the last {self.record_stale_threshold[stale_stream]}s. Restarting the ffmpeg record process..."
)
p["process"] = start_or_restart_ffmpeg(
p["cmd"],
self.logger,
p["logpipe"],
ffmpeg_process=p["process"],
)
2025-08-22 06:42:36 -06:00
for role in p["roles"]:
self.requestor.send_data(
2026-03-10 13:26:45 -06:00
f"{self.config.name}/status/{role.value}", "offline"
2025-08-22 06:42:36 -06:00
)
2026-08-25 12:51:58 -05:00
self._grant_restart_grace(recorded_streams, now_utc)
last_restart_time = now
continue
2026-08-25 12:51:58 -05:00
elif stale_stream is None:
for stream_type in recorded_streams:
self._send_record_status(stream_type, "online", now)
p["latest_segment_time"] = max(
self.latest_cache_segment_time[stream_type]
for stream_type in recorded_streams
)
if poll is None:
2020-11-29 15:55:53 -06:00
continue
2025-08-22 06:42:36 -06:00
for role in p["roles"]:
self.requestor.send_data(
2026-03-10 13:26:45 -06:00
f"{self.config.name}/status/{role.value}", "offline"
2025-08-22 06:42:36 -06:00
)
2021-02-17 07:23:32 -06:00
p["logpipe"].dump()
p["process"] = start_or_restart_ffmpeg(
p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"]
)
2026-08-25 12:51:58 -05:00
if (
self.detect_process_records_sub
and self.config.record.stream_enabled(STREAM_TYPE_SUB)
and self.capture_thread is not None
and self.capture_thread.is_alive()
):
now_utc = datetime.now().astimezone(UTC)
stale_reason = self._stream_staleness(STREAM_TYPE_SUB, now_utc)
if stale_reason is None:
self._send_record_status(STREAM_TYPE_SUB, "online", now)
elif can_restart:
self.logger.error(
f"{stale_reason} for {self.config.name} (sub, shared with detect) in the last {self.record_stale_threshold[STREAM_TYPE_SUB]}s. Restarting ffmpeg..."
)
self._send_record_status(STREAM_TYPE_SUB, "offline", now)
self.reset_capture_thread()
last_restart_time = now
2026-04-06 08:53:23 -05:00
# Prune expired reconnect timestamps
now = datetime.now().timestamp()
2026-04-06 08:53:23 -05:00
while (
self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600
):
self.reconnect_timestamps.popleft()
if self.reconnects:
self.reconnects.value = len(self.reconnect_timestamps)
# Update stall metrics based on last processed frame timestamp
processed_ts = (
float(self.detection_frame.value) if self.detection_frame else 0.0
)
if processed_ts > 0:
delta = now - processed_ts
observed_fps = (
self.camera_fps.value
if self.camera_fps.value > 0
else self.config.detect.fps
)
interval = 1.0 / max(observed_fps, 0.1)
stall_threshold = max(2.0 * interval, 2.0)
if delta > stall_threshold:
if not self._stall_active:
self._stall_timestamps.append(now)
self._stall_active = True
else:
self._stall_active = False
while self._stall_timestamps and self._stall_timestamps[0] < now - 3600:
self._stall_timestamps.popleft()
if self.stalls:
self.stalls.value = len(self._stall_timestamps)
2025-03-03 09:30:52 -06:00
self.stop_all_ffmpeg()
2021-05-21 11:39:14 -04:00
self.logpipe.close()
2025-03-03 09:30:52 -06:00
self.config_subscriber.stop()
2025-09-28 11:52:14 -05:00
self.segment_subscriber.stop()
2021-02-17 07:23:32 -06:00
2020-11-29 15:55:53 -06:00
def start_ffmpeg_detect(self):
2026-08-25 12:51:58 -05:00
detect_cmd = [c for c in self.config.ffmpeg_cmds if "detect" in c["roles"]][0]
ffmpeg_cmd = detect_cmd["cmd"]
self.detect_process_records_sub = "record_sub" in detect_cmd["roles"]
2021-02-17 07:23:32 -06:00
self.ffmpeg_detect_process = start_or_restart_ffmpeg(
ffmpeg_cmd, self.logger, self.logpipe, self.frame_size
)
2020-11-29 15:55:53 -06:00
self.ffmpeg_pid.value = self.ffmpeg_detect_process.pid
2025-06-12 12:12:34 -06:00
self.capture_thread = CameraCaptureRunner(
2024-09-03 10:22:30 -06:00
self.config,
self.shm_frame_count,
2024-11-17 10:25:49 -07:00
self.frame_index,
2021-02-17 07:23:32 -06:00
self.ffmpeg_detect_process,
self.frame_shape,
self.frame_queue,
self.camera_fps,
self.skipped_fps,
2023-02-04 08:58:45 -06:00
self.stop_event,
2021-02-17 07:23:32 -06:00
)
2020-11-01 10:55:11 -06:00
self.capture_thread.start()
2020-10-25 10:05:21 -05:00
2025-03-03 09:30:52 -06:00
def start_all_ffmpeg(self):
"""Start all ffmpeg processes (detection and others)."""
2025-05-22 12:16:51 -06:00
logger.debug(f"Starting all ffmpeg processes for {self.config.name}")
2025-03-03 09:30:52 -06:00
self.start_ffmpeg_detect()
for c in self.config.ffmpeg_cmds:
if "detect" in c["roles"]:
continue
logpipe = LogPipe(
2025-05-22 12:16:51 -06:00
f"ffmpeg.{self.config.name}.{'_'.join(sorted(c['roles']))}"
2025-03-03 09:30:52 -06:00
)
self.ffmpeg_other_processes.append(
{
"cmd": c["cmd"],
"roles": c["roles"],
"logpipe": logpipe,
"process": start_or_restart_ffmpeg(c["cmd"], self.logger, logpipe),
}
)
def stop_all_ffmpeg(self):
"""Stop all ffmpeg processes (detection and others)."""
2025-05-22 12:16:51 -06:00
logger.debug(f"Stopping all ffmpeg processes for {self.config.name}")
2025-03-03 09:30:52 -06:00
if self.capture_thread is not None and self.capture_thread.is_alive():
self.capture_thread.join(timeout=5)
if self.capture_thread.is_alive():
self.logger.warning(
2025-05-22 12:16:51 -06:00
f"Capture thread for {self.config.name} did not stop gracefully."
2025-03-03 09:30:52 -06:00
)
if self.ffmpeg_detect_process is not None:
stop_ffmpeg(self.ffmpeg_detect_process, self.logger)
self.ffmpeg_detect_process = None
for p in self.ffmpeg_other_processes[:]:
if p["process"] is not None:
stop_ffmpeg(p["process"], self.logger)
p["logpipe"].close()
self.ffmpeg_other_processes.clear()
2021-02-17 07:23:32 -06:00
2025-06-12 12:12:34 -06:00
class CameraCaptureRunner(threading.Thread):
2023-02-04 08:58:45 -06:00
def __init__(
self,
2024-09-03 10:22:30 -06:00
config: CameraConfig,
shm_frame_count: int,
2024-11-17 10:25:49 -07:00
frame_index: int,
ffmpeg_process,
2024-11-17 10:25:49 -07:00
frame_shape: tuple[int, int],
2025-04-23 17:06:06 -06:00
frame_queue: Queue,
fps: Value,
skipped_fps: Value,
stop_event: MpEvent,
2023-02-04 08:58:45 -06:00
):
threading.Thread.__init__(self)
2024-09-03 10:22:30 -06:00
self.name = f"capture:{config.name}"
self.config = config
self.shm_frame_count = shm_frame_count
2024-11-17 10:25:49 -07:00
self.frame_index = frame_index
self.frame_shape = frame_shape
self.frame_queue = frame_queue
self.fps = fps
2023-02-04 08:58:45 -06:00
self.stop_event = stop_event
self.skipped_fps = skipped_fps
self.frame_manager = SharedMemoryFrameManager()
self.ffmpeg_process = ffmpeg_process
2025-04-23 17:06:06 -06:00
self.current_frame = Value("d", 0.0)
2020-04-19 10:07:27 -05:00
self.last_frame = 0
def run(self):
2021-02-17 07:23:32 -06:00
capture_frames(
self.ffmpeg_process,
2024-09-03 10:22:30 -06:00
self.config,
self.shm_frame_count,
2024-11-17 10:25:49 -07:00
self.frame_index,
2021-02-17 07:23:32 -06:00
self.frame_shape,
self.frame_manager,
self.frame_queue,
self.fps,
self.skipped_fps,
self.current_frame,
2023-02-04 08:58:45 -06:00
self.stop_event,
2021-02-17 07:23:32 -06:00
)
2020-08-02 08:46:36 -05:00
2025-06-13 11:09:51 -06:00
class CameraCapture(FrigateProcess):
2025-06-12 12:12:34 -06:00
def __init__(
2025-06-24 11:41:11 -06:00
self,
config: CameraConfig,
shm_frame_count: int,
camera_metrics: CameraMetrics,
stop_event: MpEvent,
2025-11-17 08:12:05 -06:00
log_config: LoggerConfig | None = None,
2025-06-12 12:12:34 -06:00
) -> None:
super().__init__(
stop_event,
PROCESS_PRIORITY_HIGH,
name=f"frigate.capture:{config.name}",
daemon=True,
)
2025-06-12 12:12:34 -06:00
self.config = config
self.shm_frame_count = shm_frame_count
self.camera_metrics = camera_metrics
2025-11-17 08:12:05 -06:00
self.log_config = log_config
2021-02-17 07:23:32 -06:00
2025-06-12 12:12:34 -06:00
def run(self) -> None:
2025-11-17 08:12:05 -06:00
self.pre_run_setup(self.log_config)
2025-06-12 12:12:34 -06:00
camera_watchdog = CameraWatchdog(
self.config,
self.shm_frame_count,
self.camera_metrics.frame_queue,
self.camera_metrics.camera_fps,
self.camera_metrics.skipped_fps,
self.camera_metrics.ffmpeg_pid,
self.camera_metrics.stalls_last_hour,
self.camera_metrics.reconnects_last_hour,
self.camera_metrics.detection_frame,
2025-06-12 12:12:34 -06:00
self.stop_event,
)
camera_watchdog.start()
camera_watchdog.join()