mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-09-29 03:16:51 +03:00
* pin genai review frames to the main stream * retain previews as long as either stream has recordings * watch sub stream recording health separately from main * reject record_sub on the same input as record and document the role * derive recording paths from the cache segment timestamp Recording paths carry one second of resolution, but since sub stream recording start times are resolved to fractional wall clock, anchored to the cache file mtime and chained to the previous segment's end. A stream cutting segments faster than once a second resolves consecutive segments into the same second, so two rows collide on the unique path index and the batch insert fails. The cache segment name is unique per camera stream and second by construction because ffmpeg names segments with strftime, so the recording path is now built from that timestamp while the row keeps the resolved start time. This also restores the path semantics from before sub stream recording, when start times came straight from the cache filename. Nothing derives times from recording paths: playback offsets, stream switching, and export all use the row's start time, which is unchanged, and the recordings sync matches files by exact path string. * keep the rest of a recording batch when one row conflicts * only publish record_sub status when a sub stream is configured * don't shadow camera_cfg when publishing empty cache streams * back off restarts when a recording stream goes stale * give the shared sub stream grace on any capture thread reset * include segment details in recording discard warnings
750 lines
29 KiB
Python
750 lines
29 KiB
Python
"""Manages ffmpeg processes for camera frame capture."""
|
|
|
|
import logging
|
|
import queue
|
|
import subprocess as sp
|
|
import threading
|
|
import time
|
|
from collections import defaultdict, deque
|
|
from datetime import UTC, datetime, timedelta
|
|
from multiprocessing import Queue, Value
|
|
from multiprocessing.synchronize import Event as MpEvent
|
|
from typing import Any
|
|
|
|
from frigate.camera import CameraMetrics
|
|
from frigate.comms.inter_process import InterProcessRequestor
|
|
from frigate.comms.recordings_updater import (
|
|
RecordingsDataSubscriber,
|
|
RecordingsDataTypeEnum,
|
|
)
|
|
from frigate.config import CameraConfig, LoggerConfig
|
|
from frigate.config.camera.updater import (
|
|
CameraConfigUpdateEnum,
|
|
CameraConfigUpdateSubscriber,
|
|
)
|
|
from frigate.const import (
|
|
PROCESS_PRIORITY_HIGH,
|
|
RECORD_STREAM_TYPES,
|
|
ROLE_TO_STREAM_TYPE,
|
|
STREAM_TYPE_MAIN,
|
|
STREAM_TYPE_SUB,
|
|
STREAM_TYPE_TO_ROLE,
|
|
)
|
|
from frigate.log import LogPipe
|
|
from frigate.util.builtin import EventsPerSecond, get_record_segment_time
|
|
from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg
|
|
from frigate.util.image import (
|
|
FrameManager,
|
|
SharedMemoryFrameManager,
|
|
)
|
|
from frigate.util.process import FrigateProcess
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
RECORD_GRACE_SECONDS = 90
|
|
|
|
|
|
def capture_frames(
|
|
ffmpeg_process: sp.Popen[Any],
|
|
config: CameraConfig,
|
|
shm_frame_count: int,
|
|
frame_index: int,
|
|
frame_shape: tuple[int, int],
|
|
frame_manager: FrameManager,
|
|
frame_queue,
|
|
fps: Value,
|
|
skipped_fps: Value,
|
|
current_frame: Value,
|
|
stop_event: MpEvent,
|
|
) -> None:
|
|
frame_size = frame_shape[0] * frame_shape[1]
|
|
frame_rate = EventsPerSecond()
|
|
frame_rate.start()
|
|
skipped_eps = EventsPerSecond()
|
|
skipped_eps.start()
|
|
|
|
while not stop_event.is_set():
|
|
# CameraWatchdog applies enabled updates onto this same CameraConfig
|
|
# before it stops ffmpeg. Do not subscribe here: it would be rebuilt per
|
|
# ffmpeg restart and strand a pipe in the idle main process config PUB.
|
|
if not config.enabled:
|
|
logger.debug(f"Stopping capture thread for disabled {config.name}")
|
|
break
|
|
|
|
fps.value = frame_rate.eps()
|
|
skipped_fps.value = skipped_eps.eps()
|
|
current_frame.value = datetime.now().timestamp()
|
|
frame_name = f"{config.name}_frame{frame_index}"
|
|
frame_buffer = frame_manager.write(frame_name)
|
|
try:
|
|
frame_buffer[:] = ffmpeg_process.stdout.read(frame_size)
|
|
except Exception:
|
|
# shutdown has been initiated
|
|
if stop_event.is_set():
|
|
break
|
|
|
|
logger.error(f"{config.name}: Unable to read frames from ffmpeg process.")
|
|
|
|
if ffmpeg_process.poll() is not None:
|
|
logger.error(
|
|
f"{config.name}: ffmpeg process is not running. exiting capture thread..."
|
|
)
|
|
break
|
|
|
|
continue
|
|
|
|
frame_rate.update()
|
|
|
|
# don't lock the queue to check, just try since it should rarely be full
|
|
try:
|
|
# add to the queue
|
|
frame_queue.put((frame_name, current_frame.value), False)
|
|
frame_manager.close(frame_name)
|
|
except queue.Full:
|
|
# if the queue is full, skip this frame
|
|
skipped_eps.update()
|
|
|
|
frame_index = 0 if frame_index == shm_frame_count - 1 else frame_index + 1
|
|
|
|
|
|
class CameraWatchdog(threading.Thread):
|
|
def __init__(
|
|
self,
|
|
config: CameraConfig,
|
|
shm_frame_count: int,
|
|
frame_queue: Queue,
|
|
camera_fps,
|
|
skipped_fps,
|
|
ffmpeg_pid,
|
|
stalls,
|
|
reconnects,
|
|
detection_frame,
|
|
stop_event,
|
|
):
|
|
threading.Thread.__init__(self)
|
|
self.logger = logging.getLogger(f"watchdog.{config.name}")
|
|
self.config = config
|
|
self.shm_frame_count = shm_frame_count
|
|
self.capture_thread = None
|
|
self.ffmpeg_detect_process = None
|
|
self.logpipe = LogPipe(f"ffmpeg.{self.config.name}.detect")
|
|
self.ffmpeg_other_processes: list[dict[str, Any]] = []
|
|
self.camera_fps = camera_fps
|
|
self.skipped_fps = skipped_fps
|
|
self.ffmpeg_pid = ffmpeg_pid
|
|
self.frame_queue = frame_queue
|
|
self.frame_shape = self.config.frame_shape_yuv
|
|
self.frame_size = self.frame_shape[0] * self.frame_shape[1]
|
|
self.fps_overflow_count = 0
|
|
self.frame_index = 0
|
|
self.stop_event = stop_event
|
|
self.sleeptime = self.config.ffmpeg.retry_interval
|
|
self.reconnect_timestamps = deque()
|
|
self.stalls = stalls
|
|
self.reconnects = reconnects
|
|
self.detection_frame = detection_frame
|
|
|
|
self.config_subscriber = CameraConfigUpdateSubscriber(
|
|
None,
|
|
{config.name: config},
|
|
[
|
|
CameraConfigUpdateEnum.enabled,
|
|
CameraConfigUpdateEnum.ffmpeg,
|
|
CameraConfigUpdateEnum.record,
|
|
],
|
|
)
|
|
self.requestor = InterProcessRequestor()
|
|
self.was_enabled = self.config.enabled
|
|
self.was_record_enabled_in_config = self.config.record.enabled_in_config
|
|
self.was_record_sub_enabled = self.config.record.sub.enabled
|
|
|
|
self.segment_subscriber = RecordingsDataSubscriber(RecordingsDataTypeEnum.all)
|
|
self.latest_valid_segment_time: dict[str, float] = defaultdict(float)
|
|
self.latest_invalid_segment_time: dict[str, float] = defaultdict(float)
|
|
self.latest_cache_segment_time: dict[str, float] = defaultdict(float)
|
|
self.record_enable_time: datetime | None = None
|
|
self.stream_grace_until: dict[str, datetime] = {}
|
|
|
|
# `valid` segments are published with the segment's start time, so the
|
|
# gap between consecutive publishes can reach 2 * segment_time. Pad the
|
|
# staleness threshold so it's never tighter than that worst case.
|
|
self.record_stale_threshold: dict[str, int] = {
|
|
stream_type: max(
|
|
120, 2 * get_record_segment_time(self.config, stream_type) + 30
|
|
)
|
|
for stream_type in RECORD_STREAM_TYPES
|
|
}
|
|
|
|
# the sub stream usually shares its input, and therefore its ffmpeg
|
|
# process, with detect, so it isn't in ffmpeg_other_processes and needs
|
|
# its own staleness check
|
|
self.detect_process_records_sub = False
|
|
|
|
# Stall tracking (based on last processed frame)
|
|
self._stall_timestamps: deque[float] = deque()
|
|
self._stall_active: bool = False
|
|
|
|
# Status caching to reduce message volume
|
|
self._last_detect_status: str | None = None
|
|
self._last_record_status: dict[str, str] = {}
|
|
self._last_status_update_time: float = 0.0
|
|
|
|
def _send_detect_status(self, status: str, now: float) -> None:
|
|
"""Send detect status only if changed or retry_interval has elapsed."""
|
|
if (
|
|
status != self._last_detect_status
|
|
or (now - self._last_status_update_time) >= self.sleeptime
|
|
):
|
|
self.requestor.send_data(f"{self.config.name}/status/detect", status)
|
|
self._last_detect_status = status
|
|
self._last_status_update_time = now
|
|
|
|
def _send_record_status(self, stream_type: str, status: str, now: float) -> None:
|
|
"""Send a record stream's status only if changed or retry_interval has elapsed."""
|
|
if (
|
|
status != self._last_record_status.get(stream_type)
|
|
or (now - self._last_status_update_time) >= self.sleeptime
|
|
):
|
|
self.requestor.send_data(
|
|
f"{self.config.name}/status/{STREAM_TYPE_TO_ROLE[stream_type]}", status
|
|
)
|
|
self._last_record_status[stream_type] = status
|
|
self._last_status_update_time = now
|
|
|
|
def _reset_segment_times(self) -> None:
|
|
self.latest_valid_segment_time.clear()
|
|
self.latest_invalid_segment_time.clear()
|
|
self.latest_cache_segment_time.clear()
|
|
self.stream_grace_until.clear()
|
|
|
|
def _grant_restart_grace(self, stream_types: list[str], now_utc: datetime) -> None:
|
|
for stream_type in stream_types:
|
|
self.stream_grace_until[stream_type] = now_utc + timedelta(
|
|
seconds=RECORD_GRACE_SECONDS
|
|
)
|
|
|
|
def _stream_staleness(self, stream_type: str, now_utc: datetime) -> str | None:
|
|
"""Return why the stream's segments are stale, or None if they're healthy."""
|
|
# ffmpeg needs time to create a first segment after recording is
|
|
# enabled and after a restart, per stream
|
|
in_grace_period = (
|
|
self.record_enable_time is not None
|
|
and (now_utc - self.record_enable_time)
|
|
< timedelta(seconds=RECORD_GRACE_SECONDS)
|
|
) or now_utc < self.stream_grace_until.get(stream_type, now_utc)
|
|
|
|
if in_grace_period:
|
|
return None
|
|
|
|
latest_cache = self.latest_cache_segment_time[stream_type]
|
|
latest_valid = self.latest_valid_segment_time[stream_type]
|
|
latest_invalid = self.latest_invalid_segment_time[stream_type]
|
|
|
|
def as_dt(timestamp: float) -> datetime:
|
|
if timestamp > 0:
|
|
return datetime.fromtimestamp(timestamp, tz=UTC)
|
|
|
|
return now_utc - timedelta(seconds=1)
|
|
|
|
stale_window = timedelta(seconds=self.record_stale_threshold[stream_type])
|
|
|
|
if now_utc > (as_dt(latest_cache) + stale_window):
|
|
return "No new recording segments were created"
|
|
|
|
if now_utc > (as_dt(latest_valid) + stale_window):
|
|
return "No new valid recording segments were created"
|
|
|
|
if (
|
|
latest_invalid > 0
|
|
and now_utc > (as_dt(latest_invalid) + stale_window)
|
|
and latest_valid <= latest_invalid
|
|
):
|
|
return "No valid segments created since last invalid segment"
|
|
|
|
return None
|
|
|
|
def _recorded_streams(self, roles: list[Any]) -> list[str]:
|
|
"""Record stream types the given roles cover that are currently recording."""
|
|
return [
|
|
stream_type
|
|
for role, stream_type in ROLE_TO_STREAM_TYPE.items()
|
|
if role in roles and self.config.record.stream_enabled(stream_type)
|
|
]
|
|
|
|
def _check_config_updates(self) -> dict[str, list[str]]:
|
|
"""Check for config updates and return the update dict."""
|
|
return self.config_subscriber.check_for_updates()
|
|
|
|
def _update_enabled_state(self) -> bool:
|
|
"""Fetch the latest config and update enabled state."""
|
|
self._check_config_updates()
|
|
return self.config.enabled
|
|
|
|
def reset_capture_thread(
|
|
self, terminate: bool = True, drain_output: bool = True
|
|
) -> None:
|
|
if terminate:
|
|
self.ffmpeg_detect_process.terminate()
|
|
try:
|
|
self.logger.info("Waiting for ffmpeg to exit gracefully...")
|
|
|
|
if drain_output:
|
|
self.ffmpeg_detect_process.communicate(timeout=30)
|
|
else:
|
|
self.ffmpeg_detect_process.wait(timeout=30)
|
|
except sp.TimeoutExpired:
|
|
self.logger.info("FFmpeg did not exit. Force killing...")
|
|
self.ffmpeg_detect_process.kill()
|
|
|
|
if drain_output:
|
|
self.ffmpeg_detect_process.communicate()
|
|
else:
|
|
self.ffmpeg_detect_process.wait()
|
|
|
|
# Update reconnects
|
|
now = datetime.now().timestamp()
|
|
self.reconnect_timestamps.append(now)
|
|
while self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600:
|
|
self.reconnect_timestamps.popleft()
|
|
if self.reconnects:
|
|
self.reconnects.value = len(self.reconnect_timestamps)
|
|
|
|
# Wait for old capture thread to fully exit before starting a new one
|
|
if self.capture_thread is not None and self.capture_thread.is_alive():
|
|
self.logger.info("Waiting for capture thread to exit...")
|
|
self.capture_thread.join(timeout=5)
|
|
|
|
if self.capture_thread.is_alive():
|
|
self.logger.warning(
|
|
f"Capture thread for {self.config.name} did not exit in time"
|
|
)
|
|
|
|
self.logger.error(
|
|
"The following ffmpeg logs include the last 100 lines prior to exit."
|
|
)
|
|
self.logpipe.dump()
|
|
self.logger.info("Restarting ffmpeg...")
|
|
self.start_ffmpeg_detect()
|
|
|
|
# this process produces the sub stream's segments too, so it gets the
|
|
# same startup grace however the reset was triggered
|
|
if self.detect_process_records_sub:
|
|
self._grant_restart_grace([STREAM_TYPE_SUB], datetime.now().astimezone(UTC))
|
|
|
|
def run(self) -> None:
|
|
if self._update_enabled_state():
|
|
self.start_all_ffmpeg()
|
|
# If recording is enabled at startup, set the grace period timer
|
|
if self.config.record.enabled:
|
|
self.record_enable_time = datetime.now().astimezone(UTC)
|
|
|
|
time.sleep(self.sleeptime)
|
|
last_restart_time = datetime.now().timestamp()
|
|
|
|
# 1 second watchdog loop
|
|
while not self.stop_event.wait(1):
|
|
updates = self._check_config_updates()
|
|
|
|
# Handle ffmpeg config changes by restarting all ffmpeg processes
|
|
if "ffmpeg" in updates and self.config.enabled:
|
|
self.logger.debug(
|
|
"FFmpeg config updated for %s, restarting ffmpeg processes",
|
|
self.config.name,
|
|
)
|
|
self.stop_all_ffmpeg()
|
|
self.start_all_ffmpeg()
|
|
self._reset_segment_times()
|
|
self.record_enable_time = datetime.now().astimezone(UTC)
|
|
last_restart_time = datetime.now().timestamp()
|
|
continue
|
|
|
|
enabled = self.config.enabled
|
|
if enabled != self.was_enabled:
|
|
if enabled:
|
|
self.logger.debug(f"Enabling camera {self.config.name}")
|
|
self.start_all_ffmpeg()
|
|
|
|
# reset all timestamps and record the enable time for grace period
|
|
self._reset_segment_times()
|
|
self.record_enable_time = datetime.now().astimezone(UTC)
|
|
else:
|
|
self.logger.debug(f"Disabling camera {self.config.name}")
|
|
self.stop_all_ffmpeg()
|
|
self.record_enable_time = None
|
|
|
|
# update camera status
|
|
now = datetime.now().timestamp()
|
|
self._send_detect_status("disabled", now)
|
|
self._send_record_status(STREAM_TYPE_MAIN, "disabled", now)
|
|
# cameras without a sub stream never get a record_sub topic
|
|
if self.config.record.sub.enabled:
|
|
self._send_record_status(STREAM_TYPE_SUB, "disabled", now)
|
|
self.was_enabled = enabled
|
|
continue
|
|
|
|
record_enabled_in_config = self.config.record.enabled_in_config
|
|
if record_enabled_in_config != self.was_record_enabled_in_config:
|
|
if record_enabled_in_config and enabled:
|
|
self.logger.debug(
|
|
f"Record enabled in config for {self.config.name}, restarting ffmpeg"
|
|
)
|
|
self.stop_all_ffmpeg()
|
|
self.start_all_ffmpeg()
|
|
self._reset_segment_times()
|
|
self.record_enable_time = datetime.now().astimezone(UTC)
|
|
last_restart_time = datetime.now().timestamp()
|
|
self.was_record_enabled_in_config = record_enabled_in_config
|
|
continue
|
|
|
|
record_sub_enabled = self.config.record.sub.enabled
|
|
if record_sub_enabled != self.was_record_sub_enabled:
|
|
# adding and removing the record_sub output both require a
|
|
# restart, unlike the main record toggle
|
|
if record_enabled_in_config and enabled:
|
|
self.logger.debug(
|
|
f"Sub stream recording toggled in config for {self.config.name}, restarting ffmpeg"
|
|
)
|
|
self.stop_all_ffmpeg()
|
|
self.start_all_ffmpeg()
|
|
self._reset_segment_times()
|
|
self.record_enable_time = datetime.now().astimezone(UTC)
|
|
last_restart_time = datetime.now().timestamp()
|
|
self.was_record_sub_enabled = record_sub_enabled
|
|
continue
|
|
|
|
if not enabled:
|
|
continue
|
|
|
|
while True:
|
|
update = self.segment_subscriber.check_for_update(timeout=0)
|
|
|
|
if update == (None, None):
|
|
break
|
|
|
|
raw_topic, payload = update
|
|
if raw_topic and payload:
|
|
topic = str(raw_topic)
|
|
camera, stream_type, segment_time, _ = payload
|
|
|
|
if camera != self.config.name:
|
|
continue
|
|
|
|
if topic.endswith(RecordingsDataTypeEnum.invalid.value):
|
|
self.logger.warning(
|
|
f"Invalid recording segment detected for {camera} ({stream_type}) at {segment_time}"
|
|
)
|
|
self.latest_invalid_segment_time[stream_type] = segment_time
|
|
elif topic.endswith(RecordingsDataTypeEnum.valid.value):
|
|
self.logger.debug(
|
|
f"Latest valid recording segment time on {camera} ({stream_type}): {segment_time}"
|
|
)
|
|
self.latest_valid_segment_time[stream_type] = segment_time
|
|
elif topic.endswith(RecordingsDataTypeEnum.latest.value):
|
|
self.latest_cache_segment_time[stream_type] = (
|
|
segment_time if segment_time is not None else 0
|
|
)
|
|
|
|
now = datetime.now().timestamp()
|
|
|
|
# Check if enough time has passed to allow ffmpeg restart (backoff pacing)
|
|
time_since_last_restart = now - last_restart_time
|
|
can_restart = time_since_last_restart >= self.sleeptime
|
|
|
|
if not self.capture_thread.is_alive():
|
|
self._send_detect_status("offline", now)
|
|
self.camera_fps.value = 0
|
|
self.logger.error(
|
|
f"Ffmpeg process crashed unexpectedly for {self.config.name}."
|
|
)
|
|
if can_restart:
|
|
self.reset_capture_thread(terminate=False)
|
|
last_restart_time = now
|
|
elif self.camera_fps.value >= (self.config.detect.fps + 10):
|
|
self.fps_overflow_count += 1
|
|
|
|
if self.fps_overflow_count == 3:
|
|
self._send_detect_status("offline", now)
|
|
self.fps_overflow_count = 0
|
|
self.camera_fps.value = 0
|
|
self.logger.info(
|
|
f"{self.config.name} exceeded fps limit. Exiting ffmpeg..."
|
|
)
|
|
if can_restart:
|
|
self.reset_capture_thread(drain_output=False)
|
|
last_restart_time = now
|
|
elif now - self.capture_thread.current_frame.value > 20:
|
|
self._send_detect_status("offline", now)
|
|
self.camera_fps.value = 0
|
|
self.logger.info(
|
|
f"No frames received from {self.config.name} in 20 seconds. Exiting ffmpeg..."
|
|
)
|
|
if can_restart:
|
|
self.reset_capture_thread()
|
|
last_restart_time = now
|
|
else:
|
|
# process is running normally
|
|
self._send_detect_status("online", now)
|
|
self.fps_overflow_count = 0
|
|
|
|
for p in self.ffmpeg_other_processes:
|
|
poll = p["process"].poll()
|
|
|
|
recorded_streams = self._recorded_streams(p["roles"])
|
|
|
|
if recorded_streams:
|
|
now_utc = datetime.now().astimezone(UTC)
|
|
|
|
# ensure segments are still being created and that they have
|
|
# valid video data. each stream is tracked separately so a
|
|
# healthy one can't mask a stalled one.
|
|
stale_stream = None
|
|
stale_reason = None
|
|
for stream_type in recorded_streams:
|
|
stale_reason = self._stream_staleness(stream_type, now_utc)
|
|
|
|
if stale_reason is not None:
|
|
stale_stream = stream_type
|
|
break
|
|
|
|
if stale_stream is not None and can_restart:
|
|
self.logger.error(
|
|
f"{stale_reason} for {self.config.name} ({stale_stream}) in the last {self.record_stale_threshold[stale_stream]}s. Restarting the ffmpeg record process..."
|
|
)
|
|
p["process"] = start_or_restart_ffmpeg(
|
|
p["cmd"],
|
|
self.logger,
|
|
p["logpipe"],
|
|
ffmpeg_process=p["process"],
|
|
)
|
|
|
|
for role in p["roles"]:
|
|
self.requestor.send_data(
|
|
f"{self.config.name}/status/{role.value}", "offline"
|
|
)
|
|
|
|
self._grant_restart_grace(recorded_streams, now_utc)
|
|
last_restart_time = now
|
|
|
|
continue
|
|
elif stale_stream is None:
|
|
for stream_type in recorded_streams:
|
|
self._send_record_status(stream_type, "online", now)
|
|
|
|
p["latest_segment_time"] = max(
|
|
self.latest_cache_segment_time[stream_type]
|
|
for stream_type in recorded_streams
|
|
)
|
|
|
|
if poll is None:
|
|
continue
|
|
|
|
for role in p["roles"]:
|
|
self.requestor.send_data(
|
|
f"{self.config.name}/status/{role.value}", "offline"
|
|
)
|
|
|
|
p["logpipe"].dump()
|
|
p["process"] = start_or_restart_ffmpeg(
|
|
p["cmd"], self.logger, p["logpipe"], ffmpeg_process=p["process"]
|
|
)
|
|
|
|
if (
|
|
self.detect_process_records_sub
|
|
and self.config.record.stream_enabled(STREAM_TYPE_SUB)
|
|
and self.capture_thread is not None
|
|
and self.capture_thread.is_alive()
|
|
):
|
|
now_utc = datetime.now().astimezone(UTC)
|
|
stale_reason = self._stream_staleness(STREAM_TYPE_SUB, now_utc)
|
|
|
|
if stale_reason is None:
|
|
self._send_record_status(STREAM_TYPE_SUB, "online", now)
|
|
elif can_restart:
|
|
self.logger.error(
|
|
f"{stale_reason} for {self.config.name} (sub, shared with detect) in the last {self.record_stale_threshold[STREAM_TYPE_SUB]}s. Restarting ffmpeg..."
|
|
)
|
|
self._send_record_status(STREAM_TYPE_SUB, "offline", now)
|
|
self.reset_capture_thread()
|
|
last_restart_time = now
|
|
|
|
# Prune expired reconnect timestamps
|
|
now = datetime.now().timestamp()
|
|
while (
|
|
self.reconnect_timestamps and self.reconnect_timestamps[0] < now - 3600
|
|
):
|
|
self.reconnect_timestamps.popleft()
|
|
if self.reconnects:
|
|
self.reconnects.value = len(self.reconnect_timestamps)
|
|
|
|
# Update stall metrics based on last processed frame timestamp
|
|
processed_ts = (
|
|
float(self.detection_frame.value) if self.detection_frame else 0.0
|
|
)
|
|
if processed_ts > 0:
|
|
delta = now - processed_ts
|
|
observed_fps = (
|
|
self.camera_fps.value
|
|
if self.camera_fps.value > 0
|
|
else self.config.detect.fps
|
|
)
|
|
interval = 1.0 / max(observed_fps, 0.1)
|
|
stall_threshold = max(2.0 * interval, 2.0)
|
|
|
|
if delta > stall_threshold:
|
|
if not self._stall_active:
|
|
self._stall_timestamps.append(now)
|
|
self._stall_active = True
|
|
else:
|
|
self._stall_active = False
|
|
|
|
while self._stall_timestamps and self._stall_timestamps[0] < now - 3600:
|
|
self._stall_timestamps.popleft()
|
|
|
|
if self.stalls:
|
|
self.stalls.value = len(self._stall_timestamps)
|
|
|
|
self.stop_all_ffmpeg()
|
|
self.logpipe.close()
|
|
self.config_subscriber.stop()
|
|
self.segment_subscriber.stop()
|
|
|
|
def start_ffmpeg_detect(self):
|
|
detect_cmd = [c for c in self.config.ffmpeg_cmds if "detect" in c["roles"]][0]
|
|
ffmpeg_cmd = detect_cmd["cmd"]
|
|
self.detect_process_records_sub = "record_sub" in detect_cmd["roles"]
|
|
self.ffmpeg_detect_process = start_or_restart_ffmpeg(
|
|
ffmpeg_cmd, self.logger, self.logpipe, self.frame_size
|
|
)
|
|
self.ffmpeg_pid.value = self.ffmpeg_detect_process.pid
|
|
self.capture_thread = CameraCaptureRunner(
|
|
self.config,
|
|
self.shm_frame_count,
|
|
self.frame_index,
|
|
self.ffmpeg_detect_process,
|
|
self.frame_shape,
|
|
self.frame_queue,
|
|
self.camera_fps,
|
|
self.skipped_fps,
|
|
self.stop_event,
|
|
)
|
|
self.capture_thread.start()
|
|
|
|
def start_all_ffmpeg(self):
|
|
"""Start all ffmpeg processes (detection and others)."""
|
|
logger.debug(f"Starting all ffmpeg processes for {self.config.name}")
|
|
self.start_ffmpeg_detect()
|
|
for c in self.config.ffmpeg_cmds:
|
|
if "detect" in c["roles"]:
|
|
continue
|
|
logpipe = LogPipe(
|
|
f"ffmpeg.{self.config.name}.{'_'.join(sorted(c['roles']))}"
|
|
)
|
|
self.ffmpeg_other_processes.append(
|
|
{
|
|
"cmd": c["cmd"],
|
|
"roles": c["roles"],
|
|
"logpipe": logpipe,
|
|
"process": start_or_restart_ffmpeg(c["cmd"], self.logger, logpipe),
|
|
}
|
|
)
|
|
|
|
def stop_all_ffmpeg(self):
|
|
"""Stop all ffmpeg processes (detection and others)."""
|
|
logger.debug(f"Stopping all ffmpeg processes for {self.config.name}")
|
|
if self.capture_thread is not None and self.capture_thread.is_alive():
|
|
self.capture_thread.join(timeout=5)
|
|
if self.capture_thread.is_alive():
|
|
self.logger.warning(
|
|
f"Capture thread for {self.config.name} did not stop gracefully."
|
|
)
|
|
if self.ffmpeg_detect_process is not None:
|
|
stop_ffmpeg(self.ffmpeg_detect_process, self.logger)
|
|
self.ffmpeg_detect_process = None
|
|
for p in self.ffmpeg_other_processes[:]:
|
|
if p["process"] is not None:
|
|
stop_ffmpeg(p["process"], self.logger)
|
|
p["logpipe"].close()
|
|
self.ffmpeg_other_processes.clear()
|
|
|
|
|
|
class CameraCaptureRunner(threading.Thread):
|
|
def __init__(
|
|
self,
|
|
config: CameraConfig,
|
|
shm_frame_count: int,
|
|
frame_index: int,
|
|
ffmpeg_process,
|
|
frame_shape: tuple[int, int],
|
|
frame_queue: Queue,
|
|
fps: Value,
|
|
skipped_fps: Value,
|
|
stop_event: MpEvent,
|
|
):
|
|
threading.Thread.__init__(self)
|
|
self.name = f"capture:{config.name}"
|
|
self.config = config
|
|
self.shm_frame_count = shm_frame_count
|
|
self.frame_index = frame_index
|
|
self.frame_shape = frame_shape
|
|
self.frame_queue = frame_queue
|
|
self.fps = fps
|
|
self.stop_event = stop_event
|
|
self.skipped_fps = skipped_fps
|
|
self.frame_manager = SharedMemoryFrameManager()
|
|
self.ffmpeg_process = ffmpeg_process
|
|
self.current_frame = Value("d", 0.0)
|
|
self.last_frame = 0
|
|
|
|
def run(self):
|
|
capture_frames(
|
|
self.ffmpeg_process,
|
|
self.config,
|
|
self.shm_frame_count,
|
|
self.frame_index,
|
|
self.frame_shape,
|
|
self.frame_manager,
|
|
self.frame_queue,
|
|
self.fps,
|
|
self.skipped_fps,
|
|
self.current_frame,
|
|
self.stop_event,
|
|
)
|
|
|
|
|
|
class CameraCapture(FrigateProcess):
|
|
def __init__(
|
|
self,
|
|
config: CameraConfig,
|
|
shm_frame_count: int,
|
|
camera_metrics: CameraMetrics,
|
|
stop_event: MpEvent,
|
|
log_config: LoggerConfig | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
stop_event,
|
|
PROCESS_PRIORITY_HIGH,
|
|
name=f"frigate.capture:{config.name}",
|
|
daemon=True,
|
|
)
|
|
self.config = config
|
|
self.shm_frame_count = shm_frame_count
|
|
self.camera_metrics = camera_metrics
|
|
self.log_config = log_config
|
|
|
|
def run(self) -> None:
|
|
self.pre_run_setup(self.log_config)
|
|
camera_watchdog = CameraWatchdog(
|
|
self.config,
|
|
self.shm_frame_count,
|
|
self.camera_metrics.frame_queue,
|
|
self.camera_metrics.camera_fps,
|
|
self.camera_metrics.skipped_fps,
|
|
self.camera_metrics.ffmpeg_pid,
|
|
self.camera_metrics.stalls_last_hour,
|
|
self.camera_metrics.reconnects_last_hour,
|
|
self.camera_metrics.detection_frame,
|
|
self.stop_event,
|
|
)
|
|
camera_watchdog.start()
|
|
camera_watchdog.join()
|