mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-09-26 22:48:57 +03:00
Josh HawkinsandNicolas Mowen
7bc32fd4c9
* refactor notices * show startup message for enrichments in health pane * tweaks
144 lines
4.5 KiB
Python
144 lines
4.5 KiB
Python
import datetime
|
|
import logging
|
|
import threading
|
|
import time
|
|
from collections import deque
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass, field
|
|
from multiprocessing.synchronize import Event as MpEvent
|
|
|
|
from frigate.notices import raise_notice
|
|
from frigate.object_detection.base import ObjectDetectProcess
|
|
from frigate.util.process import FrigateProcess
|
|
from frigate.util.services import restart_frigate
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
MAX_RESTARTS = 5
|
|
RESTART_WINDOW_S = 60
|
|
|
|
|
|
@dataclass
|
|
class MonitoredProcess:
|
|
"""A process monitored by the watchdog for automatic restart."""
|
|
|
|
name: str
|
|
process: FrigateProcess
|
|
factory: Callable[[], FrigateProcess]
|
|
on_restart: Callable[[FrigateProcess], None] | None = None
|
|
restart_timestamps: deque[float] = field(
|
|
default_factory=lambda: deque(maxlen=MAX_RESTARTS)
|
|
)
|
|
clean_exit_logged: bool = False
|
|
|
|
def is_restarting_too_fast(self, now: float) -> bool:
|
|
while (
|
|
self.restart_timestamps
|
|
and now - self.restart_timestamps[0] > RESTART_WINDOW_S
|
|
):
|
|
self.restart_timestamps.popleft()
|
|
return len(self.restart_timestamps) >= MAX_RESTARTS
|
|
|
|
|
|
class FrigateWatchdog(threading.Thread):
|
|
def __init__(
|
|
self,
|
|
detectors: dict[str, ObjectDetectProcess],
|
|
stop_event: MpEvent,
|
|
):
|
|
super().__init__(name="frigate_watchdog")
|
|
self.detectors = detectors
|
|
self.stop_event = stop_event
|
|
self._monitored: list[MonitoredProcess] = []
|
|
|
|
def register(
|
|
self,
|
|
name: str,
|
|
process: FrigateProcess,
|
|
factory: Callable[[], FrigateProcess],
|
|
on_restart: Callable[[FrigateProcess], None] | None = None,
|
|
) -> None:
|
|
"""Register a FrigateProcess for monitoring and automatic restart."""
|
|
self._monitored.append(
|
|
MonitoredProcess(
|
|
name=name,
|
|
process=process,
|
|
factory=factory,
|
|
on_restart=on_restart,
|
|
)
|
|
)
|
|
|
|
def _check_process(self, entry: MonitoredProcess) -> None:
|
|
if entry.process.is_alive():
|
|
return
|
|
|
|
exitcode = entry.process.exitcode
|
|
if exitcode == 0:
|
|
if not entry.clean_exit_logged:
|
|
logger.info("Process %s exited cleanly, not restarting", entry.name)
|
|
entry.clean_exit_logged = True
|
|
return
|
|
|
|
logger.warning(
|
|
"Process %s (PID %s) exited with code %s",
|
|
entry.name,
|
|
entry.process.pid,
|
|
exitcode,
|
|
)
|
|
|
|
now = datetime.datetime.now().timestamp()
|
|
|
|
if entry.is_restarting_too_fast(now):
|
|
logger.error(
|
|
"Process %s restarting too frequently (%d times in %ds), backing off",
|
|
entry.name,
|
|
MAX_RESTARTS,
|
|
RESTART_WINDOW_S,
|
|
)
|
|
return
|
|
|
|
try:
|
|
entry.process.close()
|
|
new_process = entry.factory()
|
|
new_process.start()
|
|
|
|
entry.process = new_process
|
|
entry.restart_timestamps.append(now)
|
|
|
|
if entry.on_restart:
|
|
entry.on_restart(new_process)
|
|
|
|
logger.info("Restarted %s (PID %s)", entry.name, new_process.pid)
|
|
except Exception:
|
|
logger.exception("Failed to restart %s", entry.name)
|
|
|
|
def _check_detectors(self) -> None:
|
|
now = datetime.datetime.now().timestamp()
|
|
|
|
for name, detector in self.detectors.items():
|
|
detection_start = detector.detection_start.value # type: ignore[attr-defined]
|
|
# issue https://github.com/python/typeshed/issues/8799
|
|
# from mypy 0.981 onwards
|
|
if detection_start > 0.0 and now - detection_start > 10:
|
|
logger.info(
|
|
"Detection appears to be stuck. Restarting detection process..."
|
|
)
|
|
detector.start_or_restart()
|
|
raise_notice("detector_stuck", scope=name, params={"detector": name})
|
|
elif (
|
|
detector.detect_process is not None
|
|
and not detector.detect_process.is_alive()
|
|
):
|
|
logger.info("Detection appears to have stopped. Exiting Frigate...")
|
|
restart_frigate()
|
|
|
|
def run(self) -> None:
|
|
time.sleep(10)
|
|
while not self.stop_event.wait(10):
|
|
self._check_detectors()
|
|
|
|
for entry in self._monitored:
|
|
self._check_process(entry)
|
|
|
|
logger.info("Exiting watchdog...")
|