frigate/frigate/watchdog.py

import logging
import threading
import time
from collections import deque
from collections.abc import Callable
from dataclasses import dataclass, field
from multiprocessing.synchronize import Event as MpEvent

from frigate.notices import raise_notice
from frigate.object_detection.base import ObjectDetectProcess
from frigate.util.process import FrigateProcess
from frigate.util.services import restart_frigate

logger = logging.getLogger(__name__)

MAX_RESTARTS = 5
RESTART_WINDOW_S = 60


@dataclass
class MonitoredProcess:
    """A process monitored by the watchdog for automatic restart."""

    name: str
    process: FrigateProcess
    factory: Callable[[], FrigateProcess]
    on_restart: Callable[[FrigateProcess], None] | None = None
    restart_timestamps: deque[float] = field(
        default_factory=lambda: deque(maxlen=MAX_RESTARTS)
    )
    clean_exit_logged: bool = False

    def is_restarting_too_fast(self, now: float) -> bool:
        while (
            self.restart_timestamps
            and now - self.restart_timestamps[0] > RESTART_WINDOW_S
        ):
            self.restart_timestamps.popleft()
        return len(self.restart_timestamps) >= MAX_RESTARTS


class FrigateWatchdog(threading.Thread):
    def __init__(
        self,
        detectors: dict[str, ObjectDetectProcess],
        stop_event: MpEvent,
    ):
        super().__init__(name="frigate_watchdog")
        self.detectors = detectors
        self.stop_event = stop_event
        self._monitored: list[MonitoredProcess] = []

    def register(
        self,
        name: str,
        process: FrigateProcess,
        factory: Callable[[], FrigateProcess],
        on_restart: Callable[[FrigateProcess], None] | None = None,
    ) -> None:
        """Register a FrigateProcess for monitoring and automatic restart."""
        self._monitored.append(
            MonitoredProcess(
                name=name,
                process=process,
                factory=factory,
                on_restart=on_restart,
            )
        )

    def _check_process(self, entry: MonitoredProcess) -> None:
        if entry.process.is_alive():
            return

        exitcode = entry.process.exitcode
        if exitcode == 0:
            if not entry.clean_exit_logged:
                logger.info("Process %s exited cleanly, not restarting", entry.name)
                entry.clean_exit_logged = True
            return

        logger.warning(
            "Process %s (PID %s) exited with code %s",
            entry.name,
            entry.process.pid,
            exitcode,
        )

        now = time.monotonic()

        if entry.is_restarting_too_fast(now):
            logger.error(
                "Process %s restarting too frequently (%d times in %ds), backing off",
                entry.name,
                MAX_RESTARTS,
                RESTART_WINDOW_S,
            )
            return

        try:
            entry.process.close()
            new_process = entry.factory()
            new_process.start()

            entry.process = new_process
            entry.restart_timestamps.append(now)

            if entry.on_restart:
                entry.on_restart(new_process)

            logger.info("Restarted %s (PID %s)", entry.name, new_process.pid)
        except Exception:
            logger.exception("Failed to restart %s", entry.name)

    def _check_detectors(self) -> None:
        # detection_start is set from the monotonic clock
        now = time.monotonic()

        for name, detector in self.detectors.items():
            detection_start = detector.detection_start.value
            # issue https://github.com/python/typeshed/issues/8799
            # from mypy 0.981 onwards
            if detection_start > 0.0 and now - detection_start > 10:
                logger.info(
                    "Detection appears to be stuck. Restarting detection process..."
                )
                detector.start_or_restart()
                raise_notice("detector_stuck", scope=name, params={"detector": name})
            elif (
                detector.detect_process is not None
                and not detector.detect_process.is_alive()
            ):
                logger.info("Detection appears to have stopped. Exiting Frigate...")
                restart_frigate()

    def run(self) -> None:
        time.sleep(10)
        while not self.stop_event.wait(10):
            self._check_detectors()

            for entry in self._monitored:
                self._check_process(entry)

        logger.info("Exiting watchdog...")