diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 5e14d0d8c..7e54d9703 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -4,6 +4,7 @@ import base64 import json import logging import os +import sys import threading from json.decoder import JSONDecodeError from multiprocessing.synchronize import Event as MpEvent @@ -52,6 +53,14 @@ class EmbeddingProcess(FrigateProcess): self.stop_event, ) maintainer.start() + maintainer.join() + + # If the maintainer thread exited but no shutdown was requested, it + # crashed. Surface as a non-zero exit so the watchdog restarts us + # instead of treating the silent thread death as a clean shutdown. + if not self.stop_event.is_set(): + logger.error("Embeddings maintainer thread exited unexpectedly") + sys.exit(1) class EmbeddingsContext: diff --git a/frigate/watchdog.py b/frigate/watchdog.py index 63fd16629..7ae42d988 100644 --- a/frigate/watchdog.py +++ b/frigate/watchdog.py @@ -28,6 +28,7 @@ class MonitoredProcess: restart_timestamps: deque[float] = field( default_factory=lambda: deque(maxlen=MAX_RESTARTS) ) + clean_exit_logged: bool = False def is_restarting_too_fast(self, now: float) -> bool: while ( @@ -72,7 +73,9 @@ class FrigateWatchdog(threading.Thread): exitcode = entry.process.exitcode if exitcode == 0: - logger.info("Process %s exited cleanly, not restarting", entry.name) + if not entry.clean_exit_logged: + logger.info("Process %s exited cleanly, not restarting", entry.name) + entry.clean_exit_logged = True return logger.warning(