From 24b7653ea8da8b1015823bc0f8468460d97e5892 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 28 Apr 2026 14:29:54 -0500 Subject: [PATCH] ensure embeddings process restarts after maintainer thread crash --- frigate/embeddings/__init__.py | 9 +++++++++ frigate/watchdog.py | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 5e14d0d8c..7e54d9703 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -4,6 +4,7 @@ import base64 import json import logging import os +import sys import threading from json.decoder import JSONDecodeError from multiprocessing.synchronize import Event as MpEvent @@ -52,6 +53,14 @@ class EmbeddingProcess(FrigateProcess): self.stop_event, ) maintainer.start() + maintainer.join() + + # If the maintainer thread exited but no shutdown was requested, it + # crashed. Surface as a non-zero exit so the watchdog restarts us + # instead of treating the silent thread death as a clean shutdown. + if not self.stop_event.is_set(): + logger.error("Embeddings maintainer thread exited unexpectedly") + sys.exit(1) class EmbeddingsContext: diff --git a/frigate/watchdog.py b/frigate/watchdog.py index 63fd16629..7ae42d988 100644 --- a/frigate/watchdog.py +++ b/frigate/watchdog.py @@ -28,6 +28,7 @@ class MonitoredProcess: restart_timestamps: deque[float] = field( default_factory=lambda: deque(maxlen=MAX_RESTARTS) ) + clean_exit_logged: bool = False def is_restarting_too_fast(self, now: float) -> bool: while ( @@ -72,7 +73,9 @@ class FrigateWatchdog(threading.Thread): exitcode = entry.process.exitcode if exitcode == 0: - logger.info("Process %s exited cleanly, not restarting", entry.name) + if not entry.clean_exit_logged: + logger.info("Process %s exited cleanly, not restarting", entry.name) + entry.clean_exit_logged = True return logger.warning(