ensure embeddings process restarts after maintainer thread crash

This commit is contained in:
Josh Hawkins 2026-04-28 14:29:54 -05:00
parent 011ad8eda7
commit 24b7653ea8
2 changed files with 13 additions and 1 deletions

View File

@ -4,6 +4,7 @@ import base64
import json
import logging
import os
import sys
import threading
from json.decoder import JSONDecodeError
from multiprocessing.synchronize import Event as MpEvent
@ -52,6 +53,14 @@ class EmbeddingProcess(FrigateProcess):
self.stop_event,
)
maintainer.start()
maintainer.join()
# If the maintainer thread exited but no shutdown was requested, it
# crashed. Surface as a non-zero exit so the watchdog restarts us
# instead of treating the silent thread death as a clean shutdown.
if not self.stop_event.is_set():
logger.error("Embeddings maintainer thread exited unexpectedly")
sys.exit(1)
class EmbeddingsContext:

View File

@ -28,6 +28,7 @@ class MonitoredProcess:
restart_timestamps: deque[float] = field(
default_factory=lambda: deque(maxlen=MAX_RESTARTS)
)
clean_exit_logged: bool = False
def is_restarting_too_fast(self, now: float) -> bool:
while (
@ -72,7 +73,9 @@ class FrigateWatchdog(threading.Thread):
exitcode = entry.process.exitcode
if exitcode == 0:
logger.info("Process %s exited cleanly, not restarting", entry.name)
if not entry.clean_exit_logged:
logger.info("Process %s exited cleanly, not restarting", entry.name)
entry.clean_exit_logged = True
return
logger.warning(