ensure embeddings process restarts after maintainer thread crash

This commit is contained in:
Josh Hawkins 2026-04-28 14:29:54 -05:00
parent 011ad8eda7
commit 24b7653ea8
2 changed files with 13 additions and 1 deletions

View File

@ -4,6 +4,7 @@ import base64
import json import json
import logging import logging
import os import os
import sys
import threading import threading
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError
from multiprocessing.synchronize import Event as MpEvent from multiprocessing.synchronize import Event as MpEvent
@ -52,6 +53,14 @@ class EmbeddingProcess(FrigateProcess):
self.stop_event, self.stop_event,
) )
maintainer.start() maintainer.start()
maintainer.join()
# If the maintainer thread exited but no shutdown was requested, it
# crashed. Surface as a non-zero exit so the watchdog restarts us
# instead of treating the silent thread death as a clean shutdown.
if not self.stop_event.is_set():
logger.error("Embeddings maintainer thread exited unexpectedly")
sys.exit(1)
class EmbeddingsContext: class EmbeddingsContext:

View File

@ -28,6 +28,7 @@ class MonitoredProcess:
restart_timestamps: deque[float] = field( restart_timestamps: deque[float] = field(
default_factory=lambda: deque(maxlen=MAX_RESTARTS) default_factory=lambda: deque(maxlen=MAX_RESTARTS)
) )
clean_exit_logged: bool = False
def is_restarting_too_fast(self, now: float) -> bool: def is_restarting_too_fast(self, now: float) -> bool:
while ( while (
@ -72,7 +73,9 @@ class FrigateWatchdog(threading.Thread):
exitcode = entry.process.exitcode exitcode = entry.process.exitcode
if exitcode == 0: if exitcode == 0:
logger.info("Process %s exited cleanly, not restarting", entry.name) if not entry.clean_exit_logged:
logger.info("Process %s exited cleanly, not restarting", entry.name)
entry.clean_exit_logged = True
return return
logger.warning( logger.warning(