mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-05-03 12:07:40 +03:00
ensure embeddings process restarts after maintainer thread crash
This commit is contained in:
parent
011ad8eda7
commit
24b7653ea8
@ -4,6 +4,7 @@ import base64
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
import threading
|
import threading
|
||||||
from json.decoder import JSONDecodeError
|
from json.decoder import JSONDecodeError
|
||||||
from multiprocessing.synchronize import Event as MpEvent
|
from multiprocessing.synchronize import Event as MpEvent
|
||||||
@ -52,6 +53,14 @@ class EmbeddingProcess(FrigateProcess):
|
|||||||
self.stop_event,
|
self.stop_event,
|
||||||
)
|
)
|
||||||
maintainer.start()
|
maintainer.start()
|
||||||
|
maintainer.join()
|
||||||
|
|
||||||
|
# If the maintainer thread exited but no shutdown was requested, it
|
||||||
|
# crashed. Surface as a non-zero exit so the watchdog restarts us
|
||||||
|
# instead of treating the silent thread death as a clean shutdown.
|
||||||
|
if not self.stop_event.is_set():
|
||||||
|
logger.error("Embeddings maintainer thread exited unexpectedly")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
class EmbeddingsContext:
|
class EmbeddingsContext:
|
||||||
|
|||||||
@ -28,6 +28,7 @@ class MonitoredProcess:
|
|||||||
restart_timestamps: deque[float] = field(
|
restart_timestamps: deque[float] = field(
|
||||||
default_factory=lambda: deque(maxlen=MAX_RESTARTS)
|
default_factory=lambda: deque(maxlen=MAX_RESTARTS)
|
||||||
)
|
)
|
||||||
|
clean_exit_logged: bool = False
|
||||||
|
|
||||||
def is_restarting_too_fast(self, now: float) -> bool:
|
def is_restarting_too_fast(self, now: float) -> bool:
|
||||||
while (
|
while (
|
||||||
@ -72,7 +73,9 @@ class FrigateWatchdog(threading.Thread):
|
|||||||
|
|
||||||
exitcode = entry.process.exitcode
|
exitcode = entry.process.exitcode
|
||||||
if exitcode == 0:
|
if exitcode == 0:
|
||||||
|
if not entry.clean_exit_logged:
|
||||||
logger.info("Process %s exited cleanly, not restarting", entry.name)
|
logger.info("Process %s exited cleanly, not restarting", entry.name)
|
||||||
|
entry.clean_exit_logged = True
|
||||||
return
|
return
|
||||||
|
|
||||||
logger.warning(
|
logger.warning(
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user