From 8cd36416e2e52925fdf52fe6dae6ab3c232595c8 Mon Sep 17 00:00:00 2001 From: Li Xingyu <33626690+Li-Xingyu@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:24:01 +0800 Subject: [PATCH] Avoid killing recovered detector during restart (#24146) --- frigate/object_detection/base.py | 18 ++++++- frigate/test/test_detector_restart.py | 76 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 frigate/test/test_detector_restart.py diff --git a/frigate/object_detection/base.py b/frigate/object_detection/base.py index 858c8a5e4d..bda230348f 100644 --- a/frigate/object_detection/base.py +++ b/frigate/object_detection/base.py @@ -353,9 +353,23 @@ class ObjectDetectProcess: logging.info("Detection process has exited...") def start_or_restart(self) -> None: - self.detection_start.value = 0.0 # type: ignore[attr-defined] if (self.detect_process is not None) and self.detect_process.is_alive(): - self.stop() + logging.info("Waiting for detection process to exit gracefully...") + self.detect_process.join(timeout=30) + if self.detect_process.exitcode is None: + # detection_start is set only after detection_queue.get() + # returns. If it was reset during the grace period, the process + # recovered and may be waiting on the shared queue again. + if self.detection_start.value == 0.0: # type: ignore[attr-defined] + logging.info("Detection process recovered before restart") + return + + logging.info("Detection process didn't exit. Force killing...") + self.detect_process.kill() + self.detect_process.join() + logging.info("Detection process has exited...") + + self.detection_start.value = 0.0 # type: ignore[attr-defined] # Async path for MemryX if self.detector_config.type == "memryx": diff --git a/frigate/test/test_detector_restart.py b/frigate/test/test_detector_restart.py new file mode 100644 index 0000000000..ecb185d345 --- /dev/null +++ b/frigate/test/test_detector_restart.py @@ -0,0 +1,76 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, call, patch + +from frigate.object_detection.base import ObjectDetectProcess + + +def _make_detector( + detection_start: SimpleNamespace, + process: Mock, + stop_event: Mock, +) -> ObjectDetectProcess: + detector = ObjectDetectProcess.__new__(ObjectDetectProcess) + detector.name = "test" + detector.cameras = ["front"] + detector.detection_queue = Mock() + detector.avg_inference_speed = Mock() + detector.detection_start = detection_start + detector.detect_process = process + detector.config = Mock() + detector.detector_config = SimpleNamespace(type="onnx") + detector.stop_event = stop_event + return detector + + +class TestDetectorRestart(unittest.TestCase): + @patch("frigate.object_detection.base.DetectorRunner") + def test_restart_is_abandoned_when_inference_recovers( + self, detector_runner: Mock + ) -> None: + detection_start = SimpleNamespace(value=123.0) + process = Mock(exitcode=None) + process.is_alive.return_value = True + process.join.side_effect = lambda timeout=None: setattr( + detection_start, "value", 0.0 + ) + detector = _make_detector(detection_start, process, Mock()) + + detector.start_or_restart() + + process.join.assert_called_once_with(timeout=30) + process.kill.assert_not_called() + self.assertIs(detector.detect_process, process) + detector_runner.assert_not_called() + + @patch("frigate.object_detection.base.DetectorRunner") + def test_restart_force_kills_detector_that_remains_stuck( + self, detector_runner: Mock + ) -> None: + detection_start = SimpleNamespace(value=123.0) + process = Mock(exitcode=None) + process.is_alive.return_value = True + process.kill.side_effect = lambda: setattr(process, "exitcode", -9) + stop_event = Mock() + detector = _make_detector(detection_start, process, stop_event) + + detector.start_or_restart() + + process.join.assert_has_calls([call(timeout=30), call()]) + process.kill.assert_called_once_with() + self.assertEqual(detection_start.value, 0.0) + detector_runner.assert_called_once_with( + "frigate.detector:test", + detector.detection_queue, + detector.cameras, + detector.avg_inference_speed, + detection_start, + detector.config, + detector.detector_config, + stop_event, + ) + detector_runner.return_value.start.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main()