Tweaks and fixes (#24659)

* add last time to active notices

* Stop retrying the review lookup in the explore detail menu when it 404s

* don't let a slow websocket client block publishers

A websocket client that stopped reading its socket (for example a suspended workstation with a Frigate tab open) blocked ws4py's sendall on whatever thread was publishing, including the single inter-process responder thread, so the recording maintainer and stats hung until nginx dropped the client after 5 minutes. Each client now gets its own bounded send queue and writer thread, and a client that overflows the queue is shut down. ws4py writes close and pong frames directly on its shared manager thread, bypassing the queue, so a 10s socket send timeout keeps a stalled client from pinning that thread. The jsmpeg and birdseye outputs share the class, so their unreachable send error handling is removed.

* don't let a lost object enter a zone

When the detector lost an object, the tracker kept it at its last box and every held frame still counted toward the zone's inertia. A car detected in a zone once or twice as it drove out of view could reach an inertia of 8 and be marked as entering, which sent it to GenAI and any other required zones check even though Review never showed the zone. Inertia and loitering still count on frames without a detection, but an object can only newly enter a zone on a frame where it was actually detected.

* add test

* limit retry stop to 404 only

* add docstrings

frigate/test/test_tracked_object_zones.py

@@ -0,0 +1,120 @@
+"""Zone entry for tracked objects the detector has lost."""
+
+import unittest
+
+from frigate.config import FrigateConfig
+from frigate.track.tracked_object import TrackedObject
+
+CAMERA = "cam"
+FPS = 5
+INERTIA = 3
+OUTSIDE_X = 200
+INSIDE_X = 480
+
+
+def _config() -> FrigateConfig:
+    return FrigateConfig(
+        **{
+            "mqtt": {"enabled": False},
+            "cameras": {
+                CAMERA: {
+                    "ffmpeg": {
+                        "inputs": [
+                            {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}
+                        ]
+                    },
+                    "detect": {"width": 640, "height": 480, "fps": FPS},
+                    "objects": {"track": ["car"]},
+                    "zones": {
+                        "zone": {
+                            "coordinates": "0.5,0,1,0,1,1,0.5,1",
+                            "inertia": INERTIA,
+                        }
+                    },
+                }
+            },
+        }
+    )
+
+
+def _obj_data(x: int, frame_time: float) -> dict:
+    return {
+        "id": "1000.0-abcdef",
+        "label": "car",
+        "frame_time": frame_time,
+        "start_time": 1000.0,
+        "score": 0.9,
+        "box": (x - 40, 200, x + 40, 260),
+        "area": 4800,
+        "ratio": 1.33,
+        "region": (0, 0, 640, 480),
+        "centroid": (x, 230),
+        "position_changes": 1,
+        "motionless_count": 0,
+        "attributes": [],
+    }
+
+
+class TestTrackedObjectZones(unittest.TestCase):
+    def setUp(self):
+        config = _config()
+        camera_config = config.cameras[CAMERA]
+        self.obj = TrackedObject(
+            config.model_for_camera(CAMERA),
+            camera_config,
+            config.ui,
+            {},
+            {**_obj_data(OUTSIDE_X, 1000.0), "score_history": [0.9]},
+        )
+        self.frame = 0
+
+    def _step(self, x: int, detected: bool = True) -> None:
+        """Advance one frame. A lost object keeps the box from its last detection."""
+        self.frame += 1
+        current_frame_time = 1000.0 + self.frame / FPS
+
+        if detected:
+            self.last_detection = _obj_data(x, current_frame_time)
+
+        self.obj.update(current_frame_time, dict(self.last_detection), False)
+
+    def test_lost_object_does_not_enter_zone(self):
+        self._step(INSIDE_X)
+
+        for _ in range(INERTIA * 3):
+            self._step(INSIDE_X, detected=False)
+
+        self.assertEqual(self.obj.current_zones, [])
+        self.assertEqual(self.obj.entered_zones, [])
+
+    def test_lost_object_enters_zone_when_detected_again(self):
+        self._step(INSIDE_X)
+
+        for _ in range(INERTIA):
+            self._step(INSIDE_X, detected=False)
+
+        self._step(INSIDE_X)
+        self.assertEqual(self.obj.entered_zones, ["zone"])
+
+    def test_detected_object_enters_zone(self):
+        for _ in range(INERTIA - 1):
+            self._step(INSIDE_X)
+
+        self.assertEqual(self.obj.entered_zones, [])
+
+        self._step(INSIDE_X)
+        self.assertEqual(self.obj.current_zones, ["zone"])
+        self.assertEqual(self.obj.entered_zones, ["zone"])
+
+    def test_object_in_zone_stays_while_lost(self):
+        for _ in range(INERTIA):
+            self._step(INSIDE_X)
+
+        for _ in range(INERTIA):
+            self._step(INSIDE_X, detected=False)
+
+        self.assertEqual(self.obj.current_zones, ["zone"])
+
+
+if __name__ == "__main__":
+    unittest.main()

frigate/test/test_ws_send_queue.py

@@ -0,0 +1,131 @@
+"""Outbound websocket sends must never block the thread calling publish()."""
+
+import socket
+import threading
+import unittest
+
+from frigate.comms.ws import WS_MAX_PENDING_MESSAGES, WebSocket
+
+
+class _FakeSock:
+    """Socket stand-in; ``block`` makes sendall hang like a client that stopped reading."""
+
+    def __init__(self, block: bool = False) -> None:
+        self.block = block
+        self.released = threading.Event()
+        self.shutdown_called = threading.Event()
+        self.frames: list[bytes] = []
+
+    def sendall(self, data: bytes) -> None:
+        if self.shutdown_called.is_set():
+            raise BrokenPipeError()
+        if self.block and not self.released.is_set():
+            self.released.wait(timeout=10)
+            raise BrokenPipeError()
+        self.frames.append(data)
+
+    def shutdown(self, how: int) -> None:
+        assert how == socket.SHUT_RDWR
+        self.shutdown_called.set()
+        self.released.set()
+
+    def close(self) -> None:
+        pass
+
+    def setsockopt(self, *args) -> None:
+        pass
+
+    def fileno(self) -> int:
+        return 99
+
+
+def _wait_for(predicate, timeout: float = 2.0) -> bool:
+    deadline = threading.Event()
+    for _ in range(int(timeout / 0.01)):
+        if predicate():
+            return True
+        deadline.wait(0.01)
+    return predicate()
+
+
+class TestWebSocketSendQueue(unittest.TestCase):
+    def _open(self, sock: _FakeSock) -> WebSocket:
+        ws = WebSocket(sock)
+        ws.opened()
+        return ws
+
+    def test_stalled_client_does_not_block_publisher(self):
+        sock = _FakeSock(block=True)
+        ws = self._open(sock)
+
+        def publish_many():
+            for i in range(WS_MAX_PENDING_MESSAGES + 5):
+                ws.send(f"message {i}")
+
+        publisher = threading.Thread(target=publish_many, daemon=True)
+        publisher.start()
+        publisher.join(timeout=2.0)
+
+        self.assertFalse(publisher.is_alive(), "publish() blocked on a stalled client")
+        self.assertTrue(
+            sock.shutdown_called.wait(timeout=2.0),
+            "a client that cannot keep up should be disconnected",
+        )
+        ws._writer.join(2)
+        self.assertFalse(ws._writer.is_alive())
+
+    def test_overflow_warns_and_shuts_down_once(self):
+        sock = _FakeSock(block=True)
+        ws = self._open(sock)
+        shutdown_calls = []
+        original_shutdown = sock.shutdown
+        sock.shutdown = lambda how: (shutdown_calls.append(how), original_shutdown(how))
+
+        with self.assertLogs("frigate.comms.ws", level="WARNING") as logs:
+            # keep publishing after overflow, as the dispatcher does until the
+            # manager thread removes the connection
+            for i in range(WS_MAX_PENDING_MESSAGES * 3):
+                ws.send(f"message {i}")
+
+        self.assertEqual(len(logs.output), 1)
+        self.assertEqual(len(shutdown_calls), 1)
+
+    def test_messages_delivered_in_order(self):
+        sock = _FakeSock()
+        ws = self._open(sock)
+        for i in range(3):
+            ws.send(f"message {i}")
+
+        self.assertTrue(_wait_for(lambda: len(sock.frames) == 3))
+        for i, frame in enumerate(sock.frames):
+            self.assertIn(f"message {i}".encode(), frame)
+        self.assertFalse(sock.shutdown_called.is_set())
+
+    def test_closed_stops_writer_thread(self):
+        sock = _FakeSock()
+        ws = self._open(sock)
+        writer = ws._writer
+        ws.closed(1000, "bye")
+        writer.join(timeout=2.0)
+        self.assertFalse(writer.is_alive())
+
+    def test_closed_with_full_queue_and_blocked_writer_exits_after_shutdown(self):
+        sock = _FakeSock(block=True)
+        ws = self._open(sock)
+        ws.send("in flight")
+        self.assertTrue(_wait_for(ws._send_queue.empty))
+        for i in range(WS_MAX_PENDING_MESSAGES):
+            ws.send(f"message {i}")
+        self.assertTrue(ws._send_queue.full())
+
+        ws.closed(1006)
+        self.assertTrue(ws._writer.is_alive())
+        self.assertFalse(sock.shutdown_called.is_set())
+
+        sock.shutdown(socket.SHUT_RDWR)
+        ws._writer.join(2)
+        self.assertFalse(ws._writer.is_alive())
+
+
+if __name__ == "__main__":
+    unittest.main()