Integrate state changes with review items (#24503)

* Integrate state changes with review items

* Cleanup and fixes

* Improve report context management

frigate/review/maintainer.py

@@ -40,6 +40,10 @@ logger = logging.getLogger(__name__)
 THUMB_HEIGHT = 180
 THUMB_WIDTH = 320
 
+# seconds before a review item starts that a state classification change is
+# still attached to it, e.g. a garage door opening before the car is visible
+CLASSIFICATION_STATE_PRE_ROLL = 5
+
 
 class PendingReviewSegment:
     def __init__(
@@ -61,6 +65,7 @@ class PendingReviewSegment:
         self.sub_labels = sub_labels
         self.zones = zones
         self.audio = audio
+        self.classification_state_changes: list[dict[str, Any]] = []
         self.thumb_time: float | None = None
         self.last_alert_time: float | None = None
         self.last_detection_time: float = frame_time
@@ -162,6 +167,7 @@ class PendingReviewSegment:
                     "sub_labels": list(self.sub_labels.values()),
                     "zones": self.zones,
                     "audio": list(self.audio),
+                    "classification_state_changes": self.classification_state_changes,
                     "thumb_time": self.thumb_time,
                     "metadata": None,
                 },
@@ -293,6 +299,9 @@ class ReviewSegmentMaintainer(threading.Thread):
         # manual events
         self.indefinite_events: dict[str, dict[str, Any]] = {}
 
+        # state classification changes seen while a camera had no review item
+        self.recent_classification_state_changes: dict[str, list[dict[str, Any]]] = {}
+
         # ensure dirs
         Path(os.path.join(CLIPS_DIR, "review")).mkdir(exist_ok=True)
 
@@ -374,6 +383,43 @@ class ReviewSegmentMaintainer(threading.Thread):
         self.active_review_segments[segment.camera] = None
         return end_time
 
+    def _activate_segment(self, segment: PendingReviewSegment) -> None:
+        """Make a segment the camera's active one, attaching any state
+        classification changes seen just before it started."""
+        self.active_review_segments[segment.camera] = segment
+        recent = self.recent_classification_state_changes.pop(segment.camera, [])
+        segment.classification_state_changes.extend(
+            c
+            for c in recent
+            if c["timestamp"] >= segment.start_time - CLASSIFICATION_STATE_PRE_ROLL
+        )
+
+    def handle_classification_state_change(
+        self, camera: str, change: dict[str, Any]
+    ) -> None:
+        """Attach a verified state classification change to the active segment.
+
+        State changes never start, extend, or upgrade a segment. A change seen
+        with no active segment is held briefly for a segment starting right
+        after it.
+        """
+        segment = self.active_review_segments.get(camera)
+
+        if segment is None:
+            cutoff = change["timestamp"] - CLASSIFICATION_STATE_PRE_ROLL
+            self.recent_classification_state_changes[camera] = [
+                c
+                for c in self.recent_classification_state_changes.get(camera, [])
+                if c["timestamp"] >= cutoff
+            ] + [change]
+            return
+
+        prev_data = segment.get_data(False)
+        segment.classification_state_changes.append(change)
+        self._publish_segment_update(
+            segment, self.config.cameras[camera], None, [], prev_data
+        )
+
     def forcibly_end_segment(self, camera: str) -> Any:
         """Forcibly end the pending segment for a camera."""
         segment = self.active_review_segments.get(camera)
@@ -422,6 +468,7 @@ class ReviewSegmentMaintainer(threading.Thread):
         """Close out a deleted camera's segment so a reused name cannot inherit it."""
         self.forcibly_end_segment(camera)
         self.indefinite_events.pop(camera, None)
+        self.recent_classification_state_changes.pop(camera, None)
 
     def update_existing_segment(
         self,
@@ -579,7 +626,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                             audio=set(),
                             zones=list(new_zones),
                         )
-                        self.active_review_segments[segment.camera] = new_segment
+                        self._activate_segment(new_segment)
                         self._publish_segment_start(new_segment)
                         new_segment.last_detection_time = last_detection_time
             elif segment.severity == SeverityEnum.detection and frame_time > (
@@ -639,7 +686,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                     audio=set(),
                     zones=zones,
                 )
-                self.active_review_segments[camera] = new_segment
+                self._activate_segment(new_segment)
 
                 try:
                     yuv_frame = self.frame_manager.get(
@@ -713,6 +760,10 @@ class ReviewSegmentMaintainer(threading.Thread):
 
                 if camera not in self.indefinite_events:
                     self.indefinite_events[camera] = {}
+            elif topic == DetectionTypeEnum.classification_state.value:
+                (camera, classification_change) = data
+            else:
+                continue
 
             if camera not in self.config.cameras:
                 continue
@@ -723,6 +774,10 @@ class ReviewSegmentMaintainer(threading.Thread):
             ):
                 continue
 
+            if topic == DetectionTypeEnum.classification_state:
+                self.handle_classification_state_change(camera, classification_change)
+                continue
+
             current_segment = self.active_review_segments.get(camera)
 
             # Check if the current segment should be processed based on enabled settings
@@ -864,14 +919,16 @@ class ReviewSegmentMaintainer(threading.Thread):
                                 severity = SeverityEnum.detection
 
                     if severity:
-                        self.active_review_segments[camera] = PendingReviewSegment(
-                            camera,
-                            frame_time,
-                            severity,
-                            {},
-                            {},
-                            [],
-                            detections,
+                        self._activate_segment(
+                            PendingReviewSegment(
+                                camera,
+                                frame_time,
+                                severity,
+                                {},
+                                {},
+                                [],
+                                detections,
+                            )
                         )
                 elif topic == DetectionTypeEnum.api:
                     severity = self.get_manual_event_severity(
@@ -888,7 +945,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                             [],
                             set(),
                         )
-                        self.active_review_segments[camera] = api_segment
+                        self._activate_segment(api_segment)
 
                         if manual_info["state"] == ManualEventState.start:
                             self.indefinite_events[camera][manual_info["event_id"]] = (
@@ -915,7 +972,7 @@ class ReviewSegmentMaintainer(threading.Thread):
                             [],
                             set(),
                         )
-                        self.active_review_segments[camera] = lpr_segment
+                        self._activate_segment(lpr_segment)
 
                         if manual_info["state"] == ManualEventState.start:
                             self.indefinite_events[camera][manual_info["event_id"]] = (