Cleanup and fixes

docs/docs/configuration/custom_classification/state_classification.md

@@ -89,7 +89,7 @@ An optional config, `save_attempts`, can be set as a key under the model name. T
 
 When a model's state changes while its camera has an active review item, the change is recorded on that review item. This includes changes in the few seconds before the item starts, such as a garage door opening just before the car is detected. State changes never create or extend review items on their own, and the first state reported after Frigate starts is not recorded as a change.
 
-Recorded changes appear in the review item's data as `classification_state_changes` (see the [`frigate/reviews`](/integrations/mqtt#frigatereviews) MQTT topic) and are passed to [GenAI review summaries](/configuration/genai/review_summaries) as facts, so a description can note that a gate was opened during the activity.
+Recorded changes appear in the review item's data as `classification_state_changes` (see the [`frigate/reviews`](/integrations/mqtt#frigatereviews) MQTT topic) and are passed to [GenAI review summaries](/configuration/genai/genai_review) as facts, so a description can note that a gate was opened during the activity.
 
 Change times are most accurate with `motion: true`. A model that only runs on an `interval` notices a change at its next run, so the change may be recorded late or attached to a later review item.
 

frigate/data_processing/post/review_annotations.py

@@ -381,19 +381,19 @@ def build_frame_captions(
     if not frame_times:
         return []
 
+    span_end = frame_times[-1]
+    timeline: list[tuple[float, str]] = []
     events = get_tracked_events(detection_ids)
 
-    if not events:
-        logger.debug("No tracked events found for review item, skipping annotations")
-        return []
-
-    span_end = frame_times[-1]
-    timeline = [
-        (timestamp, f"[tracker] {note}")
-        for timestamp, note in build_timeline(
-            events, span_end, get_state_changes(detection_ids)
+    # audio and manual review items can have state changes but no tracked objects
+    if events:
+        timeline.extend(
+            (timestamp, f"[tracker] {note}")
+            for timestamp, note in build_timeline(
+                events, span_end, get_state_changes(detection_ids)
+            )
         )
-    ]
+
     timeline.extend(
         (change["timestamp"], f"[state] {describe_classification_change(change)}")
         for change in classification_changes

frigate/data_processing/post/review_descriptions.py

@@ -310,7 +310,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 primary_end = primary_seg["end_time"]
                 primary_camera = primary_seg["camera"]
                 contextual_items = []
-                seen_contextual_cameras = set()
+                contextual_by_camera: dict[str, dict[str, Any]] = {}
 
                 for seg in segments:
                     seg_camera = seg["camera"]
@@ -325,17 +325,29 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                     seg_end = seg["end_time"]
 
                     if seg_start < primary_end and primary_start < seg_end:
-                        # Avoid duplicates if same camera has multiple overlapping segments
-                        if seg_camera not in seen_contextual_cameras:
-                            contextual_item = copy.deepcopy(seg["metadata"])
-                            contextual_item["camera"] = seg_camera
-                            contextual_item["start_time"] = seg_start
-                            contextual_item["end_time"] = seg_end
+                        # Avoid duplicates if same camera has multiple overlapping
+                        # segments, but keep every segment's state changes
+                        existing_item = contextual_by_camera.get(seg_camera)
 
+                        if existing_item is not None:
                             if seg["state_changes"]:
-                                contextual_item["state_changes"] = seg["state_changes"]
-                            contextual_items.append(contextual_item)
-                            seen_contextual_cameras.add(seg_camera)
+                                existing_item.setdefault("state_changes", []).extend(
+                                    seg["state_changes"]
+                                )
+                            continue
+
+                        contextual_item = copy.deepcopy(seg["metadata"])
+                        contextual_item["camera"] = seg_camera
+                        contextual_item["start_time"] = seg_start
+                        contextual_item["end_time"] = seg_end
+
+                        if seg["state_changes"]:
+                            contextual_item["state_changes"] = list(
+                                seg["state_changes"]
+                            )
+
+                        contextual_items.append(contextual_item)
+                        contextual_by_camera[seg_camera] = contextual_item
 
                 # Add context array to primary item
                 primary_item["context"] = contextual_items

frigate/data_processing/real_time/custom_classification.py

@@ -91,8 +91,23 @@ class CustomStateClassificationProcessor(DeferredRealtimeProcessorApi):
         self.tensor_input_details = self.interpreter.get_input_details()
         self.tensor_output_details = self.interpreter.get_output_details()
         self.labelmap = load_labels(labelmap_path, prefill=0, indexed=False)
+        self._forget_unknown_states()
         self.classifications_per_second.start()
 
+    def _forget_unknown_states(self) -> None:
+        """Drop verified states that are not labels of the loaded model.
+
+        A retrained model can rename or remove labels. Keeping a state it can
+        no longer produce would report its first verified state as a change
+        from that obsolete label.
+        """
+        labels = set(self.labelmap.values())
+        self.state_history = {
+            camera: history
+            for camera, history in self.state_history.items()
+            if history["current_state"] in labels
+        }
+
     def __update_metrics(self, duration: float) -> None:
         self.classifications_per_second.update()
         if self.inference_speed:

frigate/test/test_review_annotations.py

@@ -357,6 +357,25 @@ class TestClassificationChangeCaptions(unittest.TestCase):
         )
         self.assertTrue(captions[0].splitlines()[1].startswith("[tracker] "))
 
+    def test_change_is_noted_without_tracked_objects(self):
+        with patch(
+            "frigate.data_processing.post.review_annotations.get_tracked_events",
+            return_value=[],
+        ):
+            captions = build_frame_captions(
+                [],
+                [0.0, 10.0],
+                [{"model": "gate", "from": "a", "to": "b", "timestamp": 5.0}],
+            )
+
+        self.assertEqual(
+            captions,
+            [
+                "Frame 1 of 2 (+0.0s):",
+                "Frame 2 of 2 (+10.0s):\n[state] gate changed from a to b",
+            ],
+        )
+
     def test_change_after_the_last_frame_is_dropped(self):
         captions = build_frame_captions(
             ["1789481994.684479-lpyc2z"],

frigate/test/test_review_classification_state.py

@@ -1,15 +1,18 @@
 """Tests for attaching state classification changes to review items."""
 
 import unittest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
 from frigate.config import FrigateConfig
 from frigate.data_processing.post.review_descriptions import (
+    ReviewDescriptionProcessor,
     format_classification_state_changes,
 )
 from frigate.data_processing.real_time.custom_classification import (
     CustomStateClassificationProcessor,
 )
+from frigate.models import ReviewSegment
 from frigate.review.maintainer import (
     CLASSIFICATION_STATE_PRE_ROLL,
     PendingReviewSegment,
@@ -71,6 +74,19 @@ class TestVerifyStateChange(unittest.TestCase):
         self.verify("open", 21.0)
         self.assertEqual(self.verify("open", 22.0), ("closed", 20.0))
 
+    def test_reload_forgets_states_the_model_no_longer_has(self):
+        for timestamp in (1.0, 2.0, 3.0):
+            self.verify("closed", timestamp)
+
+        self.processor.state_history["back_door"] = {"current_state": "open"}
+        self.processor.labelmap = {0: "open", 1: "shut"}
+        self.processor._forget_unknown_states()
+
+        self.assertEqual(list(self.processor.state_history), ["back_door"])
+        self.assertIsNone(self.verify("shut", 10.0))
+        self.assertIsNone(self.verify("shut", 11.0))
+        self.assertEqual(self.verify("shut", 12.0), (None, 10.0))
+
 
 class TestReviewSegmentAttachment(unittest.TestCase):
     def setUp(self):
@@ -177,5 +193,67 @@ class TestChangeTiming(unittest.TestCase):
         )
 
 
+class TestSummaryContext(unittest.TestCase):
+    def row(self, camera, start, end, threat, changes=()):
+        return {
+            "camera": camera,
+            "start_time": start,
+            "end_time": end,
+            "data": {
+                "metadata": {"title": camera, "potential_threat_level": threat},
+                "classification_state_changes": list(changes),
+            },
+        }
+
+    def summarize(self, rows):
+        processor = ReviewDescriptionProcessor.__new__(ReviewDescriptionProcessor)
+        processor.config = FrigateConfig.parse_yaml(CONFIG)
+        processor.genai_manager = MagicMock()
+        client = processor.genai_manager.description_client
+
+        with patch.object(ReviewSegment, "select") as select:
+            query = select.return_value.where.return_value.order_by.return_value
+            query.dicts.return_value.iterator.return_value = iter(rows)
+            processor.handle_request(
+                EmbeddingsRequestEnum.summarize_review.value,
+                {"start_ts": 0, "end_ts": 100},
+            )
+
+        return client.generate_review_summary.call_args.args[2]
+
+    def test_overlapping_context_reviews_keep_all_state_changes(self):
+        events = self.summarize(
+            [
+                self.row("front_door", 10, 60, 1),
+                self.row("driveway", 15, 25, 0, [gate_change(20.0)]),
+                self.row("driveway", 30, 40, 0, [gate_change(35.0, "open", "closed")]),
+            ]
+        )
+
+        self.assertEqual(len(events[0]["context"]), 1)
+        self.assertEqual(
+            events[0]["context"][0]["state_changes"],
+            [
+                "front gate changed from closed to open",
+                "front gate changed from open to closed",
+            ],
+        )
+
+    def test_merging_context_does_not_leak_between_primary_events(self):
+        events = self.summarize(
+            [
+                self.row("front_door", 10, 60, 1),
+                self.row("back_door", 12, 22, 1),
+                self.row("driveway", 15, 25, 0, [gate_change(20.0)]),
+                self.row("driveway", 30, 40, 0, [gate_change(35.0, "open", "closed")]),
+            ]
+        )
+
+        self.assertEqual(
+            events[1]["context"][0]["state_changes"],
+            ["front gate changed from closed to open"],
+        )
+
+
 if __name__ == "__main__":
     unittest.main()