diff --git a/docs/docs/configuration/custom_classification/state_classification.md b/docs/docs/configuration/custom_classification/state_classification.md index 495e0173cc..0784aa1f52 100644 --- a/docs/docs/configuration/custom_classification/state_classification.md +++ b/docs/docs/configuration/custom_classification/state_classification.md @@ -89,7 +89,7 @@ An optional config, `save_attempts`, can be set as a key under the model name. T When a model's state changes while its camera has an active review item, the change is recorded on that review item. This includes changes in the few seconds before the item starts, such as a garage door opening just before the car is detected. State changes never create or extend review items on their own, and the first state reported after Frigate starts is not recorded as a change. -Recorded changes appear in the review item's data as `classification_state_changes` (see the [`frigate/reviews`](/integrations/mqtt#frigatereviews) MQTT topic) and are passed to [GenAI review summaries](/configuration/genai/review_summaries) as facts, so a description can note that a gate was opened during the activity. +Recorded changes appear in the review item's data as `classification_state_changes` (see the [`frigate/reviews`](/integrations/mqtt#frigatereviews) MQTT topic) and are passed to [GenAI review summaries](/configuration/genai/genai_review) as facts, so a description can note that a gate was opened during the activity. Change times are most accurate with `motion: true`. A model that only runs on an `interval` notices a change at its next run, so the change may be recorded late or attached to a later review item. diff --git a/frigate/data_processing/post/review_annotations.py b/frigate/data_processing/post/review_annotations.py index 4f55a25410..a313ef50d7 100644 --- a/frigate/data_processing/post/review_annotations.py +++ b/frigate/data_processing/post/review_annotations.py @@ -381,19 +381,19 @@ def build_frame_captions( if not frame_times: return [] + span_end = frame_times[-1] + timeline: list[tuple[float, str]] = [] events = get_tracked_events(detection_ids) - if not events: - logger.debug("No tracked events found for review item, skipping annotations") - return [] - - span_end = frame_times[-1] - timeline = [ - (timestamp, f"[tracker] {note}") - for timestamp, note in build_timeline( - events, span_end, get_state_changes(detection_ids) + # audio and manual review items can have state changes but no tracked objects + if events: + timeline.extend( + (timestamp, f"[tracker] {note}") + for timestamp, note in build_timeline( + events, span_end, get_state_changes(detection_ids) + ) ) - ] + timeline.extend( (change["timestamp"], f"[state] {describe_classification_change(change)}") for change in classification_changes diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 58e105736b..adf322f56c 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -310,7 +310,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): primary_end = primary_seg["end_time"] primary_camera = primary_seg["camera"] contextual_items = [] - seen_contextual_cameras = set() + contextual_by_camera: dict[str, dict[str, Any]] = {} for seg in segments: seg_camera = seg["camera"] @@ -325,17 +325,29 @@ class ReviewDescriptionProcessor(PostProcessorApi): seg_end = seg["end_time"] if seg_start < primary_end and primary_start < seg_end: - # Avoid duplicates if same camera has multiple overlapping segments - if seg_camera not in seen_contextual_cameras: - contextual_item = copy.deepcopy(seg["metadata"]) - contextual_item["camera"] = seg_camera - contextual_item["start_time"] = seg_start - contextual_item["end_time"] = seg_end + # Avoid duplicates if same camera has multiple overlapping + # segments, but keep every segment's state changes + existing_item = contextual_by_camera.get(seg_camera) + if existing_item is not None: if seg["state_changes"]: - contextual_item["state_changes"] = seg["state_changes"] - contextual_items.append(contextual_item) - seen_contextual_cameras.add(seg_camera) + existing_item.setdefault("state_changes", []).extend( + seg["state_changes"] + ) + continue + + contextual_item = copy.deepcopy(seg["metadata"]) + contextual_item["camera"] = seg_camera + contextual_item["start_time"] = seg_start + contextual_item["end_time"] = seg_end + + if seg["state_changes"]: + contextual_item["state_changes"] = list( + seg["state_changes"] + ) + + contextual_items.append(contextual_item) + contextual_by_camera[seg_camera] = contextual_item # Add context array to primary item primary_item["context"] = contextual_items diff --git a/frigate/data_processing/real_time/custom_classification.py b/frigate/data_processing/real_time/custom_classification.py index 993618d053..4a5908fe5a 100644 --- a/frigate/data_processing/real_time/custom_classification.py +++ b/frigate/data_processing/real_time/custom_classification.py @@ -91,8 +91,23 @@ class CustomStateClassificationProcessor(DeferredRealtimeProcessorApi): self.tensor_input_details = self.interpreter.get_input_details() self.tensor_output_details = self.interpreter.get_output_details() self.labelmap = load_labels(labelmap_path, prefill=0, indexed=False) + self._forget_unknown_states() self.classifications_per_second.start() + def _forget_unknown_states(self) -> None: + """Drop verified states that are not labels of the loaded model. + + A retrained model can rename or remove labels. Keeping a state it can + no longer produce would report its first verified state as a change + from that obsolete label. + """ + labels = set(self.labelmap.values()) + self.state_history = { + camera: history + for camera, history in self.state_history.items() + if history["current_state"] in labels + } + def __update_metrics(self, duration: float) -> None: self.classifications_per_second.update() if self.inference_speed: diff --git a/frigate/test/test_review_annotations.py b/frigate/test/test_review_annotations.py index e163babaf1..1261282de1 100644 --- a/frigate/test/test_review_annotations.py +++ b/frigate/test/test_review_annotations.py @@ -357,6 +357,25 @@ class TestClassificationChangeCaptions(unittest.TestCase): ) self.assertTrue(captions[0].splitlines()[1].startswith("[tracker] ")) + def test_change_is_noted_without_tracked_objects(self): + with patch( + "frigate.data_processing.post.review_annotations.get_tracked_events", + return_value=[], + ): + captions = build_frame_captions( + [], + [0.0, 10.0], + [{"model": "gate", "from": "a", "to": "b", "timestamp": 5.0}], + ) + + self.assertEqual( + captions, + [ + "Frame 1 of 2 (+0.0s):", + "Frame 2 of 2 (+10.0s):\n[state] gate changed from a to b", + ], + ) + def test_change_after_the_last_frame_is_dropped(self): captions = build_frame_captions( ["1789481994.684479-lpyc2z"], diff --git a/frigate/test/test_review_classification_state.py b/frigate/test/test_review_classification_state.py index f0a6bfc49b..97429cd90f 100644 --- a/frigate/test/test_review_classification_state.py +++ b/frigate/test/test_review_classification_state.py @@ -1,15 +1,18 @@ """Tests for attaching state classification changes to review items.""" import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +from frigate.comms.embeddings_updater import EmbeddingsRequestEnum from frigate.config import FrigateConfig from frigate.data_processing.post.review_descriptions import ( + ReviewDescriptionProcessor, format_classification_state_changes, ) from frigate.data_processing.real_time.custom_classification import ( CustomStateClassificationProcessor, ) +from frigate.models import ReviewSegment from frigate.review.maintainer import ( CLASSIFICATION_STATE_PRE_ROLL, PendingReviewSegment, @@ -71,6 +74,19 @@ class TestVerifyStateChange(unittest.TestCase): self.verify("open", 21.0) self.assertEqual(self.verify("open", 22.0), ("closed", 20.0)) + def test_reload_forgets_states_the_model_no_longer_has(self): + for timestamp in (1.0, 2.0, 3.0): + self.verify("closed", timestamp) + + self.processor.state_history["back_door"] = {"current_state": "open"} + self.processor.labelmap = {0: "open", 1: "shut"} + self.processor._forget_unknown_states() + + self.assertEqual(list(self.processor.state_history), ["back_door"]) + self.assertIsNone(self.verify("shut", 10.0)) + self.assertIsNone(self.verify("shut", 11.0)) + self.assertEqual(self.verify("shut", 12.0), (None, 10.0)) + class TestReviewSegmentAttachment(unittest.TestCase): def setUp(self): @@ -177,5 +193,67 @@ class TestChangeTiming(unittest.TestCase): ) +class TestSummaryContext(unittest.TestCase): + def row(self, camera, start, end, threat, changes=()): + return { + "camera": camera, + "start_time": start, + "end_time": end, + "data": { + "metadata": {"title": camera, "potential_threat_level": threat}, + "classification_state_changes": list(changes), + }, + } + + def summarize(self, rows): + processor = ReviewDescriptionProcessor.__new__(ReviewDescriptionProcessor) + processor.config = FrigateConfig.parse_yaml(CONFIG) + processor.genai_manager = MagicMock() + client = processor.genai_manager.description_client + + with patch.object(ReviewSegment, "select") as select: + query = select.return_value.where.return_value.order_by.return_value + query.dicts.return_value.iterator.return_value = iter(rows) + processor.handle_request( + EmbeddingsRequestEnum.summarize_review.value, + {"start_ts": 0, "end_ts": 100}, + ) + + return client.generate_review_summary.call_args.args[2] + + def test_overlapping_context_reviews_keep_all_state_changes(self): + events = self.summarize( + [ + self.row("front_door", 10, 60, 1), + self.row("driveway", 15, 25, 0, [gate_change(20.0)]), + self.row("driveway", 30, 40, 0, [gate_change(35.0, "open", "closed")]), + ] + ) + + self.assertEqual(len(events[0]["context"]), 1) + self.assertEqual( + events[0]["context"][0]["state_changes"], + [ + "front gate changed from closed to open", + "front gate changed from open to closed", + ], + ) + + def test_merging_context_does_not_leak_between_primary_events(self): + events = self.summarize( + [ + self.row("front_door", 10, 60, 1), + self.row("back_door", 12, 22, 1), + self.row("driveway", 15, 25, 0, [gate_change(20.0)]), + self.row("driveway", 30, 40, 0, [gate_change(35.0, "open", "closed")]), + ] + ) + + self.assertEqual( + events[1]["context"][0]["state_changes"], + ["front gate changed from closed to open"], + ) + + if __name__ == "__main__": unittest.main()