Integrate state changes with review items (#24503)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s

* Integrate state changes with review items

* Cleanup and fixes

* Improve report context management
This commit is contained in:
Nicolas Mowen
2026-09-29 12:43:08 -05:00
committed by GitHub
parent d092189b8e
commit 5e87d101da
15 changed files with 658 additions and 45 deletions
+4
View File
@@ -179,12 +179,16 @@ class TestReviewMaintainerRemoval(unittest.TestCase):
maintainer = ReviewSegmentMaintainer.__new__(ReviewSegmentMaintainer)
maintainer.active_review_segments = {"deleted_cam": MagicMock()}
maintainer.indefinite_events = {"deleted_cam": {"1234.5-abcdef": 1.0}}
maintainer.recent_classification_state_changes = {
"deleted_cam": [{"model": "gate", "from": "a", "to": "b", "timestamp": 1.0}]
}
maintainer.forcibly_end_segment = MagicMock()
maintainer._handle_camera_removed("deleted_cam")
maintainer.forcibly_end_segment.assert_called_once_with("deleted_cam")
self.assertNotIn("deleted_cam", maintainer.indefinite_events)
self.assertNotIn("deleted_cam", maintainer.recent_classification_state_changes)
class TestAutotrackerMoveQueue(unittest.TestCase):
+45
View File
@@ -7,6 +7,7 @@ from frigate.genai.prompts import (
REVIEW_DESCRIPTION_FIELD_GUIDELINES,
REVIEW_RESPONSE_STYLES,
build_review_description_prompt,
build_review_summary_prompt,
get_review_field_guidelines,
)
@@ -83,5 +84,49 @@ class TestReviewResponseStyle(unittest.TestCase):
)
class TestClassificationStateChanges(unittest.TestCase):
def _build_prompt(self, **extra) -> str:
review_data = {
"camera": "Front Door",
"start": "Monday, 09:30 AM",
"duration": 25,
"zones": [],
"unified_objects": ["person"],
**extra,
}
return build_review_description_prompt(
review_data, [b"fake-image"], [], None, "activity context"
)
def test_no_changes_leaves_prompt_unchanged(self):
self.assertEqual(
self._build_prompt(classification_state_changes=[]),
self._build_prompt(),
)
self.assertNotIn("## State Changes", self._build_prompt())
def test_changes_are_listed_before_objects(self):
change = "front gate changed from closed to open, 12s into the activity"
prompt = self._build_prompt(classification_state_changes=[change])
self.assertIn(f"\n- {change}\n\n## Objects in Scene", prompt)
self.assertLess(
prompt.index("## Sequence Details"), prompt.index("## State Changes")
)
def test_summary_describes_state_changes_only_when_present(self):
event = {"title": "Person at door", "camera": "Front Door", "context": []}
without = build_review_summary_prompt(0, 3600, [event], None)
self.assertNotIn('"state_changes"', without)
context_event = {
**event,
"context": [
{"camera": "Driveway", "state_changes": ["gate changed from a to b"]}
],
}
with_changes = build_review_summary_prompt(0, 3600, [context_event], None)
self.assertIn('- "state_changes"', with_changes)
if __name__ == "__main__":
unittest.main()
+73
View File
@@ -1,10 +1,13 @@
"""Tests for tracker-derived review frame annotations."""
import unittest
from unittest.mock import patch
from frigate.data_processing.post.review_annotations import (
annotations_by_frame,
build_frame_captions,
build_timeline,
describe_classification_change,
describe_heading,
describe_position,
event_name,
@@ -312,5 +315,75 @@ class TestFrameBucketing(unittest.TestCase):
self.assertEqual(annotations_by_frame([(1.0, "x")], []), {})
class TestClassificationChangeCaptions(unittest.TestCase):
def setUp(self):
person = track(
"1789481994.684479-lpyc2z",
"person",
0.0,
straight_path((0.9, 0.6), (0.4, 0.3), 10, 0.0),
)
patcher = patch(
"frigate.data_processing.post.review_annotations.get_tracked_events",
return_value=[person],
)
patcher.start()
self.addCleanup(patcher.stop)
patcher = patch(
"frigate.data_processing.post.review_annotations.get_state_changes",
return_value=[],
)
patcher.start()
self.addCleanup(patcher.stop)
def test_change_is_phrased_without_underscores(self):
self.assertEqual(
describe_classification_change(
{"model": "trash_day", "from": "no_bins", "to": "bins_at_curb"}
),
"trash day changed from no bins to bins at curb",
)
def test_change_is_noted_before_the_frame_it_precedes(self):
change = {"model": "front_gate", "from": "closed", "to": "open"}
captions = build_frame_captions(
["1789481994.684479-lpyc2z"],
[0.0, 10.0, 20.0],
[{**change, "timestamp": 12.0}],
)
self.assertEqual(
captions[2],
"Frame 3 of 3 (+20.0s):\n[state] front gate changed from closed to open",
)
self.assertTrue(captions[0].splitlines()[1].startswith("[tracker] "))
def test_change_is_noted_without_tracked_objects(self):
with patch(
"frigate.data_processing.post.review_annotations.get_tracked_events",
return_value=[],
):
captions = build_frame_captions(
[],
[0.0, 10.0],
[{"model": "gate", "from": "a", "to": "b", "timestamp": 5.0}],
)
self.assertEqual(
captions,
[
"Frame 1 of 2 (+0.0s):",
"Frame 2 of 2 (+10.0s):\n[state] gate changed from a to b",
],
)
def test_change_after_the_last_frame_is_dropped(self):
captions = build_frame_captions(
["1789481994.684479-lpyc2z"],
[0.0, 10.0, 20.0],
[{"model": "gate", "from": "a", "to": "b", "timestamp": 25.0}],
)
self.assertFalse(any("[state]" in caption for caption in captions))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,258 @@
"""Tests for attaching state classification changes to review items."""
import unittest
from unittest.mock import MagicMock, patch
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
from frigate.config import FrigateConfig
from frigate.data_processing.post.review_descriptions import (
ReviewDescriptionProcessor,
format_classification_state_changes,
)
from frigate.data_processing.real_time.custom_classification import (
CustomStateClassificationProcessor,
)
from frigate.models import ReviewSegment
from frigate.review.maintainer import (
CLASSIFICATION_STATE_PRE_ROLL,
PendingReviewSegment,
ReviewSegmentMaintainer,
)
from frigate.review.types import SeverityEnum
CONFIG = """
mqtt:
enabled: False
cameras:
front_door:
ffmpeg:
inputs:
- path: rtsp://10.0.0.1:554/video
roles:
- detect
detect:
width: 1920
height: 1080
fps: 5
"""
def gate_change(timestamp: float, before: str = "closed", after: str = "open"):
return {"model": "front_gate", "from": before, "to": after, "timestamp": timestamp}
class TestVerifyStateChange(unittest.TestCase):
def setUp(self):
self.processor = CustomStateClassificationProcessor.__new__(
CustomStateClassificationProcessor
)
self.processor.state_history = {}
def verify(self, state: str, timestamp: float):
return self.processor.verify_state_change("front_door", state, timestamp)
def test_first_verified_state_has_no_previous_state(self):
self.assertIsNone(self.verify("closed", 1.0))
self.assertIsNone(self.verify("closed", 2.0))
self.assertEqual(self.verify("closed", 3.0), (None, 1.0))
def test_change_reports_previous_state_and_first_sighting(self):
for timestamp in (1.0, 2.0, 3.0):
self.verify("closed", timestamp)
self.assertIsNone(self.verify("open", 10.0))
self.assertIsNone(self.verify("open", 11.0))
self.assertEqual(self.verify("open", 12.0), ("closed", 10.0))
def test_interrupted_verification_restarts_the_first_sighting(self):
for timestamp in (1.0, 2.0, 3.0):
self.verify("closed", timestamp)
self.verify("open", 10.0)
self.verify("closed", 11.0)
self.verify("open", 20.0)
self.verify("open", 21.0)
self.assertEqual(self.verify("open", 22.0), ("closed", 20.0))
def test_reload_forgets_states_the_model_no_longer_has(self):
for timestamp in (1.0, 2.0, 3.0):
self.verify("closed", timestamp)
self.processor.state_history["back_door"] = {"current_state": "open"}
self.processor.labelmap = {0: "open", 1: "shut"}
self.processor._forget_unknown_states()
self.assertEqual(list(self.processor.state_history), ["back_door"])
self.assertIsNone(self.verify("shut", 10.0))
self.assertIsNone(self.verify("shut", 11.0))
self.assertEqual(self.verify("shut", 12.0), (None, 10.0))
class TestReviewSegmentAttachment(unittest.TestCase):
def setUp(self):
self.maintainer = ReviewSegmentMaintainer.__new__(ReviewSegmentMaintainer)
self.maintainer.config = FrigateConfig.parse_yaml(CONFIG)
self.maintainer.active_review_segments = {}
self.maintainer.recent_classification_state_changes = {}
self.maintainer._publish_segment_update = MagicMock()
def segment(self, start_time: float) -> PendingReviewSegment:
return PendingReviewSegment(
"front_door",
start_time,
SeverityEnum.alert,
{"1.0-abcdef": "person"},
{},
[],
set(),
)
def test_change_during_segment_is_attached_and_published(self):
segment = self.segment(100.0)
self.maintainer.active_review_segments["front_door"] = segment
self.maintainer.handle_classification_state_change(
"front_door", gate_change(110.0)
)
self.assertEqual(segment.classification_state_changes, [gate_change(110.0)])
self.maintainer._publish_segment_update.assert_called_once()
prev_data = self.maintainer._publish_segment_update.call_args.args[4]
self.assertEqual(prev_data["data"]["classification_state_changes"], [])
self.assertEqual(
segment.get_data(False)["data"]["classification_state_changes"],
[gate_change(110.0)],
)
def test_change_never_starts_a_segment(self):
self.maintainer.handle_classification_state_change(
"front_door", gate_change(110.0)
)
self.assertIsNone(self.maintainer.active_review_segments.get("front_door"))
self.maintainer._publish_segment_update.assert_not_called()
def test_change_just_before_a_segment_is_attached_when_it_starts(self):
self.maintainer.handle_classification_state_change(
"front_door", gate_change(98.0)
)
segment = self.segment(100.0)
self.maintainer._activate_segment(segment)
self.assertIs(self.maintainer.active_review_segments["front_door"], segment)
self.assertEqual(segment.classification_state_changes, [gate_change(98.0)])
self.assertNotIn(
"front_door", self.maintainer.recent_classification_state_changes
)
def test_change_long_before_a_segment_is_not_attached(self):
self.maintainer.handle_classification_state_change(
"front_door", gate_change(100.0 - CLASSIFICATION_STATE_PRE_ROLL - 1)
)
segment = self.segment(100.0)
self.maintainer._activate_segment(segment)
self.assertEqual(segment.classification_state_changes, [])
def test_held_changes_are_pruned(self):
self.maintainer.handle_classification_state_change(
"front_door", gate_change(10.0)
)
self.maintainer.handle_classification_state_change(
"front_door", gate_change(50.0, "open", "closed")
)
self.assertEqual(
self.maintainer.recent_classification_state_changes["front_door"],
[gate_change(50.0, "open", "closed")],
)
class TestChangeTiming(unittest.TestCase):
def test_changes_are_placed_relative_to_the_activity(self):
lines = format_classification_state_changes(
[
gate_change(98.0),
gate_change(112.4, "open", "closed"),
gate_change(140.0),
],
start_time=100.0,
end_time=130.0,
)
self.assertEqual(
lines,
[
"front gate changed from closed to open, "
"just before the activity started",
"front gate changed from open to closed, 12s into the activity",
"front gate changed from closed to open, after the activity ended",
],
)
class TestSummaryContext(unittest.TestCase):
def row(self, camera, start, end, threat, changes=()):
return {
"camera": camera,
"start_time": start,
"end_time": end,
"data": {
"metadata": {"title": camera, "potential_threat_level": threat},
"classification_state_changes": list(changes),
},
}
def summarize(self, rows):
processor = ReviewDescriptionProcessor.__new__(ReviewDescriptionProcessor)
processor.config = FrigateConfig.parse_yaml(CONFIG)
processor.genai_manager = MagicMock()
client = processor.genai_manager.description_client
with patch.object(ReviewSegment, "select") as select:
query = select.return_value.where.return_value.order_by.return_value
query.dicts.return_value.iterator.return_value = iter(rows)
processor.handle_request(
EmbeddingsRequestEnum.summarize_review.value,
{"start_ts": 0, "end_ts": 100},
)
return client.generate_review_summary.call_args.args[2]
def test_context_state_changes_stay_with_their_review(self):
events = self.summarize(
[
self.row("front_door", 10, 60, 1),
self.row("driveway", 15, 25, 0, [gate_change(20.0)]),
self.row("driveway", 30, 40, 0, [gate_change(35.0, "open", "closed")]),
]
)
self.assertEqual(
[
(item["start_time"], item["end_time"], item["state_changes"])
for item in events[0]["context"]
],
[
(15, 25, ["front gate changed from closed to open"]),
(30, 40, ["front gate changed from open to closed"]),
],
)
def test_later_context_review_without_changes_is_deduplicated(self):
events = self.summarize(
[
self.row("front_door", 10, 60, 1),
self.row("driveway", 15, 25, 0, [gate_change(20.0)]),
self.row("driveway", 30, 40, 0),
]
)
self.assertEqual(len(events[0]["context"]), 1)
self.assertEqual(events[0]["context"][0]["start_time"], 15)
if __name__ == "__main__":
unittest.main()