Optimize review summaries for consistency

frigate/data_processing/post/review_annotations.py

@@ -12,6 +12,8 @@ import math
 from collections.abc import Sequence
 from typing import Any
 
+from titlecase import titlecase
+
 from frigate.models import Event, Timeline
 
 logger = logging.getLogger(__name__)
@@ -84,12 +86,16 @@ def event_name(event: dict[str, Any]) -> str:
     tracked object whenever a subject is re-detected, so the tracking data
     cannot say whether two entries are the same subject, and the notes stay
     ambiguous rather than implying either answer.
+
+    Names are formatted the same way as in the prompt's "Objects in Scene"
+    list. Small models otherwise copy the raw lowercase sub label into their
+    response instead of the name they were told to use.
     """
     label = str(event["label"]).replace("_", " ").replace("-verified", "")
     sub_label = event.get("sub_label")
 
     if sub_label:
-        return f'{label} "{sub_label}"'
+        return f'{label} "{titlecase(str(sub_label).replace("_", " "))}"'
 
     article = "an" if label[:1].lower() in "aeiou" else "a"
     return f"{article} {label}"
@@ -213,6 +219,7 @@ def build_timeline(
     events: list[dict[str, Any]],
     span_end: float,
     state_changes: Sequence[dict[str, Any]] = (),
+    span_start: float | None = None,
 ) -> list[tuple[float, str]]:
     """All annotated moments across every event, in time order.
 
@@ -222,6 +229,12 @@ def build_timeline(
     describe nothing the model can see. A track ending means the object
     stopped being detected, which may or may not mean it left the frame.
 
+    `span_start` is the timestamp of the first sampled frame. An object
+    tracked since before it gets a single "already in view" note instead of
+    its earlier history: those moments would otherwise land on the first
+    frame and read as happening during the clip, turning a car that parked
+    minutes ago into one pulling in.
+
     `state_changes` are timeline rows (timestamp, source_id, class_type); the
     stationary and active ones become "has stopped moving" / "starts moving
     again". Frigate only marks an object stationary after it has been still
@@ -249,6 +262,13 @@ def build_timeline(
         if event["start_time"] > span_end:
             continue
 
+        if (
+            span_start is not None
+            and event["end_time"]
+            and event["end_time"] < span_start
+        ):
+            continue
+
         points = path_points(event.get("path_data") or [])
         legs = path_legs(points) if len(points) >= 2 else []
         name = event_name(event)
@@ -259,8 +279,33 @@ def build_timeline(
         )
         remaining = list(enumerate(legs))
         moments: list[tuple[float, str]] = []
+        event_changes = sorted(
+            (timestamp, phrase)
+            for timestamp, phrase in changes_by_event.get(event["id"], [])
+            if timestamp >= detected_at
+        )
 
-        if merges:
+        if span_start is not None and detected_at < span_start:
+            # Only whether it had come to rest is noted; where it is and which
+            # way it is heading are visible in the first frame.
+            earlier = [phrase for t, phrase in event_changes if t < span_start]
+            not_moving = (
+                bool(earlier) and earlier[-1] == STATE_CHANGE_PHRASES["stationary"]
+            )
+            timeline.append(
+                (
+                    span_start,
+                    f"{name} already in view when the sequence begins"
+                    + (", not moving" if not_moving else ""),
+                )
+            )
+            remaining = [
+                (index, leg)
+                for index, leg in remaining
+                if leg_start_time(points, leg) >= span_start
+            ]
+            event_changes = [(t, p) for t, p in event_changes if t >= span_start]
+        elif merges:
             heading = leg_heading(points, legs[0])
             timeline.append(
                 (detected_at, f"{name} first detected at {where}, moving {heading}")
@@ -277,11 +322,7 @@ def build_timeline(
                 )
             )
 
-        moments.extend(
-            (timestamp, phrase)
-            for timestamp, phrase in changes_by_event.get(event["id"], [])
-            if timestamp >= detected_at
-        )
+        moments.extend(event_changes)
 
         for timestamp, phrase in moments:
             if timestamp <= span_end:
@@ -390,7 +431,10 @@ def build_frame_captions(
         timeline.extend(
             (timestamp, f"[tracker] {note}")
             for timestamp, note in build_timeline(
-                events, span_end, get_state_changes(detection_ids)
+                events,
+                span_end,
+                get_state_changes(detection_ids),
+                span_start=frame_times[0],
             )
         )
 

frigate/genai/prompts.py

@@ -18,11 +18,17 @@ from frigate.models import Event
 
 # Base guidance per response field. `observations` is a reasoning scaffold,
 # not user-facing, so style presets never override it.
+#
+# Every scene and shortSummary guidance states its length. The response
+# schema caps both fields, and grammar-constrained providers such as llama.cpp
+# enforce the cap by cutting the text off, so a model that is never told the
+# limit ends mid-sentence. Every user-facing field also repeats the name rule:
+# models follow per-field guidance far more reliably than a single global rule.
 REVIEW_DESCRIPTION_FIELD_GUIDELINES: dict[str, str] = {
-    "observations": "Include the very start of the activity — for example, a vehicle entering the frame or pulling into the driveway — even if it lasts only a few frames and the rest of the clip is dominated by a longer activity. Include each arrival, departure, object handled, and notable change in position or state. Each item is a single concrete fact written as a complete sentence.",
-    "scene": 'Describe how the sequence begins, then the progression of events — all significant movements and actions in order. For example, if a vehicle arrives and then a person exits, describe both sequentially. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name — do not replace them with generic terms. For unnamed objects (e.g., "person", "car"), refer to them naturally with articles (e.g., "a person", "the car"). Your description should align with and support the threat level you assign.',
+    "observations": "List what happens in chronological order, starting from the first frame. Include each arrival, departure, object handled, and notable change in position or state, including brief moments at the start or end of the sequence that a longer activity could overshadow. Each item is a single concrete fact written as a complete sentence.",
+    "scene": 'Describe how the sequence begins, then the progression of events: all significant movements and actions in order. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name; do not replace them with generic terms. For unnamed objects (e.g., "person", "car"), refer to them naturally with articles (e.g., "a person", "the car"). Your description should align with and support the threat level you assign. Use no more than five sentences.',
     "title": "Name the primary activity across the observations, together with the location. An activity is what is being done with objects, tools, or surfaces; locomotion through the scene qualifies as the activity only when no other interaction is observed. For named subjects, always use their name. For unnamed objects, refer to them naturally with articles.",
-    "shortSummary": "Briefly summarize the primary activity across the observations.",
+    "shortSummary": "Briefly summarize the primary activity across the observations in one sentence. For named subjects, always use their name.",
     "potential_threat_level": "Must be consistent with your scene description and the activity patterns above.",
 }
 
@@ -31,19 +37,19 @@ REVIEW_DESCRIPTION_FIELD_GUIDELINES: dict[str, str] = {
 # competing style instructions.
 REVIEW_RESPONSE_STYLES: dict[str, dict[str, str]] = {
     "natural": {
-        "scene": 'Recount what happened the way a person would describe it to a neighbor, in plain everyday language: how the sequence begins, then each significant movement and action in the order it happens. Use flowing sentences that connect related actions, written the way people actually talk rather than like a surveillance report. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. Refer to unnamed objects naturally with articles. Stay factual, and keep the description consistent with the threat level you assign.',
+        "scene": 'Recount what happened the way a person would describe it to a neighbor, in plain everyday language: how the sequence begins, then each significant movement and action in the order it happens. Use flowing sentences that connect related actions, written the way people actually talk rather than like a surveillance report. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. Refer to unnamed objects naturally with articles. Stay factual, and keep the description consistent with the threat level you assign. Use no more than five sentences.',
         "title": 'Write the title as a short, sentence-case headline in present tense: the subject, then what they do, phrased the way you would text it to the homeowner. Describe only the action you see; do not assign the person a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens. For named subjects, always use their name.',
-        "shortSummary": "Sum up the primary activity in one short, natural sentence, as if mentioning it to someone in passing.",
+        "shortSummary": "Sum up the primary activity in one short, natural sentence, as if mentioning it to someone in passing. For named subjects, always use their name.",
     },
     "concise": {
-        "scene": 'Cover each significant movement and action in order using as few short, direct sentences as possible, omitting environmental and cosmetic detail unless it affects the assessment. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
+        "scene": 'Cover each significant movement and action in order using as few short, direct sentences as possible, omitting environmental and cosmetic detail unless it affects the assessment. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign. Use no more than five sentences.',
         "title": 'Write the title as a terse label of two to four words naming the specific activity observed and where it happened. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. For named subjects, always use their name.',
-        "shortSummary": "Summarize the primary activity in one short sentence.",
+        "shortSummary": "Summarize the primary activity in one short sentence. For named subjects, always use their name.",
     },
     "detailed": {
-        "scene": 'Describe how the sequence begins, then the progression of events — all significant movements and actions in order, including the specifics that best identify the subjects: colors, clothing, carried items, positions, and paths of movement, plus environmental details like lighting changes when they stand out. Favor the most identifying details over exhaustive coverage, and keep every added detail observational rather than speculative. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
+        "scene": 'Describe how the sequence begins, then the progression of events: all significant movements and actions in order, including the specifics that best identify the subjects: colors, clothing, carried items, positions, and paths of movement, plus environmental details like lighting changes when they stand out. Favor the most identifying details over exhaustive coverage, and keep every added detail observational rather than speculative. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign. Use no more than five sentences.',
         "title": 'Write the title as a specific description of who did what and where, in under roughly twelve words, including the most distinguishing visible detail of the subject, such as clothing or vehicle color. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens. For named subjects, always use their name.',
-        "shortSummary": "Briefly summarize the primary activity across the observations, including the most identifying visible detail, such as vehicle color or clothing.",
+        "shortSummary": "Briefly summarize the primary activity across the observations in one sentence, including the most identifying visible detail, such as vehicle color or clothing. For named subjects, always use their name.",
     },
 }
 
@@ -61,9 +67,12 @@ def get_review_field_guidelines(response_style: str = "default") -> dict[str, st
 
 # Explains the per-frame labels and tracker notes used by the annotated frame
 # mode. Neither the notes nor this guidance say whether repeated detections are
-# the same subject, since the tracking data cannot tell.
+# the same subject, since the tracking data cannot tell. The last line keeps
+# "no longer detected" from being read as the subject leaving the frame, which
+# is only one of the reasons the tracker loses an object.
 FRAME_ANNOTATION_GUIDANCE = """- Each image below is immediately preceded by a text label giving its frame number and how many seconds into the sequence it was captured. Use these labels to track the order of events and the time between them.
-- Some images below are preceded by notes from the camera's object tracker recording what changed at that point: an object being first detected, starting to move, reversing direction, stopping, or no longer being detected. These notes come from tracking data rather than from the images, and they are reliable. Use them to establish how many distinct activities occur and in what order, and describe every one of them."""
+- Some images below are preceded by notes from the camera's object tracker recording what changed at that point: an object being first detected, starting to move, reversing direction, stopping, or no longer being detected. These notes come from tracking data rather than from the images, and they are reliable. Use them to establish how many distinct activities occur and in what order, and describe every one of them.
+- The notes say when something changed, not why. In particular, an object is no longer detected whenever the tracker loses sight of it, which happens when it leaves the frame and also when it goes out of view behind or inside something. Look at the frames before and after each note to determine what actually happened."""
 
 
 def build_review_description_prompt(

frigate/test/test_genai_prompts.py

@@ -72,6 +72,19 @@ class TestReviewResponseStyle(unittest.TestCase):
                 f"unexpected override fields: {set(overrides)}",
             )
 
+    def test_every_style_states_length_and_name_rules(self):
+        # The schema caps these fields and grammar-constrained providers cut
+        # text off at the cap, and models only reliably use names when each
+        # field asks for them.
+        for style in ["default", *REVIEW_RESPONSE_STYLES]:
+            fields = get_review_field_guidelines(style)
+            self.assertIn("no more than five sentences", fields["scene"])
+            self.assertIn("one", fields["shortSummary"])
+            self.assertIn("sentence", fields["shortSummary"])
+
+            for field_name in ("scene", "title", "shortSummary"):
+                self.assertIn("always use their name", fields[field_name])
+
     def test_config_enum_matches_style_presets(self):
         enum_styles = {e.value for e in ReviewResponseStyleEnum}
         self.assertEqual(enum_styles, {"default", *REVIEW_RESPONSE_STYLES})

frigate/test/test_review_annotations.py

@@ -82,6 +82,10 @@ class TestEventNames(unittest.TestCase):
         event = {"label": "waste_bin", "sub_label": "Compost"}
         self.assertEqual(event_name(event), 'waste bin "Compost"')
 
+    def test_names_match_the_objects_in_scene_list(self):
+        event = {"label": "person", "sub_label": "known_subject"}
+        self.assertEqual(event_name(event), 'person "Known Subject"')
+
     def test_repeated_objects_are_never_numbered(self):
         # Whether these are the same subject is unknown, so the notes must not
         # imply either answer.
@@ -297,6 +301,57 @@ class TestLateEvents(unittest.TestCase):
         )
 
 
+class TestEarlyEvents(unittest.TestCase):
+    def setUp(self):
+        # A car that pulled in and parked well before the first sampled frame,
+        # then leaves during the clip.
+        self.car = track(
+            "1789481990.000000-parked",
+            "car",
+            0.0,
+            straight_path((0.1, 0.9), (0.4, 0.6), 6, 1.0),
+        )
+        self.changes = [
+            {
+                "timestamp": 12.0,
+                "source_id": self.car["id"],
+                "class_type": "stationary",
+            },
+            {"timestamp": 70.0, "source_id": self.car["id"], "class_type": "active"},
+        ]
+
+    def test_history_before_the_first_frame_is_summarized(self):
+        # Arrival notes on the first frame were read as the car pulling in
+        # during the clip.
+        timeline = build_timeline(
+            [self.car], 90.0, state_changes=self.changes, span_start=50.0
+        )
+        self.assertEqual(
+            timeline,
+            [
+                (50.0, "a car already in view when the sequence begins, not moving"),
+                (70.0, "a car starts moving again"),
+            ],
+        )
+
+    def test_object_still_moving_at_the_first_frame_is_not_called_stationary(self):
+        timeline = build_timeline([self.car], 90.0, span_start=3.0)
+        self.assertEqual(
+            timeline[0], (3.0, "a car already in view when the sequence begins")
+        )
+        self.assertFalse(any("first detected" in p for _, p in timeline))
+
+    def test_objects_gone_before_the_first_frame_are_skipped(self):
+        gone = dict(self.car, end_time=20.0)
+        self.assertEqual(build_timeline([gone], 90.0, span_start=50.0), [])
+
+    def test_objects_detected_inside_the_clip_are_unchanged(self):
+        self.assertEqual(
+            build_timeline([self.car], 90.0, self.changes, span_start=0.0),
+            build_timeline([self.car], 90.0, self.changes),
+        )
+
+
 class TestFrameBucketing(unittest.TestCase):
     def test_moment_attaches_to_the_following_frame(self):
         frame_times = [0.0, 10.0, 20.0, 30.0]

testing-scripts/genai_review_tester.py

@@ -42,6 +42,7 @@ from frigate.config.camera.genai import GenAIConfig, GenAIProviderEnum  # noqa:
 from frigate.genai.prompts import (  # noqa: E402
     REVIEW_RESPONSE_STYLES,
     build_review_description_response_format,
+    get_review_field_guidelines,
 )
 
 EXAMPLES_DIR = Path(__file__).resolve().parent / "genai-review-examples"
@@ -227,11 +228,14 @@ def list_examples() -> list[Path]:
     )
 
 
-def load_frames(example: Path) -> list[bytes]:
-    """Load the example's frames as JPEG bytes in frame order.
+def load_frames(example: Path) -> tuple[list[bytes], list[str] | None]:
+    """Load the example's frames as JPEG bytes in frame order, with captions.
 
     Frames are saved by Frigate as <index>.jpg or <index>.webp. Non-JPEG
     images are re-encoded to JPEG to match what Frigate sends to providers.
+    In annotated frame mode Frigate also saves <index>.txt, the caption sent
+    directly before that frame; captions are returned only when every frame
+    has one, matching Frigate's fallback to plain frames.
     """
     import cv2
 
@@ -242,34 +246,42 @@ def load_frames(example: Path) -> list[bytes]:
             return (1 << 30, path.name)
 
     frames: list[bytes] = []
+    captions: list[str] = []
     files = sorted(
         (f for f in example.iterdir() if f.suffix.lower() in IMAGE_EXTENSIONS),
         key=frame_order,
     )
     for file in files:
         if file.suffix.lower() in (".jpg", ".jpeg"):
-            frames.append(file.read_bytes())
-            continue
+            frame = file.read_bytes()
+        else:
+            image = cv2.imread(str(file))
+            if image is None:
+                print(f"  Skipping unreadable image {file.name}")
+                continue
+            ok, jpg = cv2.imencode(".jpg", image, [int(cv2.IMWRITE_JPEG_QUALITY), 100])
+            if not ok:
+                continue
+            frame = jpg.tobytes()
 
-        image = cv2.imread(str(file))
-        if image is None:
-            print(f"  Skipping unreadable image {file.name}")
-            continue
-        ok, jpg = cv2.imencode(".jpg", image, [int(cv2.IMWRITE_JPEG_QUALITY), 100])
-        if ok:
-            frames.append(jpg.tobytes())
-    return frames
+        frames.append(frame)
+        caption_file = file.with_suffix(".txt")
+        if caption_file.is_file():
+            captions.append(caption_file.read_text())
+
+    return frames, captions if captions and len(captions) == len(frames) else None
 
 
 def apply_style(prompt: str, style: str) -> str:
-    """Apply a style preset to a saved prompt.
+    """Apply the current field guidance for a style preset to a saved prompt.
 
-    Presets replace the per-field response guidance lines, matching what
-    Frigate builds at runtime. Handles both the current guidance format
+    Every per-field response guidance line is replaced with what Frigate
+    builds at runtime for that style, so "default" restores the default
+    guidance on a prompt saved with another style, and older saves pick up
+    current guidance. Handles both the current guidance format
     ("- `scene`: ...") and the 0.17 format ("- `scene` (string): ...").
     """
-    overrides = REVIEW_RESPONSE_STYLES.get(style, {})
-    for field_name, guidance in overrides.items():
+    for field_name, guidance in get_review_field_guidelines(style).items():
         pattern = rf"^- `{field_name}`(?: \(string\))?: .*$"
         prompt, count = re.subn(
             pattern,
@@ -292,7 +304,9 @@ def pretty_print_response(response: str) -> None:
     print(json.dumps(parsed, indent=2, ensure_ascii=False))
 
 
-def followup_loop(client: Any, base_prompt: str, frames: list[bytes]) -> None:
+def followup_loop(
+    client: Any, base_prompt: str, frames: list[bytes], captions: list[str] | None
+) -> None:
     """Let the user ask the model follow-up questions about the same frames."""
     transcript = base_prompt
     print("\nAsk follow-up questions about this sequence (blank line to finish).")
@@ -311,7 +325,7 @@ def followup_loop(client: Any, base_prompt: str, frames: list[bytes]) -> None:
             f"Question: {question}"
         )
         start = time.monotonic()
-        answer = client._send(prompt, frames)
+        answer = client._send(prompt, frames, image_captions=captions)
         elapsed = time.monotonic() - start
         if answer is None:
             print("No response from provider (see logs above).")
@@ -349,7 +363,7 @@ def run_example(settings: TesterSettings) -> None:
         return
     prompt = apply_style(prompt_file.read_text(), style)
 
-    frames = load_frames(example)
+    frames, captions = load_frames(example)
     if not frames:
         print(f"{example.name} contains no frame images, cannot run")
         return
@@ -365,10 +379,11 @@ def run_example(settings: TesterSettings) -> None:
 
     print(
         f"\nSending {len(frames)} frames to {settings.provider} "
-        f"(model={settings.model or '(none)'}, style={style})..."
+        f"(model={settings.model or '(none)'}, style={style}, "
+        f"captions={'yes' if captions else 'no'})..."
     )
     start = time.monotonic()
-    response = client._send(prompt, frames, response_format)
+    response = client._send(prompt, frames, response_format, image_captions=captions)
     elapsed = time.monotonic() - start
 
     if response is None:
@@ -377,7 +392,7 @@ def run_example(settings: TesterSettings) -> None:
 
     print(f"\nResponse ({elapsed:.1f}s):\n")
     pretty_print_response(response)
-    followup_loop(client, f"{prompt}\n\nYour analysis:\n{response}", frames)
+    followup_loop(client, f"{prompt}\n\nYour analysis:\n{response}", frames, captions)
 
 
 def main() -> None: