Implement annotated frames for GenAI Review (#24379)

* Implement annotated frames mode for GenAI reviews to improve models with lacking temporal understanding

* Updates

* Improve debug sharing

* Do not number objects

* Fix assumptions

* Remove unhelpful content

* Improve object data sent as part of prompt

* Cleanup ollama dumbness

* Bind db

* Fixes

* Cleanup

frigate/genai/__init__.py

@@ -105,8 +105,21 @@ class GenAIClient:
         debug_save: bool,
         activity_context_prompt: str,
         response_style: str = "default",
+        frame_captions: list[str] | None = None,
     ) -> ReviewMetadata | None:
-        """Generate a description for the review item activity."""
+        """Generate a description for the review item activity.
+
+        `frame_captions` holds one caption per thumbnail for the annotated
+        frame mode; each is sent directly before its frame.
+        """
+        if frame_captions and len(frame_captions) != len(thumbnails):
+            logger.warning(
+                "Got %d frame captions for %d thumbnails, sending plain frames",
+                len(frame_captions),
+                len(thumbnails),
+            )
+            frame_captions = None
+
         context_prompt = build_review_description_prompt(
             review_data,
             thumbnails,
@@ -114,6 +127,7 @@ class GenAIClient:
             preferred_language,
             activity_context_prompt,
             response_style,
+            frame_captions,
         )
 
         logger.debug(
@@ -129,9 +143,30 @@ class GenAIClient:
             ) as f:
                 f.write(context_prompt)
 
+            if frame_captions:
+                # One file per frame, numbered to match the image it precedes
+                # (0.txt goes with 0.jpg), so the debug folder replays without
+                # having to re-derive the mapping.
+                for index, caption in enumerate(frame_captions):
+                    with open(
+                        os.path.join(
+                            CLIPS_DIR,
+                            "genai-requests",
+                            review_data["id"],
+                            f"{index}.txt",
+                        ),
+                        "w",
+                    ) as f:
+                        f.write(caption)
+
         response_format = build_review_description_response_format(concerns)
 
-        response = self._send(context_prompt, thumbnails, response_format)
+        response = self._send(
+            context_prompt,
+            thumbnails,
+            response_format,
+            image_captions=frame_captions,
+        )
 
         if debug_save and response:
             with open(
@@ -269,6 +304,7 @@ class GenAIClient:
         images: list[bytes],
         response_format: dict | None = None,
         enable_thinking: bool = False,
+        image_captions: list[str] | None = None,
     ) -> str | None:
         """Submit a request to the provider.
 
@@ -276,6 +312,10 @@ class GenAIClient:
         ``supports_toggleable_thinking``. Description-style callers leave it
         at the default (off) since synthesis tasks don't benefit from
         reasoning traces.
+
+        ``image_captions`` carries one caption per image, to be placed
+        immediately before its image so the model can tell the frames apart.
+        Providers build their request order with ``interleave_images``.
         """
         return None