Tinkering with genai prompt

frigate/data_processing/post/review_descriptions.py

@@ -28,6 +28,8 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         if data_type != PostProcessDataEnum.review:
             return
 
+        logger.info(f"processing review {data['type']} on {data['after']['camera']}")
+
         id = data["after"]["id"]
 
         if data["type"] == "new" or data["type"] == "update":
@@ -38,6 +40,13 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             thumb_path = data["after"]["thumb_path"]
 
             if thumb_time and thumb_path:
+                if (
+                    len(self.tracked_review_items[id]) > 0
+                    and self.tracked_review_items[id][0] == thumb_time
+                ):
+                    # we have already processed this thumbnail
+                    return
+
                 thumb_data = cv2.imread(thumb_path)
                 ret, jpg = cv2.imencode(
                     ".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100]
@@ -46,8 +55,17 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 if ret:
                     self.tracked_review_items[id].append((thumb_time, jpg.tobytes()))
         else:
-            logger.info(
+            if id not in self.tracked_review_items:
-                f"would process review item end with {len(self.tracked_review_items[id])} items"
+                return
+
+            final_data = data["after"]
+            self.genai_client.generate_review_description(
+                {
+                    "camera": final_data["camera"],
+                    "objects": final_data["data"]["objects"] + final_data["data"]["sub_labels"],
+                    "zones": final_data["data"]["zones"],
+                },
+                [r[1] for r in self.tracked_review_items[id]],
             )
             self.tracked_review_items.pop(id)
 

frigate/genai/__init__.py

@@ -1,9 +1,10 @@
 """Generative AI module for Frigate."""
 
 import importlib
+import json
 import logging
 import os
-from typing import Optional
+from typing import Any, Optional
 
 from playhouse.shortcuts import model_to_dict
 
@@ -33,6 +34,28 @@ class GenAIClient:
         self.timeout = timeout
         self.provider = self._init_provider()
 
+    def generate_review_description(
+        self, review_data: dict[str, Any], thumbnails: list[bytes]
+    ) -> None:
+        """Generate a description for the review item activity."""
+        context_prompt = f"""
+        Here is additional context about the scene from a security camera:
+        {json.dumps(review_data, indent=2)}
+
+        Please analyze the image(s), which are in chronological order, strictly from the perspective of a security camera.
+        Your task is to provide a **neutral, factual, and objective description** of the scene.
+        **Do not make assumptions about intent, emotions, or potential threats.**
+        Focus solely on observable actions, visible entities, and the environment.
+
+        Describe:
+        - What is happening?
+        - Who or what is visible?
+        - What are the entities doing?
+        - What is the environmental context (e.g., time of day, weather, lighting)?
+    """
+        logger.info(f"processing {review_data}")
+        logger.info(f"Got GenAI review: {self._send(context_prompt, thumbnails)}")
+
     def generate_object_description(
         self,
         camera_config: CameraConfig,