Improve prompt and image selection

frigate/data_processing/post/review_descriptions.py

@@ -116,6 +116,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                     final_data,
                     thumbs,
                     camera_config.review.genai,
+                    list(self.config.model.merged_labelmap.values()),
                 ),
             ).start()
 
@@ -160,12 +161,18 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             return None
 
     def get_cache_frames(
-        self, camera: str, start_time: float, end_time: float
+        self,
+        camera: str,
+        start_time: float,
+        end_time: float,
+        desired_frame_count: int = 12,
     ) -> list[str]:
         preview_dir = os.path.join(CACHE_DIR, "preview_frames")
         file_start = f"preview_{camera}"
-        start_file = f"{file_start}-{start_time}.webp"
+
-        end_file = f"{file_start}-{end_time}.webp"
+        # 30 seconds padding to ensure that a frame before & after the activity is considered
+        start_file = f"{file_start}-{start_time - 30}.webp"
+        end_file = f"{file_start}-{end_time + 30}.webp"
         all_frames = []
 
         for file in sorted(os.listdir(preview_dir)):
@@ -181,13 +188,13 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             all_frames.append(os.path.join(preview_dir, file))
 
         frame_count = len(all_frames)
-        if frame_count <= 10:
+        if frame_count <= desired_frame_count:
             return all_frames
 
         selected_frames = []
-        step_size = (frame_count - 1) / 9
+        step_size = (frame_count - 1) / (desired_frame_count - 1)
 
-        for i in range(10):
+        for i in range(desired_frame_count):
             index = round(i * step_size)
             selected_frames.append(all_frames[index])
 
@@ -203,19 +210,33 @@ def run_analysis(
     final_data: dict[str, str],
     thumbs: list[bytes],
     genai_config: GenAIReviewConfig,
+    labelmap_objects: list[str],
 ) -> None:
     start = datetime.datetime.now().timestamp()
+    analytics_data = {
+        "id": final_data["id"],
+        "camera": camera,
+        "zones": final_data["data"]["zones"],
+        "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]),
+    }
+
+    objects = []
+    verified_objects = []
+
+    for label in set(final_data["data"]["objects"] + final_data["data"]["sub_labels"]):
+        if "-verified" in label:
+            continue
+
+        if label in labelmap_objects:
+            objects.append(label.replace("_", " ").title())
+        else:
+            verified_objects.append(label.replace("_", " ").title())
+
+    analytics_data["objects"] = objects
+    analytics_data["recognized_objects"] = verified_objects
+
     metadata = genai_client.generate_review_description(
-        {
+        analytics_data,
-            "id": final_data["id"],
-            "camera": camera,
-            "objects": list(
-                filter(lambda o: "-verified" not in o, final_data["data"]["objects"])
-            ),
-            "recognized_objects": final_data["data"]["sub_labels"],
-            "zones": final_data["data"]["zones"],
-            "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]),
-        },
         thumbs,
         genai_config.additional_concerns,
         genai_config.preferred_language,

frigate/genai/__init__.py

@@ -77,6 +77,7 @@ When forming your description:
 - Time of day should **increase suspicion only when paired with unusual or security-relevant behaviors**. Do not raise the threat level for common residential activities (e.g., residents walking pets, retrieving mail, gardening, playing with pets, supervising children) even at unusual hours, unless other suspicious indicators are present.
 - Focus on behaviors that are uncharacteristic of innocent activity: loitering without clear purpose, avoiding cameras, inspecting vehicles/doors, changing behavior when lights activate, scanning surroundings without an apparent benign reason.
 - **Benign context override**: If scanning or looking around is clearly part of an innocent activity (such as playing with a dog, gardening, supervising children, or watching for a pet), do not treat it as suspicious.
+- If any verified recognized object is a known trusted person (e.g., family member, employee, authorized visitor), assume the activity is normal **unless** there is clear evidence of immediate threat (threat level 2). In such cases, default to threat level 0.
 
 Your response MUST be a flat JSON object with:
 - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
@@ -90,10 +91,10 @@ Threat-level definitions:
 - 2 — Active or immediate threat: Breaking in, vandalism, aggression, weapon display.
 
 Sequence details:
-- Frame 1 = earliest, Frame 10 = latest
+- Frame 1 = earliest, Frame {len(thumbnails)} = latest
 - Activity occurred at {review_data["timestamp"].strftime("%A, %I:%M %p")}
-- Detected objects: {", ".join(obj.replace("_", " ").title() for obj in list(set(review_data["objects"])))}
+- Detected objects: {", ".join(review_data["objects"])}
-- Verified Recognized objects: {", ".join(r.title() for r in list(set(review_data["recognized_objects"]))) or "None"}
+- Verified Recognized objects: {", ".join(review_data["recognized_objects"]) or "None"}
 - Zones involved: {", ".join(z.replace("_", " ").title() for z in review_data["zones"]) or "None"}
 
 **IMPORTANT:**