Use contextual information from other cameras to inform report summary

frigate/data_processing/post/review_descriptions.py

@@ -209,10 +209,22 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             logger.debug(
                 f"Found GenAI Review Summary request for {start_ts} to {end_ts}"
             )
-            items: list[dict[str, Any]] = [
-                r["data"]["metadata"]
+
+            # Query all review segments with camera and time information
+            segments: list[dict[str, Any]] = [
+                {
+                    "camera": r["camera"],
+                    "start_time": r["start_time"],
+                    "end_time": r["end_time"],
+                    "metadata": r["data"]["metadata"],
+                }
                 for r in (
-                    ReviewSegment.select(ReviewSegment.data)
+                    ReviewSegment.select(
+                        ReviewSegment.camera,
+                        ReviewSegment.start_time,
+                        ReviewSegment.end_time,
+                        ReviewSegment.data,
+                    )
                     .where(
                         (ReviewSegment.data["metadata"].is_null(False))
                         & (ReviewSegment.start_time < end_ts)
@@ -224,21 +236,64 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 )
             ]
 
-            if len(items) == 0:
+            if len(segments) == 0:
                 logger.debug("No review items with metadata found during time period")
-                return "No activity was found during this time."
+                return "No activity was found during this time period."
 
-            important_items = list(
-                filter(
-                    lambda item: item.get("potential_threat_level", 0) > 0
-                    or item.get("other_concerns"),
-                    items,
-                )
-            )
+            # Identify primary items (important items that need review)
+            primary_segments = [
+                seg
+                for seg in segments
+                if seg["metadata"].get("potential_threat_level", 0) > 0
+                or seg["metadata"].get("other_concerns")
+            ]
 
-            if not important_items:
+            if not primary_segments:
                 return "No concerns were found during this time period."
 
+            # For each primary segment, find overlapping contextual items from other cameras
+            all_items_for_summary = []
+
+            for primary_seg in primary_segments:
+                # Add the primary item with marker
+                primary_item = copy.deepcopy(primary_seg["metadata"])
+                primary_item["_is_primary"] = True
+                primary_item["_camera"] = primary_seg["camera"]
+                all_items_for_summary.append(primary_item)
+
+                # Find overlapping contextual items from other cameras
+                primary_start = primary_seg["start_time"]
+                primary_end = primary_seg["end_time"]
+                primary_camera = primary_seg["camera"]
+
+                for seg in segments:
+                    if seg["camera"] == primary_camera:
+                        continue
+
+                    if seg in primary_segments:
+                        continue
+
+                    seg_start = seg["start_time"]
+                    seg_end = seg["end_time"]
+
+                    if seg_start < primary_end and primary_start < seg_end:
+                        contextual_item = copy.deepcopy(seg["metadata"])
+                        contextual_item["_is_primary"] = False
+                        contextual_item["_camera"] = seg["camera"]
+                        contextual_item["_related_to_camera"] = primary_camera
+
+                        if not any(
+                            item.get("_camera") == seg["camera"]
+                            and item.get("time") == contextual_item.get("time")
+                            for item in all_items_for_summary
+                        ):
+                            all_items_for_summary.append(contextual_item)
+
+            logger.debug(
+                f"Summary includes {len(primary_segments)} primary items and "
+                f"{len(all_items_for_summary) - len(primary_segments)} contextual items"
+            )
+
             if self.config.review.genai.debug_save_thumbnails:
                 Path(
                     os.path.join(CLIPS_DIR, "genai-requests", f"{start_ts}-{end_ts}")
@@ -247,7 +302,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             return self.genai_client.generate_review_summary(
                 start_ts,
                 end_ts,
-                important_items,
+                all_items_for_summary,
                 self.config.review.genai.debug_save_thumbnails,
             )
         else:

frigate/genai/__init__.py

@@ -185,10 +185,17 @@ Each line represents a detection state, not necessarily unique individuals. Pare
         timeline_summary_prompt = f"""
 You are a security officer.
 Time range: {time_range}.
-Input: JSON list with "title", "scene", "confidence", "potential_threat_level" (1-2), "other_concerns".
+Input: JSON list with "title", "scene", "confidence", "potential_threat_level" (0-2), "other_concerns", "_is_primary", "_camera".
 
 Task: Write a concise, human-presentable security report in markdown format.
 
+Important - Understanding Primary vs Contextual Items:
+- Items with "_is_primary": true are events that require review or attention
+- Items with "_is_primary": false are additional context from other camera perspectives that occurred at the same time
+- Contextual items (threat level 0) provide important background information to help understand primary events
+- Use contextual items to provide a more complete and accurate understanding of what actually happened
+- The "_camera" field indicates which camera captured each event
+
 Rules for the report:
 
 - Title & overview
@@ -198,31 +205,37 @@ Rules for the report:
 
 - Event details
   - Present events in chronological order as a bullet list.
+  - **When primary and contextual items occur at overlapping times, COMBINE them into a single comprehensive bullet.**
+    - Use the contextual information to provide a more complete picture of what happened.
   - **If multiple events occur within the same minute or overlapping time range, COMBINE them into a single bullet.**
     - Summarize the distinct activities as sub-points under the shared timestamp.
-  - If no timestamp is given, preserve order but label as “Time not specified.”
+  - If no timestamp is given, preserve order but label as "Time not specified."
   - Use bold timestamps for clarity.
+  - Include camera names when multiple cameras captured related activity.
   - Group bullets under subheadings when multiple events fall into the same category (e.g., Vehicle Activity, Porch Activity, Unusual Behavior).
 
 - Threat levels
-  - Always show the threat level for each event using these labels:
+  - Always show the threat level for PRIMARY events using these labels:
     - Threat level 0: "Normal"
     - Threat level 1: "Needs review"
     - Threat level 2: "Security concern"
   - Format as (threat level: Normal), (threat level: Needs review), or (threat level: Security concern).
+  - When contextual items help explain a primary event, adjust your description to reflect the fuller context.
   - If multiple events at the same time share the same threat level, only state it once.
 
 - Final assessment
   - End with a Final Assessment section.
-  - If all events are threat level 0:
+  - If all primary events are threat level 0 or explained by contextual items:
     Final assessment: Only normal residential activity observed during this period.
   - If threat level 1 events are present:
     Final assessment: Some activity requires review but no security concerns identified.
   - If threat level 2 events are present, clearly summarize them as Security concerns requiring immediate attention.
+  - Note if contextual information from other cameras provided clarity about seemingly suspicious activity.
 
 - Conciseness
   - Do not repeat benign clothing/appearance details unless they distinguish individuals.
   - Summarize similar routine events instead of restating full scene descriptions.
+  - When contextual items simply confirm what the primary event already described, keep the description brief.
 """
 
         for item in segments: