diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 60788e504..23317baba 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -209,10 +209,22 @@ class ReviewDescriptionProcessor(PostProcessorApi): logger.debug( f"Found GenAI Review Summary request for {start_ts} to {end_ts}" ) - items: list[dict[str, Any]] = [ - r["data"]["metadata"] + + # Query all review segments with camera and time information + segments: list[dict[str, Any]] = [ + { + "camera": r["camera"], + "start_time": r["start_time"], + "end_time": r["end_time"], + "metadata": r["data"]["metadata"], + } for r in ( - ReviewSegment.select(ReviewSegment.data) + ReviewSegment.select( + ReviewSegment.camera, + ReviewSegment.start_time, + ReviewSegment.end_time, + ReviewSegment.data, + ) .where( (ReviewSegment.data["metadata"].is_null(False)) & (ReviewSegment.start_time < end_ts) @@ -224,21 +236,64 @@ class ReviewDescriptionProcessor(PostProcessorApi): ) ] - if len(items) == 0: + if len(segments) == 0: logger.debug("No review items with metadata found during time period") - return "No activity was found during this time." + return "No activity was found during this time period." - important_items = list( - filter( - lambda item: item.get("potential_threat_level", 0) > 0 - or item.get("other_concerns"), - items, - ) - ) + # Identify primary items (important items that need review) + primary_segments = [ + seg + for seg in segments + if seg["metadata"].get("potential_threat_level", 0) > 0 + or seg["metadata"].get("other_concerns") + ] - if not important_items: + if not primary_segments: return "No concerns were found during this time period." + # For each primary segment, find overlapping contextual items from other cameras + all_items_for_summary = [] + + for primary_seg in primary_segments: + # Add the primary item with marker + primary_item = copy.deepcopy(primary_seg["metadata"]) + primary_item["_is_primary"] = True + primary_item["_camera"] = primary_seg["camera"] + all_items_for_summary.append(primary_item) + + # Find overlapping contextual items from other cameras + primary_start = primary_seg["start_time"] + primary_end = primary_seg["end_time"] + primary_camera = primary_seg["camera"] + + for seg in segments: + if seg["camera"] == primary_camera: + continue + + if seg in primary_segments: + continue + + seg_start = seg["start_time"] + seg_end = seg["end_time"] + + if seg_start < primary_end and primary_start < seg_end: + contextual_item = copy.deepcopy(seg["metadata"]) + contextual_item["_is_primary"] = False + contextual_item["_camera"] = seg["camera"] + contextual_item["_related_to_camera"] = primary_camera + + if not any( + item.get("_camera") == seg["camera"] + and item.get("time") == contextual_item.get("time") + for item in all_items_for_summary + ): + all_items_for_summary.append(contextual_item) + + logger.debug( + f"Summary includes {len(primary_segments)} primary items and " + f"{len(all_items_for_summary) - len(primary_segments)} contextual items" + ) + if self.config.review.genai.debug_save_thumbnails: Path( os.path.join(CLIPS_DIR, "genai-requests", f"{start_ts}-{end_ts}") @@ -247,7 +302,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): return self.genai_client.generate_review_summary( start_ts, end_ts, - important_items, + all_items_for_summary, self.config.review.genai.debug_save_thumbnails, ) else: diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index d86e3cbc5..2c9a9f027 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -185,10 +185,17 @@ Each line represents a detection state, not necessarily unique individuals. Pare timeline_summary_prompt = f""" You are a security officer. Time range: {time_range}. -Input: JSON list with "title", "scene", "confidence", "potential_threat_level" (1-2), "other_concerns". +Input: JSON list with "title", "scene", "confidence", "potential_threat_level" (0-2), "other_concerns", "_is_primary", "_camera". Task: Write a concise, human-presentable security report in markdown format. +Important - Understanding Primary vs Contextual Items: +- Items with "_is_primary": true are events that require review or attention +- Items with "_is_primary": false are additional context from other camera perspectives that occurred at the same time +- Contextual items (threat level 0) provide important background information to help understand primary events +- Use contextual items to provide a more complete and accurate understanding of what actually happened +- The "_camera" field indicates which camera captured each event + Rules for the report: - Title & overview @@ -198,31 +205,37 @@ Rules for the report: - Event details - Present events in chronological order as a bullet list. + - **When primary and contextual items occur at overlapping times, COMBINE them into a single comprehensive bullet.** + - Use the contextual information to provide a more complete picture of what happened. - **If multiple events occur within the same minute or overlapping time range, COMBINE them into a single bullet.** - Summarize the distinct activities as sub-points under the shared timestamp. - - If no timestamp is given, preserve order but label as “Time not specified.” + - If no timestamp is given, preserve order but label as "Time not specified." - Use bold timestamps for clarity. + - Include camera names when multiple cameras captured related activity. - Group bullets under subheadings when multiple events fall into the same category (e.g., Vehicle Activity, Porch Activity, Unusual Behavior). - Threat levels - - Always show the threat level for each event using these labels: + - Always show the threat level for PRIMARY events using these labels: - Threat level 0: "Normal" - Threat level 1: "Needs review" - Threat level 2: "Security concern" - Format as (threat level: Normal), (threat level: Needs review), or (threat level: Security concern). + - When contextual items help explain a primary event, adjust your description to reflect the fuller context. - If multiple events at the same time share the same threat level, only state it once. - Final assessment - End with a Final Assessment section. - - If all events are threat level 0: + - If all primary events are threat level 0 or explained by contextual items: Final assessment: Only normal residential activity observed during this period. - If threat level 1 events are present: Final assessment: Some activity requires review but no security concerns identified. - If threat level 2 events are present, clearly summarize them as Security concerns requiring immediate attention. + - Note if contextual information from other cameras provided clarity about seemingly suspicious activity. - Conciseness - Do not repeat benign clothing/appearance details unless they distinguish individuals. - Summarize similar routine events instead of restating full scene descriptions. + - When contextual items simply confirm what the primary event already described, keep the description brief. """ for item in segments: