diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 1a6ed502d..6807a3b8c 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -28,6 +28,8 @@ class ReviewDescriptionProcessor(PostProcessorApi): if data_type != PostProcessDataEnum.review: return + logger.info(f"processing review {data['type']} on {data['after']['camera']}") + id = data["after"]["id"] if data["type"] == "new" or data["type"] == "update": @@ -38,6 +40,13 @@ class ReviewDescriptionProcessor(PostProcessorApi): thumb_path = data["after"]["thumb_path"] if thumb_time and thumb_path: + if ( + len(self.tracked_review_items[id]) > 0 + and self.tracked_review_items[id][0] == thumb_time + ): + # we have already processed this thumbnail + return + thumb_data = cv2.imread(thumb_path) ret, jpg = cv2.imencode( ".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100] @@ -46,8 +55,17 @@ class ReviewDescriptionProcessor(PostProcessorApi): if ret: self.tracked_review_items[id].append((thumb_time, jpg.tobytes())) else: - logger.info( - f"would process review item end with {len(self.tracked_review_items[id])} items" + if id not in self.tracked_review_items: + return + + final_data = data["after"] + self.genai_client.generate_review_description( + { + "camera": final_data["camera"], + "objects": final_data["data"]["objects"] + final_data["data"]["sub_labels"], + "zones": final_data["data"]["zones"], + }, + [r[1] for r in self.tracked_review_items[id]], ) self.tracked_review_items.pop(id) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 4d8d7c9b6..2a9bcd151 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -1,9 +1,10 @@ """Generative AI module for Frigate.""" import importlib +import json import logging import os -from typing import Optional +from typing import Any, Optional from playhouse.shortcuts import model_to_dict @@ -33,6 +34,28 @@ class GenAIClient: self.timeout = timeout self.provider = self._init_provider() + def generate_review_description( + self, review_data: dict[str, Any], thumbnails: list[bytes] + ) -> None: + """Generate a description for the review item activity.""" + context_prompt = f""" + Here is additional context about the scene from a security camera: + {json.dumps(review_data, indent=2)} + + Please analyze the image(s), which are in chronological order, strictly from the perspective of a security camera. + Your task is to provide a **neutral, factual, and objective description** of the scene. + **Do not make assumptions about intent, emotions, or potential threats.** + Focus solely on observable actions, visible entities, and the environment. + + Describe: + - What is happening? + - Who or what is visible? + - What are the entities doing? + - What is the environmental context (e.g., time of day, weather, lighting)? + """ + logger.info(f"processing {review_data}") + logger.info(f"Got GenAI review: {self._send(context_prompt, thumbnails)}") + def generate_object_description( self, camera_config: CameraConfig,