diff --git a/frigate/config/camera/review.py b/frigate/config/camera/review.py index cb01e5107..3cb8d2bda 100644 --- a/frigate/config/camera/review.py +++ b/frigate/config/camera/review.py @@ -69,6 +69,10 @@ class GenAIReviewConfig(FrigateBaseModel): ) alerts: bool = Field(default=True, title="Enable GenAI for alerts.") detections: bool = Field(default=False, title="Enable GenAI for detections.") + additional_concerns: list[str] = Field( + default=[], + title="Additional concerns that GenAI should make note of on this camera.", + ) debug_save_thumbnails: bool = Field( default=False, title="Save thumbnails sent to generative AI for debugging purposes.", diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 59f47340d..19063dc9d 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -46,8 +46,9 @@ class ReviewDescriptionProcessor(PostProcessorApi): return camera = data["after"]["camera"] + camera_config = self.config.cameras[camera] - if not self.config.cameras[camera].review.genai.enabled: + if not camera_config.review.genai.enabled: return id = data["after"]["id"] @@ -59,12 +60,12 @@ class ReviewDescriptionProcessor(PostProcessorApi): if ( final_data["severity"] == "alert" - and not self.config.cameras[camera].review.genai.alerts + and not camera_config.review.genai.alerts ): return elif ( final_data["severity"] == "detection" - and not self.config.cameras[camera].review.genai.detections + and not camera_config.review.genai.detections ): return @@ -86,9 +87,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): if ret: thumbs.append(jpg.tobytes()) - if self.config.cameras[ - data["after"]["camera"] - ].review.genai.debug_save_thumbnails: + if camera_config.review.genai.debug_save_thumbnails: id = data["after"]["id"] Path(os.path.join(CLIPS_DIR, f"genai-requests/{id}")).mkdir( parents=True, exist_ok=True @@ -112,6 +111,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): camera, final_data, thumbs, + camera_config.review.genai.additional_concerns, ), ).start() @@ -161,6 +161,7 @@ def run_analysis( camera: str, final_data: dict[str, str], thumbs: list[bytes], + concerns: list[str], ) -> None: start = datetime.datetime.now().timestamp() metadata = genai_client.generate_review_description( @@ -172,6 +173,7 @@ def run_analysis( "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]), }, thumbs, + concerns, ) review_inference_speed.update(datetime.datetime.now().timestamp() - start) diff --git a/frigate/data_processing/post/types.py b/frigate/data_processing/post/types.py index d79a063b5..4e0534a8c 100644 --- a/frigate/data_processing/post/types.py +++ b/frigate/data_processing/post/types.py @@ -13,3 +13,7 @@ class ReviewMetadata(BaseModel): le=3, description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.", ) + other_concerns: list[str] | None = Field( + default=None, + description="Other concerns highlighted by the user that are observed.", + ) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 0e68f0562..b883268d7 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -37,39 +37,53 @@ class GenAIClient: self.provider = self._init_provider() def generate_review_description( - self, review_data: dict[str, Any], thumbnails: list[bytes] + self, + review_data: dict[str, Any], + thumbnails: list[bytes], + concerns: list[str], ) -> ReviewMetadata | None: """Generate a description for the review item activity.""" + if concerns: + concern_list = "\n - ".join(concerns) + other_concerns = f""" +- `other_concerns` (list of strings): Include a list of any of the following concerns that are occurring: + - {concern_list} +""" + + else: + other_concerns = None + context_prompt = f""" - Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera. +Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera. - Your task is to provide a **neutral, factual, and objective description** of the scene, while also: - - Clearly stating **what is happening** based on observable actions and movements. - - Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details. +Your task is to provide a **neutral, factual, and objective description** of the scene, while also: +- Clearly stating **what is happening** based on observable actions and movements. +- Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details. - When forming your description: - - **Facts first**: Describe the time, physical setting, people, and objects exactly as seen. - - **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door). - - Clearly separate certain facts (“A person is holding an object with horizontal rungs”) from reasonable inferences (“likely a ladder”). - - Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence. +When forming your description: +- **Facts first**: Describe the time, physical setting, people, and objects exactly as seen. +- **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door). +- Clearly separate certain facts (“A person is holding an object with horizontal rungs”) from reasonable inferences (“likely a ladder”). +- Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence. - Here is information already known: - - Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")} - - Detected objects: {review_data["objects"]} - - Recognized objects: {review_data["recognized_objects"]} - - Zones involved: {review_data["zones"]} +Here is information already known: +- Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")} +- Detected objects: {review_data["objects"]} +- Recognized objects: {review_data["recognized_objects"]} +- Zones involved: {review_data["zones"]} - Your response **MUST** be a flat JSON object with: - - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. - - `confidence` (float): A number 0–1 for overall confidence in the analysis. - - `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern: - - 0 = Normal activity is occurring - - 1 = Unusual but not overtly threatening - - 2 = Suspicious or potentially harmful - - 3 = Clear and immediate threat +Your response **MUST** be a flat JSON object with: +- `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. +- `confidence` (float): A number 0-1 for overall confidence in the analysis. +- `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern: + - 0 = Normal activity is occurring + - 1 = Unusual but not overtly threatening + - 2 = Suspicious or potentially harmful + - 3 = Clear and immediate threat +{other_concerns} - **IMPORTANT:** - - Values must be plain strings, floats, or integers — no nested objects, no extra commentary. +**IMPORTANT:** +- Values must be plain strings, floats, or integers — no nested objects, no extra commentary. """ logger.debug( f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}" diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx index c0fbaac54..cfa01b4f6 100644 --- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx +++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx @@ -76,20 +76,35 @@ export default function ReviewDetailDialog({ const aiAnalysis = useMemo(() => review?.data?.metadata, [review]); const aiThreatLevel = useMemo(() => { - if (!aiAnalysis?.potential_threat_level) { + console.log( + `${aiAnalysis?.potential_threat_level} || ${aiAnalysis?.other_concerns}`, + ); + + if ( + !aiAnalysis || + (!aiAnalysis.potential_threat_level && !aiAnalysis.other_concerns) + ) { return "None"; } + let concerns = ""; switch (aiAnalysis.potential_threat_level) { case ThreatLevel.UNUSUAL: - return "Unusual Activity"; + concerns = "• Unusual Activity\n"; + break; case ThreatLevel.SUSPICIOUS: - return "Suspicious Activity"; + concerns = "• Suspicious Activity\n"; + break; case ThreatLevel.DANGER: - return "Danger"; + concerns = "• Danger\n"; + break; } - return "Unknown"; + (aiAnalysis.other_concerns ?? []).forEach((c) => { + concerns += `• ${c}\n`; + }); + + return concerns || "None"; }, [aiAnalysis]); const hasMismatch = useMemo(() => { @@ -267,7 +282,7 @@ export default function ReviewDetailDialog({