From 6e447543a154c016cf8e1bd042dcae39e4702a23 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 11 Aug 2025 18:18:51 -0600 Subject: [PATCH] Adjust prompt to be more specific --- frigate/genai/__init__.py | 41 +++++++++++-------- frigate/genai/ollama.py | 4 +- .../overlay/detail/ReviewDetailDialog.tsx | 3 -- web/src/types/review.ts | 5 +-- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 39e80a0dd..2e2ac5842 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -62,31 +62,36 @@ class GenAIClient: context_prompt = f""" Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera. -Your task is to provide a **neutral, factual, and objective description** of the scene, while also: -- Clearly stating **what is happening** based on observable actions and movements. -- Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details. +Your task is to provide a clear, security-focused description of the scene that: +1. States exactly what is happening based on observable actions and movements. +2. Identifies and emphasizes behaviors that match patterns of suspicious activity. +3. Assigns a potential_threat_level based on the definitions below, applying them consistently. + +Facts come first, but identifying security risks is the primary goal. When forming your description: -- **Facts first**: Describe the time, physical setting, people, and objects exactly as seen. -- **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door). -- Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence. +- Describe the time, physical setting, people, and objects exactly as seen. Include any observable environmental changes (e.g., lighting changes triggered by activity). +- Time of day should **increase suspicion only when paired with unusual or security-relevant behaviors**. Do not raise the threat level for common residential activities (e.g., residents walking pets, retrieving mail) even at unusual hours, unless other suspicious indicators are present. +- Context: Note plausible purposes or activities, especially considering location zones and time of day. +- Focus on behaviors that are uncharacteristic of innocent activity: loitering without clear purpose, avoiding cameras, inspecting vehicles/doors, changing behavior when lights activate. + +Your response MUST be a flat JSON object with: +- `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. +- `confidence` (float): 0–1 confidence in the analysis. +- `potential_threat_level` (integer): 0, 1, or 2 as defined below. +{concern_prompt} + +Threat-level definitions: +- 0 — Typical or expected activity for this location/time (includes residents, guests, or known animals moving normally, even at unusual hours). +- 1 — Unusual or suspicious activity: At least one security-relevant behavior present (e.g., loitering, scanning surroundings, changing behavior after security light activation, inspecting vehicles/entry points), but no active threat observed. +- 2 — Active or immediate threat: Breaking in, vandalism, aggression, weapon display. Here is information already known: - Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")} -- Detected objects: {review_data["objects"]} -- Recognized objects: {review_data["recognized_objects"]} +- Detected objects: {list(set(review_data["objects"]))} +- Recognized objects: {list(set(review_data["recognized_objects"])) or "None"} - Zones involved: {review_data["zones"]} -Your response **MUST** be a flat JSON object with: -- `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences. -- `confidence` (float): A number 0-1 for overall confidence in the analysis. -- `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern: - - 0 = Normal activity is occurring - - 1 = Unusual but not overtly threatening - - 2 = Suspicious or potentially harmful - - 3 = Clear and immediate threat -{concern_prompt} - **IMPORTANT:** - Values must be plain strings, floats, or integers — no nested objects, no extra commentary. {language_prompt} diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py index ea88579cb..c392fadb9 100644 --- a/frigate/genai/ollama.py +++ b/frigate/genai/ollama.py @@ -47,8 +47,8 @@ class OllamaClient(GenAIClient): result = self.provider.generate( self.genai_config.model, prompt, - images=images, - options={"keep_alive": "1h"}, + images=images if images else None, + keep_alive="1h", ) return result["response"].strip() except (TimeoutException, ResponseError) as e: diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx index 70eaa5872..1f647fb04 100644 --- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx +++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx @@ -85,9 +85,6 @@ export default function ReviewDetailDialog({ let concerns = ""; switch (aiAnalysis.potential_threat_level) { - case ThreatLevel.UNUSUAL: - concerns = "• Unusual Activity\n"; - break; case ThreatLevel.SUSPICIOUS: concerns = "• Suspicious Activity\n"; break; diff --git a/web/src/types/review.ts b/web/src/types/review.ts index bf2af7d1c..e0385ae83 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -81,7 +81,6 @@ export type ConsolidatedSegmentData = { export type TimelineZoomDirection = "in" | "out" | null; export enum ThreatLevel { - UNUSUAL = 1, - SUSPICIOUS = 2, - DANGER = 3, + SUSPICIOUS = 1, + DANGER = 2, }