From ede8b74371c006c889f4ad17ab9594ae4043a170 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 19 Mar 2026 10:39:24 -0600 Subject: [PATCH] Review Summary Optimizations (#22533) * Use different association method * Clarify * Remove extra details from ollama schema * Fix Gemini Chat * Fix incorrect instructions * Improve name handling * Change order of information for llama.cpp * Simplify prompt * Fix formatting --- .../post/review_descriptions.py | 2 +- frigate/genai/__init__.py | 10 +++--- frigate/genai/gemini.py | 4 +-- frigate/genai/llama_cpp.py | 13 ++++--- frigate/genai/ollama.py | 35 ++++++++++++++++++- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 0a2754468..0d1223219 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -521,7 +521,7 @@ def run_analysis( for i, verified_label in enumerate(final_data["data"]["verified_objects"]): object_type = verified_label.replace("-verified", "").replace("_", " ") name = titlecase(sub_labels_list[i].replace("_", " ")) - unified_objects.append(f"{name} ({object_type})") + unified_objects.append(f"{name} ← {object_type}") for label in objects_list: if "-verified" in label: diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 3e939d28d..f799931ec 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -106,7 +106,7 @@ When forming your description: ## Response Field Guidelines Respond with a JSON object matching the provided schema. Field-specific guidance: -- `scene`: Describe how the sequence begins, then the progression of events — all significant movements and actions in order. For example, if a vehicle arrives and then a person exits, describe both sequentially. Your description should align with and support the threat level you assign. +- `scene`: Describe how the sequence begins, then the progression of events — all significant movements and actions in order. For example, if a vehicle arrives and then a person exits, describe both sequentially. Always use subject names from "Objects in Scene" — do not replace named subjects with generic terms like "a person" or "the individual". Your description should align with and support the threat level you assign. - `title`: Characterize **what took place and where** — interpret the overall purpose or outcome, do not simply compress the scene description into fewer words. Include the relevant location (zone, area, or entry point). Always include subject names from "Objects in Scene" — do not replace named subjects with generic terms. No editorial qualifiers like "routine" or "suspicious." - `potential_threat_level`: Must be consistent with your scene description and the activity patterns above. {get_concern_prompt()} @@ -120,9 +120,7 @@ Respond with a JSON object matching the provided schema. Field-specific guidance ## Objects in Scene -Each line represents a detection state, not necessarily unique individuals. Parentheses indicate object type or category, use only the name/label in your response, not the parentheses. - -**CRITICAL: When you see both recognized and unrecognized entries of the same type (e.g., "Joe (person)" and "Person"), visually count how many distinct people/objects you actually see based on appearance and clothing. If you observe only ONE person throughout the sequence, use ONLY the recognized name (e.g., "Joe"). The same person may be recognized in some frames but not others. Only describe both if you visually see MULTIPLE distinct people with clearly different appearances.** +Each line represents a detection state, not necessarily unique individuals. The `←` symbol separates a recognized subject's name from their object type — use only the name (before the `←`) in your response, not the type after it. The same subject may appear across multiple lines if detected multiple times. **Note: Unidentified objects (without names) are NOT indicators of suspicious activity—they simply mean the system hasn't identified that object.** {get_objects_list()} @@ -188,8 +186,8 @@ Each line represents a detection state, not necessarily unique individuals. Pare if metadata.confidence > 1.0: metadata.confidence = min(metadata.confidence / 100.0, 1.0) - # If any verified objects (contain parentheses with name), set to 0 - if any("(" in obj for obj in review_data["unified_objects"]): + # If any verified objects (contain ← separator), set to 0 + if any("←" in obj for obj in review_data["unified_objects"]): metadata.potential_threat_level = 0 metadata.time = review_data["start"] diff --git a/frigate/genai/gemini.py b/frigate/genai/gemini.py index 9e01192dc..f32d37e80 100644 --- a/frigate/genai/gemini.py +++ b/frigate/genai/gemini.py @@ -397,13 +397,13 @@ class GeminiClient(GenAIClient): tool_calls_by_index: dict[int, dict[str, Any]] = {} finish_reason = "stop" - response = self.provider.models.generate_content_stream( + stream = await self.provider.aio.models.generate_content_stream( model=self.genai_config.model, contents=gemini_messages, config=types.GenerateContentConfig(**config_params), ) - async for chunk in response: + async for chunk in stream: if not chunk or not chunk.candidates: continue diff --git a/frigate/genai/llama_cpp.py b/frigate/genai/llama_cpp.py index ac698b3b6..48ea9747c 100644 --- a/frigate/genai/llama_cpp.py +++ b/frigate/genai/llama_cpp.py @@ -64,7 +64,12 @@ class LlamaCppClient(GenAIClient): return None try: - content = [] + content = [ + { + "type": "text", + "text": prompt, + } + ] for image in images: encoded_image = base64.b64encode(image).decode("utf-8") content.append( @@ -75,12 +80,6 @@ class LlamaCppClient(GenAIClient): }, } ) - content.append( - { - "type": "text", - "text": prompt, - } - ) # Build request payload with llama.cpp native options payload = { diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py index 90bf3f05e..74c5b7a89 100644 --- a/frigate/genai/ollama.py +++ b/frigate/genai/ollama.py @@ -53,6 +53,39 @@ class OllamaClient(GenAIClient): logger.warning("Error initializing Ollama: %s", str(e)) return None + @staticmethod + def _clean_schema_for_ollama(schema: dict) -> dict: + """Strip Pydantic metadata from a JSON schema for Ollama compatibility. + + Ollama's grammar-based constrained generation works best with minimal + schemas. Pydantic adds title/description/constraint fields that can + cause the grammar generator to silently skip required fields. + """ + STRIP_KEYS = { + "title", + "description", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + } + result = {} + for key, value in schema.items(): + if key in STRIP_KEYS: + continue + if isinstance(value, dict): + result[key] = OllamaClient._clean_schema_for_ollama(value) + elif isinstance(value, list): + result[key] = [ + OllamaClient._clean_schema_for_ollama(item) + if isinstance(item, dict) + else item + for item in value + ] + else: + result[key] = value + return result + def _send( self, prompt: str, @@ -73,7 +106,7 @@ class OllamaClient(GenAIClient): if response_format and response_format.get("type") == "json_schema": schema = response_format.get("json_schema", {}).get("schema") if schema: - ollama_options["format"] = schema + ollama_options["format"] = self._clean_schema_for_ollama(schema) result = self.provider.generate( self.genai_config.model, prompt,