diff --git a/frigate/data_processing/post/types.py b/frigate/data_processing/post/types.py index 44bb09fb02..beb746658c 100644 --- a/frigate/data_processing/post/types.py +++ b/frigate/data_processing/post/types.py @@ -4,20 +4,24 @@ from pydantic import BaseModel, ConfigDict, Field class ReviewMetadata(BaseModel): model_config = ConfigDict(extra="ignore", protected_namespaces=()) - title: str = Field(description="A concise title for the activity.") + title: str = Field( + description="A short title characterizing what took place and where, under 10 words." + ) scene: str = Field( - description="A comprehensive description of the setting and entities, including relevant context and plausible inferences if supported by visual evidence." + description="A chronological narrative of what happens from start to finish." ) shortSummary: str = Field( - description="A brief 2-sentence summary of the scene, suitable for notifications. Should capture the key activity and context without full detail." + description="A brief 2-sentence summary of the scene, suitable for notifications." ) confidence: float = Field( - description="A float between 0 and 1 representing your overall confidence in this analysis." + ge=0.0, + le=1.0, + description="Confidence in the analysis, from 0 to 1.", ) potential_threat_level: int = Field( ge=0, - le=3, - description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.", + le=2, + description="Threat level: 0 = normal, 1 = suspicious, 2 = critical threat.", ) other_concerns: list[str] | None = Field( default=None, diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index de5289c64e..172a0af083 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -146,7 +146,27 @@ Each line represents a detection state, not necessarily unique individuals. Pare ) as f: f.write(context_prompt) - response = self._send(context_prompt, thumbnails) + # Build JSON schema for structured output from ReviewMetadata model + schema = ReviewMetadata.model_json_schema() + schema.get("properties", {}).pop("time", None) + + if "time" in schema.get("required", []): + schema["required"].remove("time") + if not concerns: + schema.get("properties", {}).pop("other_concerns", None) + if "other_concerns" in schema.get("required", []): + schema["required"].remove("other_concerns") + + response_format = { + "type": "json_schema", + "json_schema": { + "name": "review_metadata", + "strict": True, + "schema": schema, + }, + } + + response = self._send(context_prompt, thumbnails, response_format) if debug_save and response: with open( @@ -290,7 +310,12 @@ Guidelines: """Initialize the client.""" return None - def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + def _send( + self, + prompt: str, + images: list[bytes], + response_format: Optional[dict] = None, + ) -> Optional[str]: """Submit a request to the provider.""" return None diff --git a/frigate/genai/llama_cpp.py b/frigate/genai/llama_cpp.py index f9c2517906..87443ac4fe 100644 --- a/frigate/genai/llama_cpp.py +++ b/frigate/genai/llama_cpp.py @@ -57,7 +57,12 @@ class LlamaCppClient(GenAIClient): else None ) - def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + def _send( + self, + prompt: str, + images: list[bytes], + response_format: Optional[dict] = None, + ) -> Optional[str]: """Submit a request to llama.cpp server.""" if self.provider is None: logger.warning( @@ -96,6 +101,9 @@ class LlamaCppClient(GenAIClient): **self.provider_options, } + if response_format: + payload["response_format"] = response_format + response = requests.post( f"{self.provider}/v1/chat/completions", json=payload,