Use json format for llama.cpp

frigate/data_processing/post/types.py

@@ -4,20 +4,24 @@ from pydantic import BaseModel, ConfigDict, Field
 class ReviewMetadata(BaseModel):
     model_config = ConfigDict(extra="ignore", protected_namespaces=())
 
-    title: str = Field(description="A concise title for the activity.")
+    title: str = Field(
+        description="A short title characterizing what took place and where, under 10 words."
+    )
     scene: str = Field(
-        description="A comprehensive description of the setting and entities, including relevant context and plausible inferences if supported by visual evidence."
+        description="A chronological narrative of what happens from start to finish."
     )
     shortSummary: str = Field(
-        description="A brief 2-sentence summary of the scene, suitable for notifications. Should capture the key activity and context without full detail."
+        description="A brief 2-sentence summary of the scene, suitable for notifications."
     )
     confidence: float = Field(
-        description="A float between 0 and 1 representing your overall confidence in this analysis."
+        ge=0.0,
+        le=1.0,
+        description="Confidence in the analysis, from 0 to 1.",
     )
     potential_threat_level: int = Field(
         ge=0,
-        le=3,
+        le=2,
-        description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.",
+        description="Threat level: 0 = normal, 1 = suspicious, 2 = critical threat.",
     )
     other_concerns: list[str] | None = Field(
         default=None,

frigate/genai/__init__.py

@@ -146,7 +146,27 @@ Each line represents a detection state, not necessarily unique individuals. Pare
             ) as f:
                 f.write(context_prompt)
 
-        response = self._send(context_prompt, thumbnails)
+        # Build JSON schema for structured output from ReviewMetadata model
+        schema = ReviewMetadata.model_json_schema()
+        schema.get("properties", {}).pop("time", None)
+
+        if "time" in schema.get("required", []):
+            schema["required"].remove("time")
+        if not concerns:
+            schema.get("properties", {}).pop("other_concerns", None)
+            if "other_concerns" in schema.get("required", []):
+                schema["required"].remove("other_concerns")
+
+        response_format = {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "review_metadata",
+                "strict": True,
+                "schema": schema,
+            },
+        }
+
+        response = self._send(context_prompt, thumbnails, response_format)
 
         if debug_save and response:
             with open(
@@ -290,7 +310,12 @@ Guidelines:
         """Initialize the client."""
         return None
 
-    def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
+    def _send(
+        self,
+        prompt: str,
+        images: list[bytes],
+        response_format: Optional[dict] = None,
+    ) -> Optional[str]:
         """Submit a request to the provider."""
         return None
 

frigate/genai/llama_cpp.py

@@ -57,7 +57,12 @@ class LlamaCppClient(GenAIClient):
             else None
         )
 
-    def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
+    def _send(
+        self,
+        prompt: str,
+        images: list[bytes],
+        response_format: Optional[dict] = None,
+    ) -> Optional[str]:
         """Submit a request to llama.cpp server."""
         if self.provider is None:
             logger.warning(
@@ -96,6 +101,9 @@ class LlamaCppClient(GenAIClient):
                 **self.provider_options,
             }
 
+            if response_format:
+                payload["response_format"] = response_format
+
             response = requests.post(
                 f"{self.provider}/v1/chat/completions",
                 json=payload,