Switch to a feature-based roles so it is easier to choose models for different tasks

frigate/api/chat.py

@@ -520,45 +520,14 @@ async def _execute_get_live_context(
             "detections": list(tracked_objects_dict.values()),
         }
 
-        # Grab live frame and handle based on provider configuration
+        # Grab live frame when the chat model supports vision
         image_url = await _get_live_frame_image_url(request, camera, allowed_cameras)
         if image_url:
-            genai_manager = request.app.genai_manager
+            chat_client = request.app.genai_manager.chat_client
-            if genai_manager.tool_client is genai_manager.vision_client:
+            if chat_client is not None and chat_client.supports_vision:
-                # Same provider handles both roles — pass image URL so it can
+                # Pass image URL so it can be injected as a user message
-                # be injected as a user message (images can't be in tool results)
+                # (images can't be in tool results)
                 result["_image_url"] = image_url
-            elif genai_manager.vision_client is not None:
-                # Separate vision provider — have it describe the image,
-                # providing detection context so it knows what to focus on
-                frame_bytes = _decode_data_url(image_url)
-                if frame_bytes:
-                    detections = result.get("detections", [])
-                    if detections:
-                        detection_lines = []
-                        for d in detections:
-                            parts = [d.get("label", "unknown")]
-                            if d.get("sub_label"):
-                                parts.append(f"({d['sub_label']})")
-                            if d.get("zones"):
-                                parts.append(f"in {', '.join(d['zones'])}")
-                            detection_lines.append(" ".join(parts))
-                        context = (
-                            "The following objects are currently being tracked: "
-                            + "; ".join(detection_lines)
-                            + "."
-                        )
-                    else:
-                        context = "No objects are currently being tracked."
-
-                    description = genai_manager.vision_client._send(
-                        f"Describe what you see in this security camera image. "
-                        f"{context} Focus on the scene, any visible activity, "
-                        f"and details about the tracked objects.",
-                        [frame_bytes],
-                    )
-                    if description:
-                        result["image_description"] = description
 
         return result
 
@@ -609,17 +578,6 @@ async def _get_live_frame_image_url(
         return None
 
 
-def _decode_data_url(data_url: str) -> Optional[bytes]:
-    """Decode a base64 data URL to raw bytes."""
-    try:
-        # Format: data:image/jpeg;base64,<data>
-        _, encoded = data_url.split(",", 1)
-        return base64.b64decode(encoded)
-    except (ValueError, Exception) as e:
-        logger.debug("Failed to decode data URL: %s", e)
-        return None
-
-
 async def _execute_set_camera_state(
     request: Request,
     arguments: Dict[str, Any],
@@ -734,9 +692,9 @@ async def _execute_start_camera_watch(
     await require_camera_access(camera, request=request)
 
     genai_manager = request.app.genai_manager
-    vision_client = genai_manager.vision_client or genai_manager.tool_client
+    chat_client = genai_manager.chat_client
-    if vision_client is None:
+    if chat_client is None or not chat_client.supports_vision:
-        return {"error": "No vision/GenAI provider configured."}
+        return {"error": "VLM watch requires a chat model with vision support."}
 
     try:
         job_id = start_vlm_watch_job(
@@ -1070,7 +1028,7 @@ async def chat_completion(
     6. Repeats until final answer
     7. Returns response to user
     """
-    genai_client = request.app.genai_manager.tool_client
+    genai_client = request.app.genai_manager.chat_client
     if not genai_client:
         return JSONResponse(
             content={
@@ -1381,12 +1339,12 @@ async def start_vlm_monitor(
 
     await require_camera_access(body.camera, request=request)
 
-    vision_client = genai_manager.vision_client or genai_manager.tool_client
+    chat_client = genai_manager.chat_client
-    if vision_client is None:
+    if chat_client is None or not chat_client.supports_vision:
         return JSONResponse(
             content={
                 "success": False,
-                "message": "No vision/GenAI provider configured.",
+                "message": "VLM watch requires a chat model with vision support.",
             },
             status_code=400,
         )

frigate/api/review.py

@@ -746,7 +746,7 @@ async def set_not_reviewed(
     description="Use GenAI to summarize review items over a period of time.",
 )
 def generate_review_summary(request: Request, start_ts: float, end_ts: float):
-    if not request.app.genai_manager.vision_client:
+    if not request.app.genai_manager.description_client:
         return JSONResponse(
             content=(
                 {

frigate/config/camera/genai.py

@@ -18,8 +18,8 @@ class GenAIProviderEnum(str, Enum):
 
 
 class GenAIRoleEnum(str, Enum):
-    tools = "tools"
+    chat = "chat"
-    vision = "vision"
+    descriptions = "descriptions"
     embeddings = "embeddings"
 
 
@@ -49,11 +49,11 @@ class GenAIConfig(FrigateBaseModel):
     roles: list[GenAIRoleEnum] = Field(
         default_factory=lambda: [
             GenAIRoleEnum.embeddings,
-            GenAIRoleEnum.vision,
+            GenAIRoleEnum.descriptions,
-            GenAIRoleEnum.tools,
+            GenAIRoleEnum.chat,
         ],
         title="Roles",
-        description="GenAI roles (tools, vision, embeddings); one provider per role.",
+        description="GenAI roles (chat, descriptions, embeddings); one provider per role.",
     )
     provider_options: dict[str, Any] = Field(
         default={},

frigate/embeddings/maintainer.py

@@ -202,7 +202,7 @@ class EmbeddingMaintainer(threading.Thread):
         # post processors
         self.post_processors: list[PostProcessorApi] = []
 
-        if self.genai_manager.vision_client is not None and any(
+        if self.genai_manager.description_client is not None and any(
             c.review.genai.enabled_in_config for c in self.config.cameras.values()
         ):
             self.post_processors.append(
@@ -210,7 +210,7 @@ class EmbeddingMaintainer(threading.Thread):
                     self.config,
                     self.requestor,
                     self.metrics,
-                    self.genai_manager.vision_client,
+                    self.genai_manager.description_client,
                 )
             )
 
@@ -248,7 +248,7 @@ class EmbeddingMaintainer(threading.Thread):
             )
             self.post_processors.append(semantic_trigger_processor)
 
-        if self.genai_manager.vision_client is not None and any(
+        if self.genai_manager.description_client is not None and any(
             c.objects.genai.enabled_in_config for c in self.config.cameras.values()
         ):
             self.post_processors.append(
@@ -257,7 +257,7 @@ class EmbeddingMaintainer(threading.Thread):
                     self.embeddings,
                     self.requestor,
                     self.metrics,
-                    self.genai_manager.vision_client,
+                    self.genai_manager.description_client,
                     semantic_trigger_processor,
                 )
             )

frigate/genai/__init__.py

@@ -320,6 +320,15 @@ Guidelines:
         """Submit a request to the provider."""
         return None
 
+    @property
+    def supports_vision(self) -> bool:
+        """Whether the model supports vision/image input.
+
+        Defaults to True for cloud providers. Providers that can detect
+        capability at runtime (e.g. llama.cpp) should override this.
+        """
+        return True
+
     def get_context_size(self) -> int:
         """Get the context window size for this provider in tokens."""
         return 4096

frigate/genai/llama_cpp.py

@@ -126,7 +126,7 @@ class LlamaCppClient(GenAIClient):
             chat_caps = props.get("chat_template_caps", {})
             self._supports_tools = chat_caps.get("supports_tools", False)
 
-            logger.debug(
+            logger.info(
                 "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s",
                 configured_model,
                 self._context_size or "unknown",

frigate/genai/manager.py

@@ -2,7 +2,7 @@
 
 Manages GenAI provider clients from Frigate config. Configuration is read only
 in _update_config(); no other code should read config.genai. Exposes clients
-by role: tool_client, vision_client, embeddings_client.
+by role: chat_client, description_client, embeddings_client.
 """
 
 import logging
@@ -21,8 +21,8 @@ class GenAIClientManager:
     """Manages GenAI provider clients from Frigate config."""
 
     def __init__(self, config: FrigateConfig) -> None:
-        self._tool_client: Optional[GenAIClient] = None
+        self._chat_client: Optional[GenAIClient] = None
-        self._vision_client: Optional[GenAIClient] = None
+        self._description_client: Optional[GenAIClient] = None
         self._embeddings_client: Optional[GenAIClient] = None
         self.update_config(config)
 
@@ -30,13 +30,13 @@ class GenAIClientManager:
         """Build role clients from current Frigate config.genai.
 
         Called from __init__ and can be called again when config is reloaded.
-        Each role (tools, vision, embeddings) gets the client for the provider
+        Each role (chat, descriptions, embeddings) gets the client for the
-        that has that role in its roles list.
+        provider that has that role in its roles list.
         """
         from frigate.genai import PROVIDERS, load_providers
 
-        self._tool_client = None
+        self._chat_client = None
-        self._vision_client = None
+        self._description_client = None
         self._embeddings_client = None
 
         if not config.genai:
@@ -65,22 +65,22 @@ class GenAIClientManager:
                 continue
 
             for role in genai_cfg.roles:
-                if role == GenAIRoleEnum.tools:
+                if role == GenAIRoleEnum.chat:
-                    self._tool_client = client
+                    self._chat_client = client
-                elif role == GenAIRoleEnum.vision:
+                elif role == GenAIRoleEnum.descriptions:
-                    self._vision_client = client
+                    self._description_client = client
                 elif role == GenAIRoleEnum.embeddings:
                     self._embeddings_client = client
 
     @property
-    def tool_client(self) -> "Optional[GenAIClient]":
+    def chat_client(self) -> "Optional[GenAIClient]":
-        """Client configured for the tools role (e.g. chat with function calling)."""
+        """Client configured for the chat role (e.g. chat with function calling)."""
-        return self._tool_client
+        return self._chat_client
 
     @property
-    def vision_client(self) -> "Optional[GenAIClient]":
+    def description_client(self) -> "Optional[GenAIClient]":
-        """Client configured for the vision role (e.g. review descriptions, object descriptions)."""
+        """Client configured for the descriptions role (e.g. review descriptions, object descriptions)."""
-        return self._vision_client
+        return self._description_client
 
     @property
     def embeddings_client(self) -> "Optional[GenAIClient]":

frigate/jobs/vlm_watch.py

@@ -121,11 +121,12 @@ class VLMWatchRunner(threading.Thread):
 
     def _run_iteration(self) -> float:
         """Run one VLM analysis iteration. Returns seconds until next run."""
-        vision_client = (
+        chat_client = self.genai_manager.chat_client
-            self.genai_manager.vision_client or self.genai_manager.tool_client
+        if chat_client is None or not chat_client.supports_vision:
+            logger.warning(
+                "VLM watch job %s: no chat client with vision support available",
+                self.job.id,
             )
-        if vision_client is None:
-            logger.warning("VLM watch job %s: no vision client available", self.job.id)
             return 30
 
         frame = self.frame_processor.get_current_frame(self.job.camera, {})
@@ -163,7 +164,7 @@ class VLMWatchRunner(threading.Thread):
             }
         )
 
-        response = vision_client.chat_with_tools(
+        response = chat_client.chat_with_tools(
             messages=self.conversation,
             tools=None,
             tool_choice=None,