From bae30211d280db4121a0108e7b230a33f8e48094 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 3 Apr 2026 15:29:28 -0600 Subject: [PATCH] Switch to a feature-based roles so it is easier to choose models for different tasks --- frigate/api/chat.py | 66 ++++++-------------------------- frigate/api/review.py | 2 +- frigate/config/camera/genai.py | 10 ++--- frigate/embeddings/maintainer.py | 8 ++-- frigate/genai/__init__.py | 9 +++++ frigate/genai/llama_cpp.py | 2 +- frigate/genai/manager.py | 34 ++++++++-------- frigate/jobs/vlm_watch.py | 13 ++++--- 8 files changed, 56 insertions(+), 88 deletions(-) diff --git a/frigate/api/chat.py b/frigate/api/chat.py index 60bbeb8e9d..3e10714cac 100644 --- a/frigate/api/chat.py +++ b/frigate/api/chat.py @@ -520,45 +520,14 @@ async def _execute_get_live_context( "detections": list(tracked_objects_dict.values()), } - # Grab live frame and handle based on provider configuration + # Grab live frame when the chat model supports vision image_url = await _get_live_frame_image_url(request, camera, allowed_cameras) if image_url: - genai_manager = request.app.genai_manager - if genai_manager.tool_client is genai_manager.vision_client: - # Same provider handles both roles — pass image URL so it can - # be injected as a user message (images can't be in tool results) + chat_client = request.app.genai_manager.chat_client + if chat_client is not None and chat_client.supports_vision: + # Pass image URL so it can be injected as a user message + # (images can't be in tool results) result["_image_url"] = image_url - elif genai_manager.vision_client is not None: - # Separate vision provider — have it describe the image, - # providing detection context so it knows what to focus on - frame_bytes = _decode_data_url(image_url) - if frame_bytes: - detections = result.get("detections", []) - if detections: - detection_lines = [] - for d in detections: - parts = [d.get("label", "unknown")] - if d.get("sub_label"): - parts.append(f"({d['sub_label']})") - if d.get("zones"): - parts.append(f"in {', '.join(d['zones'])}") - detection_lines.append(" ".join(parts)) - context = ( - "The following objects are currently being tracked: " - + "; ".join(detection_lines) - + "." - ) - else: - context = "No objects are currently being tracked." - - description = genai_manager.vision_client._send( - f"Describe what you see in this security camera image. " - f"{context} Focus on the scene, any visible activity, " - f"and details about the tracked objects.", - [frame_bytes], - ) - if description: - result["image_description"] = description return result @@ -609,17 +578,6 @@ async def _get_live_frame_image_url( return None -def _decode_data_url(data_url: str) -> Optional[bytes]: - """Decode a base64 data URL to raw bytes.""" - try: - # Format: data:image/jpeg;base64, - _, encoded = data_url.split(",", 1) - return base64.b64decode(encoded) - except (ValueError, Exception) as e: - logger.debug("Failed to decode data URL: %s", e) - return None - - async def _execute_set_camera_state( request: Request, arguments: Dict[str, Any], @@ -734,9 +692,9 @@ async def _execute_start_camera_watch( await require_camera_access(camera, request=request) genai_manager = request.app.genai_manager - vision_client = genai_manager.vision_client or genai_manager.tool_client - if vision_client is None: - return {"error": "No vision/GenAI provider configured."} + chat_client = genai_manager.chat_client + if chat_client is None or not chat_client.supports_vision: + return {"error": "VLM watch requires a chat model with vision support."} try: job_id = start_vlm_watch_job( @@ -1070,7 +1028,7 @@ async def chat_completion( 6. Repeats until final answer 7. Returns response to user """ - genai_client = request.app.genai_manager.tool_client + genai_client = request.app.genai_manager.chat_client if not genai_client: return JSONResponse( content={ @@ -1381,12 +1339,12 @@ async def start_vlm_monitor( await require_camera_access(body.camera, request=request) - vision_client = genai_manager.vision_client or genai_manager.tool_client - if vision_client is None: + chat_client = genai_manager.chat_client + if chat_client is None or not chat_client.supports_vision: return JSONResponse( content={ "success": False, - "message": "No vision/GenAI provider configured.", + "message": "VLM watch requires a chat model with vision support.", }, status_code=400, ) diff --git a/frigate/api/review.py b/frigate/api/review.py index 06480dd8ce..cb114db2a0 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -746,7 +746,7 @@ async def set_not_reviewed( description="Use GenAI to summarize review items over a period of time.", ) def generate_review_summary(request: Request, start_ts: float, end_ts: float): - if not request.app.genai_manager.vision_client: + if not request.app.genai_manager.description_client: return JSONResponse( content=( { diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index fae0ae577f..2adf223c19 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -18,8 +18,8 @@ class GenAIProviderEnum(str, Enum): class GenAIRoleEnum(str, Enum): - tools = "tools" - vision = "vision" + chat = "chat" + descriptions = "descriptions" embeddings = "embeddings" @@ -49,11 +49,11 @@ class GenAIConfig(FrigateBaseModel): roles: list[GenAIRoleEnum] = Field( default_factory=lambda: [ GenAIRoleEnum.embeddings, - GenAIRoleEnum.vision, - GenAIRoleEnum.tools, + GenAIRoleEnum.descriptions, + GenAIRoleEnum.chat, ], title="Roles", - description="GenAI roles (tools, vision, embeddings); one provider per role.", + description="GenAI roles (chat, descriptions, embeddings); one provider per role.", ) provider_options: dict[str, Any] = Field( default={}, diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 18b7577437..f924a8dbbf 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -202,7 +202,7 @@ class EmbeddingMaintainer(threading.Thread): # post processors self.post_processors: list[PostProcessorApi] = [] - if self.genai_manager.vision_client is not None and any( + if self.genai_manager.description_client is not None and any( c.review.genai.enabled_in_config for c in self.config.cameras.values() ): self.post_processors.append( @@ -210,7 +210,7 @@ class EmbeddingMaintainer(threading.Thread): self.config, self.requestor, self.metrics, - self.genai_manager.vision_client, + self.genai_manager.description_client, ) ) @@ -248,7 +248,7 @@ class EmbeddingMaintainer(threading.Thread): ) self.post_processors.append(semantic_trigger_processor) - if self.genai_manager.vision_client is not None and any( + if self.genai_manager.description_client is not None and any( c.objects.genai.enabled_in_config for c in self.config.cameras.values() ): self.post_processors.append( @@ -257,7 +257,7 @@ class EmbeddingMaintainer(threading.Thread): self.embeddings, self.requestor, self.metrics, - self.genai_manager.vision_client, + self.genai_manager.description_client, semantic_trigger_processor, ) ) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 96e956242f..fdbf406091 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -320,6 +320,15 @@ Guidelines: """Submit a request to the provider.""" return None + @property + def supports_vision(self) -> bool: + """Whether the model supports vision/image input. + + Defaults to True for cloud providers. Providers that can detect + capability at runtime (e.g. llama.cpp) should override this. + """ + return True + def get_context_size(self) -> int: """Get the context window size for this provider in tokens.""" return 4096 diff --git a/frigate/genai/llama_cpp.py b/frigate/genai/llama_cpp.py index 03bd2e9630..ed939d0c63 100644 --- a/frigate/genai/llama_cpp.py +++ b/frigate/genai/llama_cpp.py @@ -126,7 +126,7 @@ class LlamaCppClient(GenAIClient): chat_caps = props.get("chat_template_caps", {}) self._supports_tools = chat_caps.get("supports_tools", False) - logger.debug( + logger.info( "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s", configured_model, self._context_size or "unknown", diff --git a/frigate/genai/manager.py b/frigate/genai/manager.py index 01daa35e09..e236f28032 100644 --- a/frigate/genai/manager.py +++ b/frigate/genai/manager.py @@ -2,7 +2,7 @@ Manages GenAI provider clients from Frigate config. Configuration is read only in _update_config(); no other code should read config.genai. Exposes clients -by role: tool_client, vision_client, embeddings_client. +by role: chat_client, description_client, embeddings_client. """ import logging @@ -21,8 +21,8 @@ class GenAIClientManager: """Manages GenAI provider clients from Frigate config.""" def __init__(self, config: FrigateConfig) -> None: - self._tool_client: Optional[GenAIClient] = None - self._vision_client: Optional[GenAIClient] = None + self._chat_client: Optional[GenAIClient] = None + self._description_client: Optional[GenAIClient] = None self._embeddings_client: Optional[GenAIClient] = None self.update_config(config) @@ -30,13 +30,13 @@ class GenAIClientManager: """Build role clients from current Frigate config.genai. Called from __init__ and can be called again when config is reloaded. - Each role (tools, vision, embeddings) gets the client for the provider - that has that role in its roles list. + Each role (chat, descriptions, embeddings) gets the client for the + provider that has that role in its roles list. """ from frigate.genai import PROVIDERS, load_providers - self._tool_client = None - self._vision_client = None + self._chat_client = None + self._description_client = None self._embeddings_client = None if not config.genai: @@ -65,22 +65,22 @@ class GenAIClientManager: continue for role in genai_cfg.roles: - if role == GenAIRoleEnum.tools: - self._tool_client = client - elif role == GenAIRoleEnum.vision: - self._vision_client = client + if role == GenAIRoleEnum.chat: + self._chat_client = client + elif role == GenAIRoleEnum.descriptions: + self._description_client = client elif role == GenAIRoleEnum.embeddings: self._embeddings_client = client @property - def tool_client(self) -> "Optional[GenAIClient]": - """Client configured for the tools role (e.g. chat with function calling).""" - return self._tool_client + def chat_client(self) -> "Optional[GenAIClient]": + """Client configured for the chat role (e.g. chat with function calling).""" + return self._chat_client @property - def vision_client(self) -> "Optional[GenAIClient]": - """Client configured for the vision role (e.g. review descriptions, object descriptions).""" - return self._vision_client + def description_client(self) -> "Optional[GenAIClient]": + """Client configured for the descriptions role (e.g. review descriptions, object descriptions).""" + return self._description_client @property def embeddings_client(self) -> "Optional[GenAIClient]": diff --git a/frigate/jobs/vlm_watch.py b/frigate/jobs/vlm_watch.py index 7884d86377..cd64325d0d 100644 --- a/frigate/jobs/vlm_watch.py +++ b/frigate/jobs/vlm_watch.py @@ -121,11 +121,12 @@ class VLMWatchRunner(threading.Thread): def _run_iteration(self) -> float: """Run one VLM analysis iteration. Returns seconds until next run.""" - vision_client = ( - self.genai_manager.vision_client or self.genai_manager.tool_client - ) - if vision_client is None: - logger.warning("VLM watch job %s: no vision client available", self.job.id) + chat_client = self.genai_manager.chat_client + if chat_client is None or not chat_client.supports_vision: + logger.warning( + "VLM watch job %s: no chat client with vision support available", + self.job.id, + ) return 30 frame = self.frame_processor.get_current_frame(self.job.camera, {}) @@ -163,7 +164,7 @@ class VLMWatchRunner(threading.Thread): } ) - response = vision_client.chat_with_tools( + response = chat_client.chat_with_tools( messages=self.conversation, tools=None, tool_choice=None,