administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-07-15 00:11:15 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Switch to a feature-based roles so it is easier to choose models for different tasks

Browse Source
This commit is contained in:
Nicolas Mowen 2026-04-03 15:29:28 -06:00
parent 68dfb157ea
commit bae30211d2
8 changed files with 56 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
frigate/api/chat.py
View File

@ -520,45 +520,14 @@ async def _execute_get_live_context(
"detections": list(tracked_objects_dict.values()),
}
# Grab live frame and handle based on provider configuration
# Grab live frame when the chat model supports vision
image_url = await _get_live_frame_image_url(request, camera, allowed_cameras)
if image_url:
genai_manager = request.app.genai_manager
if genai_manager.tool_client is genai_manager.vision_client:
# Same provider handles both roles — pass image URL so it can
# be injected as a user message (images can't be in tool results)
chat_client = request.app.genai_manager.chat_client
if chat_client is not None and chat_client.supports_vision:
# Pass image URL so it can be injected as a user message
# (images can't be in tool results)
result["_image_url"] = image_url
elif genai_manager.vision_client is not None:
# Separate vision provider — have it describe the image,
# providing detection context so it knows what to focus on
frame_bytes = _decode_data_url(image_url)
if frame_bytes:
detections = result.get("detections", [])
if detections:
detection_lines = []
for d in detections:
parts = [d.get("label", "unknown")]
if d.get("sub_label"):
parts.append(f"({d['sub_label']})")
if d.get("zones"):
parts.append(f"in {', '.join(d['zones'])}")
detection_lines.append(" ".join(parts))
context = (
"The following objects are currently being tracked: "
+ "; ".join(detection_lines)
+ "."
)
else:
context = "No objects are currently being tracked."
description = genai_manager.vision_client._send(
f"Describe what you see in this security camera image. "
f"{context} Focus on the scene, any visible activity, "
f"and details about the tracked objects.",
[frame_bytes],
)
if description:
result["image_description"] = description
return result
@ -609,17 +578,6 @@ async def _get_live_frame_image_url(
return None
def _decode_data_url(data_url: str) -> Optional[bytes]:
"""Decode a base64 data URL to raw bytes."""
try:
# Format: data:image/jpeg;base64,<data>
_, encoded = data_url.split(",", 1)
return base64.b64decode(encoded)
except (ValueError, Exception) as e:
logger.debug("Failed to decode data URL: %s", e)
return None
async def _execute_set_camera_state(
request: Request,
arguments: Dict[str, Any],
@ -734,9 +692,9 @@ async def _execute_start_camera_watch(
await require_camera_access(camera, request=request)
genai_manager = request.app.genai_manager
vision_client = genai_manager.vision_client or genai_manager.tool_client
if vision_client is None:
return {"error": "No vision/GenAI provider configured."}
chat_client = genai_manager.chat_client
if chat_client is None or not chat_client.supports_vision:
return {"error": "VLM watch requires a chat model with vision support."}
try:
job_id = start_vlm_watch_job(
@ -1070,7 +1028,7 @@ async def chat_completion(
6. Repeats until final answer
7. Returns response to user
"""
genai_client = request.app.genai_manager.tool_client
genai_client = request.app.genai_manager.chat_client
if not genai_client:
return JSONResponse(
content={
@ -1381,12 +1339,12 @@ async def start_vlm_monitor(
await require_camera_access(body.camera, request=request)
vision_client = genai_manager.vision_client or genai_manager.tool_client
if vision_client is None:
chat_client = genai_manager.chat_client
if chat_client is None or not chat_client.supports_vision:
return JSONResponse(
content={
"success": False,
"message": "No vision/GenAI provider configured.",
"message": "VLM watch requires a chat model with vision support.",
},
status_code=400,
)

2
frigate/api/review.py
View File

@ -746,7 +746,7 @@ async def set_not_reviewed(
description="Use GenAI to summarize review items over a period of time.",
)
def generate_review_summary(request: Request, start_ts: float, end_ts: float):
if not request.app.genai_manager.vision_client:
if not request.app.genai_manager.description_client:
return JSONResponse(
content=(
{

10
frigate/config/camera/genai.py
View File

@ -18,8 +18,8 @@ class GenAIProviderEnum(str, Enum):
class GenAIRoleEnum(str, Enum):
tools = "tools"
vision = "vision"
chat = "chat"
descriptions = "descriptions"
embeddings = "embeddings"
@ -49,11 +49,11 @@ class GenAIConfig(FrigateBaseModel):
roles: list[GenAIRoleEnum] = Field(
default_factory=lambda: [
GenAIRoleEnum.embeddings,
GenAIRoleEnum.vision,
GenAIRoleEnum.tools,
GenAIRoleEnum.descriptions,
GenAIRoleEnum.chat,
],
title="Roles",
description="GenAI roles (tools, vision, embeddings); one provider per role.",
description="GenAI roles (chat, descriptions, embeddings); one provider per role.",
)
provider_options: dict[str, Any] = Field(
default={},

8
frigate/embeddings/maintainer.py
View File

@ -202,7 +202,7 @@ class EmbeddingMaintainer(threading.Thread):
# post processors
self.post_processors: list[PostProcessorApi] = []
if self.genai_manager.vision_client is not None and any(
if self.genai_manager.description_client is not None and any(
c.review.genai.enabled_in_config for c in self.config.cameras.values()
):
self.post_processors.append(
@ -210,7 +210,7 @@ class EmbeddingMaintainer(threading.Thread):
self.config,
self.requestor,
self.metrics,
self.genai_manager.vision_client,
self.genai_manager.description_client,
)
)
@ -248,7 +248,7 @@ class EmbeddingMaintainer(threading.Thread):
)
self.post_processors.append(semantic_trigger_processor)
if self.genai_manager.vision_client is not None and any(
if self.genai_manager.description_client is not None and any(
c.objects.genai.enabled_in_config for c in self.config.cameras.values()
):
self.post_processors.append(
@ -257,7 +257,7 @@ class EmbeddingMaintainer(threading.Thread):
self.embeddings,
self.requestor,
self.metrics,
self.genai_manager.vision_client,
self.genai_manager.description_client,
semantic_trigger_processor,
)
)

9
frigate/genai/__init__.py
View File

@ -320,6 +320,15 @@ Guidelines:
"""Submit a request to the provider."""
return None
@property
def supports_vision(self) -> bool:
"""Whether the model supports vision/image input.
Defaults to True for cloud providers. Providers that can detect
capability at runtime (e.g. llama.cpp) should override this.
"""
return True
def get_context_size(self) -> int:
"""Get the context window size for this provider in tokens."""
return 4096

2
frigate/genai/llama_cpp.py
View File

@ -126,7 +126,7 @@ class LlamaCppClient(GenAIClient):
chat_caps = props.get("chat_template_caps", {})
self._supports_tools = chat_caps.get("supports_tools", False)
logger.debug(
logger.info(
"llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s",
configured_model,
self._context_size or "unknown",

34
frigate/genai/manager.py
View File

@ -2,7 +2,7 @@
Manages GenAI provider clients from Frigate config. Configuration is read only
in _update_config(); no other code should read config.genai. Exposes clients
by role: tool_client, vision_client, embeddings_client.
by role: chat_client, description_client, embeddings_client.
"""
import logging
@ -21,8 +21,8 @@ class GenAIClientManager:
"""Manages GenAI provider clients from Frigate config."""
def __init__(self, config: FrigateConfig) -> None:
self._tool_client: Optional[GenAIClient] = None
self._vision_client: Optional[GenAIClient] = None
self._chat_client: Optional[GenAIClient] = None
self._description_client: Optional[GenAIClient] = None
self._embeddings_client: Optional[GenAIClient] = None
self.update_config(config)
@ -30,13 +30,13 @@ class GenAIClientManager:
"""Build role clients from current Frigate config.genai.
Called from __init__ and can be called again when config is reloaded.
Each role (tools, vision, embeddings) gets the client for the provider
that has that role in its roles list.
Each role (chat, descriptions, embeddings) gets the client for the
provider that has that role in its roles list.
"""
from frigate.genai import PROVIDERS, load_providers
self._tool_client = None
self._vision_client = None
self._chat_client = None
self._description_client = None
self._embeddings_client = None
if not config.genai:
@ -65,22 +65,22 @@ class GenAIClientManager:
continue
for role in genai_cfg.roles:
if role == GenAIRoleEnum.tools:
self._tool_client = client
elif role == GenAIRoleEnum.vision:
self._vision_client = client
if role == GenAIRoleEnum.chat:
self._chat_client = client
elif role == GenAIRoleEnum.descriptions:
self._description_client = client
elif role == GenAIRoleEnum.embeddings:
self._embeddings_client = client
@property
def tool_client(self) -> "Optional[GenAIClient]":
"""Client configured for the tools role (e.g. chat with function calling)."""
return self._tool_client
def chat_client(self) -> "Optional[GenAIClient]":
"""Client configured for the chat role (e.g. chat with function calling)."""
return self._chat_client
@property
def vision_client(self) -> "Optional[GenAIClient]":
"""Client configured for the vision role (e.g. review descriptions, object descriptions)."""
return self._vision_client
def description_client(self) -> "Optional[GenAIClient]":
"""Client configured for the descriptions role (e.g. review descriptions, object descriptions)."""
return self._description_client
@property
def embeddings_client(self) -> "Optional[GenAIClient]":

13
frigate/jobs/vlm_watch.py
View File

@ -121,11 +121,12 @@ class VLMWatchRunner(threading.Thread):
def _run_iteration(self) -> float:
"""Run one VLM analysis iteration. Returns seconds until next run."""
vision_client = (
self.genai_manager.vision_client or self.genai_manager.tool_client
)
if vision_client is None:
logger.warning("VLM watch job %s: no vision client available", self.job.id)
chat_client = self.genai_manager.chat_client
if chat_client is None or not chat_client.supports_vision:
logger.warning(
"VLM watch job %s: no chat client with vision support available",
self.job.id,
)
return 30
frame = self.frame_processor.get_current_frame(self.job.camera, {})
@ -163,7 +164,7 @@ class VLMWatchRunner(threading.Thread):
}
)
response = vision_client.chat_with_tools(
response = chat_client.chat_with_tools(
messages=self.conversation,
tools=None,
tool_choice=None,