administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-08-02 17:12:16 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

GenAI Refactor (#23253)

Browse Source 
* Ensure runtime options are passed

* Add attribute info to prompt when configured

* Move GenAI plugins to dedicated directory

* Migrate prompts to dedicated folder

* Move chat prompts to prompts

* Implement reasoning traces in the UI

* Cleanup

* Make azure a subclass of openai

* Implement reasoning for other providers

* mypy

* Cleanup
This commit is contained in:
Nicolas Mowen
2026-05-19 13:03:57 -05:00
committed by GitHub
parent b1de5e2290
commit b0b00fe1d0
16 changed files with 1108 additions and 930 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frigate/genai/plugins/__init__.py
+1
View File
@@ -0,0 +1 @@
"""GenAI provider plugins."""
frigate/genai/plugins/azure-openai.py
+53
View File
@@ -0,0 +1,53 @@
"""Azure OpenAI Provider for Frigate AI.
Azure OpenAI exposes the same chat completions API as OpenAI once the
client is constructed, so this provider inherits all transport, streaming,
reasoning, and tool-calling logic from :class:`OpenAIClient` and only
overrides what is genuinely Azure-specific:
- Client construction: parses ``api-version`` out of the configured
``base_url`` query string and instantiates :class:`openai.AzureOpenAI`
with ``azure_endpoint`` instead of ``base_url``. Raises if the URL is
malformed; :class:`GenAIClientManager` catches the exception and
disables the provider.
- Context size: Azure does not expose a per-model ``max_model_len`` field
reliably, so we keep the historical 128K default rather than the
model-name heuristic used by OpenAI.
"""
import logging
from urllib.parse import parse_qs, urlparse
from openai import AzureOpenAI
from frigate.config import GenAIProviderEnum
from frigate.genai import register_genai_provider
from frigate.genai.plugins.openai import OpenAIClient
logger = logging.getLogger(__name__)
@register_genai_provider(GenAIProviderEnum.azure_openai)
class AzureOpenAIClient(OpenAIClient):
"""Generative AI client for Frigate using Azure OpenAI."""
def _init_provider(self) -> AzureOpenAI:
"""Initialize the AzureOpenAI client from the configured base_url."""
parsed_url = urlparse(self.genai_config.base_url or "")
query_params = parse_qs(parsed_url.query)
api_version = query_params.get("api-version", [None])[0]
if not api_version:
raise ValueError("Azure OpenAI base_url is missing api-version.")
azure_endpoint = f"{parsed_url.scheme}://{parsed_url.netloc}/"
return AzureOpenAI(
api_key=self.genai_config.api_key,
api_version=api_version,
azure_endpoint=azure_endpoint,
)
def get_context_size(self) -> int:
"""Azure does not reliably surface per-model context size; use 128K."""
return 128000
frigate/genai/plugins/gemini.py
+654
View File
@@ -0,0 +1,654 @@
"""Gemini Provider for Frigate AI."""
import json
import logging
from typing import Any, AsyncGenerator, Optional
from google import genai
from google.genai import errors, types
from google.genai.types import FunctionCallingConfigMode
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
logger = logging.getLogger(__name__)
def _stats_from_gemini_usage(usage: Any) -> Optional[dict[str, Any]]:
"""Build a stats dict from a Gemini usage_metadata object."""
prompt_tokens = getattr(usage, "prompt_token_count", None)
completion_tokens = getattr(usage, "candidates_token_count", None)
if prompt_tokens is None and completion_tokens is None:
return None
stats: dict[str, Any] = {}
if isinstance(prompt_tokens, int):
stats["prompt_tokens"] = prompt_tokens
if isinstance(completion_tokens, int):
stats["completion_tokens"] = completion_tokens
return stats or None
@register_genai_provider(GenAIProviderEnum.gemini)
class GeminiClient(GenAIClient):
"""Generative AI client for Frigate using Gemini."""
provider: genai.Client
def _init_provider(self) -> genai.Client:
"""Initialize the client."""
# Merge provider_options into HttpOptions
http_options_dict: dict[str, Any] = {
"timeout": int(self.timeout * 1000), # requires milliseconds
"retry_options": types.HttpRetryOptions(
attempts=3,
initial_delay=1.0,
max_delay=60.0,
exp_base=2.0,
jitter=1.0,
http_status_codes=[429, 500, 502, 503, 504],
),
}
if isinstance(self.genai_config.provider_options, dict):
http_options_dict.update(self.genai_config.provider_options)
return genai.Client(
api_key=self.genai_config.api_key,
http_options=types.HttpOptions(**http_options_dict),
)
def _send(
self,
prompt: str,
images: list[bytes],
response_format: Optional[dict] = None,
) -> Optional[str]:
"""Submit a request to Gemini."""
contents = [prompt] + [
types.Part.from_bytes(data=img, mime_type="image/jpeg") for img in images
]
try:
# Merge runtime_options into generation_config if provided
generation_config_dict: dict[str, Any] = {"candidate_count": 1}
generation_config_dict.update(self.genai_config.runtime_options)
if response_format and response_format.get("type") == "json_schema":
generation_config_dict["response_mime_type"] = "application/json"
schema = response_format.get("json_schema", {}).get("schema")
if schema:
generation_config_dict["response_schema"] = schema
response = self.provider.models.generate_content(
model=self.genai_config.model,
contents=contents, # type: ignore[arg-type]
config=types.GenerateContentConfig(
**generation_config_dict,
),
)
except errors.APIError as e:
logger.warning("Gemini returned an error: %s", str(e))
return None
except Exception as e:
logger.warning("An unexpected error occurred with Gemini: %s", str(e))
return None
try:
if response.text is None:
return None
description = response.text.strip()
except (ValueError, AttributeError):
# No description was generated
return None
return description
def list_models(self) -> list[str]:
"""Return available model names from Gemini."""
try:
return sorted(m.name or "" for m in self.provider.models.list())
except Exception as e:
logger.warning("Failed to list Gemini models: %s", e)
return []
def get_context_size(self) -> int:
"""Get the context window size for Gemini."""
# Gemini Pro Vision has a 1M token context window
return 1000000
def chat_with_tools(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> dict[str, Any]:
"""
Send chat messages to Gemini with optional tool definitions.
Implements function calling/tool usage for Gemini models.
"""
try:
# Convert messages to Gemini format
gemini_messages: list[types.Content] = []
for msg in messages:
role = msg.get("role", "user")
content = msg.get("content", "")
# Map roles to Gemini format
if role == "system":
# Gemini doesn't have system role, prepend to first user message
if (
gemini_messages
and gemini_messages[0].role == "user"
and gemini_messages[0].parts
):
gemini_messages[0].parts[
0
].text = f"{content}\n\n{gemini_messages[0].parts[0].text}"
else:
gemini_messages.append(
types.Content(
role="user", parts=[types.Part.from_text(text=content)]
)
)
elif role == "assistant":
parts: list[types.Part] = []
if content:
parts.append(types.Part.from_text(text=content))
for tc in msg.get("tool_calls") or []:
func = tc.get("function") or {}
tc_name = func.get("name") or ""
tc_args: Any = func.get("arguments")
if isinstance(tc_args, str):
try:
tc_args = json.loads(tc_args)
except (json.JSONDecodeError, TypeError):
tc_args = {}
if not isinstance(tc_args, dict):
tc_args = {}
if tc_name:
parts.append(
types.Part.from_function_call(
name=tc_name, args=tc_args
)
)
if not parts:
parts.append(types.Part.from_text(text=" "))
gemini_messages.append(types.Content(role="model", parts=parts))
elif role == "tool":
# Handle tool response
response_payload = (
content if isinstance(content, dict) else {"result": content}
)
gemini_messages.append(
types.Content(
role="function",
parts=[
types.Part.from_function_response(
name=msg.get("name")
or msg.get("tool_call_id")
or "",
response=response_payload,
)
],
)
)
else: # user
gemini_messages.append(
types.Content(
role="user", parts=[types.Part.from_text(text=content)]
)
)
# Convert tools to Gemini format
gemini_tools = None
if tools:
gemini_tools = []
for tool in tools:
if tool.get("type") == "function":
func = tool.get("function", {})
gemini_tools.append(
types.Tool(
function_declarations=[
types.FunctionDeclaration(
name=func.get("name", ""),
description=func.get("description", ""),
parameters=func.get("parameters", {}),
)
]
)
)
# Configure tool choice
tool_config = None
if tool_choice:
if tool_choice == "none":
tool_config = types.ToolConfig(
function_calling_config=types.FunctionCallingConfig(
mode=FunctionCallingConfigMode.NONE
)
)
elif tool_choice == "auto":
tool_config = types.ToolConfig(
function_calling_config=types.FunctionCallingConfig(
mode=FunctionCallingConfigMode.AUTO
)
)
elif tool_choice == "required":
tool_config = types.ToolConfig(
function_calling_config=types.FunctionCallingConfig(
mode=FunctionCallingConfigMode.ANY
)
)
# Build request config
config_params: dict[str, Any] = {"candidate_count": 1}
if gemini_tools:
config_params["tools"] = gemini_tools
if tool_config:
config_params["tool_config"] = tool_config
# Ask thinking-capable models (Gemini 2.5+) to include their
# reasoning trace as separate `thought` parts so we can surface
# it on the reasoning channel. Older models ignore this field.
config_params["thinking_config"] = types.ThinkingConfig(
include_thoughts=True
)
# Merge runtime_options
if isinstance(self.genai_config.runtime_options, dict):
config_params.update(self.genai_config.runtime_options)
response = self.provider.models.generate_content(
model=self.genai_config.model,
contents=gemini_messages, # type: ignore[arg-type]
config=types.GenerateContentConfig(**config_params),
)
# Check if response is valid
if not response or not response.candidates:
return {
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
}
candidate = response.candidates[0]
content = None
reasoning_parts: list[str] = []
tool_calls = None
# Extract content, reasoning, and tool calls from response
if candidate.content and candidate.content.parts:
for part in candidate.content.parts:
if part.text:
if getattr(part, "thought", False):
reasoning_parts.append(part.text)
else:
content = part.text.strip()
elif part.function_call:
# Handle function call
if tool_calls is None:
tool_calls = []
try:
arguments = (
dict(part.function_call.args)
if part.function_call.args
else {}
)
except Exception:
arguments = {}
tool_calls.append(
{
"id": part.function_call.name or "",
"name": part.function_call.name or "",
"arguments": arguments,
}
)
reasoning = "".join(reasoning_parts).strip() or None
# Determine finish reason
finish_reason = "error"
if hasattr(candidate, "finish_reason") and candidate.finish_reason:
from google.genai.types import FinishReason
if candidate.finish_reason == FinishReason.STOP:
finish_reason = "stop"
elif candidate.finish_reason == FinishReason.MAX_TOKENS:
finish_reason = "length"
elif candidate.finish_reason in [
FinishReason.SAFETY,
FinishReason.RECITATION,
]:
finish_reason = "error"
elif tool_calls:
finish_reason = "tool_calls"
elif content:
finish_reason = "stop"
elif tool_calls:
finish_reason = "tool_calls"
elif content:
finish_reason = "stop"
return {
"content": content,
"reasoning": reasoning,
"tool_calls": tool_calls,
"finish_reason": finish_reason,
}
except errors.APIError as e:
logger.warning("Gemini API error during chat_with_tools: %s", str(e))
return {
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
}
except Exception as e:
logger.warning(
"Gemini returned an error during chat_with_tools: %s", str(e)
)
return {
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
}
async def chat_with_tools_stream(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> AsyncGenerator[tuple[str, Any], None]:
"""
Stream chat with tools; yields content deltas then final message.
Implements streaming function calling/tool usage for Gemini models.
"""
try:
# Convert messages to Gemini format
gemini_messages: list[types.Content] = []
for msg in messages:
role = msg.get("role", "user")
content = msg.get("content", "")
# Map roles to Gemini format
if role == "system":
# Gemini doesn't have system role, prepend to first user message
if (
gemini_messages
and gemini_messages[0].role == "user"
and gemini_messages[0].parts
):
gemini_messages[0].parts[
0
].text = f"{content}\n\n{gemini_messages[0].parts[0].text}"
else:
gemini_messages.append(
types.Content(
role="user", parts=[types.Part.from_text(text=content)]
)
)
elif role == "assistant":
parts: list[types.Part] = []
if content:
parts.append(types.Part.from_text(text=content))
for tc in msg.get("tool_calls") or []:
func = tc.get("function") or {}
tc_name = func.get("name") or ""
tc_args: Any = func.get("arguments")
if isinstance(tc_args, str):
try:
tc_args = json.loads(tc_args)
except (json.JSONDecodeError, TypeError):
tc_args = {}
if not isinstance(tc_args, dict):
tc_args = {}
if tc_name:
parts.append(
types.Part.from_function_call(
name=tc_name, args=tc_args
)
)
if not parts:
parts.append(types.Part.from_text(text=" "))
gemini_messages.append(types.Content(role="model", parts=parts))
elif role == "tool":
# Handle tool response
response_payload = (
content if isinstance(content, dict) else {"result": content}
)
gemini_messages.append(
types.Content(
role="function",
parts=[
types.Part.from_function_response(
name=msg.get("name")
or msg.get("tool_call_id")
or "",
response=response_payload,
)
],
)
)
else: # user
gemini_messages.append(
types.Content(
role="user", parts=[types.Part.from_text(text=content)]
)
)
# Convert tools to Gemini format
gemini_tools = None
if tools:
gemini_tools = []
for tool in tools:
if tool.get("type") == "function":
func = tool.get("function", {})
gemini_tools.append(
types.Tool(
function_declarations=[
types.FunctionDeclaration(
name=func.get("name", ""),
description=func.get("description", ""),
parameters=func.get("parameters", {}),
)
]
)
)
# Configure tool choice
tool_config = None
if tool_choice:
if tool_choice == "none":
tool_config = types.ToolConfig(
function_calling_config=types.FunctionCallingConfig(
mode=FunctionCallingConfigMode.NONE
)
)
elif tool_choice == "auto":
tool_config = types.ToolConfig(
function_calling_config=types.FunctionCallingConfig(
mode=FunctionCallingConfigMode.AUTO
)
)
elif tool_choice == "required":
tool_config = types.ToolConfig(
function_calling_config=types.FunctionCallingConfig(
mode=FunctionCallingConfigMode.ANY
)
)
# Build request config
config_params: dict[str, Any] = {"candidate_count": 1}
if gemini_tools:
config_params["tools"] = gemini_tools
if tool_config:
config_params["tool_config"] = tool_config
# Ask thinking-capable models to include their reasoning trace
# as separate `thought` parts (Gemini 2.5+; ignored elsewhere).
config_params["thinking_config"] = types.ThinkingConfig(
include_thoughts=True
)
# Merge runtime_options
if isinstance(self.genai_config.runtime_options, dict):
config_params.update(self.genai_config.runtime_options)
# Use streaming API
content_parts: list[str] = []
reasoning_parts: list[str] = []
tool_calls_by_index: dict[int, dict[str, Any]] = {}
finish_reason = "stop"
usage_stats: Optional[dict[str, Any]] = None
stream = await self.provider.aio.models.generate_content_stream(
model=self.genai_config.model,
contents=gemini_messages, # type: ignore[arg-type]
config=types.GenerateContentConfig(**config_params),
)
async for chunk in stream:
chunk_usage = getattr(chunk, "usage_metadata", None)
if chunk_usage is not None:
maybe_stats = _stats_from_gemini_usage(chunk_usage)
if maybe_stats is not None:
usage_stats = maybe_stats
if not chunk or not chunk.candidates:
continue
candidate = chunk.candidates[0]
# Check for finish reason
if hasattr(candidate, "finish_reason") and candidate.finish_reason:
from google.genai.types import FinishReason
if candidate.finish_reason == FinishReason.STOP:
finish_reason = "stop"
elif candidate.finish_reason == FinishReason.MAX_TOKENS:
finish_reason = "length"
elif candidate.finish_reason in [
FinishReason.SAFETY,
FinishReason.RECITATION,
]:
finish_reason = "error"
# Extract content, reasoning, and tool calls from chunk
if candidate.content and candidate.content.parts:
for part in candidate.content.parts:
if part.text:
if getattr(part, "thought", False):
reasoning_parts.append(part.text)
yield ("reasoning_delta", part.text)
else:
content_parts.append(part.text)
yield ("content_delta", part.text)
elif part.function_call:
# Handle function call
try:
arguments = (
dict(part.function_call.args)
if part.function_call.args
else {}
)
except Exception:
arguments = {}
# Store tool call
tool_call_id = part.function_call.name or ""
tool_call_name = part.function_call.name or ""
# Check if we already have this tool call
found_index = None
for idx, tc in tool_calls_by_index.items():
if tc["name"] == tool_call_name:
found_index = idx
break
if found_index is None:
found_index = len(tool_calls_by_index)
tool_calls_by_index[found_index] = {
"id": tool_call_id,
"name": tool_call_name,
"arguments": "",
}
# Accumulate arguments
if arguments:
tool_calls_by_index[found_index]["arguments"] += (
json.dumps(arguments)
if isinstance(arguments, dict)
else str(arguments)
)
# Build final message
full_content = "".join(content_parts).strip() or None
full_reasoning = "".join(reasoning_parts).strip() or None
# Convert tool calls to list format
tool_calls_list = None
if tool_calls_by_index:
tool_calls_list = []
for tc in tool_calls_by_index.values():
try:
# Try to parse accumulated arguments as JSON
parsed_args = json.loads(tc["arguments"])
except (json.JSONDecodeError, Exception):
parsed_args = tc["arguments"]
tool_calls_list.append(
{
"id": tc["id"],
"name": tc["name"],
"arguments": parsed_args,
}
)
finish_reason = "tool_calls"
if usage_stats is not None:
yield ("stats", usage_stats)
yield (
"message",
{
"content": full_content,
"reasoning": full_reasoning,
"tool_calls": tool_calls_list,
"finish_reason": finish_reason,
},
)
except errors.APIError as e:
logger.warning("Gemini API error during streaming: %s", str(e))
yield (
"message",
{
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
},
)
except Exception as e:
logger.warning(
"Gemini returned an error during chat_with_tools_stream: %s", str(e)
)
yield (
"message",
{
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
},
)
frigate/genai/plugins/llama_cpp.py
+912
View File
@@ -0,0 +1,912 @@
"""llama.cpp Provider for Frigate AI."""
import base64
import io
import json
import logging
from typing import Any, AsyncGenerator, Optional, cast
import httpx
import numpy as np
import requests
from PIL import Image
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
from frigate.genai.utils import parse_tool_calls_from_message
logger = logging.getLogger(__name__)
def _stats_from_llama_cpp_chunk(data: dict[str, Any]) -> Optional[dict[str, Any]]:
"""Build a stats dict from a llama.cpp streaming chunk.
Final-chunk `usage` carries authoritative token counts. Per-chunk
`timings` (enabled via timings_per_token) carries the running token
counts (prompt_n, predicted_n) and generation rate, so live updates
work mid-stream.
"""
usage = data.get("usage") or {}
timings = data.get("timings") or {}
prompt_tokens = usage.get("prompt_tokens")
completion_tokens = usage.get("completion_tokens")
predicted_ms = timings.get("predicted_ms")
tps = timings.get("predicted_per_second")
stats: dict[str, Any] = {}
if not isinstance(prompt_tokens, int):
prompt_n = timings.get("prompt_n")
if isinstance(prompt_n, int):
prompt_tokens = prompt_n
if not isinstance(completion_tokens, int):
predicted_n = timings.get("predicted_n")
if isinstance(predicted_n, int):
completion_tokens = predicted_n
if not isinstance(prompt_tokens, int) and not isinstance(completion_tokens, int):
return None
if isinstance(prompt_tokens, int):
stats["prompt_tokens"] = prompt_tokens
if isinstance(completion_tokens, int):
stats["completion_tokens"] = completion_tokens
if isinstance(predicted_ms, (int, float)) and predicted_ms > 0:
stats["completion_duration_ms"] = float(predicted_ms)
if isinstance(tps, (int, float)) and tps > 0:
stats["tokens_per_second"] = float(tps)
return stats or None
def _parse_launch_arg(args: list[str], flag: str) -> str | None:
"""Return the value following `flag` in a positional argv list, or None."""
try:
idx = args.index(flag)
except ValueError:
return None
if idx + 1 >= len(args):
return None
return args[idx + 1]
def _fetch_llama_props(base_url: str, model: str) -> dict[str, Any]:
"""Fetch /props from a llama.cpp server, with llama-swap fallback.
Raises the underlying RequestException if both endpoints fail; callers
decide how to surface the failure.
"""
try:
response = requests.get(
f"{base_url}/props",
params={"model": model},
timeout=10,
)
response.raise_for_status()
return cast(dict[str, Any], response.json())
except Exception:
response = requests.get(
f"{base_url}/upstream/{model}/props",
timeout=10,
)
response.raise_for_status()
return cast(dict[str, Any], response.json())
def _to_jpeg(img_bytes: bytes) -> bytes | None:
"""Convert image bytes to JPEG. llama.cpp/STB does not support WebP."""
try:
img = Image.open(io.BytesIO(img_bytes))
if img.mode != "RGB":
img = img.convert("RGB") # type: ignore[assignment]
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85)
return buf.getvalue()
except Exception as e:
logger.warning("Failed to convert image to JPEG: %s", e)
return None
@register_genai_provider(GenAIProviderEnum.llamacpp)
class LlamaCppClient(GenAIClient):
"""Generative AI client for Frigate using llama.cpp server."""
provider: str | None # base_url
provider_options: dict[str, Any]
_context_size: int | None
_supports_vision: bool
_supports_audio: bool
_supports_tools: bool
_image_token_cache: dict[tuple[int, int], int]
_text_baseline_tokens: int | None
_media_marker: str
def _init_provider(self) -> str | None:
"""Initialize the client and query model metadata from the server."""
self.provider_options = {
**self.genai_config.provider_options,
}
self._context_size = None
self._supports_vision = False
self._supports_audio = False
self._supports_tools = False
self._image_token_cache = {}
self._text_baseline_tokens = None
self._media_marker = "<__media__>"
base_url = (
self.genai_config.base_url.rstrip("/")
if self.genai_config.base_url
else None
)
if base_url is None:
return None
else:
base_url = base_url.replace("/v1", "") # Strip /v1 if included in base_url
configured_model = self.genai_config.model
info = self._get_model_info(base_url, configured_model)
if info is None:
return None
self._context_size = info["context_size"]
self._supports_vision = info["supports_vision"]
self._supports_audio = info["supports_audio"]
self._supports_tools = info["supports_tools"]
self._media_marker = info["media_marker"]
logger.info(
"llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s",
configured_model,
self._context_size or "unknown",
self._supports_vision,
self._supports_audio,
self._supports_tools,
)
return base_url
def _get_model_info(
self, base_url: str, configured_model: str
) -> dict[str, Any] | None:
"""Resolve model metadata from /v1/models with /props fallback.
Returns a dict of capability fields, or None if the server's model
registry was reachable and reported the configured model as missing.
A reachable-but-unparseable /v1/models is treated as soft-pass and
falls through to /props, matching prior behavior.
After ggml-org/llama.cpp#22952, /v1/models exposes per-model
`architecture.input_modalities` (text/image/audio) — the primary
source. When proxied through llama-swap, the same entry carries
`status.args` (server launch argv) and, for the loaded model,
`meta.n_ctx`. /props remains the only source for `media_marker`,
which the server randomizes per startup unless LLAMA_MEDIA_MARKER
is set.
"""
info: dict[str, Any] = {
"context_size": None,
"supports_vision": False,
"supports_audio": False,
"supports_tools": False,
"media_marker": "<__media__>",
}
model_entry: dict[str, Any] | None = None
try:
response = requests.get(f"{base_url}/v1/models", timeout=10)
response.raise_for_status()
models_data = response.json()
for model in models_data.get("data", []):
model_ids = {model.get("id")}
for alias in model.get("aliases", []):
model_ids.add(alias)
if configured_model in model_ids:
model_entry = model
break
if model_entry is None:
available = []
for m in models_data.get("data", []):
available.append(m.get("id", "unknown"))
for alias in m.get("aliases", []):
available.append(alias)
logger.error(
"Model '%s' not found on llama.cpp server. Available models: %s",
configured_model,
available,
)
return None
except Exception as e:
logger.warning(
"Failed to query llama.cpp /v1/models endpoint: %s. "
"Model validation skipped.",
e,
)
if model_entry is not None:
architecture = model_entry.get("architecture") or {}
input_modalities = architecture.get("input_modalities") or []
if isinstance(input_modalities, list):
info["supports_vision"] = "image" in input_modalities
info["supports_audio"] = "audio" in input_modalities
status = model_entry.get("status") or {}
launch_args = status.get("args") if isinstance(status, dict) else None
if not isinstance(launch_args, list):
launch_args = []
meta = model_entry.get("meta") if isinstance(model_entry, dict) else None
n_ctx = meta.get("n_ctx") if isinstance(meta, dict) else None
if not n_ctx:
n_ctx = _parse_launch_arg(launch_args, "--ctx-size")
if n_ctx:
try:
info["context_size"] = int(n_ctx)
except (TypeError, ValueError):
pass
# Tool calling on llama-server requires --jinja.
if "--jinja" in launch_args:
info["supports_tools"] = True
try:
props = _fetch_llama_props(base_url, configured_model)
if info["context_size"] is None:
default_settings = props.get("default_generation_settings", {})
n_ctx = default_settings.get("n_ctx")
if n_ctx:
info["context_size"] = int(n_ctx)
if not (info["supports_vision"] or info["supports_audio"]):
modalities = props.get("modalities", {})
info["supports_vision"] = bool(modalities.get("vision", False))
info["supports_audio"] = bool(modalities.get("audio", False))
if not info["supports_tools"]:
chat_caps = props.get("chat_template_caps", {})
info["supports_tools"] = bool(chat_caps.get("supports_tools", False))
media_marker = props.get("media_marker")
if isinstance(media_marker, str) and media_marker:
info["media_marker"] = media_marker
except Exception as e:
logger.warning(
"Failed to query llama.cpp /props endpoint: %s. "
"Image embeddings may fail if the server randomized its media marker.",
e,
)
return info
def _send(
self,
prompt: str,
images: list[bytes],
response_format: Optional[dict] = None,
) -> Optional[str]:
"""Submit a request to llama.cpp server."""
if self.provider is None:
logger.warning(
"llama.cpp provider has not been initialized, a description will not be generated. Check your llama.cpp configuration."
)
return None
try:
content = [
{
"type": "text",
"text": prompt,
}
]
for image in images:
encoded_image = base64.b64encode(image).decode("utf-8")
content.append(
{
"type": "image_url",
"image_url": { # type: ignore[dict-item]
"url": f"data:image/jpeg;base64,{encoded_image}",
},
}
)
# Build request payload with llama.cpp native options
payload = {
"model": self.genai_config.model,
"messages": [
{
"role": "user",
"content": content,
},
],
**self.provider_options,
}
if response_format:
payload["response_format"] = response_format
response = requests.post(
f"{self.provider}/v1/chat/completions",
json=payload,
timeout=self.timeout,
)
response.raise_for_status()
result = response.json()
if (
result is not None
and "choices" in result
and len(result["choices"]) > 0
):
choice = result["choices"][0]
if "message" in choice and "content" in choice["message"]:
return str(choice["message"]["content"].strip())
return None
except Exception as e:
logger.warning("llama.cpp returned an error: %s", str(e))
return None
@property
def supports_vision(self) -> bool:
"""Whether the loaded model supports vision/image input."""
return self._supports_vision
@property
def supports_audio(self) -> bool:
"""Whether the loaded model supports audio input."""
return self._supports_audio
@property
def supports_tools(self) -> bool:
"""Whether the loaded model supports tool/function calling."""
return self._supports_tools
def list_models(self) -> list[str]:
"""Return available model IDs from the llama.cpp server."""
base_url = self.provider or (
self.genai_config.base_url.rstrip("/")
if self.genai_config.base_url
else None
)
if base_url is None:
return []
try:
response = requests.get(f"{base_url}/v1/models", timeout=10)
response.raise_for_status()
models = []
for m in response.json().get("data", []):
models.append(m.get("id", "unknown"))
for alias in m.get("aliases", []):
models.append(alias)
return sorted(models)
except Exception as e:
logger.warning("Failed to list llama.cpp models: %s", e)
return []
def get_context_size(self) -> int:
"""Get the context window size for llama.cpp.
Resolution order:
1. provider_options["context_size"] (user override)
2. Value queried from llama.cpp server at init
3. Default fallback of 4096
"""
if "context_size" in self.provider_options:
return int(self.provider_options["context_size"])
if self._context_size is not None:
return self._context_size
return 4096
def estimate_image_tokens(self, width: int, height: int) -> float:
"""Probe the llama.cpp server to learn the model's image-token cost at the
requested dimensions.
llama.cpp's image tokenization is a deterministic function of dimensions and
the loaded mmproj, so the result is cached per (width, height) for the
lifetime of the process. Falls back to the base pixel heuristic if the
server is unreachable or the response is malformed.
"""
if self.provider is None:
return super().estimate_image_tokens(width, height)
cached = self._image_token_cache.get((width, height))
if cached is not None:
return cached
try:
baseline = self._probe_baseline_tokens()
with_image = self._probe_image_prompt_tokens(width, height)
tokens = max(1, with_image - baseline)
except Exception as e:
logger.debug(
"llama.cpp image-token probe failed for %dx%d (%s); using heuristic",
width,
height,
e,
)
return super().estimate_image_tokens(width, height)
self._image_token_cache[(width, height)] = tokens
logger.debug(
"llama.cpp model '%s' uses ~%d tokens for %dx%d images",
self.genai_config.model,
tokens,
width,
height,
)
return tokens
def _probe_baseline_tokens(self) -> int:
"""Return prompt_tokens for a minimal text-only request. Cached after first call."""
if self._text_baseline_tokens is not None:
return self._text_baseline_tokens
self._text_baseline_tokens = self._probe_prompt_tokens(
[{"type": "text", "text": "."}]
)
return self._text_baseline_tokens
def _probe_image_prompt_tokens(self, width: int, height: int) -> int:
"""Return prompt_tokens for a single synthetic image plus minimal text."""
img = Image.new("RGB", (width, height), (128, 128, 128))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=60)
encoded = base64.b64encode(buf.getvalue()).decode("utf-8")
return self._probe_prompt_tokens(
[
{"type": "text", "text": "."},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
},
]
)
def _probe_prompt_tokens(self, content: list[dict[str, Any]]) -> int:
"""POST a 1-token chat completion and return reported prompt_tokens.
Uses a generous timeout to absorb a cold model load on the first probe
when the server lazily loads models on demand (e.g. llama-swap).
"""
payload = {
"model": self.genai_config.model,
"messages": [{"role": "user", "content": content}],
"max_tokens": 1,
}
response = requests.post(
f"{self.provider}/v1/chat/completions",
json=payload,
timeout=60,
)
response.raise_for_status()
return int(response.json()["usage"]["prompt_tokens"])
def _build_payload(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]],
tool_choice: Optional[str],
stream: bool = False,
) -> dict[str, Any]:
"""Build request payload for chat completions (sync or stream)."""
openai_tool_choice = None
if tool_choice:
if tool_choice == "none":
openai_tool_choice = "none"
elif tool_choice == "auto":
openai_tool_choice = "auto"
elif tool_choice == "required":
openai_tool_choice = "required"
payload: dict[str, Any] = {
"messages": messages,
"model": self.genai_config.model,
}
if stream:
payload["stream"] = True
payload["stream_options"] = {"include_usage": True}
payload["timings_per_token"] = True
if tools:
payload["tools"] = tools
if openai_tool_choice is not None:
payload["tool_choice"] = openai_tool_choice
provider_opts = {
k: v for k, v in self.provider_options.items() if k != "context_size"
}
payload.update(provider_opts)
payload.update(self.genai_config.runtime_options)
return payload
def _message_from_choice(self, choice: dict[str, Any]) -> dict[str, Any]:
"""Parse OpenAI-style choice into {content, reasoning, tool_calls, finish_reason}.
llama.cpp's `--reasoning-format` puts the trace in
`message.reasoning_content` (preferred) or `message.thinking`; both
keys are accepted so different builds work without configuration.
"""
message = choice.get("message", {})
content = message.get("content")
content = content.strip() if content else None
reasoning = message.get("reasoning_content") or message.get("thinking")
reasoning = reasoning.strip() if reasoning else None
tool_calls = parse_tool_calls_from_message(message)
finish_reason = choice.get("finish_reason") or (
"tool_calls" if tool_calls else "stop" if content else "error"
)
return {
"content": content,
"reasoning": reasoning,
"tool_calls": tool_calls,
"finish_reason": finish_reason,
}
@staticmethod
def _streamed_tool_calls_to_list(
tool_calls_by_index: dict[int, dict[str, Any]],
) -> Optional[list[dict[str, Any]]]:
"""Convert streamed tool_calls index map to list of {id, name, arguments}."""
if not tool_calls_by_index:
return None
result = []
for idx in sorted(tool_calls_by_index.keys()):
t = tool_calls_by_index[idx]
args_str = t.get("arguments") or "{}"
try:
arguments = json.loads(args_str)
except json.JSONDecodeError:
arguments = {}
result.append(
{
"id": t.get("id", ""),
"name": t.get("name", ""),
"arguments": arguments,
}
)
return result if result else None
def _refresh_media_marker(self) -> bool:
"""Re-fetch /props and update the cached media marker if it changed.
The server randomizes the marker per startup (unless LLAMA_MEDIA_MARKER
is set), so a stale marker indicates a restart. Returns True iff the
marker was updated to a new value — used to gate a one-shot retry of
a failed embeddings request.
"""
if self.provider is None:
return False
try:
props = _fetch_llama_props(self.provider, self.genai_config.model)
except Exception as e:
logger.warning("Failed to refresh llama.cpp media marker: %s", e)
return False
marker = props.get("media_marker")
if not isinstance(marker, str) or not marker or marker == self._media_marker:
return False
logger.info("llama.cpp media marker changed (server restart); refreshed")
self._media_marker = marker
return True
def embed(
self,
texts: list[str] | None = None,
images: list[bytes] | None = None,
) -> list[np.ndarray]:
"""Generate embeddings via llama.cpp /embeddings endpoint.
Supports batch requests. Uses content format with prompt_string and
multimodal_data for images (PR #15108). Server must be started with
--embeddings and --mmproj for multimodal support.
"""
if self.provider is None:
logger.warning(
"llama.cpp provider has not been initialized. Check your llama.cpp configuration."
)
return []
texts = texts or []
images = images or []
if not texts and not images:
return []
EMBEDDING_DIM = 768
encoded_images: list[str] = []
for img in images:
# llama.cpp uses STB which does not support WebP; convert to JPEG
jpeg_bytes = _to_jpeg(img)
to_encode = jpeg_bytes if jpeg_bytes is not None else img
encoded_images.append(base64.b64encode(to_encode).decode("utf-8"))
def build_content() -> list[dict[str, Any]]:
# prompt_string must contain the server's media marker placeholder
# for each image. The marker is randomized per server startup.
content: list[dict[str, Any]] = []
for text in texts:
content.append({"prompt_string": text})
for encoded in encoded_images:
content.append(
{
"prompt_string": f"{self._media_marker}\n",
"multimodal_data": [encoded],
}
)
return content
def post_embeddings() -> requests.Response:
return requests.post(
f"{self.provider}/embeddings",
json={"model": self.genai_config.model, "content": build_content()},
timeout=self.timeout,
)
try:
try:
response = post_embeddings()
response.raise_for_status()
except requests.exceptions.RequestException:
# The server may have restarted with a new media marker.
# Refresh from /props; only retry if the marker actually changed.
if not encoded_images or not self._refresh_media_marker():
raise
response = post_embeddings()
response.raise_for_status()
result = response.json()
items = result.get("data", result) if isinstance(result, dict) else result
if not isinstance(items, list):
logger.warning("llama.cpp embeddings returned unexpected format")
return []
embeddings = []
for item in items:
emb = item.get("embedding") if isinstance(item, dict) else None
if emb is None:
logger.warning("llama.cpp embeddings item missing embedding field")
continue
arr = np.array(emb, dtype=np.float32)
if arr.ndim > 1:
# llama.cpp can return token-level embeddings; pool per item
arr = arr.mean(axis=0)
arr = arr.flatten()
orig_dim = arr.size
if orig_dim != EMBEDDING_DIM:
if orig_dim > EMBEDDING_DIM:
arr = arr[:EMBEDDING_DIM]
logger.debug(
"Truncated llama.cpp embedding from %d to %d dimensions",
orig_dim,
EMBEDDING_DIM,
)
else:
arr = np.pad(
arr,
(0, EMBEDDING_DIM - orig_dim),
mode="constant",
constant_values=0,
)
logger.debug(
"Padded llama.cpp embedding from %d to %d dimensions",
orig_dim,
EMBEDDING_DIM,
)
embeddings.append(arr)
return embeddings
except requests.exceptions.Timeout:
logger.warning("llama.cpp embeddings request timed out")
return []
except requests.exceptions.RequestException as e:
error_detail = str(e)
if hasattr(e, "response") and e.response is not None:
try:
error_detail = f"{str(e)} - Response: {e.response.text[:500]}"
except Exception:
pass
logger.warning("llama.cpp embeddings error: %s", error_detail)
return []
except Exception as e:
logger.warning("Unexpected error in llama.cpp embeddings: %s", str(e))
return []
def chat_with_tools(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> dict[str, Any]:
"""
Send chat messages to llama.cpp server with optional tool definitions.
Uses the OpenAI-compatible endpoint but passes through all native llama.cpp
parameters (like slot_id, temperature, etc.) via provider_options.
"""
if self.provider is None:
logger.warning(
"llama.cpp provider has not been initialized. Check your llama.cpp configuration."
)
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
try:
payload = self._build_payload(messages, tools, tool_choice, stream=False)
response = requests.post(
f"{self.provider}/v1/chat/completions",
json=payload,
timeout=self.timeout,
)
response.raise_for_status()
result = response.json()
if result is None or "choices" not in result or len(result["choices"]) == 0:
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
return self._message_from_choice(result["choices"][0])
except requests.exceptions.Timeout as e:
logger.warning("llama.cpp request timed out: %s", str(e))
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
except requests.exceptions.RequestException as e:
error_detail = str(e)
if hasattr(e, "response") and e.response is not None:
try:
error_detail = f"{str(e)} - Response: {e.response.text[:500]}"
except Exception:
pass
logger.warning("llama.cpp returned an error: %s", error_detail)
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
except Exception as e:
logger.warning("Unexpected error in llama.cpp chat_with_tools: %s", str(e))
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
async def chat_with_tools_stream(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> AsyncGenerator[tuple[str, Any], None]:
"""Stream chat with tools via OpenAI-compatible streaming API."""
if self.provider is None:
logger.warning(
"llama.cpp provider has not been initialized. Check your llama.cpp configuration."
)
yield (
"message",
{
"content": None,
"tool_calls": None,
"finish_reason": "error",
},
)
return
try:
payload = self._build_payload(messages, tools, tool_choice, stream=True)
content_parts: list[str] = []
reasoning_parts: list[str] = []
tool_calls_by_index: dict[int, dict[str, Any]] = {}
finish_reason = "stop"
async with httpx.AsyncClient(timeout=float(self.timeout)) as client:
async with client.stream(
"POST",
f"{self.provider}/v1/chat/completions",
json=payload,
) as response:
response.raise_for_status()
async for line in response.aiter_lines():
if not line.startswith("data: "):
continue
data_str = line[6:].strip()
if data_str == "[DONE]":
break
try:
data = json.loads(data_str)
except json.JSONDecodeError:
continue
maybe_stats = _stats_from_llama_cpp_chunk(data)
if maybe_stats is not None:
yield ("stats", maybe_stats)
choices = data.get("choices") or []
if not choices:
continue
delta = choices[0].get("delta", {})
if choices[0].get("finish_reason"):
finish_reason = choices[0]["finish_reason"]
# llama.cpp emits separated thinking under
# reasoning_content (preferred) or thinking before any
# content tokens arrive
reasoning_delta = delta.get("reasoning_content") or delta.get(
"thinking"
)
if reasoning_delta:
reasoning_parts.append(reasoning_delta)
yield ("reasoning_delta", reasoning_delta)
if delta.get("content"):
content_parts.append(delta["content"])
yield ("content_delta", delta["content"])
for tc in delta.get("tool_calls") or []:
idx = tc.get("index", 0)
fn = tc.get("function") or {}
if idx not in tool_calls_by_index:
tool_calls_by_index[idx] = {
"id": tc.get("id", ""),
"name": tc.get("name") or fn.get("name", ""),
"arguments": "",
}
t = tool_calls_by_index[idx]
if tc.get("id"):
t["id"] = tc["id"]
name = tc.get("name") or fn.get("name")
if name:
t["name"] = name
arg = tc.get("arguments") or fn.get("arguments")
if arg is not None:
t["arguments"] += (
arg if isinstance(arg, str) else json.dumps(arg)
)
full_content = "".join(content_parts).strip() or None
full_reasoning = "".join(reasoning_parts).strip() or None
tool_calls_list = self._streamed_tool_calls_to_list(tool_calls_by_index)
if tool_calls_list:
finish_reason = "tool_calls"
yield (
"message",
{
"content": full_content,
"reasoning": full_reasoning,
"tool_calls": tool_calls_list,
"finish_reason": finish_reason,
},
)
except httpx.HTTPStatusError as e:
logger.warning("llama.cpp streaming HTTP error: %s", e)
yield (
"message",
{
"content": None,
"tool_calls": None,
"finish_reason": "error",
},
)
except Exception as e:
logger.warning(
"Unexpected error in llama.cpp chat_with_tools_stream: %s", str(e)
)
yield (
"message",
{
"content": None,
"tool_calls": None,
"finish_reason": "error",
},
)
frigate/genai/plugins/ollama.py
+525
View File
@@ -0,0 +1,525 @@
"""Ollama Provider for Frigate AI."""
import base64
import binascii
import json
import logging
from typing import Any, AsyncGenerator, Optional
from httpx import RemoteProtocolError, TimeoutException
from ollama import AsyncClient as OllamaAsyncClient
from ollama import Client as ApiClient
from ollama import ResponseError
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
from frigate.genai.utils import parse_tool_calls_from_message
logger = logging.getLogger(__name__)
def _extract_ollama_stats(response: Any) -> Optional[dict[str, Any]]:
"""Build a stats dict from Ollama's response metadata.
Ollama reports eval_count/eval_duration (generation) and
prompt_eval_count (context size). Durations are nanoseconds.
"""
if not response:
return None
if hasattr(response, "get"):
getter = response.get
else:
getter = lambda key: getattr(response, key, None) # noqa: E731
eval_count = getter("eval_count")
eval_duration_ns = getter("eval_duration")
prompt_eval_count = getter("prompt_eval_count")
if eval_count is None and prompt_eval_count is None:
return None
stats: dict[str, Any] = {}
if isinstance(prompt_eval_count, int):
stats["prompt_tokens"] = prompt_eval_count
if isinstance(eval_count, int):
stats["completion_tokens"] = eval_count
if isinstance(eval_duration_ns, int) and eval_duration_ns > 0:
stats["completion_duration_ms"] = eval_duration_ns / 1_000_000
if isinstance(eval_count, int) and eval_count > 0:
stats["tokens_per_second"] = eval_count / (eval_duration_ns / 1_000_000_000)
return stats or None
def _normalize_multimodal_content(
content: Any,
) -> tuple[Optional[str], Optional[list[bytes]]]:
"""Convert OpenAI-style multimodal content to Ollama's (text, images) shape.
The chat API constructs user messages with content as a list of
``{"type": "text"}`` and ``{"type": "image_url"}`` parts when a tool
returns a live frame. Ollama's SDK requires content to be a string and
images to be passed in a separate field, so we extract each.
"""
if not isinstance(content, list):
return content, None
text_parts: list[str] = []
images: list[bytes] = []
for part in content:
if not isinstance(part, dict):
continue
part_type = part.get("type")
if part_type == "text":
text = part.get("text")
if text:
text_parts.append(str(text))
elif part_type == "image_url":
url = (part.get("image_url") or {}).get("url", "")
if isinstance(url, str) and url.startswith("data:"):
try:
encoded = url.split(",", 1)[1]
images.append(base64.b64decode(encoded, validate=True))
except (ValueError, IndexError, binascii.Error) as e:
logger.debug("Failed to decode multimodal image url: %s", e)
return ("\n".join(text_parts) if text_parts else None), (images or None)
@register_genai_provider(GenAIProviderEnum.ollama)
class OllamaClient(GenAIClient):
"""Generative AI client for Frigate using Ollama."""
LOCAL_OPTIMIZED_OPTIONS = {
"options": {
"temperature": 0.5,
"repeat_penalty": 1.05,
"presence_penalty": 0.3,
},
}
provider: ApiClient | None
provider_options: dict[str, Any]
def _auth_headers(self) -> dict | None:
if self.genai_config.api_key:
return {"Authorization": "Bearer " + self.genai_config.api_key}
return None
def _init_provider(self) -> ApiClient | None:
"""Initialize the client."""
self.provider_options = {
**self.LOCAL_OPTIMIZED_OPTIONS,
**self.genai_config.provider_options,
}
try:
client = ApiClient(
host=self.genai_config.base_url,
timeout=self.timeout,
headers=self._auth_headers(),
)
# ensure the model is available locally
response = client.show(self.genai_config.model)
if response.get("error"):
logger.error(
"Ollama error: %s",
response["error"],
)
return None
return client
except Exception as e:
logger.warning("Error initializing Ollama: %s", str(e))
return None
@staticmethod
def _clean_schema_for_ollama(schema: dict, *, _is_properties: bool = False) -> dict:
"""Strip Pydantic metadata from a JSON schema for Ollama compatibility.
Ollama's grammar-based constrained generation works best with minimal
schemas. Pydantic adds title/description/constraint fields that can
cause the grammar generator to silently skip required fields.
Keys inside a ``properties`` dict are actual field names and must never
be stripped, even if they collide with a metadata key name (e.g. a
model field called ``title``).
"""
STRIP_KEYS = {
"title",
"description",
"minimum",
"maximum",
"exclusiveMinimum",
"exclusiveMaximum",
}
result: dict[str, Any] = {}
for key, value in schema.items():
if not _is_properties and key in STRIP_KEYS:
continue
if isinstance(value, dict):
result[key] = OllamaClient._clean_schema_for_ollama(
value, _is_properties=(key == "properties")
)
elif isinstance(value, list):
result[key] = [
OllamaClient._clean_schema_for_ollama(item)
if isinstance(item, dict)
else item
for item in value
]
else:
result[key] = value
return result
def _send(
self,
prompt: str,
images: list[bytes],
response_format: Optional[dict] = None,
) -> Optional[str]:
"""Submit a request to Ollama"""
if self.provider is None:
logger.warning(
"Ollama provider has not been initialized, a description will not be generated. Check your Ollama configuration."
)
return None
try:
ollama_options = {
**self.provider_options,
**self.genai_config.runtime_options,
}
if response_format and response_format.get("type") == "json_schema":
schema = response_format.get("json_schema", {}).get("schema")
if schema:
ollama_options["format"] = self._clean_schema_for_ollama(schema)
logger.debug(
"Ollama generate request: model=%s, prompt_len=%s, image_count=%s, "
"has_format=%s, options=%s",
self.genai_config.model,
len(prompt),
len(images) if images else 0,
"format" in ollama_options,
{k: v for k, v in ollama_options.items() if k != "format"},
)
result = self.provider.generate(
self.genai_config.model,
prompt,
images=images if images else None,
**ollama_options,
)
logger.debug(
"Ollama generate response: done=%s, done_reason=%s, eval_count=%s, "
"prompt_eval_count=%s, response_len=%s",
result.get("done"),
result.get("done_reason"),
result.get("eval_count"),
result.get("prompt_eval_count"),
len(result.get("response", "") or ""),
)
response_text = str(result["response"]).strip()
if not response_text:
logger.warning(
"Ollama returned a blank response for model %s (done_reason=%s, "
"eval_count=%s). Check model output, ensure thinking is disabled.",
self.genai_config.model,
result.get("done_reason"),
result.get("eval_count"),
)
return response_text
except (
TimeoutException,
ResponseError,
RemoteProtocolError,
ConnectionError,
) as e:
logger.warning("Ollama returned an error: %s", str(e))
return None
def list_models(self) -> list[str]:
"""Return available model names from the Ollama server."""
client = self.provider
if client is None:
# Provider init may have failed due to invalid model, but we can
# still list available models with a fresh client.
if not self.genai_config.base_url:
return []
try:
client = ApiClient(
host=self.genai_config.base_url,
timeout=self.timeout,
headers=self._auth_headers(),
)
except Exception:
return []
try:
response = client.list()
return sorted(
m.get("name", m.get("model", "")) for m in response.get("models", [])
)
except Exception as e:
logger.warning("Failed to list Ollama models: %s", e)
return []
def get_context_size(self) -> int:
"""Get the context window size for Ollama."""
return int(
self.genai_config.provider_options.get("options", {}).get("num_ctx", 4096)
)
def _build_request_params(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]],
tool_choice: Optional[str],
stream: bool = False,
) -> dict[str, Any]:
"""Build request_messages and params for chat (sync or stream)."""
request_messages = []
for msg in messages:
content, images = _normalize_multimodal_content(msg.get("content", ""))
msg_dict: dict[str, Any] = {
"role": msg.get("role"),
"content": content if content is not None else "",
}
if images:
msg_dict["images"] = images
if msg.get("tool_call_id"):
msg_dict["tool_call_id"] = msg["tool_call_id"]
if msg.get("name"):
msg_dict["name"] = msg["name"]
if msg.get("tool_calls"):
# Ollama requires tool call arguments as dicts, but the
# conversation format (OpenAI-style) stores them as JSON
# strings. Convert back to dicts for Ollama.
ollama_tool_calls = []
for tc in msg["tool_calls"]:
func = tc.get("function") or {}
args = func.get("arguments") or {}
if isinstance(args, str):
try:
args = json.loads(args)
except (json.JSONDecodeError, TypeError):
args = {}
ollama_tool_calls.append(
{"function": {"name": func.get("name", ""), "arguments": args}}
)
msg_dict["tool_calls"] = ollama_tool_calls
request_messages.append(msg_dict)
request_params: dict[str, Any] = {
"model": self.genai_config.model,
"messages": request_messages,
**self.provider_options,
**self.genai_config.runtime_options,
}
if stream:
request_params["stream"] = True
if tools:
request_params["tools"] = tools
return request_params
def _message_from_response(self, response: dict[str, Any]) -> dict[str, Any]:
"""Parse Ollama chat response into {content, tool_calls, finish_reason}."""
if not response or "message" not in response:
logger.debug("Ollama response empty or missing 'message' key")
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
message = response["message"]
logger.debug(
"Ollama response message keys: %s, content_len=%s, thinking_len=%s, "
"tool_calls=%s, done=%s",
list(message.keys()) if hasattr(message, "keys") else "N/A",
len(message.get("content", "") or "") if message.get("content") else 0,
len(message.get("thinking", "") or "") if message.get("thinking") else 0,
bool(message.get("tool_calls")),
response.get("done"),
)
content = message.get("content", "").strip() if message.get("content") else None
reasoning = (
message.get("thinking", "").strip() if message.get("thinking") else None
)
tool_calls = parse_tool_calls_from_message(message)
finish_reason = "error"
if response.get("done"):
finish_reason = (
"tool_calls" if tool_calls else "stop" if content else "error"
)
elif tool_calls:
finish_reason = "tool_calls"
elif content:
finish_reason = "stop"
return {
"content": content,
"reasoning": reasoning,
"tool_calls": tool_calls,
"finish_reason": finish_reason,
}
def chat_with_tools(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> dict[str, Any]:
if self.provider is None:
logger.warning(
"Ollama provider has not been initialized. Check your Ollama configuration."
)
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
try:
request_params = self._build_request_params(
messages, tools, tool_choice, stream=False
)
response = self.provider.chat(**request_params)
return self._message_from_response(response)
except (TimeoutException, ResponseError, ConnectionError) as e:
logger.warning("Ollama returned an error: %s", str(e))
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
except Exception as e:
logger.warning("Unexpected error in Ollama chat_with_tools: %s", str(e))
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
async def chat_with_tools_stream(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> AsyncGenerator[tuple[str, Any], None]:
"""Stream chat with tools; yields content deltas then final message.
When tools are provided, Ollama streaming does not include tool_calls
in the response chunks. To work around this, we use a non-streaming
call when tools are present to ensure tool calls are captured, then
emit the content as a single delta followed by the final message.
"""
if self.provider is None:
logger.warning(
"Ollama provider has not been initialized. Check your Ollama configuration."
)
yield (
"message",
{
"content": None,
"tool_calls": None,
"finish_reason": "error",
},
)
return
try:
# Ollama does not return tool_calls in streaming mode, so fall
# back to a non-streaming call when tools are provided.
if tools:
logger.debug(
"Ollama: tools provided, using non-streaming call for tool support"
)
request_params = self._build_request_params(
messages, tools, tool_choice, stream=False
)
async_client = OllamaAsyncClient(
host=self.genai_config.base_url,
timeout=self.timeout,
headers=self._auth_headers(),
)
response = await async_client.chat(**request_params)
result = self._message_from_response(response)
reasoning = result.get("reasoning")
if reasoning:
yield ("reasoning_delta", reasoning)
content = result.get("content")
if content:
yield ("content_delta", content)
stats = _extract_ollama_stats(response)
if stats is not None:
yield ("stats", stats)
yield ("message", result)
return
request_params = self._build_request_params(
messages, tools, tool_choice, stream=True
)
async_client = OllamaAsyncClient(
host=self.genai_config.base_url,
timeout=self.timeout,
headers=self._auth_headers(),
)
content_parts: list[str] = []
reasoning_parts: list[str] = []
final_message: dict[str, Any] | None = None
final_chunk: Any = None
stream = await async_client.chat(**request_params)
async for chunk in stream:
if not chunk or "message" not in chunk:
continue
msg = chunk.get("message", {})
reasoning_delta = msg.get("thinking") or ""
if reasoning_delta:
reasoning_parts.append(reasoning_delta)
yield ("reasoning_delta", reasoning_delta)
delta = msg.get("content") or ""
if delta:
content_parts.append(delta)
yield ("content_delta", delta)
if chunk.get("done"):
final_chunk = chunk
full_content = "".join(content_parts).strip() or None
full_reasoning = "".join(reasoning_parts).strip() or None
final_message = {
"content": full_content,
"reasoning": full_reasoning,
"tool_calls": None,
"finish_reason": "stop",
}
break
stats = _extract_ollama_stats(final_chunk)
if stats is not None:
yield ("stats", stats)
if final_message is not None:
yield ("message", final_message)
else:
yield (
"message",
{
"content": "".join(content_parts).strip() or None,
"reasoning": "".join(reasoning_parts).strip() or None,
"tool_calls": None,
"finish_reason": "stop",
},
)
except (TimeoutException, ResponseError, ConnectionError) as e:
logger.warning("Ollama streaming error: %s", str(e))
yield (
"message",
{
"content": None,
"tool_calls": None,
"finish_reason": "error",
},
)
except Exception as e:
logger.warning(
"Unexpected error in Ollama chat_with_tools_stream: %s", str(e)
)
yield (
"message",
{
"content": None,
"tool_calls": None,
"finish_reason": "error",
},
)
frigate/genai/plugins/openai.py
+464
View File
@@ -0,0 +1,464 @@
"""OpenAI Provider for Frigate AI."""
import base64
import json
import logging
from typing import Any, AsyncGenerator, Optional
from httpx import TimeoutException
from openai import OpenAI
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
logger = logging.getLogger(__name__)
def _stats_from_openai_usage(usage: Any) -> Optional[dict[str, Any]]:
"""Build a stats dict from an OpenAI-compatible usage object."""
if usage is None:
return None
prompt_tokens = getattr(usage, "prompt_tokens", None)
completion_tokens = getattr(usage, "completion_tokens", None)
if prompt_tokens is None and completion_tokens is None:
return None
stats: dict[str, Any] = {}
if isinstance(prompt_tokens, int):
stats["prompt_tokens"] = prompt_tokens
if isinstance(completion_tokens, int):
stats["completion_tokens"] = completion_tokens
return stats or None
@register_genai_provider(GenAIProviderEnum.openai)
class OpenAIClient(GenAIClient):
"""Generative AI client for Frigate using OpenAI."""
provider: OpenAI
context_size: Optional[int] = None
def _init_provider(self) -> OpenAI:
"""Initialize the client.
Subclasses (e.g. Azure) should raise on configuration errors; the
manager catches construction failures and disables the provider.
"""
# Extract context_size from provider_options as it's not a valid OpenAI client parameter
# It will be used in get_context_size() instead
provider_opts = {
k: v
for k, v in self.genai_config.provider_options.items()
if k != "context_size"
}
if self.genai_config.base_url:
provider_opts["base_url"] = self.genai_config.base_url
return OpenAI(api_key=self.genai_config.api_key, **provider_opts)
def _send(
self,
prompt: str,
images: list[bytes],
response_format: Optional[dict] = None,
) -> Optional[str]:
"""Submit a request to OpenAI."""
encoded_images = [base64.b64encode(image).decode("utf-8") for image in images]
messages_content: list[dict] = [
{
"type": "text",
"text": prompt,
}
]
for image in encoded_images:
messages_content.append(
{
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{image}",
"detail": "low",
},
}
)
try:
request_params = {
"model": self.genai_config.model,
"messages": [
{
"role": "user",
"content": messages_content,
},
],
"timeout": self.timeout,
**self.genai_config.runtime_options,
}
if response_format:
# OpenAI strict mode requires additionalProperties: false on the schema
if response_format.get("type") == "json_schema" and response_format.get(
"json_schema", {}
).get("strict"):
schema = response_format.get("json_schema", {}).get("schema")
if isinstance(schema, dict):
schema["additionalProperties"] = False
request_params["response_format"] = response_format
result = self.provider.chat.completions.create(**request_params)
if (
result is not None
and hasattr(result, "choices")
and len(result.choices) > 0
):
message = result.choices[0].message
content = message.content
if not content:
# When reasoning is enabled for some OpenAI backends the actual response
# is incorrectly placed in reasoning_content instead of content.
# This is buggy/incorrect behavior — reasoning should not be
# enabled for these models.
reasoning_content = getattr(message, "reasoning_content", None)
if reasoning_content:
logger.warning(
"Response content was empty but reasoning_content was provided; "
"reasoning appears to be enabled and should be disabled for this model."
)
content = reasoning_content
return str(content.strip()) if content else None
return None
except (TimeoutException, Exception) as e:
logger.warning("OpenAI returned an error: %s", str(e))
return None
def list_models(self) -> list[str]:
"""Return available model IDs from the OpenAI-compatible API."""
try:
return sorted(m.id for m in self.provider.models.list().data)
except Exception as e:
logger.warning("Failed to list OpenAI models: %s", e)
return []
def get_context_size(self) -> int:
"""Get the context window size for OpenAI."""
if self.context_size is not None:
return self.context_size
# First check provider_options for manually specified context size
# This is necessary for llama.cpp and other OpenAI-compatible servers
# that don't expose the configured runtime context size in the API response
if "context_size" in self.genai_config.provider_options:
self.context_size = self.genai_config.provider_options["context_size"]
logger.debug(
f"Using context size {self.context_size} from provider_options for model {self.genai_config.model}"
)
return self.context_size
try:
models = self.provider.models.list()
for model in models.data:
if model.id == self.genai_config.model:
if hasattr(model, "max_model_len") and model.max_model_len:
self.context_size = model.max_model_len
logger.debug(
f"Retrieved context size {self.context_size} for model {self.genai_config.model}"
)
return self.context_size
except Exception as e:
logger.debug(
f"Failed to fetch model context size from API: {e}, using default"
)
# Default to 128K for ChatGPT models, 8K for others
model_name = self.genai_config.model.lower()
if "gpt" in model_name:
self.context_size = 128000
else:
self.context_size = 8192
logger.debug(
f"Using default context size {self.context_size} for model {self.genai_config.model}"
)
return self.context_size
def chat_with_tools(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> dict[str, Any]:
"""
Send chat messages to OpenAI with optional tool definitions.
Implements function calling/tool usage for OpenAI models.
"""
try:
openai_tool_choice = None
if tool_choice:
if tool_choice == "none":
openai_tool_choice = "none"
elif tool_choice == "auto":
openai_tool_choice = "auto"
elif tool_choice == "required":
openai_tool_choice = "required"
request_params = {
"model": self.genai_config.model,
"messages": messages,
"timeout": self.timeout,
**self.genai_config.runtime_options,
}
if tools:
request_params["tools"] = tools
if openai_tool_choice is not None:
request_params["tool_choice"] = openai_tool_choice
if isinstance(self.genai_config.provider_options, dict):
excluded_options = {"context_size"}
provider_opts = {
k: v
for k, v in self.genai_config.provider_options.items()
if k not in excluded_options
}
request_params.update(provider_opts)
result = self.provider.chat.completions.create(**request_params)
if (
result is None
or not hasattr(result, "choices")
or len(result.choices) == 0
):
return {
"content": None,
"tool_calls": None,
"finish_reason": "error",
}
choice = result.choices[0]
message = choice.message
content = message.content.strip() if message.content else None
raw_reasoning = getattr(message, "reasoning_content", None) or getattr(
message, "reasoning", None
)
reasoning = raw_reasoning.strip() if raw_reasoning else None
tool_calls = None
if message.tool_calls:
tool_calls = []
for tool_call in message.tool_calls:
try:
arguments = json.loads(tool_call.function.arguments)
except (json.JSONDecodeError, AttributeError) as e:
logger.warning(
f"Failed to parse tool call arguments: {e}, "
f"tool: {tool_call.function.name if hasattr(tool_call.function, 'name') else 'unknown'}"
)
arguments = {}
tool_calls.append(
{
"id": tool_call.id if hasattr(tool_call, "id") else "",
"name": tool_call.function.name
if hasattr(tool_call.function, "name")
else "",
"arguments": arguments,
}
)
finish_reason = "error"
if hasattr(choice, "finish_reason") and choice.finish_reason:
finish_reason = choice.finish_reason
elif tool_calls:
finish_reason = "tool_calls"
elif content:
finish_reason = "stop"
return {
"content": content,
"reasoning": reasoning,
"tool_calls": tool_calls,
"finish_reason": finish_reason,
}
except TimeoutException as e:
logger.warning("OpenAI request timed out: %s", str(e))
return {
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
}
except Exception as e:
logger.warning("OpenAI returned an error: %s", str(e))
return {
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
}
async def chat_with_tools_stream(
self,
messages: list[dict[str, Any]],
tools: Optional[list[dict[str, Any]]] = None,
tool_choice: Optional[str] = "auto",
) -> AsyncGenerator[tuple[str, Any], None]:
"""
Stream chat with tools; yields content deltas then final message.
Implements streaming function calling/tool usage for OpenAI models.
"""
try:
openai_tool_choice = None
if tool_choice:
if tool_choice == "none":
openai_tool_choice = "none"
elif tool_choice == "auto":
openai_tool_choice = "auto"
elif tool_choice == "required":
openai_tool_choice = "required"
request_params = {
"model": self.genai_config.model,
"messages": messages,
"timeout": self.timeout,
"stream": True,
"stream_options": {"include_usage": True},
**self.genai_config.runtime_options,
}
if tools:
request_params["tools"] = tools
if openai_tool_choice is not None:
request_params["tool_choice"] = openai_tool_choice
if isinstance(self.genai_config.provider_options, dict):
excluded_options = {"context_size"}
provider_opts = {
k: v
for k, v in self.genai_config.provider_options.items()
if k not in excluded_options
}
request_params.update(provider_opts)
# Use streaming API
content_parts: list[str] = []
reasoning_parts: list[str] = []
tool_calls_by_index: dict[int, dict[str, Any]] = {}
finish_reason = "stop"
usage_stats: Optional[dict[str, Any]] = None
stream = self.provider.chat.completions.create(**request_params)
for chunk in stream:
chunk_usage = getattr(chunk, "usage", None)
if chunk_usage is not None:
usage_stats = _stats_from_openai_usage(chunk_usage)
if not chunk or not chunk.choices:
continue
choice = chunk.choices[0]
delta = choice.delta
# Check for finish reason
if choice.finish_reason:
finish_reason = choice.finish_reason
# Extract reasoning deltas (reasoning_content or reasoning,
# depending on the server)
reasoning_delta = getattr(delta, "reasoning_content", None) or getattr(
delta, "reasoning", None
)
if reasoning_delta:
reasoning_parts.append(reasoning_delta)
yield ("reasoning_delta", reasoning_delta)
# Extract content deltas
if delta.content:
content_parts.append(delta.content)
yield ("content_delta", delta.content)
# Extract tool calls
if delta.tool_calls:
for tc in delta.tool_calls:
idx = tc.index
fn = tc.function
if idx not in tool_calls_by_index:
tool_calls_by_index[idx] = {
"id": tc.id or "",
"name": fn.name if fn and fn.name else "",
"arguments": "",
}
t = tool_calls_by_index[idx]
if tc.id:
t["id"] = tc.id
if fn and fn.name:
t["name"] = fn.name
if fn and fn.arguments:
t["arguments"] += fn.arguments
# Build final message
full_content = "".join(content_parts).strip() or None
full_reasoning = "".join(reasoning_parts).strip() or None
# Convert tool calls to list format
tool_calls_list = None
if tool_calls_by_index:
tool_calls_list = []
for tc in tool_calls_by_index.values():
try:
# Parse accumulated arguments as JSON
parsed_args = json.loads(tc["arguments"])
except (json.JSONDecodeError, Exception):
parsed_args = tc["arguments"]
tool_calls_list.append(
{
"id": tc["id"],
"name": tc["name"],
"arguments": parsed_args,
}
)
finish_reason = "tool_calls"
if usage_stats is not None:
yield ("stats", usage_stats)
yield (
"message",
{
"content": full_content,
"reasoning": full_reasoning,
"tool_calls": tool_calls_list,
"finish_reason": finish_reason,
},
)
except TimeoutException as e:
logger.warning("OpenAI streaming request timed out: %s", str(e))
yield (
"message",
{
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
},
)
except Exception as e:
logger.warning("OpenAI streaming returned an error: %s", str(e))
yield (
"message",
{
"content": None,
"reasoning": None,
"tool_calls": None,
"finish_reason": "error",
},
)