mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-19 03:53:03 +03:00
Handle other fields for context size from api providers
This commit is contained in:
@@ -225,7 +225,7 @@ Frigate supports any provider that implements the OpenAI API standard. This incl
|
||||
|
||||
:::tip
|
||||
|
||||
For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`:
|
||||
Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`:
|
||||
|
||||
```yaml
|
||||
genai:
|
||||
@@ -427,7 +427,7 @@ To use a different OpenAI-compatible API endpoint, set the `OPENAI_BASE_URL` env
|
||||
|
||||
:::tip
|
||||
|
||||
For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`:
|
||||
Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`:
|
||||
|
||||
```yaml {6,7}
|
||||
genai:
|
||||
|
||||
@@ -39,6 +39,28 @@ def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None:
|
||||
return stats or None
|
||||
|
||||
|
||||
def _context_size_from_model(model: Any) -> int | None:
|
||||
"""Read the context window from an OpenAI-compatible /models entry.
|
||||
|
||||
vLLM reports ``max_model_len``. OpenRouter reports ``context_length`` as the
|
||||
largest window any of its providers offers, and ``top_provider.context_length``
|
||||
for the provider it routes to first. The latter can be smaller, so it is
|
||||
preferred to avoid sending more than the serving provider accepts.
|
||||
"""
|
||||
top_provider = getattr(model, "top_provider", None)
|
||||
candidates = (
|
||||
getattr(model, "max_model_len", None),
|
||||
top_provider.get("context_length") if isinstance(top_provider, dict) else None,
|
||||
getattr(model, "context_length", None),
|
||||
)
|
||||
|
||||
for value in candidates:
|
||||
if isinstance(value, int) and value > 0:
|
||||
return value
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@register_genai_provider(GenAIProviderEnum.openai)
|
||||
class OpenAIClient(GenAIClient):
|
||||
"""Generative AI client for Frigate using OpenAI."""
|
||||
@@ -225,8 +247,9 @@ class OpenAIClient(GenAIClient):
|
||||
models = self.provider.models.list()
|
||||
for model in models.data:
|
||||
if model.id == self.genai_config.model:
|
||||
if hasattr(model, "max_model_len") and model.max_model_len:
|
||||
self.context_size = model.max_model_len
|
||||
context_size = _context_size_from_model(model)
|
||||
if context_size:
|
||||
self.context_size = context_size
|
||||
logger.debug(
|
||||
f"Retrieved context size {self.context_size} for model {self.genai_config.model}"
|
||||
)
|
||||
|
||||
@@ -327,6 +327,76 @@ class TestOpenAIProvider(unittest.TestCase):
|
||||
async_client.close.assert_awaited_once()
|
||||
|
||||
|
||||
class TestOpenAIContextSize(unittest.TestCase):
|
||||
"""Context size lookup from an OpenAI-compatible /models endpoint."""
|
||||
|
||||
# Neither model name contains "gpt", so the name-based default is 8192 and
|
||||
# any other value must have come from the /models response.
|
||||
MODEL = "google/gemini-2.5-flash"
|
||||
|
||||
def _client_with_models(self, *entries, provider_options=None):
|
||||
client = _make_client(
|
||||
"openai",
|
||||
model=self.MODEL,
|
||||
api_key="k",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
provider_options=provider_options or {},
|
||||
)
|
||||
client.provider.models.list = MagicMock(
|
||||
return_value=SimpleNamespace(data=list(entries))
|
||||
)
|
||||
return client
|
||||
|
||||
def test_vllm_max_model_len(self):
|
||||
client = self._client_with_models(
|
||||
SimpleNamespace(id=self.MODEL, max_model_len=32768)
|
||||
)
|
||||
self.assertEqual(client.get_context_size(), 32768)
|
||||
|
||||
def test_openrouter_prefers_top_provider_context_length(self):
|
||||
# The top-level value is the largest any provider offers; the routed
|
||||
# provider's smaller window is the one requests must fit in.
|
||||
client = self._client_with_models(
|
||||
SimpleNamespace(id="other/model", context_length=4096),
|
||||
SimpleNamespace(
|
||||
id=self.MODEL,
|
||||
context_length=1048576,
|
||||
top_provider={"context_length": 262144},
|
||||
),
|
||||
)
|
||||
self.assertEqual(client.get_context_size(), 262144)
|
||||
|
||||
def test_openrouter_falls_back_to_context_length(self):
|
||||
client = self._client_with_models(
|
||||
SimpleNamespace(
|
||||
id=self.MODEL,
|
||||
context_length=1048576,
|
||||
top_provider={"context_length": None},
|
||||
)
|
||||
)
|
||||
self.assertEqual(client.get_context_size(), 1048576)
|
||||
|
||||
def test_default_when_model_reports_no_context(self):
|
||||
client = self._client_with_models(SimpleNamespace(id=self.MODEL))
|
||||
self.assertEqual(client.get_context_size(), 8192)
|
||||
|
||||
def test_provider_options_override_api(self):
|
||||
client = self._client_with_models(
|
||||
SimpleNamespace(id=self.MODEL, context_length=1048576),
|
||||
provider_options={"context_size": 16000},
|
||||
)
|
||||
self.assertEqual(client.get_context_size(), 16000)
|
||||
client.provider.models.list.assert_not_called()
|
||||
|
||||
def test_result_is_cached(self):
|
||||
client = self._client_with_models(
|
||||
SimpleNamespace(id=self.MODEL, context_length=1048576)
|
||||
)
|
||||
client.get_context_size()
|
||||
client.get_context_size()
|
||||
client.provider.models.list.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gemini
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user