Handle other fields for context size from api providers

This commit is contained in:
Nicolas Mowen
2026-11-18 14:19:07 -07:00
parent f4e9fe6d3b
commit 313009aa11
3 changed files with 97 additions and 4 deletions
+2 -2
View File
@@ -225,7 +225,7 @@ Frigate supports any provider that implements the OpenAI API standard. This incl
:::tip :::tip
For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`:
```yaml ```yaml
genai: genai:
@@ -427,7 +427,7 @@ To use a different OpenAI-compatible API endpoint, set the `OPENAI_BASE_URL` env
:::tip :::tip
For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`:
```yaml {6,7} ```yaml {6,7}
genai: genai:
+25 -2
View File
@@ -39,6 +39,28 @@ def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None:
return stats or None return stats or None
def _context_size_from_model(model: Any) -> int | None:
"""Read the context window from an OpenAI-compatible /models entry.
vLLM reports ``max_model_len``. OpenRouter reports ``context_length`` as the
largest window any of its providers offers, and ``top_provider.context_length``
for the provider it routes to first. The latter can be smaller, so it is
preferred to avoid sending more than the serving provider accepts.
"""
top_provider = getattr(model, "top_provider", None)
candidates = (
getattr(model, "max_model_len", None),
top_provider.get("context_length") if isinstance(top_provider, dict) else None,
getattr(model, "context_length", None),
)
for value in candidates:
if isinstance(value, int) and value > 0:
return value
return None
@register_genai_provider(GenAIProviderEnum.openai) @register_genai_provider(GenAIProviderEnum.openai)
class OpenAIClient(GenAIClient): class OpenAIClient(GenAIClient):
"""Generative AI client for Frigate using OpenAI.""" """Generative AI client for Frigate using OpenAI."""
@@ -225,8 +247,9 @@ class OpenAIClient(GenAIClient):
models = self.provider.models.list() models = self.provider.models.list()
for model in models.data: for model in models.data:
if model.id == self.genai_config.model: if model.id == self.genai_config.model:
if hasattr(model, "max_model_len") and model.max_model_len: context_size = _context_size_from_model(model)
self.context_size = model.max_model_len if context_size:
self.context_size = context_size
logger.debug( logger.debug(
f"Retrieved context size {self.context_size} for model {self.genai_config.model}" f"Retrieved context size {self.context_size} for model {self.genai_config.model}"
) )
+70
View File
@@ -327,6 +327,76 @@ class TestOpenAIProvider(unittest.TestCase):
async_client.close.assert_awaited_once() async_client.close.assert_awaited_once()
class TestOpenAIContextSize(unittest.TestCase):
"""Context size lookup from an OpenAI-compatible /models endpoint."""
# Neither model name contains "gpt", so the name-based default is 8192 and
# any other value must have come from the /models response.
MODEL = "google/gemini-2.5-flash"
def _client_with_models(self, *entries, provider_options=None):
client = _make_client(
"openai",
model=self.MODEL,
api_key="k",
base_url="https://openrouter.ai/api/v1",
provider_options=provider_options or {},
)
client.provider.models.list = MagicMock(
return_value=SimpleNamespace(data=list(entries))
)
return client
def test_vllm_max_model_len(self):
client = self._client_with_models(
SimpleNamespace(id=self.MODEL, max_model_len=32768)
)
self.assertEqual(client.get_context_size(), 32768)
def test_openrouter_prefers_top_provider_context_length(self):
# The top-level value is the largest any provider offers; the routed
# provider's smaller window is the one requests must fit in.
client = self._client_with_models(
SimpleNamespace(id="other/model", context_length=4096),
SimpleNamespace(
id=self.MODEL,
context_length=1048576,
top_provider={"context_length": 262144},
),
)
self.assertEqual(client.get_context_size(), 262144)
def test_openrouter_falls_back_to_context_length(self):
client = self._client_with_models(
SimpleNamespace(
id=self.MODEL,
context_length=1048576,
top_provider={"context_length": None},
)
)
self.assertEqual(client.get_context_size(), 1048576)
def test_default_when_model_reports_no_context(self):
client = self._client_with_models(SimpleNamespace(id=self.MODEL))
self.assertEqual(client.get_context_size(), 8192)
def test_provider_options_override_api(self):
client = self._client_with_models(
SimpleNamespace(id=self.MODEL, context_length=1048576),
provider_options={"context_size": 16000},
)
self.assertEqual(client.get_context_size(), 16000)
client.provider.models.list.assert_not_called()
def test_result_is_cached(self):
client = self._client_with_models(
SimpleNamespace(id=self.MODEL, context_length=1048576)
)
client.get_context_size()
client.get_context_size()
client.provider.models.list.assert_called_once()
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Gemini # Gemini
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------