From 313009aa114014b4cd59a13bd1e82c2ca3429bba Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 18 Nov 2026 14:18:58 -0700 Subject: [PATCH] Handle other fields for context size from api providers --- docs/docs/configuration/genai/config.md | 4 +- frigate/genai/plugins/openai.py | 27 +++++++++- frigate/test/test_genai_providers.py | 70 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) diff --git a/docs/docs/configuration/genai/config.md b/docs/docs/configuration/genai/config.md index bfcb9d1373..d0005fb59d 100644 --- a/docs/docs/configuration/genai/config.md +++ b/docs/docs/configuration/genai/config.md @@ -225,7 +225,7 @@ Frigate supports any provider that implements the OpenAI API standard. This incl :::tip -For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: +Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: ```yaml genai: @@ -427,7 +427,7 @@ To use a different OpenAI-compatible API endpoint, set the `OPENAI_BASE_URL` env :::tip -For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: +Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: ```yaml {6,7} genai: diff --git a/frigate/genai/plugins/openai.py b/frigate/genai/plugins/openai.py index fe4b63a636..70ae6a8e28 100644 --- a/frigate/genai/plugins/openai.py +++ b/frigate/genai/plugins/openai.py @@ -39,6 +39,28 @@ def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None: return stats or None +def _context_size_from_model(model: Any) -> int | None: + """Read the context window from an OpenAI-compatible /models entry. + + vLLM reports ``max_model_len``. OpenRouter reports ``context_length`` as the + largest window any of its providers offers, and ``top_provider.context_length`` + for the provider it routes to first. The latter can be smaller, so it is + preferred to avoid sending more than the serving provider accepts. + """ + top_provider = getattr(model, "top_provider", None) + candidates = ( + getattr(model, "max_model_len", None), + top_provider.get("context_length") if isinstance(top_provider, dict) else None, + getattr(model, "context_length", None), + ) + + for value in candidates: + if isinstance(value, int) and value > 0: + return value + + return None + + @register_genai_provider(GenAIProviderEnum.openai) class OpenAIClient(GenAIClient): """Generative AI client for Frigate using OpenAI.""" @@ -225,8 +247,9 @@ class OpenAIClient(GenAIClient): models = self.provider.models.list() for model in models.data: if model.id == self.genai_config.model: - if hasattr(model, "max_model_len") and model.max_model_len: - self.context_size = model.max_model_len + context_size = _context_size_from_model(model) + if context_size: + self.context_size = context_size logger.debug( f"Retrieved context size {self.context_size} for model {self.genai_config.model}" ) diff --git a/frigate/test/test_genai_providers.py b/frigate/test/test_genai_providers.py index f4cdccaac3..7ff601e17d 100644 --- a/frigate/test/test_genai_providers.py +++ b/frigate/test/test_genai_providers.py @@ -327,6 +327,76 @@ class TestOpenAIProvider(unittest.TestCase): async_client.close.assert_awaited_once() +class TestOpenAIContextSize(unittest.TestCase): + """Context size lookup from an OpenAI-compatible /models endpoint.""" + + # Neither model name contains "gpt", so the name-based default is 8192 and + # any other value must have come from the /models response. + MODEL = "google/gemini-2.5-flash" + + def _client_with_models(self, *entries, provider_options=None): + client = _make_client( + "openai", + model=self.MODEL, + api_key="k", + base_url="https://openrouter.ai/api/v1", + provider_options=provider_options or {}, + ) + client.provider.models.list = MagicMock( + return_value=SimpleNamespace(data=list(entries)) + ) + return client + + def test_vllm_max_model_len(self): + client = self._client_with_models( + SimpleNamespace(id=self.MODEL, max_model_len=32768) + ) + self.assertEqual(client.get_context_size(), 32768) + + def test_openrouter_prefers_top_provider_context_length(self): + # The top-level value is the largest any provider offers; the routed + # provider's smaller window is the one requests must fit in. + client = self._client_with_models( + SimpleNamespace(id="other/model", context_length=4096), + SimpleNamespace( + id=self.MODEL, + context_length=1048576, + top_provider={"context_length": 262144}, + ), + ) + self.assertEqual(client.get_context_size(), 262144) + + def test_openrouter_falls_back_to_context_length(self): + client = self._client_with_models( + SimpleNamespace( + id=self.MODEL, + context_length=1048576, + top_provider={"context_length": None}, + ) + ) + self.assertEqual(client.get_context_size(), 1048576) + + def test_default_when_model_reports_no_context(self): + client = self._client_with_models(SimpleNamespace(id=self.MODEL)) + self.assertEqual(client.get_context_size(), 8192) + + def test_provider_options_override_api(self): + client = self._client_with_models( + SimpleNamespace(id=self.MODEL, context_length=1048576), + provider_options={"context_size": 16000}, + ) + self.assertEqual(client.get_context_size(), 16000) + client.provider.models.list.assert_not_called() + + def test_result_is_cached(self): + client = self._client_with_models( + SimpleNamespace(id=self.MODEL, context_length=1048576) + ) + client.get_context_size() + client.get_context_size() + client.provider.models.list.assert_called_once() + + # --------------------------------------------------------------------------- # Gemini # ---------------------------------------------------------------------------