Support Dynamic Thinking Models (#23281)

* Add ability to toggle thinking

* Disable thinking for descriptions automatically

* mypy

* Cleanup

frigate/genai/__init__.py

@@ -222,8 +222,15 @@ class GenAIClient:
         prompt: str,
         images: list[bytes],
         response_format: Optional[dict] = None,
+        enable_thinking: bool = False,
     ) -> Optional[str]:
-        """Submit a request to the provider."""
+        """Submit a request to the provider.
+
+        ``enable_thinking`` is honored only by providers that report
+        ``supports_toggleable_thinking``. Description-style callers leave it
+        at the default (off) since synthesis tasks don't benefit from
+        reasoning traces.
+        """
         return None
 
     @property
@@ -235,6 +242,11 @@ class GenAIClient:
         """
         return True
 
+    @property
+    def supports_toggleable_thinking(self) -> bool:
+        """Whether the configured model exposes a per-request thinking toggle."""
+        return False
+
     def list_models(self) -> list[str]:
         """Return the list of model names available from this provider.
 
@@ -278,6 +290,7 @@ class GenAIClient:
         messages: list[dict[str, Any]],
         tools: Optional[list[dict[str, Any]]] = None,
         tool_choice: Optional[str] = "auto",
+        enable_thinking: Optional[bool] = None,
     ) -> dict[str, Any]:
         """
         Send chat messages to LLM with optional tool definitions.
@@ -301,7 +314,9 @@ class GenAIClient:
                 - 'none': Model must not call tools
                 - 'required': Model must call at least one tool
                 - Or a dict specifying a specific tool to call
-            **kwargs: Additional provider-specific parameters.
+            enable_thinking: Per-request thinking toggle. None means use the
+                provider default. Ignored by providers without a per-request
+                toggle (see `supports_toggleable_thinking`).
 
         Returns:
             Dictionary with:

frigate/genai/manager.py

@@ -6,7 +6,7 @@ no chat feature is active) are never initialized.
 """
 
 import logging
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 from frigate.config import FrigateConfig
 from frigate.config.camera.genai import GenAIConfig, GenAIRoleEnum
@@ -108,11 +108,16 @@ class GenAIClientManager:
         name = self._role_map.get(GenAIRoleEnum.embeddings)
         return self._get_client(name) if name else None
 
-    def list_models(self) -> dict[str, list[str]]:
-        """Return available models keyed by config entry name."""
-        result: dict[str, list[str]] = {}
-        for name in self._configs:
+    def list_models(self) -> dict[str, dict[str, Any]]:
+        """Return per-entry model lists and capabilities, keyed by config entry name."""
+        result: dict[str, dict[str, Any]] = {}
+        for name, genai_cfg in self._configs.items():
             client = self._get_client(name)
-            if client:
-                result[name] = client.list_models()
+            if not client:
+                continue
+            result[name] = {
+                "models": client.list_models(),
+                "roles": [r.value for r in genai_cfg.roles],
+                "supports_toggleable_thinking": client.supports_toggleable_thinking,
+            }
         return result

frigate/genai/plugins/gemini.py

@@ -62,6 +62,7 @@ class GeminiClient(GenAIClient):
         prompt: str,
         images: list[bytes],
         response_format: Optional[dict] = None,
+        enable_thinking: bool = False,
     ) -> Optional[str]:
         """Submit a request to Gemini."""
         contents = [prompt] + [
@@ -119,11 +120,14 @@ class GeminiClient(GenAIClient):
         messages: list[dict[str, Any]],
         tools: Optional[list[dict[str, Any]]] = None,
         tool_choice: Optional[str] = "auto",
+        enable_thinking: Optional[bool] = None,
     ) -> dict[str, Any]:
         """
         Send chat messages to Gemini with optional tool definitions.
 
-        Implements function calling/tool usage for Gemini models.
+        Implements function calling/tool usage for Gemini models. Thinking is
+        configured at the model level for Gemini, so ``enable_thinking`` is
+        accepted for interface parity and ignored.
         """
         try:
             # Convert messages to Gemini format

frigate/genai/plugins/llama_cpp.py

@@ -122,6 +122,7 @@ class LlamaCppClient(GenAIClient):
     _supports_vision: bool
     _supports_audio: bool
     _supports_tools: bool
+    _supports_reasoning: bool
     _image_token_cache: dict[tuple[int, int], int]
     _text_baseline_tokens: int | None
     _media_marker: str
@@ -135,6 +136,7 @@ class LlamaCppClient(GenAIClient):
         self._supports_vision = False
         self._supports_audio = False
         self._supports_tools = False
+        self._supports_reasoning = False
         self._image_token_cache = {}
         self._text_baseline_tokens = None
         self._media_marker = "<__media__>"
@@ -164,15 +166,17 @@ class LlamaCppClient(GenAIClient):
         self._supports_vision = info["supports_vision"]
         self._supports_audio = info["supports_audio"]
         self._supports_tools = info["supports_tools"]
+        self._supports_reasoning = info["supports_reasoning"]
         self._media_marker = info["media_marker"]
 
         logger.info(
-            "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s",
+            "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s, reasoning: %s",
             configured_model,
             self._context_size or "unknown",
             self._supports_vision,
             self._supports_audio,
             self._supports_tools,
+            self._supports_reasoning,
         )
 
         return base_url
@@ -200,6 +204,7 @@ class LlamaCppClient(GenAIClient):
             "supports_vision": False,
             "supports_audio": False,
             "supports_tools": False,
+            "supports_reasoning": False,
             "media_marker": "<__media__>",
         }
 
@@ -279,10 +284,17 @@ class LlamaCppClient(GenAIClient):
                 info["supports_vision"] = bool(modalities.get("vision", False))
                 info["supports_audio"] = bool(modalities.get("audio", False))
 
+            chat_caps = props.get("chat_template_caps") or {}
+
             if not info["supports_tools"]:
-                chat_caps = props.get("chat_template_caps", {})
                 info["supports_tools"] = bool(chat_caps.get("supports_tools", False))
 
+            # llama.cpp does not advertise per-template reasoning support, so
+            # detect it by looking for the `enable_thinking` toggle variable
+            # in the Jinja chat template itself.
+            chat_template = props.get("chat_template") or ""
+            info["supports_reasoning"] = "enable_thinking" in chat_template
+
             media_marker = props.get("media_marker")
             if isinstance(media_marker, str) and media_marker:
                 info["media_marker"] = media_marker
@@ -300,6 +312,7 @@ class LlamaCppClient(GenAIClient):
         prompt: str,
         images: list[bytes],
         response_format: Optional[dict] = None,
+        enable_thinking: bool = False,
     ) -> Optional[str]:
         """Submit a request to llama.cpp server."""
         if self.provider is None:
@@ -327,7 +340,7 @@ class LlamaCppClient(GenAIClient):
                 )
 
             # Build request payload with llama.cpp native options
-            payload = {
+            payload: dict[str, Any] = {
                 "model": self.genai_config.model,
                 "messages": [
                     {
@@ -341,6 +354,9 @@ class LlamaCppClient(GenAIClient):
             if response_format:
                 payload["response_format"] = response_format
 
+            if self.supports_toggleable_thinking:
+                payload["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
+
             response = requests.post(
                 f"{self.provider}/v1/chat/completions",
                 json=payload,
@@ -377,6 +393,10 @@ class LlamaCppClient(GenAIClient):
         """Whether the loaded model supports tool/function calling."""
         return self._supports_tools
 
+    @property
+    def supports_toggleable_thinking(self) -> bool:
+        return self._supports_reasoning
+
     def list_models(self) -> list[str]:
         """Return available model IDs from the llama.cpp server."""
         base_url = self.provider or (
@@ -504,6 +524,7 @@ class LlamaCppClient(GenAIClient):
         tools: Optional[list[dict[str, Any]]],
         tool_choice: Optional[str],
         stream: bool = False,
+        enable_thinking: Optional[bool] = None,
     ) -> dict[str, Any]:
         """Build request payload for chat completions (sync or stream)."""
         openai_tool_choice = None
@@ -519,14 +540,21 @@ class LlamaCppClient(GenAIClient):
             "messages": messages,
             "model": self.genai_config.model,
         }
+
         if stream:
             payload["stream"] = True
             payload["stream_options"] = {"include_usage": True}
             payload["timings_per_token"] = True
+
         if tools:
             payload["tools"] = tools
+
             if openai_tool_choice is not None:
                 payload["tool_choice"] = openai_tool_choice
+
+        if enable_thinking is not None and self._supports_reasoning:
+            payload["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
+
         provider_opts = {
             k: v for k, v in self.provider_options.items() if k != "context_size"
         }
@@ -732,6 +760,7 @@ class LlamaCppClient(GenAIClient):
         messages: list[dict[str, Any]],
         tools: Optional[list[dict[str, Any]]] = None,
         tool_choice: Optional[str] = "auto",
+        enable_thinking: Optional[bool] = None,
     ) -> dict[str, Any]:
         """
         Send chat messages to llama.cpp server with optional tool definitions.
@@ -749,7 +778,13 @@ class LlamaCppClient(GenAIClient):
                 "finish_reason": "error",
             }
         try:
-            payload = self._build_payload(messages, tools, tool_choice, stream=False)
+            payload = self._build_payload(
+                messages,
+                tools,
+                tool_choice,
+                stream=False,
+                enable_thinking=enable_thinking,
+            )
             response = requests.post(
                 f"{self.provider}/v1/chat/completions",
                 json=payload,
@@ -797,6 +832,7 @@ class LlamaCppClient(GenAIClient):
         messages: list[dict[str, Any]],
         tools: Optional[list[dict[str, Any]]] = None,
         tool_choice: Optional[str] = "auto",
+        enable_thinking: Optional[bool] = None,
     ) -> AsyncGenerator[tuple[str, Any], None]:
         """Stream chat with tools via OpenAI-compatible streaming API."""
         if self.provider is None:
@@ -813,7 +849,13 @@ class LlamaCppClient(GenAIClient):
             )
             return
         try:
-            payload = self._build_payload(messages, tools, tool_choice, stream=True)
+            payload = self._build_payload(
+                messages,
+                tools,
+                tool_choice,
+                stream=True,
+                enable_thinking=enable_thinking,
+            )
             content_parts: list[str] = []
             reasoning_parts: list[str] = []
             tool_calls_by_index: dict[int, dict[str, Any]] = {}

frigate/genai/plugins/ollama.py

@@ -98,6 +98,22 @@ class OllamaClient(GenAIClient):
 
     provider: ApiClient | None
     provider_options: dict[str, Any]
+    _supports_thinking_cache: Optional[bool] = None
+
+    @property
+    def supports_toggleable_thinking(self) -> bool:
+        if self._supports_thinking_cache is not None:
+            return self._supports_thinking_cache
+        if self.provider is None:
+            return False
+        try:
+            response = self.provider.show(self.genai_config.model)
+            capabilities = response.get("capabilities") or []
+            self._supports_thinking_cache = "thinking" in capabilities
+        except Exception as e:
+            logger.debug("Failed to query Ollama model capabilities: %s", e)
+            self._supports_thinking_cache = False
+        return self._supports_thinking_cache
 
     def _auth_headers(self) -> dict | None:
         if self.genai_config.api_key:
@@ -178,6 +194,7 @@ class OllamaClient(GenAIClient):
         prompt: str,
         images: list[bytes],
         response_format: Optional[dict] = None,
+        enable_thinking: bool = False,
     ) -> Optional[str]:
         """Submit a request to Ollama"""
         if self.provider is None:
@@ -194,6 +211,8 @@ class OllamaClient(GenAIClient):
                 schema = response_format.get("json_schema", {}).get("schema")
                 if schema:
                     ollama_options["format"] = self._clean_schema_for_ollama(schema)
+            if self.supports_toggleable_thinking:
+                ollama_options["think"] = enable_thinking
             logger.debug(
                 "Ollama generate request: model=%s, prompt_len=%s, image_count=%s, "
                 "has_format=%s, options=%s",
@@ -274,6 +293,7 @@ class OllamaClient(GenAIClient):
         tools: Optional[list[dict[str, Any]]],
         tool_choice: Optional[str],
         stream: bool = False,
+        enable_thinking: Optional[bool] = None,
     ) -> dict[str, Any]:
         """Build request_messages and params for chat (sync or stream)."""
         request_messages = []
@@ -318,6 +338,8 @@ class OllamaClient(GenAIClient):
             request_params["stream"] = True
         if tools:
             request_params["tools"] = tools
+        if enable_thinking is not None and self.supports_toggleable_thinking:
+            request_params["think"] = enable_thinking
         return request_params
 
     def _message_from_response(self, response: dict[str, Any]) -> dict[str, Any]:
@@ -365,6 +387,7 @@ class OllamaClient(GenAIClient):
         messages: list[dict[str, Any]],
         tools: Optional[list[dict[str, Any]]] = None,
         tool_choice: Optional[str] = "auto",
+        enable_thinking: Optional[bool] = None,
     ) -> dict[str, Any]:
         if self.provider is None:
             logger.warning(
@@ -377,7 +400,11 @@ class OllamaClient(GenAIClient):
             }
         try:
             request_params = self._build_request_params(
-                messages, tools, tool_choice, stream=False
+                messages,
+                tools,
+                tool_choice,
+                stream=False,
+                enable_thinking=enable_thinking,
             )
             response = self.provider.chat(**request_params)
             return self._message_from_response(response)
@@ -401,6 +428,7 @@ class OllamaClient(GenAIClient):
         messages: list[dict[str, Any]],
         tools: Optional[list[dict[str, Any]]] = None,
         tool_choice: Optional[str] = "auto",
+        enable_thinking: Optional[bool] = None,
     ) -> AsyncGenerator[tuple[str, Any], None]:
         """Stream chat with tools; yields content deltas then final message.
 
@@ -430,7 +458,11 @@ class OllamaClient(GenAIClient):
                     "Ollama: tools provided, using non-streaming call for tool support"
                 )
                 request_params = self._build_request_params(
-                    messages, tools, tool_choice, stream=False
+                    messages,
+                    tools,
+                    tool_choice,
+                    stream=False,
+                    enable_thinking=enable_thinking,
                 )
                 async_client = OllamaAsyncClient(
                     host=self.genai_config.base_url,
@@ -452,7 +484,11 @@ class OllamaClient(GenAIClient):
                 return
 
             request_params = self._build_request_params(
-                messages, tools, tool_choice, stream=True
+                messages,
+                tools,
+                tool_choice,
+                stream=True,
+                enable_thinking=enable_thinking,
             )
             async_client = OllamaAsyncClient(
                 host=self.genai_config.base_url,

frigate/genai/plugins/openai.py

@@ -61,6 +61,7 @@ class OpenAIClient(GenAIClient):
         prompt: str,
         images: list[bytes],
         response_format: Optional[dict] = None,
+        enable_thinking: bool = False,
     ) -> Optional[str]:
         """Submit a request to OpenAI."""
         encoded_images = [base64.b64encode(image).decode("utf-8") for image in images]
@@ -187,11 +188,14 @@ class OpenAIClient(GenAIClient):
         messages: list[dict[str, Any]],
         tools: Optional[list[dict[str, Any]]] = None,
         tool_choice: Optional[str] = "auto",
+        enable_thinking: Optional[bool] = None,
     ) -> dict[str, Any]:
         """
         Send chat messages to OpenAI with optional tool definitions.
 
-        Implements function calling/tool usage for OpenAI models.
+        Implements function calling/tool usage for OpenAI models. The OpenAI
+        chat completions API does not expose a per-request thinking toggle,
+        so ``enable_thinking`` is accepted for interface parity and ignored.
         """
         try:
             openai_tool_choice = None