GenAI Fixes (#23708)

* Fix Gemini tool calling

* Catch openai bug

* Implement tool calling tests for GenAI

* Expose if embeddings are supported for a given provider

frigate/genai/__init__.py

@@ -281,6 +281,11 @@ class GenAIClient:
         """Whether the configured model exposes a per-request thinking toggle."""
         return False
 
+    @property
+    def supports_embeddings(self) -> bool:
+        """Whether the configured model can generate embeddings via embed()."""
+        return False
+
     def list_models(self) -> list[str]:
         """Return the list of model names available from this provider.
 

frigate/genai/manager.py

@@ -121,5 +121,6 @@ class GenAIClientManager:
                 "models": client.list_models(),
                 "roles": [r.value for r in genai_cfg.roles],
                 "supports_toggleable_thinking": client.supports_toggleable_thinking,
+                "supports_embeddings": client.supports_embeddings,
             }
         return result

frigate/genai/plugins/gemini.py

@@ -38,6 +38,37 @@ def _encode_thought_signature(signature: bytes | None) -> str | None:
     return base64.b64encode(signature).decode("ascii")
 
 
+def _decode_data_uri(url: str) -> tuple[str, bytes] | None:
+    """Decode a ``data:`` URI into ``(mime_type, bytes)``; None if not a data URI."""
+    if not isinstance(url, str) or not url.startswith("data:"):
+        return None
+    try:
+        header, b64 = url.split(",", 1)
+        mime = header[len("data:") :].split(";")[0] or "image/jpeg"
+        return mime, base64.b64decode(b64)
+    except (ValueError, binascii.Error):
+        return None
+
+
+def _parts_from_content(content: Any) -> list[types.Part]:
+    """Convert OpenAI-style message content (str or multimodal list) to Gemini parts."""
+    if isinstance(content, list):
+        parts: list[types.Part] = []
+        for item in content:
+            if not isinstance(item, dict):
+                continue
+            if item.get("type") == "text":
+                parts.append(types.Part.from_text(text=item.get("text") or ""))
+            elif item.get("type") == "image_url":
+                decoded = _decode_data_uri((item.get("image_url") or {}).get("url", ""))
+                if decoded is not None:
+                    mime, data = decoded
+                    parts.append(types.Part.from_bytes(data=data, mime_type=mime))
+        # Gemini rejects empty parts; fall back to a single space.
+        return parts or [types.Part.from_text(text=" ")]
+    return [types.Part.from_text(text=content or "")]
+
+
 def _stats_from_gemini_usage(usage: Any) -> dict[str, Any] | None:
     """Build a stats dict from a Gemini usage_metadata object."""
     prompt_tokens = getattr(usage, "prompt_token_count", None)
@@ -227,9 +258,7 @@ class GeminiClient(GenAIClient):
                     )
                 else:  # user
                     gemini_messages.append(
-                        types.Content(
+                        types.Content(role="user", parts=_parts_from_content(content))
-                            role="user", parts=[types.Part.from_text(text=content)]
-                        )
                     )
 
             # Convert tools to Gemini format
@@ -485,9 +514,7 @@ class GeminiClient(GenAIClient):
                     )
                 else:  # user
                     gemini_messages.append(
-                        types.Content(
+                        types.Content(role="user", parts=_parts_from_content(content))
-                            role="user", parts=[types.Part.from_text(text=content)]
-                        )
                     )
 
             # Convert tools to Gemini format
@@ -553,7 +580,7 @@ class GeminiClient(GenAIClient):
             # Use streaming API
             content_parts: list[str] = []
             reasoning_parts: list[str] = []
-            tool_calls_by_index: dict[int, dict[str, Any]] = {}
+            tool_calls_accum: list[dict[str, Any]] = []
             finish_reason = "stop"
             usage_stats: dict[str, Any] | None = None
 
@@ -600,7 +627,11 @@ class GeminiClient(GenAIClient):
                                 content_parts.append(part.text)
                                 yield ("content_delta", part.text)
                         elif part.function_call:
-                            # Handle function call
+                            # Gemini streams complete function calls (not partial
+                            # argument deltas), so each part is a distinct tool
+                            # call. Append rather than accumulate by name — the
+                            # latter concatenated parallel/repeated calls into one
+                            # invalid arguments string (e.g. `{...}{...}`).
                             try:
                                 arguments = (
                                     dict(part.function_call.args)
@@ -610,40 +641,16 @@ class GeminiClient(GenAIClient):
                             except Exception:
                                 arguments = {}
 
-                            # Store tool call
+                            tool_calls_accum.append(
-                            tool_call_id = part.function_call.name or ""
+                                {
-                            tool_call_name = part.function_call.name or ""
+                                    "id": part.function_call.name or "",
-
+                                    "name": part.function_call.name or "",
-                            # Check if we already have this tool call
+                                    "arguments": arguments,
-                            found_index = None
+                                    "thought_signature": getattr(
-                            for idx, tc in tool_calls_by_index.items():
+                                        part, "thought_signature", None
-                                if tc["name"] == tool_call_name:
+                                    ),
-                                    found_index = idx
-                                    break
-
-                            if found_index is None:
-                                found_index = len(tool_calls_by_index)
-                                tool_calls_by_index[found_index] = {
-                                    "id": tool_call_id,
-                                    "name": tool_call_name,
-                                    "arguments": "",
-                                    "thought_signature": None,
                                 }
-
+                            )
-                            # Accumulate arguments
-                            if arguments:
-                                tool_calls_by_index[found_index]["arguments"] += (
-                                    json.dumps(arguments)
-                                    if isinstance(arguments, dict)
-                                    else str(arguments)
-                                )
-
-                            # Capture latest thought_signature for this call
-                            chunk_sig = getattr(part, "thought_signature", None)
-                            if chunk_sig:
-                                tool_calls_by_index[found_index][
-                                    "thought_signature"
-                                ] = chunk_sig
 
             # Build final message
             full_content = "".join(content_parts).strip() or None
@@ -651,25 +658,20 @@ class GeminiClient(GenAIClient):
 
             # Convert tool calls to list format
             tool_calls_list = None
-            if tool_calls_by_index:
+            if tool_calls_accum:
-                tool_calls_list = []
+                tool_calls_list = [
-                for tc in tool_calls_by_index.values():
+                    {
-                    try:
+                        "id": tc["id"],
-                        # Try to parse accumulated arguments as JSON
+                        "name": tc["name"],
-                        parsed_args = json.loads(tc["arguments"])
+                        "arguments": tc["arguments"]
-                    except (json.JSONDecodeError, Exception):
+                        if isinstance(tc["arguments"], dict)
-                        parsed_args = tc["arguments"]
+                        else {},
-
+                        "thought_signature": _encode_thought_signature(
-                    tool_calls_list.append(
+                            tc.get("thought_signature")
-                        {
+                        ),
-                            "id": tc["id"],
+                    }
-                            "name": tc["name"],
+                    for tc in tool_calls_accum
-                            "arguments": parsed_args,
+                ]
-                            "thought_signature": _encode_thought_signature(
-                                tc.get("thought_signature")
-                            ),
-                        }
-                    )
                 finish_reason = "tool_calls"
 
             if usage_stats is not None:

frigate/genai/plugins/llama_cpp.py

@@ -128,6 +128,11 @@ class LlamaCppClient(GenAIClient):
     _text_baseline_tokens: int | None
     _media_marker: str
 
+    @property
+    def supports_embeddings(self) -> bool:
+        """llama.cpp exposes an /embeddings endpoint for any loaded model."""
+        return True
+
     def _init_provider(self) -> str | None:
         """Initialize the client and query model metadata from the server."""
         self.provider_options = {

frigate/genai/plugins/openai.py

@@ -423,9 +423,18 @@ class OpenAIClient(GenAIClient):
                 for tc in tool_calls_by_index.values():
                     try:
                         # Parse accumulated arguments as JSON
-                        parsed_args = json.loads(tc["arguments"])
+                        parsed_args = json.loads(tc["arguments"] or "{}")
-                    except (json.JSONDecodeError, Exception):
+                    except (json.JSONDecodeError, ValueError):
-                        parsed_args = tc["arguments"]
+                        logger.warning(
+                            "Failed to parse streamed tool call arguments for %s",
+                            tc["name"],
+                        )
+                        parsed_args = {}
+
+                    # Downstream (ToolCall model) requires a dict; never leak a
+                    # partial/invalid arguments string.
+                    if not isinstance(parsed_args, dict):
+                        parsed_args = {}
 
                     tool_calls_list.append(
                         {

frigate/test/test_genai_providers.py

@@ -0,0 +1,496 @@
+"""Smoke tests for GenAI chat providers.
+
+Each provider's ``chat_with_tools_stream`` is driven with a canned "test
+response" so the two conversion layers are exercised without any network:
+
+  1. Frigate (OpenAI-style) messages -> provider-native request format
+  2. provider-native response -> Frigate ``("kind", value)`` stream events
+
+These guard against regressions such as tool-call arguments arriving as raw
+strings instead of dicts (which crash the ``ToolCall`` model), and multimodal
+user content (a list of text/image parts, as injected by ``get_live_context``)
+crashing message conversion.
+"""
+
+import asyncio
+import base64
+import json
+import unittest
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from frigate.config import GenAIConfig, GenAIProviderEnum
+from frigate.genai import PROVIDERS, load_providers
+
+load_providers()
+
+# A minimal but valid JPEG data URI, mirroring what get_live_context injects.
+_TINY_JPEG = base64.b64encode(b"\xff\xd8\xff\xd9").decode("ascii")
+_IMAGE_DATA_URI = f"data:image/jpeg;base64,{_TINY_JPEG}"
+
+# Conversation ending in a multimodal user message (text + live image), the
+# exact shape the chat endpoint builds after a get_live_context tool result.
+MULTIMODAL_MESSAGES = [
+    {"role": "system", "content": "You are a test assistant."},
+    {"role": "user", "content": "what do you see on the front camera?"},
+    {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [
+            {
+                "id": "call_1",
+                "type": "function",
+                "function": {
+                    "name": "get_live_context",
+                    "arguments": json.dumps({"camera": "front"}),
+                },
+            }
+        ],
+    },
+    {
+        "role": "tool",
+        "tool_call_id": "call_1",
+        "name": "get_live_context",
+        "content": json.dumps({"camera": "front"}),
+    },
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "Here is the current live image from camera 'front'.",
+            },
+            {"type": "image_url", "image_url": {"url": _IMAGE_DATA_URI}},
+        ],
+    },
+]
+
+SIMPLE_MESSAGES = [
+    {"role": "system", "content": "You are a test assistant."},
+    {"role": "user", "content": "hello"},
+]
+
+TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "search_objects",
+            "description": "Search tracked objects",
+            "parameters": {
+                "type": "object",
+                "properties": {"label": {"type": "string"}},
+            },
+        },
+    }
+]
+
+
+def _make_client(provider: str, **cfg_overrides):
+    """Build a provider client offline (no model validation, no network)."""
+    cfg = GenAIConfig(provider=provider, **cfg_overrides)
+    cls = PROVIDERS[GenAIProviderEnum(provider)]
+    return cls(cfg, timeout=5, validate_model=False)
+
+
+def _collect(client, messages, tools=TOOLS):
+    """Drain chat_with_tools_stream into a list of (kind, value) events."""
+
+    async def _run():
+        events = []
+        async for event in client.chat_with_tools_stream(
+            messages=messages, tools=tools, tool_choice="auto"
+        ):
+            events.append(event)
+        return events
+
+    return asyncio.run(_run())
+
+
+def _final_message(events) -> dict:
+    messages = [value for (kind, value) in events if kind == "message"]
+    assert messages, f"stream produced no final message: {events}"
+    return messages[-1]
+
+
+def _assert_tool_args_are_dicts(final: dict) -> None:
+    """Every returned tool call must expose arguments as a dict, never a string."""
+    for tool_call in final.get("tool_calls") or []:
+        assert isinstance(tool_call["arguments"], dict), (
+            f"tool call arguments must be a dict, got "
+            f"{type(tool_call['arguments']).__name__}: {tool_call['arguments']!r}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# OpenAI
+# ---------------------------------------------------------------------------
+def _openai_tc(index, id=None, name=None, arguments=None):
+    return SimpleNamespace(
+        index=index,
+        id=id,
+        function=SimpleNamespace(name=name, arguments=arguments),
+    )
+
+
+def _openai_chunk(content=None, tool_calls=None, finish_reason=None, usage=None):
+    delta = SimpleNamespace(
+        content=content,
+        tool_calls=tool_calls,
+        reasoning_content=None,
+        reasoning=None,
+    )
+    choice = SimpleNamespace(delta=delta, finish_reason=finish_reason)
+    return SimpleNamespace(choices=[choice], usage=usage)
+
+
+class TestOpenAIProvider(unittest.TestCase):
+    def _client(self):
+        return _make_client(
+            "openai", model="gpt-4o", api_key="k", base_url="http://localhost:9999/v1"
+        )
+
+    def test_stream_tool_call_arguments_are_dict(self):
+        # Arguments arrive split across chunks, as the real API streams them.
+        chunks = [
+            _openai_chunk(
+                tool_calls=[
+                    _openai_tc(0, id="c1", name="search_objects", arguments='{"label":')
+                ]
+            ),
+            _openai_chunk(tool_calls=[_openai_tc(0, arguments=' "person"}')]),
+            _openai_chunk(finish_reason="tool_calls"),
+        ]
+        client = self._client()
+        client.provider.chat.completions.create = MagicMock(return_value=iter(chunks))
+
+        final = _final_message(_collect(client, SIMPLE_MESSAGES))
+        self.assertEqual(final["finish_reason"], "tool_calls")
+        self.assertEqual(len(final["tool_calls"]), 1)
+        _assert_tool_args_are_dicts(final)
+        self.assertEqual(final["tool_calls"][0]["arguments"], {"label": "person"})
+
+    def test_stream_content_response(self):
+        chunks = [
+            _openai_chunk(content="hel"),
+            _openai_chunk(content="lo"),
+            _openai_chunk(finish_reason="stop"),
+        ]
+        client = self._client()
+        client.provider.chat.completions.create = MagicMock(return_value=iter(chunks))
+
+        events = _collect(client, SIMPLE_MESSAGES)
+        deltas = [v for (k, v) in events if k == "content_delta"]
+        self.assertEqual("".join(deltas), "hello")
+        self.assertEqual(_final_message(events)["content"], "hello")
+
+    def test_multimodal_message_does_not_crash(self):
+        client = self._client()
+        client.provider.chat.completions.create = MagicMock(
+            return_value=iter([_openai_chunk(content="ok", finish_reason="stop")])
+        )
+        # Passing the OpenAI-native multimodal list through must not raise.
+        final = _final_message(_collect(client, MULTIMODAL_MESSAGES))
+        self.assertEqual(final["content"], "ok")
+
+
+# ---------------------------------------------------------------------------
+# Gemini
+# ---------------------------------------------------------------------------
+def _gemini_part(text=None, thought=False, function_call=None, thought_signature=None):
+    return SimpleNamespace(
+        text=text,
+        thought=thought,
+        function_call=function_call,
+        thought_signature=thought_signature,
+    )
+
+
+def _gemini_chunk(parts, finish_reason=None, usage_metadata=None):
+    candidate = SimpleNamespace(
+        content=SimpleNamespace(parts=parts), finish_reason=finish_reason
+    )
+    return SimpleNamespace(candidates=[candidate], usage_metadata=usage_metadata)
+
+
+def _gemini_stream(chunks):
+    async def _agen(*args, **kwargs):
+        for chunk in chunks:
+            yield chunk
+
+    return _agen
+
+
+class TestGeminiProvider(unittest.TestCase):
+    def _client(self):
+        return _make_client("gemini", model="gemini-2.5-flash", api_key="k")
+
+    def _patch_stream(self, client, chunks):
+        client.provider = MagicMock()
+        client.provider.aio.models.generate_content_stream = AsyncMock(
+            side_effect=_gemini_stream(chunks)
+        )
+
+    def test_stream_parallel_tool_calls_stay_separate_dicts(self):
+        # Regression: Gemini streams complete function calls. Two calls to the
+        # same tool must NOT be merged into one concatenated arguments string.
+        from google.genai.types import FinishReason
+
+        chunks = [
+            _gemini_chunk(
+                parts=[
+                    _gemini_part(
+                        function_call=SimpleNamespace(
+                            name="search_objects", args={"label": "person"}
+                        )
+                    ),
+                    _gemini_part(
+                        function_call=SimpleNamespace(
+                            name="search_objects", args={"limit": 1}
+                        )
+                    ),
+                ],
+                finish_reason=FinishReason.STOP,
+            ),
+        ]
+        client = self._client()
+        self._patch_stream(client, chunks)
+
+        final = _final_message(_collect(client, SIMPLE_MESSAGES))
+        self.assertEqual(final["finish_reason"], "tool_calls")
+        self.assertEqual(len(final["tool_calls"]), 2)
+        _assert_tool_args_are_dicts(final)
+        self.assertEqual(final["tool_calls"][0]["arguments"], {"label": "person"})
+        self.assertEqual(final["tool_calls"][1]["arguments"], {"limit": 1})
+
+    def test_stream_content_response(self):
+        from google.genai.types import FinishReason
+
+        chunks = [
+            _gemini_chunk(parts=[_gemini_part(text="hel")]),
+            _gemini_chunk(
+                parts=[_gemini_part(text="lo")], finish_reason=FinishReason.STOP
+            ),
+        ]
+        client = self._client()
+        self._patch_stream(client, chunks)
+
+        events = _collect(client, SIMPLE_MESSAGES)
+        deltas = [v for (k, v) in events if k == "content_delta"]
+        self.assertEqual("".join(deltas), "hello")
+        self.assertEqual(_final_message(events)["content"], "hello")
+
+    def test_multimodal_message_converts_without_crash(self):
+        # Regression: a user message with list content (text + image_url) used
+        # to be handed to Part.from_text(text=<list>) and raise ValidationError.
+        from google.genai.types import FinishReason
+
+        client = self._client()
+        self._patch_stream(
+            client,
+            [
+                _gemini_chunk(
+                    parts=[_gemini_part(text="ok")], finish_reason=FinishReason.STOP
+                )
+            ],
+        )
+        final = _final_message(_collect(client, MULTIMODAL_MESSAGES))
+        self.assertEqual(final["content"], "ok")
+
+
+# ---------------------------------------------------------------------------
+# Ollama
+# ---------------------------------------------------------------------------
+class TestOllamaProvider(unittest.TestCase):
+    def _client(self):
+        return _make_client("ollama", model="llama3", base_url="http://localhost:9999")
+
+    def _run_with_response(self, client, response, messages):
+        # Ollama uses a non-streaming call when tools are present, via an
+        # internally-constructed async client.
+        fake_async = MagicMock()
+        fake_async.chat = AsyncMock(return_value=response)
+        with patch(
+            "frigate.genai.plugins.ollama.OllamaAsyncClient",
+            return_value=fake_async,
+        ):
+            return _collect(client, messages)
+
+    def test_tool_call_arguments_are_dict(self):
+        response = {
+            "message": {
+                "content": "",
+                "tool_calls": [
+                    {
+                        "function": {
+                            "name": "search_objects",
+                            "arguments": {"label": "person"},
+                        }
+                    }
+                ],
+            },
+            "done": True,
+            "done_reason": "stop",
+            "eval_count": 5,
+            "prompt_eval_count": 3,
+            "eval_duration": 1_000_000,
+        }
+        client = self._client()
+        final = _final_message(
+            self._run_with_response(client, response, SIMPLE_MESSAGES)
+        )
+        self.assertEqual(final["finish_reason"], "tool_calls")
+        _assert_tool_args_are_dicts(final)
+        self.assertEqual(final["tool_calls"][0]["arguments"], {"label": "person"})
+
+    def test_multimodal_message_normalizes_image(self):
+        # Ollama needs content as a string with images pulled into a separate
+        # field; the normalizer must extract both without crashing.
+        response = {
+            "message": {"content": "ok"},
+            "done": True,
+            "done_reason": "stop",
+        }
+        client = self._client()
+        final = _final_message(
+            self._run_with_response(client, response, MULTIMODAL_MESSAGES)
+        )
+        self.assertEqual(final["content"], "ok")
+
+    def test_normalize_multimodal_content(self):
+        from frigate.genai.plugins.ollama import _normalize_multimodal_content
+
+        text, images = _normalize_multimodal_content(MULTIMODAL_MESSAGES[-1]["content"])
+        self.assertIn("live image", text)
+        self.assertEqual(len(images), 1)
+        self.assertEqual(images[0], b"\xff\xd8\xff\xd9")
+
+
+# ---------------------------------------------------------------------------
+# llama.cpp
+# ---------------------------------------------------------------------------
+class _FakeStreamResponse:
+    def __init__(self, lines):
+        self._lines = lines
+
+    def raise_for_status(self):
+        return None
+
+    async def aiter_lines(self):
+        for line in self._lines:
+            yield line
+
+
+class _FakeStreamCtx:
+    def __init__(self, lines):
+        self._resp = _FakeStreamResponse(lines)
+
+    async def __aenter__(self):
+        return self._resp
+
+    async def __aexit__(self, *exc):
+        return False
+
+
+class _FakeAsyncClient:
+    def __init__(self, lines):
+        self._lines = lines
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return False
+
+    def stream(self, method, url, json=None):
+        return _FakeStreamCtx(self._lines)
+
+
+class TestLlamaCppProvider(unittest.TestCase):
+    def _client(self):
+        return _make_client("llamacpp", model="m", base_url="http://localhost:9999")
+
+    def _run_with_lines(self, client, lines, messages):
+        with patch(
+            "frigate.genai.plugins.llama_cpp.httpx.AsyncClient",
+            return_value=_FakeAsyncClient(lines),
+        ):
+            return _collect(client, messages)
+
+    def test_stream_tool_call_arguments_are_dict(self):
+        lines = [
+            "data: "
+            + json.dumps(
+                {
+                    "choices": [
+                        {
+                            "delta": {
+                                "tool_calls": [
+                                    {
+                                        "index": 0,
+                                        "id": "c1",
+                                        "function": {
+                                            "name": "search_objects",
+                                            "arguments": '{"label":',
+                                        },
+                                    }
+                                ]
+                            }
+                        }
+                    ]
+                }
+            ),
+            "data: "
+            + json.dumps(
+                {
+                    "choices": [
+                        {
+                            "delta": {
+                                "tool_calls": [
+                                    {
+                                        "index": 0,
+                                        "function": {"arguments": ' "person"}'},
+                                    }
+                                ]
+                            }
+                        }
+                    ]
+                }
+            ),
+            "data: "
+            + json.dumps({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}),
+            "data: [DONE]",
+        ]
+        client = self._client()
+        final = _final_message(self._run_with_lines(client, lines, SIMPLE_MESSAGES))
+        self.assertEqual(final["finish_reason"], "tool_calls")
+        _assert_tool_args_are_dicts(final)
+        self.assertEqual(final["tool_calls"][0]["arguments"], {"label": "person"})
+
+    def test_stream_content_response(self):
+        lines = [
+            "data: " + json.dumps({"choices": [{"delta": {"content": "hel"}}]}),
+            "data: " + json.dumps({"choices": [{"delta": {"content": "lo"}}]}),
+            "data: "
+            + json.dumps({"choices": [{"delta": {}, "finish_reason": "stop"}]}),
+            "data: [DONE]",
+        ]
+        client = self._client()
+        events = self._run_with_lines(client, lines, SIMPLE_MESSAGES)
+        deltas = [v for (k, v) in events if k == "content_delta"]
+        self.assertEqual("".join(deltas), "hello")
+        self.assertEqual(_final_message(events)["content"], "hello")
+
+    def test_multimodal_message_does_not_crash(self):
+        lines = [
+            "data: " + json.dumps({"choices": [{"delta": {"content": "ok"}}]}),
+            "data: "
+            + json.dumps({"choices": [{"delta": {}, "finish_reason": "stop"}]}),
+            "data: [DONE]",
+        ]
+        client = self._client()
+        final = _final_message(self._run_with_lines(client, lines, MULTIMODAL_MESSAGES))
+        self.assertEqual(final["content"], "ok")
+
+
+if __name__ == "__main__":
+    unittest.main()

web/src/components/config-form/theme/widgets/GenAIRolesWidget.tsx

@@ -1,8 +1,10 @@
 import type { WidgetProps } from "@rjsf/utils";
-import { useMemo } from "react";
+import { useEffect, useMemo } from "react";
 import { useTranslation } from "react-i18next";
+import useSWR from "swr";
 import { Switch } from "@/components/ui/switch";
 import type { ConfigFormContext } from "@/types/configForm";
+import type { GenAIModelsResponse } from "@/types/chat";
 
 const GENAI_ROLES = ["embeddings", "descriptions", "chat"] as const;
 
@@ -37,10 +39,24 @@ export function GenAIRolesWidget(props: WidgetProps) {
   const selectedRoles = useMemo(() => normalizeValue(value), [value]);
   const providerKey = useMemo(() => getProviderKey(id), [id]);
 
-  // Compute occupied roles directly from formData. The computation is
+  const { data: genaiInfo } = useSWR<GenAIModelsResponse>("genai/models", {
-  // trivially cheap (iterate providers × 3 roles max) so we skip an
+    revalidateOnFocus: false,
-  // intermediate memoization layer whose formData dependency would
+  });
-  // never produce a cache hit (new object reference on every change).
+
+  const embeddingsSupported = useMemo(() => {
+    if (!providerKey) return true;
+    const info = genaiInfo?.[providerKey];
+    return info ? info.supports_embeddings : true;
+  }, [genaiInfo, providerKey]);
+
+  const availableRoles = useMemo(
+    () =>
+      embeddingsSupported
+        ? GENAI_ROLES
+        : GENAI_ROLES.filter((role) => role !== "embeddings"),
+    [embeddingsSupported],
+  );
+
   const occupiedRoles = useMemo(() => {
     const occupied = new Set<string>();
     const fd = formContext?.formData;
@@ -64,6 +80,12 @@ export function GenAIRolesWidget(props: WidgetProps) {
     return occupied;
   }, [formContext?.formData, providerKey]);
 
+  useEffect(() => {
+    if (!embeddingsSupported && selectedRoles.includes("embeddings")) {
+      onChange(selectedRoles.filter((role) => role !== "embeddings"));
+    }
+  }, [embeddingsSupported, selectedRoles, onChange]);
+
   const toggleRole = (role: string, enabled: boolean) => {
     if (enabled) {
       if (!selectedRoles.includes(role)) {
@@ -78,7 +100,7 @@ export function GenAIRolesWidget(props: WidgetProps) {
   return (
     <div className="rounded-lg border border-secondary-highlight bg-background_alt p-2 pr-0 md:max-w-md">
       <div className="grid gap-2">
-        {GENAI_ROLES.map((role) => {
+        {availableRoles.map((role) => {
           const checked = selectedRoles.includes(role);
           const roleDisabled = !checked && occupiedRoles.has(role);
           const label = t(`configForm.genaiRoles.options.${role}`, {

web/src/types/chat.ts

@@ -43,6 +43,7 @@ export type GenAIProviderInfo = {
   models: string[];
   roles: string[];
   supports_toggleable_thinking: boolean;
+  supports_embeddings: boolean;
 };
 
 export type GenAIModelsResponse = Record<string, GenAIProviderInfo>;