diff --git a/docs/docs/configuration/genai/config.md b/docs/docs/configuration/genai/config.md index bfcb9d1373..922f24a7ff 100644 --- a/docs/docs/configuration/genai/config.md +++ b/docs/docs/configuration/genai/config.md @@ -53,7 +53,7 @@ Each provider handles one or more **roles**: `chat`, `descriptions`, `embeddings | Ollama (`ollama`) | ✅ | ✅ | ✅ | ❌ | | OpenAI (`openai`) | ✅ | ✅ | ❌ | ✅ | | Azure OpenAI (`azure_openai`) | ✅ | ✅ | ❌ | ✅ | -| Google Gemini (`gemini`) | ✅ | ✅ | ❌ | ✅ | +| Google Gemini (`gemini`) | ✅ | ✅ | ✅ | ✅ | A ✅ means Frigate can use the provider for that feature. The configured model must also support it: a vision model for descriptions and chat, a multimodal embedding model for embeddings (see [Embedding models](#embedding-models)), and an audio-capable model for transcription. Some features also need extra provider setup, covered in each provider's section below. OpenAI-compatible servers use the `openai` provider, so they follow the OpenAI row. @@ -86,10 +86,11 @@ You must use a vision-capable model with Frigate. The following models are recom The `embeddings` role needs a different kind of model. Text queries are matched against the stored image embeddings, so the model must be trained to place images and text into the same vector space. A chat or description model will still return vectors when asked, but those vectors are not trained for retrieval and text searches will return poor matches with no error to indicate why. -| Model | Notes | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `embeddinggemma-2` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Strong semantic search accuracy with efficient inference on a small model. | -| `qwen3-vl-embedding` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Good performance, large model that requires strong hardware for inference. | +| Model | Type | Notes | +| -------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `embeddinggemma-2` | `local` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Strong semantic search accuracy with efficient inference on a small model. | +| `qwen3-vl-embedding` | `local` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider). Good performance, large model that requires strong hardware for inference. | +| `gemini-embedding-2` | `cloud` | Multimodal embeddings for [Semantic Search](/configuration/semantic_search#genai-provider) via [Google Gemini](#google-gemini). Sends thumbnails and search text to Google. | #### Transcription models @@ -383,6 +384,26 @@ Other HTTP options are available, see the [python-genai documentation](https://g ::: +#### Embeddings + +Gemini can serve the `embeddings` role for [Semantic Search](/configuration/semantic_search#genai-provider) with a multimodal embedding model such as `gemini-embedding-2`. Give that model its own provider entry. + +Reindexing sends every stored thumbnail to Google and uses API quota. See [Network Requirements](/frigate/network_requirements#generative-ai). + +```yaml +genai: + gemini_embeddings: + provider: gemini + api_key: "{FRIGATE_GEMINI_API_KEY}" + model: gemini-embedding-2 + roles: + - embeddings + +semantic_search: + enabled: True + model: gemini_embeddings +``` + ### OpenAI OpenAI does not have a free tier for their API. diff --git a/docs/docs/frigate/network_requirements.md b/docs/docs/frigate/network_requirements.md index 65e15089ba..5f5349c6f1 100644 --- a/docs/docs/frigate/network_requirements.md +++ b/docs/docs/frigate/network_requirements.md @@ -117,7 +117,7 @@ See [Frigate+](/integrations/plus) for details. ### Generative AI -When a Generative AI provider is configured, Frigate sends images and prompts to the configured provider for event descriptions, chat, and camera monitoring. Available providers: +When a Generative AI provider is configured, Frigate sends images and prompts to the configured provider for the various features that are enabled. Available providers: | Provider | Internet Required | | ------------- | --------------------------------------------------------------- | diff --git a/frigate/genai/plugins/gemini.py b/frigate/genai/plugins/gemini.py index d55fbeb402..e4caef6f92 100644 --- a/frigate/genai/plugins/gemini.py +++ b/frigate/genai/plugins/gemini.py @@ -9,6 +9,7 @@ import time from collections.abc import AsyncGenerator from typing import Any +import numpy as np from google import genai from google.genai import errors, types from google.genai.types import FunctionCallingConfigMode @@ -16,7 +17,7 @@ from PIL import Image from frigate.config import GenAIProviderEnum from frigate.genai import GenAIClient, register_genai_provider -from frigate.genai.utils import interleave_images +from frigate.genai.utils import interleave_images, to_jpeg logger = logging.getLogger(__name__) @@ -29,6 +30,9 @@ GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024 COUNT_TOKENS_TIMEOUT_MS = 5000 COUNT_TOKENS_RETRY_INTERVAL = 300 +# sqlite-vec stores thumbnail and description embeddings as FLOAT[768]. +GEMINI_EMBEDDING_DIM = 768 + def _decode_thought_signature(value: Any) -> bytes | None: """Decode a base64-encoded thought_signature carried across conversation turns.""" @@ -103,11 +107,13 @@ class GeminiClient(GenAIClient): provider: genai.Client _image_token_cache: dict[tuple[int, int], int] _image_token_retry_at: float + _capabilities_cache: dict[str, dict[str, bool]] | None = None def _init_provider(self) -> genai.Client: """Initialize the client.""" self._image_token_cache = {} self._image_token_retry_at = 0.0 + self._capabilities_cache = None # Merge provider_options into HttpOptions http_options_dict: dict[str, Any] = { @@ -232,13 +238,41 @@ class GeminiClient(GenAIClient): return transcript or None + @staticmethod + def _capabilities_from_models(models: Any) -> dict[str, dict[str, bool]]: + """Build a per-model capability map from Gemini's models.list() entries.""" + capabilities: dict[str, dict[str, bool]] = {} + for m in models: + name = getattr(m, "name", None) or "" + if not name: + continue + actions = getattr(m, "supported_actions", None) or [] + flags = {"supports_embeddings": "embedContent" in actions} + capabilities[name] = flags + capabilities[name.removeprefix("models/")] = flags + return capabilities + def list_models(self) -> list[str]: """Return available model names from Gemini.""" try: - return sorted(m.name or "" for m in self.provider.models.list()) + models = list(self.provider.models.list()) except Exception as e: logger.warning("Failed to list Gemini models: %s", e) return [] + self._capabilities_cache = self._capabilities_from_models(models) + return sorted(m.name or "" for m in models) + + def list_model_capabilities(self) -> dict[str, dict[str, bool]]: + """Report per-model capabilities from Gemini's model list.""" + if self._capabilities_cache is not None: + return self._capabilities_cache + try: + models = list(self.provider.models.list()) + except Exception as e: + logger.debug("Failed to query Gemini model capabilities: %s", e) + return {} + self._capabilities_cache = self._capabilities_from_models(models) + return self._capabilities_cache def get_context_size(self) -> int: """Get the context window size for Gemini.""" @@ -310,6 +344,103 @@ class GeminiClient(GenAIClient): return total + @property + def supports_embeddings(self) -> bool: + """Whether Gemini reports embedContent support for the configured model.""" + capabilities = self.list_model_capabilities().get(self._embedding_model_name()) + return bool(capabilities and capabilities.get("supports_embeddings")) + + def _embedding_model_name(self) -> str: + """Model id for embed_content, without a ``models/`` prefix.""" + return self.genai_config.model.removeprefix("models/") + + def _vector_from_embed(self, result: Any) -> np.ndarray: + """Read the single embedding vector from an embed_content response.""" + embeddings = getattr(result, "embeddings", None) or [] + values = embeddings[0].values if embeddings else None + if not values: + raise ValueError("Gemini embeddings response had no vector") + return np.asarray(values, dtype=np.float32) + + def _embed_one(self, model: str, contents: Any) -> np.ndarray: + """One embed_content call. API failures become ValueError. + + ``_embed_thumbnail`` only catches ValueError. Returning an empty + list makes the caller index ``[0]`` and can stop the embeddings + process. + """ + try: + result = self.provider.models.embed_content( + model=model, + contents=contents, + config=types.EmbedContentConfig( + output_dimensionality=GEMINI_EMBEDDING_DIM + ), + ) + except Exception as e: + logger.warning("Gemini embeddings request failed: %s", e) + raise ValueError("Gemini embeddings request failed") from e + + return self._vector_from_embed(result) + + def embed( + self, + texts: list[str] | None = None, + images: list[bytes] | None = None, + ) -> list[np.ndarray]: + """Embed each text or image on its own. + + gemini-embedding-2 returns one vector per request. Several parts + in one contents list are aggregated into that single vector, so + each description, search query, and thumbnail is its own call. + + Stored descriptions and search queries share this text path, so + both get the search-query prefix and stay in one format. A text + that already starts with ``task:`` is sent unchanged. Images are + not prefixed. The API accepts PNG and JPEG only, and Frigate + stores thumbnails as WebP, so each image is transcoded first. + Requested length is 768, matching the semantic search tables. + + Args: + texts: Strings to embed. + images: Encoded images, typically WebP thumbnails. + + Returns: + One float32 vector per input, length 768. + + Raises: + ValueError: An image cannot be converted, the response has + no vector, or the request fails. + """ + texts = texts or [] + images = images or [] + if not texts and not images: + return [] + + model = self._embedding_model_name() + vectors: list[np.ndarray] = [] + + for text in texts: + content = ( + text + if text.startswith("task:") + else f"task: search result | query: {text}" + ) + vectors.append(self._embed_one(model, content)) + + for image in images: + jpeg = to_jpeg(image) + if jpeg is None: + raise ValueError("thumbnail could not be converted to JPEG") + vectors.append( + self._embed_one( + model, + [types.Part.from_bytes(data=jpeg, mime_type="image/jpeg")], + ) + ) + + return vectors + def chat_with_tools( self, messages: list[dict[str, Any]], diff --git a/frigate/test/test_genai_providers.py b/frigate/test/test_genai_providers.py index f4cdccaac3..0af70c75fc 100644 --- a/frigate/test/test_genai_providers.py +++ b/frigate/test/test_genai_providers.py @@ -1286,6 +1286,154 @@ class TestImageTokenEstimate(unittest.TestCase): self.assertEqual(client._image_token_cache, {}) +class TestGeminiEmbeddings(unittest.TestCase): + """gemini-embedding-2 via embed_content. No network, no real images.""" + + @staticmethod + def _webp_bytes(): + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (8, 8), (10, 20, 30)).save(buf, format="WEBP") + return buf.getvalue() + + @staticmethod + def _vector(value: float): + return SimpleNamespace(embeddings=[SimpleNamespace(values=[value] * 768)]) + + def _client(self, model="models/gemini-embedding-2"): + client = _make_client("gemini", model=model, api_key="k") + client.provider = MagicMock() + return client + + def test_supports_embeddings_checks_supported_actions(self): + models = [ + SimpleNamespace( + name="models/gemini-2.5-flash", + supported_actions=["generateContent"], + ), + SimpleNamespace( + name="models/gemini-embedding-2", + supported_actions=["embedContent", "countTokens"], + ), + ] + + client = self._client(model="models/gemini-embedding-2") + client.provider.models.list.return_value = models + self.assertTrue(client.supports_embeddings) + self.assertEqual( + client.list_model_capabilities(), + { + "models/gemini-2.5-flash": {"supports_embeddings": False}, + "gemini-2.5-flash": {"supports_embeddings": False}, + "models/gemini-embedding-2": {"supports_embeddings": True}, + "gemini-embedding-2": {"supports_embeddings": True}, + }, + ) + client.provider.models.list.assert_called_once() + + unprefixed = self._client(model="gemini-embedding-2") + unprefixed.provider.models.list.return_value = models + self.assertTrue(unprefixed.supports_embeddings) + + chat_client = self._client(model="models/gemini-2.5-flash") + chat_client.provider.models.list.return_value = models + self.assertFalse(chat_client.supports_embeddings) + + def test_capability_lookup_failure_is_not_cached(self): + client = self._client() + client.provider.models.list.side_effect = [ + RuntimeError("offline"), + [ + SimpleNamespace( + name="models/gemini-embedding-2", + supported_actions=["embedContent"], + ) + ], + ] + + self.assertFalse(client.supports_embeddings) + self.assertTrue(client.supports_embeddings) + + def test_one_request_per_text_and_image(self): + client = self._client() + client.provider.models.embed_content.side_effect = [ + self._vector(0.1), + self._vector(0.2), + ] + + result = client.embed(texts=["a person walking"], images=[self._webp_bytes()]) + + calls = client.provider.models.embed_content.call_args_list + self.assertEqual(len(calls), 2) + self.assertEqual(calls[0].kwargs["model"], "gemini-embedding-2") + self.assertEqual( + calls[0].kwargs["contents"], + "task: search result | query: a person walking", + ) + self.assertEqual(calls[0].kwargs["config"].output_dimensionality, 768) + parts = calls[1].kwargs["contents"] + self.assertEqual(len(parts), 1) + self.assertEqual(parts[0].inline_data.mime_type, "image/jpeg") + self.assertTrue(parts[0].inline_data.data.startswith(b"\xff\xd8\xff")) + self.assertEqual(calls[1].kwargs["model"], "gemini-embedding-2") + self.assertEqual(len(result), 2) + self.assertEqual(len(result[0]), 768) + self.assertEqual(result[0].dtype.name, "float32") + self.assertAlmostEqual(float(result[1][0]), 0.2, places=5) + + def test_prepared_task_prefix_is_kept(self): + client = self._client(model="gemini-embedding-2") + client.provider.models.embed_content.return_value = self._vector(0.3) + prepared = "task: search result | query: already prepared" + + client.embed(texts=[prepared]) + + contents = client.provider.models.embed_content.call_args.kwargs["contents"] + self.assertEqual(contents, prepared) + self.assertEqual( + client.provider.models.embed_content.call_args.kwargs["model"], + "gemini-embedding-2", + ) + + def test_empty_input_makes_no_request(self): + client = self._client() + + self.assertEqual(client.embed(), []) + client.provider.models.embed_content.assert_not_called() + + def test_api_error_raises_value_error(self): + client = self._client() + client.provider.models.embed_content.side_effect = RuntimeError("rejected") + + with self.assertRaises(ValueError) as caught: + client.embed(texts=["hello"]) + + self.assertEqual(str(caught.exception), "Gemini embeddings request failed") + + def test_bad_image_is_not_sent(self): + client = self._client() + + with self.assertRaises(ValueError) as caught: + client.embed(images=[b"this is not an image"]) + + self.assertEqual( + str(caught.exception), "thumbnail could not be converted to JPEG" + ) + client.provider.models.embed_content.assert_not_called() + + def test_missing_vector_raises(self): + client = self._client() + client.provider.models.embed_content.return_value = SimpleNamespace( + embeddings=[] + ) + + with self.assertRaises(ValueError): + client.embed(texts=["hello"]) + + class TestBaseClientTranscribe(unittest.TestCase): """Providers that don't implement the role must be inert, not broken."""