Add Gemini embeddings for semantic search (#24690)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s

* Add Gemini embeddings for semantic search

gemini-embedding-2 maps text and images into one space. Request 768 dimensions, send one input per call, and transcode WebP thumbnails to JPEG. Check model supported_actions for embedContent before advertising embeddings support, and raise ValueError on failure so the embeddings process stays up.

* Fix unreachable Gemini provider checks
This commit is contained in:
Rhácius Castelo
2026-11-18 15:36:55 -07:00
committed by GitHub
parent 3c122150ae
commit 1c30924ff1
4 changed files with 308 additions and 8 deletions
+133 -2
View File
@@ -9,6 +9,7 @@ import time
from collections.abc import AsyncGenerator
from typing import Any
import numpy as np
from google import genai
from google.genai import errors, types
from google.genai.types import FunctionCallingConfigMode
@@ -16,7 +17,7 @@ from PIL import Image
from frigate.config import GenAIProviderEnum
from frigate.genai import GenAIClient, register_genai_provider
from frigate.genai.utils import interleave_images
from frigate.genai.utils import interleave_images, to_jpeg
logger = logging.getLogger(__name__)
@@ -29,6 +30,9 @@ GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024
COUNT_TOKENS_TIMEOUT_MS = 5000
COUNT_TOKENS_RETRY_INTERVAL = 300
# sqlite-vec stores thumbnail and description embeddings as FLOAT[768].
GEMINI_EMBEDDING_DIM = 768
def _decode_thought_signature(value: Any) -> bytes | None:
"""Decode a base64-encoded thought_signature carried across conversation turns."""
@@ -103,11 +107,13 @@ class GeminiClient(GenAIClient):
provider: genai.Client
_image_token_cache: dict[tuple[int, int], int]
_image_token_retry_at: float
_capabilities_cache: dict[str, dict[str, bool]] | None = None
def _init_provider(self) -> genai.Client:
"""Initialize the client."""
self._image_token_cache = {}
self._image_token_retry_at = 0.0
self._capabilities_cache = None
# Merge provider_options into HttpOptions
http_options_dict: dict[str, Any] = {
@@ -232,13 +238,41 @@ class GeminiClient(GenAIClient):
return transcript or None
@staticmethod
def _capabilities_from_models(models: Any) -> dict[str, dict[str, bool]]:
"""Build a per-model capability map from Gemini's models.list() entries."""
capabilities: dict[str, dict[str, bool]] = {}
for m in models:
name = getattr(m, "name", None) or ""
if not name:
continue
actions = getattr(m, "supported_actions", None) or []
flags = {"supports_embeddings": "embedContent" in actions}
capabilities[name] = flags
capabilities[name.removeprefix("models/")] = flags
return capabilities
def list_models(self) -> list[str]:
"""Return available model names from Gemini."""
try:
return sorted(m.name or "" for m in self.provider.models.list())
models = list(self.provider.models.list())
except Exception as e:
logger.warning("Failed to list Gemini models: %s", e)
return []
self._capabilities_cache = self._capabilities_from_models(models)
return sorted(m.name or "" for m in models)
def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
"""Report per-model capabilities from Gemini's model list."""
if self._capabilities_cache is not None:
return self._capabilities_cache
try:
models = list(self.provider.models.list())
except Exception as e:
logger.debug("Failed to query Gemini model capabilities: %s", e)
return {}
self._capabilities_cache = self._capabilities_from_models(models)
return self._capabilities_cache
def get_context_size(self) -> int:
"""Get the context window size for Gemini."""
@@ -310,6 +344,103 @@ class GeminiClient(GenAIClient):
return total
@property
def supports_embeddings(self) -> bool:
"""Whether Gemini reports embedContent support for the configured model."""
capabilities = self.list_model_capabilities().get(self._embedding_model_name())
return bool(capabilities and capabilities.get("supports_embeddings"))
def _embedding_model_name(self) -> str:
"""Model id for embed_content, without a ``models/`` prefix."""
return self.genai_config.model.removeprefix("models/")
def _vector_from_embed(self, result: Any) -> np.ndarray:
"""Read the single embedding vector from an embed_content response."""
embeddings = getattr(result, "embeddings", None) or []
values = embeddings[0].values if embeddings else None
if not values:
raise ValueError("Gemini embeddings response had no vector")
return np.asarray(values, dtype=np.float32)
def _embed_one(self, model: str, contents: Any) -> np.ndarray:
"""One embed_content call. API failures become ValueError.
``_embed_thumbnail`` only catches ValueError. Returning an empty
list makes the caller index ``[0]`` and can stop the embeddings
process.
"""
try:
result = self.provider.models.embed_content(
model=model,
contents=contents,
config=types.EmbedContentConfig(
output_dimensionality=GEMINI_EMBEDDING_DIM
),
)
except Exception as e:
logger.warning("Gemini embeddings request failed: %s", e)
raise ValueError("Gemini embeddings request failed") from e
return self._vector_from_embed(result)
def embed(
self,
texts: list[str] | None = None,
images: list[bytes] | None = None,
) -> list[np.ndarray]:
"""Embed each text or image on its own.
gemini-embedding-2 returns one vector per request. Several parts
in one contents list are aggregated into that single vector, so
each description, search query, and thumbnail is its own call.
Stored descriptions and search queries share this text path, so
both get the search-query prefix and stay in one format. A text
that already starts with ``task:`` is sent unchanged. Images are
not prefixed. The API accepts PNG and JPEG only, and Frigate
stores thumbnails as WebP, so each image is transcoded first.
Requested length is 768, matching the semantic search tables.
Args:
texts: Strings to embed.
images: Encoded images, typically WebP thumbnails.
Returns:
One float32 vector per input, length 768.
Raises:
ValueError: An image cannot be converted, the response has
no vector, or the request fails.
"""
texts = texts or []
images = images or []
if not texts and not images:
return []
model = self._embedding_model_name()
vectors: list[np.ndarray] = []
for text in texts:
content = (
text
if text.startswith("task:")
else f"task: search result | query: {text}"
)
vectors.append(self._embed_one(model, content))
for image in images:
jpeg = to_jpeg(image)
if jpeg is None:
raise ValueError("thumbnail could not be converted to JPEG")
vectors.append(
self._embed_one(
model,
[types.Part.from_bytes(data=jpeg, mime_type="image/jpeg")],
)
)
return vectors
def chat_with_tools(
self,
messages: list[dict[str, Any]],
+148
View File
@@ -1286,6 +1286,154 @@ class TestImageTokenEstimate(unittest.TestCase):
self.assertEqual(client._image_token_cache, {})
class TestGeminiEmbeddings(unittest.TestCase):
"""gemini-embedding-2 via embed_content. No network, no real images."""
@staticmethod
def _webp_bytes():
import io
from PIL import Image
buf = io.BytesIO()
Image.new("RGB", (8, 8), (10, 20, 30)).save(buf, format="WEBP")
return buf.getvalue()
@staticmethod
def _vector(value: float):
return SimpleNamespace(embeddings=[SimpleNamespace(values=[value] * 768)])
def _client(self, model="models/gemini-embedding-2"):
client = _make_client("gemini", model=model, api_key="k")
client.provider = MagicMock()
return client
def test_supports_embeddings_checks_supported_actions(self):
models = [
SimpleNamespace(
name="models/gemini-2.5-flash",
supported_actions=["generateContent"],
),
SimpleNamespace(
name="models/gemini-embedding-2",
supported_actions=["embedContent", "countTokens"],
),
]
client = self._client(model="models/gemini-embedding-2")
client.provider.models.list.return_value = models
self.assertTrue(client.supports_embeddings)
self.assertEqual(
client.list_model_capabilities(),
{
"models/gemini-2.5-flash": {"supports_embeddings": False},
"gemini-2.5-flash": {"supports_embeddings": False},
"models/gemini-embedding-2": {"supports_embeddings": True},
"gemini-embedding-2": {"supports_embeddings": True},
},
)
client.provider.models.list.assert_called_once()
unprefixed = self._client(model="gemini-embedding-2")
unprefixed.provider.models.list.return_value = models
self.assertTrue(unprefixed.supports_embeddings)
chat_client = self._client(model="models/gemini-2.5-flash")
chat_client.provider.models.list.return_value = models
self.assertFalse(chat_client.supports_embeddings)
def test_capability_lookup_failure_is_not_cached(self):
client = self._client()
client.provider.models.list.side_effect = [
RuntimeError("offline"),
[
SimpleNamespace(
name="models/gemini-embedding-2",
supported_actions=["embedContent"],
)
],
]
self.assertFalse(client.supports_embeddings)
self.assertTrue(client.supports_embeddings)
def test_one_request_per_text_and_image(self):
client = self._client()
client.provider.models.embed_content.side_effect = [
self._vector(0.1),
self._vector(0.2),
]
result = client.embed(texts=["a person walking"], images=[self._webp_bytes()])
calls = client.provider.models.embed_content.call_args_list
self.assertEqual(len(calls), 2)
self.assertEqual(calls[0].kwargs["model"], "gemini-embedding-2")
self.assertEqual(
calls[0].kwargs["contents"],
"task: search result | query: a person walking",
)
self.assertEqual(calls[0].kwargs["config"].output_dimensionality, 768)
parts = calls[1].kwargs["contents"]
self.assertEqual(len(parts), 1)
self.assertEqual(parts[0].inline_data.mime_type, "image/jpeg")
self.assertTrue(parts[0].inline_data.data.startswith(b"\xff\xd8\xff"))
self.assertEqual(calls[1].kwargs["model"], "gemini-embedding-2")
self.assertEqual(len(result), 2)
self.assertEqual(len(result[0]), 768)
self.assertEqual(result[0].dtype.name, "float32")
self.assertAlmostEqual(float(result[1][0]), 0.2, places=5)
def test_prepared_task_prefix_is_kept(self):
client = self._client(model="gemini-embedding-2")
client.provider.models.embed_content.return_value = self._vector(0.3)
prepared = "task: search result | query: already prepared"
client.embed(texts=[prepared])
contents = client.provider.models.embed_content.call_args.kwargs["contents"]
self.assertEqual(contents, prepared)
self.assertEqual(
client.provider.models.embed_content.call_args.kwargs["model"],
"gemini-embedding-2",
)
def test_empty_input_makes_no_request(self):
client = self._client()
self.assertEqual(client.embed(), [])
client.provider.models.embed_content.assert_not_called()
def test_api_error_raises_value_error(self):
client = self._client()
client.provider.models.embed_content.side_effect = RuntimeError("rejected")
with self.assertRaises(ValueError) as caught:
client.embed(texts=["hello"])
self.assertEqual(str(caught.exception), "Gemini embeddings request failed")
def test_bad_image_is_not_sent(self):
client = self._client()
with self.assertRaises(ValueError) as caught:
client.embed(images=[b"this is not an image"])
self.assertEqual(
str(caught.exception), "thumbnail could not be converted to JPEG"
)
client.provider.models.embed_content.assert_not_called()
def test_missing_vector_raises(self):
client = self._client()
client.provider.models.embed_content.return_value = SimpleNamespace(
embeddings=[]
)
with self.assertRaises(ValueError):
client.embed(texts=["hello"])
class TestBaseClientTranscribe(unittest.TestCase):
"""Providers that don't implement the role must be inert, not broken."""