mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-22 21:41:50 +03:00
Add Gemini embeddings for semantic search (#24690)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
* Add Gemini embeddings for semantic search gemini-embedding-2 maps text and images into one space. Request 768 dimensions, send one input per call, and transcode WebP thumbnails to JPEG. Check model supported_actions for embedContent before advertising embeddings support, and raise ValueError on failure so the embeddings process stays up. * Fix unreachable Gemini provider checks
This commit is contained in:
@@ -9,6 +9,7 @@ import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from google import genai
|
||||
from google.genai import errors, types
|
||||
from google.genai.types import FunctionCallingConfigMode
|
||||
@@ -16,7 +17,7 @@ from PIL import Image
|
||||
|
||||
from frigate.config import GenAIProviderEnum
|
||||
from frigate.genai import GenAIClient, register_genai_provider
|
||||
from frigate.genai.utils import interleave_images
|
||||
from frigate.genai.utils import interleave_images, to_jpeg
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,6 +30,9 @@ GEMINI_MAX_INLINE_BYTES = 15 * 1024 * 1024
|
||||
COUNT_TOKENS_TIMEOUT_MS = 5000
|
||||
COUNT_TOKENS_RETRY_INTERVAL = 300
|
||||
|
||||
# sqlite-vec stores thumbnail and description embeddings as FLOAT[768].
|
||||
GEMINI_EMBEDDING_DIM = 768
|
||||
|
||||
|
||||
def _decode_thought_signature(value: Any) -> bytes | None:
|
||||
"""Decode a base64-encoded thought_signature carried across conversation turns."""
|
||||
@@ -103,11 +107,13 @@ class GeminiClient(GenAIClient):
|
||||
provider: genai.Client
|
||||
_image_token_cache: dict[tuple[int, int], int]
|
||||
_image_token_retry_at: float
|
||||
_capabilities_cache: dict[str, dict[str, bool]] | None = None
|
||||
|
||||
def _init_provider(self) -> genai.Client:
|
||||
"""Initialize the client."""
|
||||
self._image_token_cache = {}
|
||||
self._image_token_retry_at = 0.0
|
||||
self._capabilities_cache = None
|
||||
|
||||
# Merge provider_options into HttpOptions
|
||||
http_options_dict: dict[str, Any] = {
|
||||
@@ -232,13 +238,41 @@ class GeminiClient(GenAIClient):
|
||||
|
||||
return transcript or None
|
||||
|
||||
@staticmethod
|
||||
def _capabilities_from_models(models: Any) -> dict[str, dict[str, bool]]:
|
||||
"""Build a per-model capability map from Gemini's models.list() entries."""
|
||||
capabilities: dict[str, dict[str, bool]] = {}
|
||||
for m in models:
|
||||
name = getattr(m, "name", None) or ""
|
||||
if not name:
|
||||
continue
|
||||
actions = getattr(m, "supported_actions", None) or []
|
||||
flags = {"supports_embeddings": "embedContent" in actions}
|
||||
capabilities[name] = flags
|
||||
capabilities[name.removeprefix("models/")] = flags
|
||||
return capabilities
|
||||
|
||||
def list_models(self) -> list[str]:
|
||||
"""Return available model names from Gemini."""
|
||||
try:
|
||||
return sorted(m.name or "" for m in self.provider.models.list())
|
||||
models = list(self.provider.models.list())
|
||||
except Exception as e:
|
||||
logger.warning("Failed to list Gemini models: %s", e)
|
||||
return []
|
||||
self._capabilities_cache = self._capabilities_from_models(models)
|
||||
return sorted(m.name or "" for m in models)
|
||||
|
||||
def list_model_capabilities(self) -> dict[str, dict[str, bool]]:
|
||||
"""Report per-model capabilities from Gemini's model list."""
|
||||
if self._capabilities_cache is not None:
|
||||
return self._capabilities_cache
|
||||
try:
|
||||
models = list(self.provider.models.list())
|
||||
except Exception as e:
|
||||
logger.debug("Failed to query Gemini model capabilities: %s", e)
|
||||
return {}
|
||||
self._capabilities_cache = self._capabilities_from_models(models)
|
||||
return self._capabilities_cache
|
||||
|
||||
def get_context_size(self) -> int:
|
||||
"""Get the context window size for Gemini."""
|
||||
@@ -310,6 +344,103 @@ class GeminiClient(GenAIClient):
|
||||
|
||||
return total
|
||||
|
||||
@property
|
||||
def supports_embeddings(self) -> bool:
|
||||
"""Whether Gemini reports embedContent support for the configured model."""
|
||||
capabilities = self.list_model_capabilities().get(self._embedding_model_name())
|
||||
return bool(capabilities and capabilities.get("supports_embeddings"))
|
||||
|
||||
def _embedding_model_name(self) -> str:
|
||||
"""Model id for embed_content, without a ``models/`` prefix."""
|
||||
return self.genai_config.model.removeprefix("models/")
|
||||
|
||||
def _vector_from_embed(self, result: Any) -> np.ndarray:
|
||||
"""Read the single embedding vector from an embed_content response."""
|
||||
embeddings = getattr(result, "embeddings", None) or []
|
||||
values = embeddings[0].values if embeddings else None
|
||||
if not values:
|
||||
raise ValueError("Gemini embeddings response had no vector")
|
||||
return np.asarray(values, dtype=np.float32)
|
||||
|
||||
def _embed_one(self, model: str, contents: Any) -> np.ndarray:
|
||||
"""One embed_content call. API failures become ValueError.
|
||||
|
||||
``_embed_thumbnail`` only catches ValueError. Returning an empty
|
||||
list makes the caller index ``[0]`` and can stop the embeddings
|
||||
process.
|
||||
"""
|
||||
try:
|
||||
result = self.provider.models.embed_content(
|
||||
model=model,
|
||||
contents=contents,
|
||||
config=types.EmbedContentConfig(
|
||||
output_dimensionality=GEMINI_EMBEDDING_DIM
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Gemini embeddings request failed: %s", e)
|
||||
raise ValueError("Gemini embeddings request failed") from e
|
||||
|
||||
return self._vector_from_embed(result)
|
||||
|
||||
def embed(
|
||||
self,
|
||||
texts: list[str] | None = None,
|
||||
images: list[bytes] | None = None,
|
||||
) -> list[np.ndarray]:
|
||||
"""Embed each text or image on its own.
|
||||
|
||||
gemini-embedding-2 returns one vector per request. Several parts
|
||||
in one contents list are aggregated into that single vector, so
|
||||
each description, search query, and thumbnail is its own call.
|
||||
|
||||
Stored descriptions and search queries share this text path, so
|
||||
both get the search-query prefix and stay in one format. A text
|
||||
that already starts with ``task:`` is sent unchanged. Images are
|
||||
not prefixed. The API accepts PNG and JPEG only, and Frigate
|
||||
stores thumbnails as WebP, so each image is transcoded first.
|
||||
Requested length is 768, matching the semantic search tables.
|
||||
|
||||
Args:
|
||||
texts: Strings to embed.
|
||||
images: Encoded images, typically WebP thumbnails.
|
||||
|
||||
Returns:
|
||||
One float32 vector per input, length 768.
|
||||
|
||||
Raises:
|
||||
ValueError: An image cannot be converted, the response has
|
||||
no vector, or the request fails.
|
||||
"""
|
||||
texts = texts or []
|
||||
images = images or []
|
||||
if not texts and not images:
|
||||
return []
|
||||
|
||||
model = self._embedding_model_name()
|
||||
vectors: list[np.ndarray] = []
|
||||
|
||||
for text in texts:
|
||||
content = (
|
||||
text
|
||||
if text.startswith("task:")
|
||||
else f"task: search result | query: {text}"
|
||||
)
|
||||
vectors.append(self._embed_one(model, content))
|
||||
|
||||
for image in images:
|
||||
jpeg = to_jpeg(image)
|
||||
if jpeg is None:
|
||||
raise ValueError("thumbnail could not be converted to JPEG")
|
||||
vectors.append(
|
||||
self._embed_one(
|
||||
model,
|
||||
[types.Part.from_bytes(data=jpeg, mime_type="image/jpeg")],
|
||||
)
|
||||
)
|
||||
|
||||
return vectors
|
||||
|
||||
def chat_with_tools(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
|
||||
@@ -1286,6 +1286,154 @@ class TestImageTokenEstimate(unittest.TestCase):
|
||||
self.assertEqual(client._image_token_cache, {})
|
||||
|
||||
|
||||
class TestGeminiEmbeddings(unittest.TestCase):
|
||||
"""gemini-embedding-2 via embed_content. No network, no real images."""
|
||||
|
||||
@staticmethod
|
||||
def _webp_bytes():
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (8, 8), (10, 20, 30)).save(buf, format="WEBP")
|
||||
return buf.getvalue()
|
||||
|
||||
@staticmethod
|
||||
def _vector(value: float):
|
||||
return SimpleNamespace(embeddings=[SimpleNamespace(values=[value] * 768)])
|
||||
|
||||
def _client(self, model="models/gemini-embedding-2"):
|
||||
client = _make_client("gemini", model=model, api_key="k")
|
||||
client.provider = MagicMock()
|
||||
return client
|
||||
|
||||
def test_supports_embeddings_checks_supported_actions(self):
|
||||
models = [
|
||||
SimpleNamespace(
|
||||
name="models/gemini-2.5-flash",
|
||||
supported_actions=["generateContent"],
|
||||
),
|
||||
SimpleNamespace(
|
||||
name="models/gemini-embedding-2",
|
||||
supported_actions=["embedContent", "countTokens"],
|
||||
),
|
||||
]
|
||||
|
||||
client = self._client(model="models/gemini-embedding-2")
|
||||
client.provider.models.list.return_value = models
|
||||
self.assertTrue(client.supports_embeddings)
|
||||
self.assertEqual(
|
||||
client.list_model_capabilities(),
|
||||
{
|
||||
"models/gemini-2.5-flash": {"supports_embeddings": False},
|
||||
"gemini-2.5-flash": {"supports_embeddings": False},
|
||||
"models/gemini-embedding-2": {"supports_embeddings": True},
|
||||
"gemini-embedding-2": {"supports_embeddings": True},
|
||||
},
|
||||
)
|
||||
client.provider.models.list.assert_called_once()
|
||||
|
||||
unprefixed = self._client(model="gemini-embedding-2")
|
||||
unprefixed.provider.models.list.return_value = models
|
||||
self.assertTrue(unprefixed.supports_embeddings)
|
||||
|
||||
chat_client = self._client(model="models/gemini-2.5-flash")
|
||||
chat_client.provider.models.list.return_value = models
|
||||
self.assertFalse(chat_client.supports_embeddings)
|
||||
|
||||
def test_capability_lookup_failure_is_not_cached(self):
|
||||
client = self._client()
|
||||
client.provider.models.list.side_effect = [
|
||||
RuntimeError("offline"),
|
||||
[
|
||||
SimpleNamespace(
|
||||
name="models/gemini-embedding-2",
|
||||
supported_actions=["embedContent"],
|
||||
)
|
||||
],
|
||||
]
|
||||
|
||||
self.assertFalse(client.supports_embeddings)
|
||||
self.assertTrue(client.supports_embeddings)
|
||||
|
||||
def test_one_request_per_text_and_image(self):
|
||||
client = self._client()
|
||||
client.provider.models.embed_content.side_effect = [
|
||||
self._vector(0.1),
|
||||
self._vector(0.2),
|
||||
]
|
||||
|
||||
result = client.embed(texts=["a person walking"], images=[self._webp_bytes()])
|
||||
|
||||
calls = client.provider.models.embed_content.call_args_list
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(calls[0].kwargs["model"], "gemini-embedding-2")
|
||||
self.assertEqual(
|
||||
calls[0].kwargs["contents"],
|
||||
"task: search result | query: a person walking",
|
||||
)
|
||||
self.assertEqual(calls[0].kwargs["config"].output_dimensionality, 768)
|
||||
parts = calls[1].kwargs["contents"]
|
||||
self.assertEqual(len(parts), 1)
|
||||
self.assertEqual(parts[0].inline_data.mime_type, "image/jpeg")
|
||||
self.assertTrue(parts[0].inline_data.data.startswith(b"\xff\xd8\xff"))
|
||||
self.assertEqual(calls[1].kwargs["model"], "gemini-embedding-2")
|
||||
self.assertEqual(len(result), 2)
|
||||
self.assertEqual(len(result[0]), 768)
|
||||
self.assertEqual(result[0].dtype.name, "float32")
|
||||
self.assertAlmostEqual(float(result[1][0]), 0.2, places=5)
|
||||
|
||||
def test_prepared_task_prefix_is_kept(self):
|
||||
client = self._client(model="gemini-embedding-2")
|
||||
client.provider.models.embed_content.return_value = self._vector(0.3)
|
||||
prepared = "task: search result | query: already prepared"
|
||||
|
||||
client.embed(texts=[prepared])
|
||||
|
||||
contents = client.provider.models.embed_content.call_args.kwargs["contents"]
|
||||
self.assertEqual(contents, prepared)
|
||||
self.assertEqual(
|
||||
client.provider.models.embed_content.call_args.kwargs["model"],
|
||||
"gemini-embedding-2",
|
||||
)
|
||||
|
||||
def test_empty_input_makes_no_request(self):
|
||||
client = self._client()
|
||||
|
||||
self.assertEqual(client.embed(), [])
|
||||
client.provider.models.embed_content.assert_not_called()
|
||||
|
||||
def test_api_error_raises_value_error(self):
|
||||
client = self._client()
|
||||
client.provider.models.embed_content.side_effect = RuntimeError("rejected")
|
||||
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
client.embed(texts=["hello"])
|
||||
|
||||
self.assertEqual(str(caught.exception), "Gemini embeddings request failed")
|
||||
|
||||
def test_bad_image_is_not_sent(self):
|
||||
client = self._client()
|
||||
|
||||
with self.assertRaises(ValueError) as caught:
|
||||
client.embed(images=[b"this is not an image"])
|
||||
|
||||
self.assertEqual(
|
||||
str(caught.exception), "thumbnail could not be converted to JPEG"
|
||||
)
|
||||
client.provider.models.embed_content.assert_not_called()
|
||||
|
||||
def test_missing_vector_raises(self):
|
||||
client = self._client()
|
||||
client.provider.models.embed_content.return_value = SimpleNamespace(
|
||||
embeddings=[]
|
||||
)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
client.embed(texts=["hello"])
|
||||
|
||||
|
||||
class TestBaseClientTranscribe(unittest.TestCase):
|
||||
"""Providers that don't implement the role must be inert, not broken."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user