From 85e16eb31bd4a574d6150d7491c16d120db6bf80 Mon Sep 17 00:00:00 2001 From: Mohit Varikuti Date: Thu, 25 Jun 2026 00:43:04 -0700 Subject: [PATCH 1/2] feat(genai): add TwelveLabs Marengo embeddings provider Adds an opt-in 'twelvelabs' GenAI provider implementing the embeddings role using TwelveLabs' Marengo multimodal model. Text and image inputs are embedded into a shared vector space, powering semantic search over event thumbnails. The provider only implements the embeddings role; descriptions/chat are left to other providers since Marengo is an embedding model. Marengo returns 512-dim vectors, which the existing GenAIEmbedding adapter pads to Frigate's 768-dim search schema. Includes no-network unit tests and a live test gated on TWELVELABS_API_KEY. --- docker/main/requirements-wheels.txt | 1 + docs/docs/configuration/genai/config.md | 36 +++++- frigate/config/camera/genai.py | 1 + frigate/genai/plugins/twelvelabs.py | 120 ++++++++++++++++++++ frigate/test/test_genai_twelvelabs.py | 144 ++++++++++++++++++++++++ 5 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 frigate/genai/plugins/twelvelabs.py create mode 100644 frigate/test/test_genai_twelvelabs.py diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index 7bd098454d..f856b6cfb3 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -50,6 +50,7 @@ transformers == 4.45.* google-genai == 1.58.* ollama == 0.6.* openai == 1.65.* +twelvelabs == 1.2.* # push notifications py-vapid == 1.9.* pywebpush == 2.0.* diff --git a/docs/docs/configuration/genai/config.md b/docs/docs/configuration/genai/config.md index 9f396d3ccc..46ab77ceba 100644 --- a/docs/docs/configuration/genai/config.md +++ b/docs/docs/configuration/genai/config.md @@ -9,7 +9,7 @@ import NavPath from "@site/src/components/NavPath"; ## Configuration -A Generative AI provider can be configured in the global config, which will make the Generative AI features available for use. There are currently 4 native providers available to integrate with Frigate. Other providers that support the OpenAI standard API can also be used. See the OpenAI-Compatible section below. +A Generative AI provider can be configured in the global config, which will make the Generative AI features available for use. There are currently several native providers available to integrate with Frigate. Other providers that support the OpenAI standard API can also be used. See the OpenAI-Compatible section below. To use Generative AI, you must define a single provider at the global level of your Frigate configuration. If the provider you choose requires an API key, you may either directly paste it in your configuration, or store it in an environment variable prefixed with `FRIGATE_`. @@ -386,3 +386,37 @@ genai: + +### TwelveLabs (Marengo embeddings) + +[TwelveLabs](https://twelvelabs.io) provides the Marengo multimodal model, which embeds text and images into a shared vector space. This makes it usable as the `embeddings` provider for Frigate's [Semantic Search](/configuration/semantic_search), letting a natural-language query match stored event thumbnails directly. + +This provider implements only the `embeddings` role. Use a separate provider (such as Ollama, Gemini, or OpenAI) for the `descriptions` and `chat` roles if you want generated descriptions as well. + +#### Get API Key + +Create an API key from the [TwelveLabs dashboard](https://playground.twelvelabs.io/dashboard/api-key). There is a generous free tier. + +#### Configuration + + + + +1. Navigate to . + - Set **Provider** to `twelvelabs` + - Set **API key** to your TwelveLabs API key (or use an environment variable such as `{FRIGATE_TWELVELABS_API_KEY}`) + - Optionally set **Model** to override the default Marengo model (`marengo3.0`) + + + + +```yaml +genai: + provider: twelvelabs + api_key: "{FRIGATE_TWELVELABS_API_KEY}" + roles: + - embeddings +``` + + + diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index 5b94755723..9db6c4b085 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -15,6 +15,7 @@ class GenAIProviderEnum(str, Enum): gemini = "gemini" ollama = "ollama" llamacpp = "llamacpp" + twelvelabs = "twelvelabs" class GenAIRoleEnum(str, Enum): diff --git a/frigate/genai/plugins/twelvelabs.py b/frigate/genai/plugins/twelvelabs.py new file mode 100644 index 0000000000..979e1e2444 --- /dev/null +++ b/frigate/genai/plugins/twelvelabs.py @@ -0,0 +1,120 @@ +"""TwelveLabs provider for Frigate AI. + +Provides multimodal embeddings for Frigate's semantic search via TwelveLabs' +Marengo model. Marengo produces text and image embeddings in a shared vector +space, so a natural-language query and a stored event thumbnail can be matched +directly — which is exactly what the ``embeddings`` GenAI role drives. + +This provider is opt-in: it is only used when a ``genai`` config entry sets +``provider: twelvelabs`` and includes the ``embeddings`` role. It does not +implement the ``descriptions``/``chat`` roles — Marengo is an embedding model, +and TwelveLabs' Pegasus description model operates on whole video clips rather +than the per-frame thumbnails Frigate hands to ``_send``. + +Marengo embeddings are 512-dimensional; Frigate's semantic search schema +expects 768 dimensions. The shared :class:`~frigate.embeddings.genai_embedding.GenAIEmbedding` +adapter zero-pads shorter vectors, so the dimension difference is handled +upstream and is consistent for both text and image inputs. +""" + +import logging + +import numpy as np + +from frigate.config import GenAIProviderEnum +from frigate.genai import GenAIClient, register_genai_provider + +logger = logging.getLogger(__name__) + +# Default Marengo model. Overridable via the `model` config field. +DEFAULT_MODEL = "marengo3.0" + + +@register_genai_provider(GenAIProviderEnum.twelvelabs) +class TwelveLabsClient(GenAIClient): + """GenAI client for Frigate using TwelveLabs Marengo embeddings.""" + + def _init_provider(self): + """Initialize the TwelveLabs SDK client.""" + try: + from twelvelabs import TwelveLabs + except ImportError: + logger.error( + "The twelvelabs package is required for the TwelveLabs provider." + ) + return None + + if not self.genai_config.api_key: + logger.error("TwelveLabs provider requires an api_key.") + return None + + return TwelveLabs(api_key=self.genai_config.api_key) + + @property + def _model(self) -> str: + return self.genai_config.model or DEFAULT_MODEL + + def list_models(self) -> list[str]: + """Marengo is the embedding model exposed by this provider.""" + return [DEFAULT_MODEL] + + def embed( + self, + texts: list[str] | None = None, + images: list[bytes] | None = None, + ) -> list[np.ndarray]: + """Generate Marengo embeddings for text and/or images. + + The TwelveLabs embed API embeds a single input per call, so inputs are + sent one at a time. Returns one 512-dim float32 vector per input, in + order (texts first, then images). The shared GenAIEmbedding adapter + pads these to Frigate's 768-dim search schema. + """ + if self.provider is None: + logger.warning( + "TwelveLabs provider has not been initialized. Check your configuration." + ) + return [] + + results: list[np.ndarray] = [] + + for text in texts or []: + vector = self._embed_one(text=text) + if vector is not None: + results.append(vector) + + for image in images or []: + vector = self._embed_one(image=image) + if vector is not None: + results.append(vector) + + return results + + def _embed_one( + self, text: str | None = None, image: bytes | None = None + ) -> np.ndarray | None: + """Embed a single text or image input, returning a float32 vector.""" + try: + if text is not None: + response = self.provider.embed.create( + model_name=self._model, + text=text, + request_options={"timeout_in_seconds": self.timeout}, + ) + result = response.text_embedding + else: + response = self.provider.embed.create( + model_name=self._model, + image_file=image, + request_options={"timeout_in_seconds": self.timeout}, + ) + result = response.image_embedding + + if result is None or not result.segments: + logger.warning("TwelveLabs returned no embedding for input.") + return None + + return np.array(result.segments[0].float_, dtype=np.float32) + except Exception as e: + logger.warning("TwelveLabs returned an error: %s", e) + return None diff --git a/frigate/test/test_genai_twelvelabs.py b/frigate/test/test_genai_twelvelabs.py new file mode 100644 index 0000000000..c4c6860cb6 --- /dev/null +++ b/frigate/test/test_genai_twelvelabs.py @@ -0,0 +1,144 @@ +"""Tests for the TwelveLabs GenAI provider (Marengo embeddings).""" + +import io +import os +import unittest +from unittest.mock import MagicMock + +import numpy as np + +from frigate.config.camera.genai import ( + GenAIConfig, + GenAIProviderEnum, + GenAIRoleEnum, +) +from frigate.genai.plugins.twelvelabs import DEFAULT_MODEL, TwelveLabsClient + + +def _make_config(model: str = "") -> GenAIConfig: + return GenAIConfig( + provider=GenAIProviderEnum.twelvelabs, + api_key="test-key", + model=model, + roles=[GenAIRoleEnum.embeddings], + ) + + +def _segment(values): + """Mimic the SDK BaseSegment shape (a `float_` list per segment).""" + seg = MagicMock() + seg.float_ = values + return seg + + +class TestTwelveLabsEmbedNoNetwork(unittest.TestCase): + """Unit tests with the SDK client mocked — no network access.""" + + def _client_with_provider(self, provider) -> TwelveLabsClient: + client = TwelveLabsClient.__new__(TwelveLabsClient) + client.genai_config = _make_config() + client.timeout = 120 + client.provider = provider + return client + + def test_text_embedding_returns_vector(self): + provider = MagicMock() + response = MagicMock() + response.text_embedding.segments = [_segment([0.1, 0.2, 0.3])] + provider.embed.create.return_value = response + + client = self._client_with_provider(provider) + out = client.embed(texts=["a person walking a dog"]) + + self.assertEqual(len(out), 1) + self.assertIsInstance(out[0], np.ndarray) + self.assertEqual(out[0].dtype, np.float32) + np.testing.assert_allclose(out[0], [0.1, 0.2, 0.3], rtol=1e-6) + + _, kwargs = provider.embed.create.call_args + self.assertEqual(kwargs["model_name"], DEFAULT_MODEL) + self.assertEqual(kwargs["text"], "a person walking a dog") + + def test_image_embedding_uses_image_file(self): + provider = MagicMock() + response = MagicMock() + response.image_embedding.segments = [_segment([1.0, 2.0])] + provider.embed.create.return_value = response + + client = self._client_with_provider(provider) + out = client.embed(images=[b"\xff\xd8\xff jpeg bytes"]) + + self.assertEqual(len(out), 1) + _, kwargs = provider.embed.create.call_args + self.assertEqual(kwargs["image_file"], b"\xff\xd8\xff jpeg bytes") + self.assertNotIn("text", kwargs) + + def test_custom_model_name_is_used(self): + provider = MagicMock() + response = MagicMock() + response.text_embedding.segments = [_segment([0.0])] + provider.embed.create.return_value = response + + client = self._client_with_provider(provider) + client.genai_config = _make_config(model="marengo-custom") + client.embed(texts=["x"]) + + _, kwargs = provider.embed.create.call_args + self.assertEqual(kwargs["model_name"], "marengo-custom") + + def test_empty_segments_are_skipped(self): + provider = MagicMock() + response = MagicMock() + response.text_embedding = None + provider.embed.create.return_value = response + + client = self._client_with_provider(provider) + self.assertEqual(client.embed(texts=["x"]), []) + + def test_api_error_is_swallowed(self): + provider = MagicMock() + provider.embed.create.side_effect = RuntimeError("boom") + + client = self._client_with_provider(provider) + self.assertEqual(client.embed(texts=["x"]), []) + + def test_no_provider_returns_empty(self): + client = self._client_with_provider(None) + self.assertEqual(client.embed(texts=["x"]), []) + + def test_no_inputs_returns_empty(self): + client = self._client_with_provider(MagicMock()) + self.assertEqual(client.embed(), []) + + +@unittest.skipUnless( + os.environ.get("TWELVELABS_API_KEY"), + "TWELVELABS_API_KEY not set; skipping live TwelveLabs API test", +) +class TestTwelveLabsEmbedLive(unittest.TestCase): + """Live smoke test against the real TwelveLabs API (Marengo).""" + + def _client(self) -> TwelveLabsClient: + config = _make_config() + config.api_key = os.environ["TWELVELABS_API_KEY"] + return TwelveLabsClient(config) + + def test_text_embedding_dim(self): + out = self._client().embed(texts=["a delivery person at the front door"]) + self.assertEqual(len(out), 1) + self.assertEqual(out[0].shape, (512,)) + + def test_image_embedding_dim(self): + from PIL import Image + + arr = (np.random.rand(224, 224, 3) * 255).astype("uint8") + buf = io.BytesIO() + Image.fromarray(arr, "RGB").save(buf, format="JPEG") + + out = self._client().embed(images=[buf.getvalue()]) + self.assertEqual(len(out), 1) + self.assertEqual(out[0].shape, (512,)) + + +if __name__ == "__main__": + unittest.main() From ee823f8488cb752fe5c94875578279176fa5d72b Mon Sep 17 00:00:00 2001 From: Mohit Varikuti Date: Thu, 25 Jun 2026 16:45:21 -0700 Subject: [PATCH 2/2] fix(genai): drop twelvelabs SDK dep, call Marengo via REST --- docker/main/requirements-wheels.txt | 1 - frigate/genai/plugins/twelvelabs.py | 73 ++++++++++++-------- frigate/test/test_genai_twelvelabs.py | 98 +++++++++++++-------------- 3 files changed, 92 insertions(+), 80 deletions(-) diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index f856b6cfb3..7bd098454d 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -50,7 +50,6 @@ transformers == 4.45.* google-genai == 1.58.* ollama == 0.6.* openai == 1.65.* -twelvelabs == 1.2.* # push notifications py-vapid == 1.9.* pywebpush == 2.0.* diff --git a/frigate/genai/plugins/twelvelabs.py b/frigate/genai/plugins/twelvelabs.py index 979e1e2444..09f8b6b661 100644 --- a/frigate/genai/plugins/twelvelabs.py +++ b/frigate/genai/plugins/twelvelabs.py @@ -20,6 +20,7 @@ upstream and is consistent for both text and image inputs. import logging import numpy as np +import requests from frigate.config import GenAIProviderEnum from frigate.genai import GenAIClient, register_genai_provider @@ -29,26 +30,28 @@ logger = logging.getLogger(__name__) # Default Marengo model. Overridable via the `model` config field. DEFAULT_MODEL = "marengo3.0" +# Marengo embed REST endpoint. No SDK is needed — this is a plain multipart POST +# made through Frigate's existing `requests` dependency. +EMBED_URL = "https://api.twelvelabs.io/v1.3/embed" + @register_genai_provider(GenAIProviderEnum.twelvelabs) class TwelveLabsClient(GenAIClient): """GenAI client for Frigate using TwelveLabs Marengo embeddings.""" def _init_provider(self): - """Initialize the TwelveLabs SDK client.""" - try: - from twelvelabs import TwelveLabs - except ImportError: - logger.error( - "The twelvelabs package is required for the TwelveLabs provider." - ) - return None + """Validate config for the TwelveLabs REST provider. + The provider is just an HTTPS API, so there is no client object to + build — the API key is the only thing required. A non-None sentinel is + returned so the shared ``ensure_provider``/initialization machinery + treats the provider as available. + """ if not self.genai_config.api_key: logger.error("TwelveLabs provider requires an api_key.") return None - return TwelveLabs(api_key=self.genai_config.api_key) + return self.genai_config.api_key @property def _model(self) -> str: @@ -69,6 +72,8 @@ class TwelveLabsClient(GenAIClient): sent one at a time. Returns one 512-dim float32 vector per input, in order (texts first, then images). The shared GenAIEmbedding adapter pads these to Frigate's 768-dim search schema. + + Calls the Marengo REST endpoint directly via ``requests`` — no SDK. """ if self.provider is None: logger.warning( @@ -93,28 +98,42 @@ class TwelveLabsClient(GenAIClient): def _embed_one( self, text: str | None = None, image: bytes | None = None ) -> np.ndarray | None: - """Embed a single text or image input, returning a float32 vector.""" - try: - if text is not None: - response = self.provider.embed.create( - model_name=self._model, - text=text, - request_options={"timeout_in_seconds": self.timeout}, - ) - result = response.text_embedding - else: - response = self.provider.embed.create( - model_name=self._model, - image_file=image, - request_options={"timeout_in_seconds": self.timeout}, - ) - result = response.image_embedding + """Embed a single text or image input, returning a float32 vector. - if result is None or not result.segments: + Posts a multipart form to the Marengo embed endpoint (``model_name`` plus + either a ``text`` or an ``image_file`` part). The endpoint requires + multipart/form-data, so every field — including text — is passed via + ``files`` (the ``(None, value)`` form makes requests emit a multipart + text part). ``self.provider`` holds the validated API key. The 512-dim + vector is at ``_embedding.segments[0].float`` in the JSON + response. + """ + headers = {"x-api-key": self.provider} + files: dict = {"model_name": (None, self._model)} + + if text is not None: + files["text"] = (None, text) + result_key = "text_embedding" + else: + files["image_file"] = ("image.jpg", image, "image/jpeg") + result_key = "image_embedding" + + try: + response = requests.post( + EMBED_URL, + headers=headers, + files=files, + timeout=self.timeout, + ) + response.raise_for_status() + result = response.json().get(result_key) or {} + segments = result.get("segments") or [] + + if not segments: logger.warning("TwelveLabs returned no embedding for input.") return None - return np.array(result.segments[0].float_, dtype=np.float32) + return np.array(segments[0]["float"], dtype=np.float32) except Exception as e: logger.warning("TwelveLabs returned an error: %s", e) return None diff --git a/frigate/test/test_genai_twelvelabs.py b/frigate/test/test_genai_twelvelabs.py index c4c6860cb6..fab9023f50 100644 --- a/frigate/test/test_genai_twelvelabs.py +++ b/frigate/test/test_genai_twelvelabs.py @@ -3,7 +3,7 @@ import io import os import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import numpy as np @@ -24,91 +24,85 @@ def _make_config(model: str = "") -> GenAIConfig: ) -def _segment(values): - """Mimic the SDK BaseSegment shape (a `float_` list per segment).""" - seg = MagicMock() - seg.float_ = values - return seg +def _response(key: str, values): + """Mimic the Marengo REST JSON: ``{: {segments: [{float: [...]}]}}``.""" + resp = MagicMock() + resp.raise_for_status.return_value = None + resp.json.return_value = {key: {"segments": [{"float": values}]}} + return resp class TestTwelveLabsEmbedNoNetwork(unittest.TestCase): - """Unit tests with the SDK client mocked — no network access.""" + """Unit tests with ``requests`` mocked — no network access, no SDK.""" - def _client_with_provider(self, provider) -> TwelveLabsClient: + def _client(self) -> TwelveLabsClient: client = TwelveLabsClient.__new__(TwelveLabsClient) client.genai_config = _make_config() client.timeout = 120 - client.provider = provider + client.provider = "test-key" return client - def test_text_embedding_returns_vector(self): - provider = MagicMock() - response = MagicMock() - response.text_embedding.segments = [_segment([0.1, 0.2, 0.3])] - provider.embed.create.return_value = response + @patch("frigate.genai.plugins.twelvelabs.requests.post") + def test_text_embedding_returns_vector(self, post): + post.return_value = _response("text_embedding", [0.1, 0.2, 0.3]) - client = self._client_with_provider(provider) - out = client.embed(texts=["a person walking a dog"]) + out = self._client().embed(texts=["a person walking a dog"]) self.assertEqual(len(out), 1) self.assertIsInstance(out[0], np.ndarray) self.assertEqual(out[0].dtype, np.float32) np.testing.assert_allclose(out[0], [0.1, 0.2, 0.3], rtol=1e-6) - _, kwargs = provider.embed.create.call_args - self.assertEqual(kwargs["model_name"], DEFAULT_MODEL) - self.assertEqual(kwargs["text"], "a person walking a dog") + _, kwargs = post.call_args + self.assertEqual(kwargs["files"]["model_name"][1], DEFAULT_MODEL) + self.assertEqual(kwargs["files"]["text"][1], "a person walking a dog") + self.assertEqual(kwargs["headers"]["x-api-key"], "test-key") + self.assertNotIn("image_file", kwargs["files"]) - def test_image_embedding_uses_image_file(self): - provider = MagicMock() - response = MagicMock() - response.image_embedding.segments = [_segment([1.0, 2.0])] - provider.embed.create.return_value = response + @patch("frigate.genai.plugins.twelvelabs.requests.post") + def test_image_embedding_uses_image_file(self, post): + post.return_value = _response("image_embedding", [1.0, 2.0]) - client = self._client_with_provider(provider) - out = client.embed(images=[b"\xff\xd8\xff jpeg bytes"]) + out = self._client().embed(images=[b"\xff\xd8\xff jpeg bytes"]) self.assertEqual(len(out), 1) - _, kwargs = provider.embed.create.call_args - self.assertEqual(kwargs["image_file"], b"\xff\xd8\xff jpeg bytes") - self.assertNotIn("text", kwargs) + _, kwargs = post.call_args + self.assertEqual(kwargs["files"]["image_file"][1], b"\xff\xd8\xff jpeg bytes") + self.assertNotIn("text", kwargs["files"]) - def test_custom_model_name_is_used(self): - provider = MagicMock() - response = MagicMock() - response.text_embedding.segments = [_segment([0.0])] - provider.embed.create.return_value = response + @patch("frigate.genai.plugins.twelvelabs.requests.post") + def test_custom_model_name_is_used(self, post): + post.return_value = _response("text_embedding", [0.0]) - client = self._client_with_provider(provider) + client = self._client() client.genai_config = _make_config(model="marengo-custom") client.embed(texts=["x"]) - _, kwargs = provider.embed.create.call_args - self.assertEqual(kwargs["model_name"], "marengo-custom") + _, kwargs = post.call_args + self.assertEqual(kwargs["files"]["model_name"][1], "marengo-custom") - def test_empty_segments_are_skipped(self): - provider = MagicMock() - response = MagicMock() - response.text_embedding = None - provider.embed.create.return_value = response + @patch("frigate.genai.plugins.twelvelabs.requests.post") + def test_empty_segments_are_skipped(self, post): + resp = MagicMock() + resp.raise_for_status.return_value = None + resp.json.return_value = {"text_embedding": {"segments": []}} + post.return_value = resp - client = self._client_with_provider(provider) - self.assertEqual(client.embed(texts=["x"]), []) + self.assertEqual(self._client().embed(texts=["x"]), []) - def test_api_error_is_swallowed(self): - provider = MagicMock() - provider.embed.create.side_effect = RuntimeError("boom") + @patch("frigate.genai.plugins.twelvelabs.requests.post") + def test_api_error_is_swallowed(self, post): + post.side_effect = RuntimeError("boom") - client = self._client_with_provider(provider) - self.assertEqual(client.embed(texts=["x"]), []) + self.assertEqual(self._client().embed(texts=["x"]), []) def test_no_provider_returns_empty(self): - client = self._client_with_provider(None) + client = self._client() + client.provider = None self.assertEqual(client.embed(texts=["x"]), []) def test_no_inputs_returns_empty(self): - client = self._client_with_provider(MagicMock()) - self.assertEqual(client.embed(), []) + self.assertEqual(self._client().embed(), []) @unittest.skipUnless(