From dc97f294a05c090d3f926f80f9c0cd50f81ae8fc Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 28 Sep 2026 14:22:59 -0600 Subject: [PATCH] Support newer and cleaner llama.cpp embeddings api (#24499) --- frigate/genai/plugins/llama_cpp.py | 125 ++++++++------------------- frigate/test/test_genai_providers.py | 60 ++++++++++++- 2 files changed, 94 insertions(+), 91 deletions(-) diff --git a/frigate/genai/plugins/llama_cpp.py b/frigate/genai/plugins/llama_cpp.py index 9f50dea676..c33013935a 100644 --- a/frigate/genai/plugins/llama_cpp.py +++ b/frigate/genai/plugins/llama_cpp.py @@ -103,11 +103,10 @@ class LlamaCppClient(GenAIClient): _supports_reasoning: bool _image_token_cache: dict[tuple[int, int], int] _text_baseline_tokens: int | None - _media_marker: str @property def supports_embeddings(self) -> bool: - """llama.cpp exposes an /embeddings endpoint for any loaded model.""" + """llama.cpp exposes a /v1/embeddings endpoint for any loaded model.""" return True def _auth_headers(self) -> dict | None: @@ -159,7 +158,6 @@ class LlamaCppClient(GenAIClient): self._supports_reasoning = False self._image_token_cache = {} self._text_baseline_tokens = None - self._media_marker = "<__media__>" base_url = ( self.genai_config.base_url.rstrip("/") @@ -187,7 +185,6 @@ class LlamaCppClient(GenAIClient): self._supports_audio = info["supports_audio"] self._supports_tools = info["supports_tools"] self._supports_reasoning = info["supports_reasoning"] - self._media_marker = info["media_marker"] logger.info( "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s, reasoning: %s", @@ -215,9 +212,7 @@ class LlamaCppClient(GenAIClient): `architecture.input_modalities` (text/image/audio) — the primary source. When proxied through llama-swap, the same entry carries `status.args` (server launch argv) and, for the loaded model, - `meta.n_ctx`. /props remains the only source for `media_marker`, - which the server randomizes per startup unless LLAMA_MEDIA_MARKER - is set. + `meta.n_ctx`. """ info: dict[str, Any] = { "context_size": None, @@ -225,7 +220,6 @@ class LlamaCppClient(GenAIClient): "supports_audio": False, "supports_tools": False, "supports_reasoning": False, - "media_marker": "<__media__>", } model_entry: dict[str, Any] | None = None @@ -314,16 +308,8 @@ class LlamaCppClient(GenAIClient): # in the Jinja chat template itself. chat_template = props.get("chat_template") or "" info["supports_reasoning"] = "enable_thinking" in chat_template - - media_marker = props.get("media_marker") - if isinstance(media_marker, str) and media_marker: - info["media_marker"] = media_marker except Exception as e: - logger.warning( - "Failed to query llama.cpp /props endpoint: %s. " - "Image embeddings may fail if the server randomized its media marker.", - e, - ) + logger.warning("Failed to query llama.cpp /props endpoint: %s", e) return info @@ -474,9 +460,6 @@ class LlamaCppClient(GenAIClient): def _transcribe_via_chat(self, audio: bytes, language: str | None) -> str | None: """Transcribe through /v1/chat/completions, for servers without the transcriptions route. - - The _media_marker / multimodal_data convention is an /embeddings-only - protocol, so no marker-refresh retry is needed here. """ prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech." @@ -794,41 +777,16 @@ class LlamaCppClient(GenAIClient): ) return result if result else None - def _refresh_media_marker(self) -> bool: - """Re-fetch /props and update the cached media marker if it changed. - - The server randomizes the marker per startup (unless LLAMA_MEDIA_MARKER - is set), so a stale marker indicates a restart. Returns True iff the - marker was updated to a new value — used to gate a one-shot retry of - a failed embeddings request. - """ - if self.provider is None: - return False - try: - props = self._fetch_llama_props(self.provider, self.genai_config.model) - except Exception as e: - logger.warning("Failed to refresh llama.cpp media marker: %s", e) - return False - - marker = props.get("media_marker") - - if not isinstance(marker, str) or not marker or marker == self._media_marker: - return False - - logger.info("llama.cpp media marker changed (server restart); refreshed") - self._media_marker = marker - return True - def embed( self, texts: list[str] | None = None, images: list[bytes] | None = None, ) -> list[np.ndarray]: - """Generate embeddings via llama.cpp /embeddings endpoint. + """Generate embeddings via llama.cpp /v1/embeddings endpoint. - Supports batch requests. Uses content format with prompt_string and - multimodal_data for images (PR #15108). Server must be started with - --embeddings and --mmproj for multimodal support. + Each text or image is one entry in `input`, using the chat-style + content array from ggml-org/llama.cpp#29556. Server must be started + with --embeddings, and --mmproj for image support. """ if self.provider is None: logger.warning( @@ -843,49 +801,42 @@ class LlamaCppClient(GenAIClient): EMBEDDING_DIM = 768 - encoded_images: list[str] = [] + inputs: list[dict[str, Any]] = [ + {"content": [{"type": "text", "text": text}]} for text in texts + ] + for img in images: # llama.cpp uses STB which does not support WebP; convert to JPEG jpeg_bytes = _to_jpeg(img) to_encode = jpeg_bytes if jpeg_bytes is not None else img - encoded_images.append(base64.b64encode(to_encode).decode("utf-8")) - - def build_content() -> list[dict[str, Any]]: - # prompt_string must contain the server's media marker placeholder - # for each image. The marker is randomized per server startup. - content: list[dict[str, Any]] = [] - for text in texts: - content.append({"prompt_string": text}) - for encoded in encoded_images: - content.append( - { - "prompt_string": f"{self._media_marker}\n", - "multimodal_data": [encoded], - } - ) - return content - - def post_embeddings() -> requests.Response: - return self._post( - f"{self.provider}/embeddings", - json={"model": self.genai_config.model, "content": build_content()}, - timeout=self.timeout, + encoded = base64.b64encode(to_encode).decode("utf-8") + # The trailing newline keeps tokenization identical to the older + # "<__media__>\n" prompt_string format, so indexed vectors stay valid + inputs.append( + { + "content": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{encoded}"}, + }, + {"type": "text", "text": "\n"}, + ] + } ) try: - try: - response = post_embeddings() - response.raise_for_status() - except requests.exceptions.RequestException: - # The server may have restarted with a new media marker. - # Refresh from /props; only retry if the marker actually changed. - if not encoded_images or not self._refresh_media_marker(): - raise - response = post_embeddings() - response.raise_for_status() - result = response.json() + response = self._post( + f"{self.provider}/v1/embeddings", + json={ + "model": self.genai_config.model, + "input": inputs, + "encoding_format": "float", + }, + timeout=self.timeout, + ) + response.raise_for_status() + items = response.json().get("data") - items = result.get("data", result) if isinstance(result, dict) else result if not isinstance(items, list): logger.warning("llama.cpp embeddings returned unexpected format") return [] @@ -896,11 +847,7 @@ class LlamaCppClient(GenAIClient): if emb is None: logger.warning("llama.cpp embeddings item missing embedding field") continue - arr = np.array(emb, dtype=np.float32) - if arr.ndim > 1: - # llama.cpp can return token-level embeddings; pool per item - arr = arr.mean(axis=0) - arr = arr.flatten() + arr = np.array(emb, dtype=np.float32).flatten() orig_dim = arr.size if orig_dim != EMBEDDING_DIM: if orig_dim > EMBEDDING_DIM: diff --git a/frigate/test/test_genai_providers.py b/frigate/test/test_genai_providers.py index 6b81dd0395..d2c7ae85ed 100644 --- a/frigate/test/test_genai_providers.py +++ b/frigate/test/test_genai_providers.py @@ -19,6 +19,8 @@ import unittest from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch +import requests + from frigate.config import GenAIConfig, GenAIProviderEnum from frigate.genai import PROVIDERS, load_providers @@ -566,7 +568,6 @@ class TestLlamaCppProvider(unittest.TestCase): "supports_audio": False, "supports_tools": False, "supports_reasoning": False, - "media_marker": "<__media__>", } cls = PROVIDERS[GenAIProviderEnum.llamacpp] with patch.object(cls, "_get_model_info", return_value=info): @@ -589,6 +590,62 @@ class TestLlamaCppProvider(unittest.TestCase): with patch.object(client, "_fetch_models_data", return_value=models_data): self.assertEqual(client.list_models(), ["g4", "gemma", "qwen3-asr"]) + @staticmethod + def _embeddings_response(vectors): + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "object": "list", + "data": [ + {"object": "embedding", "index": i, "embedding": v} + for i, v in enumerate(vectors) + ], + } + return response + + def test_embed_posts_content_arrays_to_v1_embeddings(self): + client = self._client() + response = self._embeddings_response([[0.1] * 768, [0.2] * 768]) + + with patch.object(client, "_post", return_value=response) as post: + result = client.embed(texts=["a person"], images=[b"not an image"]) + + url = post.call_args.args[0] + payload = post.call_args.kwargs["json"] + self.assertEqual(url, "http://localhost:9999/v1/embeddings") + self.assertEqual(payload["model"], "m") + self.assertEqual(payload["encoding_format"], "float") + self.assertEqual( + payload["input"][0], {"content": [{"type": "text", "text": "a person"}]} + ) + image_parts = payload["input"][1]["content"] + self.assertEqual(image_parts[0]["type"], "image_url") + self.assertEqual( + image_parts[0]["image_url"]["url"], + "data:image/jpeg;base64," + base64.b64encode(b"not an image").decode(), + ) + self.assertEqual(image_parts[1], {"type": "text", "text": "\n"}) + self.assertEqual(len(result), 2) + self.assertAlmostEqual(float(result[1][0]), 0.2, places=5) + + def test_embed_normalizes_dimension(self): + client = self._client() + response = self._embeddings_response([[1.0] * 1024, [1.0] * 512]) + + with patch.object(client, "_post", return_value=response): + result = client.embed(texts=["long", "short"]) + + self.assertEqual([r.shape for r in result], [(768,), (768,)]) + self.assertEqual(float(result[1][-1]), 0.0) + + def test_embed_request_error_returns_empty(self): + client = self._client() + response = MagicMock() + response.raise_for_status.side_effect = requests.exceptions.HTTPError("400") + + with patch.object(client, "_post", return_value=response): + self.assertEqual(client.embed(texts=["a"]), []) + # --------------------------------------------------------------------------- # transcribe role @@ -762,7 +819,6 @@ class TestLlamaCppTranscribe(unittest.TestCase): "supports_audio": supports_audio, "supports_tools": False, "supports_reasoning": False, - "media_marker": "<__media__>", } cls = PROVIDERS[GenAIProviderEnum.llamacpp] with patch.object(cls, "_get_model_info", return_value=info):