Support newer and cleaner llama.cpp embeddings api (#24499)

frigate/genai/plugins/llama_cpp.py

@@ -103,11 +103,10 @@ class LlamaCppClient(GenAIClient):
     _supports_reasoning: bool
     _image_token_cache: dict[tuple[int, int], int]
     _text_baseline_tokens: int | None
-    _media_marker: str
 
     @property
     def supports_embeddings(self) -> bool:
-        """llama.cpp exposes an /embeddings endpoint for any loaded model."""
+        """llama.cpp exposes a /v1/embeddings endpoint for any loaded model."""
         return True
 
     def _auth_headers(self) -> dict | None:
@@ -159,7 +158,6 @@ class LlamaCppClient(GenAIClient):
         self._supports_reasoning = False
         self._image_token_cache = {}
         self._text_baseline_tokens = None
-        self._media_marker = "<__media__>"
 
         base_url = (
             self.genai_config.base_url.rstrip("/")
@@ -187,7 +185,6 @@ class LlamaCppClient(GenAIClient):
         self._supports_audio = info["supports_audio"]
         self._supports_tools = info["supports_tools"]
         self._supports_reasoning = info["supports_reasoning"]
-        self._media_marker = info["media_marker"]
 
         logger.info(
             "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s, reasoning: %s",
@@ -215,9 +212,7 @@ class LlamaCppClient(GenAIClient):
         `architecture.input_modalities` (text/image/audio) — the primary
         source. When proxied through llama-swap, the same entry carries
         `status.args` (server launch argv) and, for the loaded model,
-        `meta.n_ctx`. /props remains the only source for `media_marker`,
+        `meta.n_ctx`.
-        which the server randomizes per startup unless LLAMA_MEDIA_MARKER
-        is set.
         """
         info: dict[str, Any] = {
             "context_size": None,
@@ -225,7 +220,6 @@ class LlamaCppClient(GenAIClient):
             "supports_audio": False,
             "supports_tools": False,
             "supports_reasoning": False,
-            "media_marker": "<__media__>",
         }
 
         model_entry: dict[str, Any] | None = None
@@ -314,16 +308,8 @@ class LlamaCppClient(GenAIClient):
             # in the Jinja chat template itself.
             chat_template = props.get("chat_template") or ""
             info["supports_reasoning"] = "enable_thinking" in chat_template
-
-            media_marker = props.get("media_marker")
-            if isinstance(media_marker, str) and media_marker:
-                info["media_marker"] = media_marker
         except Exception as e:
-            logger.warning(
+            logger.warning("Failed to query llama.cpp /props endpoint: %s", e)
-                "Failed to query llama.cpp /props endpoint: %s. "
-                "Image embeddings may fail if the server randomized its media marker.",
-                e,
-            )
 
         return info
 
@@ -474,9 +460,6 @@ class LlamaCppClient(GenAIClient):
     def _transcribe_via_chat(self, audio: bytes, language: str | None) -> str | None:
         """Transcribe through /v1/chat/completions, for servers without the
         transcriptions route.
-
-        The _media_marker / multimodal_data convention is an /embeddings-only
-        protocol, so no marker-refresh retry is needed here.
         """
         prompt = "Transcribe the speech in this audio verbatim. Respond with the transcript only, and with nothing at all if there is no speech."
 
@@ -794,41 +777,16 @@ class LlamaCppClient(GenAIClient):
             )
         return result if result else None
 
-    def _refresh_media_marker(self) -> bool:
-        """Re-fetch /props and update the cached media marker if it changed.
-
-        The server randomizes the marker per startup (unless LLAMA_MEDIA_MARKER
-        is set), so a stale marker indicates a restart. Returns True iff the
-        marker was updated to a new value — used to gate a one-shot retry of
-        a failed embeddings request.
-        """
-        if self.provider is None:
-            return False
-        try:
-            props = self._fetch_llama_props(self.provider, self.genai_config.model)
-        except Exception as e:
-            logger.warning("Failed to refresh llama.cpp media marker: %s", e)
-            return False
-
-        marker = props.get("media_marker")
-
-        if not isinstance(marker, str) or not marker or marker == self._media_marker:
-            return False
-
-        logger.info("llama.cpp media marker changed (server restart); refreshed")
-        self._media_marker = marker
-        return True
-
     def embed(
         self,
         texts: list[str] | None = None,
         images: list[bytes] | None = None,
     ) -> list[np.ndarray]:
-        """Generate embeddings via llama.cpp /embeddings endpoint.
+        """Generate embeddings via llama.cpp /v1/embeddings endpoint.
 
-        Supports batch requests. Uses content format with prompt_string and
+        Each text or image is one entry in `input`, using the chat-style
-        multimodal_data for images (PR #15108). Server must be started with
+        content array from ggml-org/llama.cpp#29556. Server must be started
-        --embeddings and --mmproj for multimodal support.
+        with --embeddings, and --mmproj for image support.
         """
         if self.provider is None:
             logger.warning(
@@ -843,49 +801,42 @@ class LlamaCppClient(GenAIClient):
 
         EMBEDDING_DIM = 768
 
-        encoded_images: list[str] = []
+        inputs: list[dict[str, Any]] = [
+            {"content": [{"type": "text", "text": text}]} for text in texts
+        ]
+
         for img in images:
             # llama.cpp uses STB which does not support WebP; convert to JPEG
             jpeg_bytes = _to_jpeg(img)
             to_encode = jpeg_bytes if jpeg_bytes is not None else img
-            encoded_images.append(base64.b64encode(to_encode).decode("utf-8"))
+            encoded = base64.b64encode(to_encode).decode("utf-8")
-
+            # The trailing newline keeps tokenization identical to the older
-        def build_content() -> list[dict[str, Any]]:
+            # "<__media__>\n" prompt_string format, so indexed vectors stay valid
-            # prompt_string must contain the server's media marker placeholder
+            inputs.append(
-            # for each image. The marker is randomized per server startup.
+                {
-            content: list[dict[str, Any]] = []
+                    "content": [
-            for text in texts:
+                        {
-                content.append({"prompt_string": text})
+                            "type": "image_url",
-            for encoded in encoded_images:
+                            "image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
-                content.append(
+                        },
-                    {
+                        {"type": "text", "text": "\n"},
-                        "prompt_string": f"{self._media_marker}\n",
+                    ]
-                        "multimodal_data": [encoded],
+                }
-                    }
-                )
-            return content
-
-        def post_embeddings() -> requests.Response:
-            return self._post(
-                f"{self.provider}/embeddings",
-                json={"model": self.genai_config.model, "content": build_content()},
-                timeout=self.timeout,
             )
 
         try:
-            try:
+            response = self._post(
-                response = post_embeddings()
+                f"{self.provider}/v1/embeddings",
-                response.raise_for_status()
+                json={
-            except requests.exceptions.RequestException:
+                    "model": self.genai_config.model,
-                # The server may have restarted with a new media marker.
+                    "input": inputs,
-                # Refresh from /props; only retry if the marker actually changed.
+                    "encoding_format": "float",
-                if not encoded_images or not self._refresh_media_marker():
+                },
-                    raise
+                timeout=self.timeout,
-                response = post_embeddings()
+            )
-                response.raise_for_status()
+            response.raise_for_status()
-            result = response.json()
+            items = response.json().get("data")
 
-            items = result.get("data", result) if isinstance(result, dict) else result
             if not isinstance(items, list):
                 logger.warning("llama.cpp embeddings returned unexpected format")
                 return []
@@ -896,11 +847,7 @@ class LlamaCppClient(GenAIClient):
                 if emb is None:
                     logger.warning("llama.cpp embeddings item missing embedding field")
                     continue
-                arr = np.array(emb, dtype=np.float32)
+                arr = np.array(emb, dtype=np.float32).flatten()
-                if arr.ndim > 1:
-                    # llama.cpp can return token-level embeddings; pool per item
-                    arr = arr.mean(axis=0)
-                arr = arr.flatten()
                 orig_dim = arr.size
                 if orig_dim != EMBEDDING_DIM:
                     if orig_dim > EMBEDDING_DIM:

frigate/test/test_genai_providers.py

@@ -19,6 +19,8 @@ import unittest
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import requests
+
 from frigate.config import GenAIConfig, GenAIProviderEnum
 from frigate.genai import PROVIDERS, load_providers
 
@@ -566,7 +568,6 @@ class TestLlamaCppProvider(unittest.TestCase):
             "supports_audio": False,
             "supports_tools": False,
             "supports_reasoning": False,
-            "media_marker": "<__media__>",
         }
         cls = PROVIDERS[GenAIProviderEnum.llamacpp]
         with patch.object(cls, "_get_model_info", return_value=info):
@@ -589,6 +590,62 @@ class TestLlamaCppProvider(unittest.TestCase):
         with patch.object(client, "_fetch_models_data", return_value=models_data):
             self.assertEqual(client.list_models(), ["g4", "gemma", "qwen3-asr"])
 
+    @staticmethod
+    def _embeddings_response(vectors):
+        response = MagicMock()
+        response.status_code = 200
+        response.json.return_value = {
+            "object": "list",
+            "data": [
+                {"object": "embedding", "index": i, "embedding": v}
+                for i, v in enumerate(vectors)
+            ],
+        }
+        return response
+
+    def test_embed_posts_content_arrays_to_v1_embeddings(self):
+        client = self._client()
+        response = self._embeddings_response([[0.1] * 768, [0.2] * 768])
+
+        with patch.object(client, "_post", return_value=response) as post:
+            result = client.embed(texts=["a person"], images=[b"not an image"])
+
+        url = post.call_args.args[0]
+        payload = post.call_args.kwargs["json"]
+        self.assertEqual(url, "http://localhost:9999/v1/embeddings")
+        self.assertEqual(payload["model"], "m")
+        self.assertEqual(payload["encoding_format"], "float")
+        self.assertEqual(
+            payload["input"][0], {"content": [{"type": "text", "text": "a person"}]}
+        )
+        image_parts = payload["input"][1]["content"]
+        self.assertEqual(image_parts[0]["type"], "image_url")
+        self.assertEqual(
+            image_parts[0]["image_url"]["url"],
+            "data:image/jpeg;base64," + base64.b64encode(b"not an image").decode(),
+        )
+        self.assertEqual(image_parts[1], {"type": "text", "text": "\n"})
+        self.assertEqual(len(result), 2)
+        self.assertAlmostEqual(float(result[1][0]), 0.2, places=5)
+
+    def test_embed_normalizes_dimension(self):
+        client = self._client()
+        response = self._embeddings_response([[1.0] * 1024, [1.0] * 512])
+
+        with patch.object(client, "_post", return_value=response):
+            result = client.embed(texts=["long", "short"])
+
+        self.assertEqual([r.shape for r in result], [(768,), (768,)])
+        self.assertEqual(float(result[1][-1]), 0.0)
+
+    def test_embed_request_error_returns_empty(self):
+        client = self._client()
+        response = MagicMock()
+        response.raise_for_status.side_effect = requests.exceptions.HTTPError("400")
+
+        with patch.object(client, "_post", return_value=response):
+            self.assertEqual(client.embed(texts=["a"]), [])
+
 
 # ---------------------------------------------------------------------------
 # transcribe role
@@ -762,7 +819,6 @@ class TestLlamaCppTranscribe(unittest.TestCase):
             "supports_audio": supports_audio,
             "supports_tools": False,
             "supports_reasoning": False,
-            "media_marker": "<__media__>",
         }
         cls = PROVIDERS[GenAIProviderEnum.llamacpp]
         with patch.object(cls, "_get_model_info", return_value=info):