Ollama image improvements (#24605)

* Support embeddinggemma for ollama

* Refactor docs

* Allow Ollama to probe cost of image tokens

* Cleanup

frigate/genai/__init__.py

@@ -22,6 +22,7 @@ from frigate.genai.prompts import (
     build_review_description_response_format,
     build_review_summary_prompt,
 )
+from frigate.genai.utils import synthetic_jpeg
 from frigate.models import Event
 from frigate.util.builtin import has_non_finite_number
 
@@ -66,6 +67,8 @@ class GenAIClient:
         self.genai_config: GenAIConfig = genai_config
         self.timeout = timeout
         self.validate_model = validate_model
+        self._image_token_cache: dict[tuple[int, int], int] = {}
+        self._text_baseline_tokens: int | None = None
         self.provider = self._init_provider()
         self._last_init_attempt = time.monotonic()
 
@@ -372,10 +375,63 @@ class GenAIClient:
     def estimate_image_tokens(self, width: int, height: int) -> float:
         """Estimate prompt tokens consumed by a single image of the given dimensions.
 
-        Default heuristic: ~1 token per 1250 pixels. Providers that can measure or
-        know their model's exact image-token cost should override.
+        Providers that implement ``_count_prompt_tokens`` are probed for the
+        model's real cost: the same minimal prompt is counted with and without a
+        synthetic image, and the difference is cached per (width, height) since
+        image tokenization depends only on the dimensions and the loaded model.
+        Otherwise, or if probing fails, falls back to ~1 token per 1250 pixels.
         """
-        return (width * height) / 1250
+        heuristic = (width * height) / 1250
+
+        if self.provider is None:
+            return heuristic
+
+        cached = self._image_token_cache.get((width, height))
+
+        if cached is not None:
+            return cached
+
+        try:
+            if self._text_baseline_tokens is None:
+                self._text_baseline_tokens = self._count_prompt_tokens(None)
+
+            if self._text_baseline_tokens is None:
+                return heuristic
+
+            with_image = self._count_prompt_tokens(synthetic_jpeg(width, height))
+        except Exception as e:
+            logger.debug(
+                "%s image-token probe failed for %dx%d (%s); using heuristic",
+                self.__class__.__name__,
+                width,
+                height,
+                e,
+            )
+            return heuristic
+
+        if with_image is None:
+            return heuristic
+
+        tokens = max(1, with_image - self._text_baseline_tokens)
+        self._image_token_cache[(width, height)] = tokens
+        logger.debug(
+            "%s model '%s' uses ~%d tokens for %dx%d images",
+            self.__class__.__name__,
+            self.genai_config.model,
+            tokens,
+            width,
+            height,
+        )
+        return tokens
+
+    def _count_prompt_tokens(self, image: bytes | None) -> int | None:
+        """Prompt tokens the provider reports for a minimal "." request, with
+        ``image`` attached when given.
+
+        Return None when the provider cannot report prompt tokens; raise on
+        request failures. Used by estimate_image_tokens.
+        """
+        return None
 
     def embed(
         self,

frigate/genai/plugins/llama_cpp.py

@@ -1,7 +1,6 @@
 """llama.cpp Provider for Frigate AI."""
 
 import base64
-import io
 import json
 import logging
 from collections.abc import AsyncGenerator
@@ -10,11 +9,14 @@ from typing import Any, cast
 import httpx
 import numpy as np
 import requests
-from PIL import Image
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
-from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
+from frigate.genai.utils import (
+    interleave_images,
+    parse_tool_calls_from_message,
+    to_jpeg,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -76,20 +78,6 @@ def _parse_launch_arg(args: list[str], flag: str) -> str | None:
     return args[idx + 1]
 
 
-def _to_jpeg(img_bytes: bytes) -> bytes | None:
-    """Convert image bytes to JPEG. llama.cpp/STB does not support WebP."""
-    try:
-        img = Image.open(io.BytesIO(img_bytes))
-        if img.mode != "RGB":
-            img = img.convert("RGB")  # type: ignore[assignment]
-        buf = io.BytesIO()
-        img.save(buf, format="JPEG", quality=85)
-        return buf.getvalue()
-    except Exception as e:
-        logger.warning("Failed to convert image to JPEG: %s", e)
-        return None
-
-
 @register_genai_provider(GenAIProviderEnum.llamacpp)
 class LlamaCppClient(GenAIClient):
     """Generative AI client for Frigate using llama.cpp server."""
@@ -101,8 +89,6 @@ class LlamaCppClient(GenAIClient):
     _supports_audio: bool
     _supports_tools: bool
     _supports_reasoning: bool
-    _image_token_cache: dict[tuple[int, int], int]
-    _text_baseline_tokens: int | None
 
     @property
     def supports_embeddings(self) -> bool:
@@ -156,8 +142,6 @@ class LlamaCppClient(GenAIClient):
         self._supports_audio = False
         self._supports_tools = False
         self._supports_reasoning = False
-        self._image_token_cache = {}
-        self._text_baseline_tokens = None
 
         base_url = (
             self.genai_config.base_url.rstrip("/")
@@ -601,78 +585,23 @@ class LlamaCppClient(GenAIClient):
             return self._context_size
         return 4096
 
-    def estimate_image_tokens(self, width: int, height: int) -> float:
-        """Probe the llama.cpp server to learn the model's image-token cost at the
-        requested dimensions.
-
-        llama.cpp's image tokenization is a deterministic function of dimensions and
-        the loaded mmproj, so the result is cached per (width, height) for the
-        lifetime of the process. Falls back to the base pixel heuristic if the
-        server is unreachable or the response is malformed.
-        """
-        if self.provider is None:
-            return super().estimate_image_tokens(width, height)
-
-        cached = self._image_token_cache.get((width, height))
-
-        if cached is not None:
-            return cached
-
-        try:
-            baseline = self._probe_baseline_tokens()
-            with_image = self._probe_image_prompt_tokens(width, height)
-            tokens = max(1, with_image - baseline)
-        except Exception as e:
-            logger.debug(
-                "llama.cpp image-token probe failed for %dx%d (%s); using heuristic",
-                width,
-                height,
-                e,
-            )
-            return super().estimate_image_tokens(width, height)
-
-        self._image_token_cache[(width, height)] = tokens
-        logger.debug(
-            "llama.cpp model '%s' uses ~%d tokens for %dx%d images",
-            self.genai_config.model,
-            tokens,
-            width,
-            height,
-        )
-        return tokens
-
-    def _probe_baseline_tokens(self) -> int:
-        """Return prompt_tokens for a minimal text-only request. Cached after first call."""
-        if self._text_baseline_tokens is not None:
-            return self._text_baseline_tokens
-
-        self._text_baseline_tokens = self._probe_prompt_tokens(
-            [{"type": "text", "text": "."}]
-        )
-        return self._text_baseline_tokens
-
-    def _probe_image_prompt_tokens(self, width: int, height: int) -> int:
-        """Return prompt_tokens for a single synthetic image plus minimal text."""
-        img = Image.new("RGB", (width, height), (128, 128, 128))
-        buf = io.BytesIO()
-        img.save(buf, format="JPEG", quality=60)
-        encoded = base64.b64encode(buf.getvalue()).decode("utf-8")
-        return self._probe_prompt_tokens(
-            [
-                {"type": "text", "text": "."},
-                {
-                    "type": "image_url",
-                    "image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
-                },
-            ]
-        )
-
-    def _probe_prompt_tokens(self, content: list[dict[str, Any]]) -> int:
+    def _count_prompt_tokens(self, image: bytes | None) -> int | None:
         """POST a 1-token chat completion and return reported prompt_tokens.
 
         Uses a generous timeout to absorb a cold model load on the first probe
         when the server lazily loads models on demand (e.g. llama-swap).
         """
+        content: list[dict[str, Any]] = [{"type": "text", "text": "."}]
+
+        if image is not None:
+            encoded = base64.b64encode(image).decode("utf-8")
+            content.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
+                }
+            )
+
         payload = {
             "model": self.genai_config.model,
             "messages": [{"role": "user", "content": content}],
@@ -807,7 +736,7 @@ class LlamaCppClient(GenAIClient):
 
         for img in images:
             # llama.cpp uses STB which does not support WebP; convert to JPEG
-            jpeg_bytes = _to_jpeg(img)
+            jpeg_bytes = to_jpeg(img)
             to_encode = jpeg_bytes if jpeg_bytes is not None else img
             encoded = base64.b64encode(to_encode).decode("utf-8")
             # The trailing newline keeps tokenization identical to the older

frigate/genai/plugins/ollama.py

@@ -7,6 +7,7 @@ import logging
 from collections.abc import AsyncGenerator
 from typing import Any
 
+import numpy as np
 from httpx import RemoteProtocolError, TimeoutException
 from ollama import AsyncClient as OllamaAsyncClient
 from ollama import Client as ApiClient
@@ -14,7 +15,11 @@ from ollama import ResponseError
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
-from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
+from frigate.genai.utils import (
+    interleave_images,
+    parse_tool_calls_from_message,
+    to_jpeg,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -124,23 +129,39 @@ class OllamaClient(GenAIClient):
 
     provider: ApiClient | None
     provider_options: dict[str, Any]
+    _capabilities_cache: list[str] | None = None
     _supports_thinking_cache: bool | None = None
 
+    def _model_capabilities(self) -> list[str] | None:
+        """Capabilities Ollama reports for the configured model, or None when
+        they could not be fetched. Only successful lookups are cached."""
+        if self._capabilities_cache is not None:
+            return self._capabilities_cache
+        if self.provider is None:
+            return None
+        try:
+            response = self.provider.show(self.genai_config.model)
+        except Exception as e:
+            logger.debug("Failed to query Ollama model capabilities: %s", e)
+            return None
+        self._capabilities_cache = list(response.get("capabilities") or [])
+        return self._capabilities_cache
+
     @property
     def supports_toggleable_thinking(self) -> bool:
         if self._supports_thinking_cache is not None:
             return self._supports_thinking_cache
-        if self.provider is None:
+        capabilities = self._model_capabilities()
+        if capabilities is None:
             return False
-        try:
-            response = self.provider.show(self.genai_config.model)
-            capabilities = response.get("capabilities") or []
-            self._supports_thinking_cache = "thinking" in capabilities
-        except Exception as e:
-            logger.debug("Failed to query Ollama model capabilities: %s", e)
-            self._supports_thinking_cache = False
+        self._supports_thinking_cache = "thinking" in capabilities
         return self._supports_thinking_cache
 
+    @property
+    def supports_embeddings(self) -> bool:
+        """Whether Ollama reports the configured model as an embedding model."""
+        return "embedding" in (self._model_capabilities() or [])
+
     def _auth_headers(self) -> dict | None:
         if self.genai_config.api_key:
             return {"Authorization": "Bearer " + self.genai_config.api_key}
@@ -322,6 +343,93 @@ class OllamaClient(GenAIClient):
             self.genai_config.provider_options.get("options", {}).get("num_ctx", 4096)
         )
 
+    def _count_prompt_tokens(self, image: bytes | None) -> int | None:
+        """Send a 1-token chat request and return Ollama's prompt_eval_count.
+
+        Reuses the description request options so the probe runs with the same
+        num_ctx; a different value would make Ollama reload the model.
+        """
+        if self.provider is None:
+            return None
+
+        message: dict[str, Any] = {"role": "user", "content": "."}
+
+        if image is not None:
+            message["images"] = [image]
+
+        request_params = self._build_request_params(
+            [message], None, None, enable_thinking=False
+        )
+        request_params["options"] = {
+            **(request_params.get("options") or {}),
+            "num_predict": 1,
+        }
+        response = self.provider.chat(**request_params)
+        count = response.get("prompt_eval_count")
+        return int(count) if count is not None else None
+
+    def embed(
+        self,
+        texts: list[str] | None = None,
+        images: list[bytes] | None = None,
+    ) -> list[np.ndarray]:
+        """Generate embeddings via Ollama's /api/embed endpoint.
+
+        Each text is a plain string in `input` and each image is an
+        ``{"image": <base64>}`` item. Image input requires Ollama 0.40.1 or
+        newer and a model with a vision encoder (e.g. embeddinggemma-2:440m).
+        """
+        if self.provider is None:
+            logger.warning(
+                "Ollama provider has not been initialized. Check your Ollama configuration."
+            )
+            return []
+
+        texts = texts or []
+        images = images or []
+        if not texts and not images:
+            return []
+
+        inputs: list[str | dict[str, str]] = list(texts)
+        for img in images:
+            jpeg_bytes = to_jpeg(img)
+            to_encode = jpeg_bytes if jpeg_bytes is not None else img
+            inputs.append({"image": base64.b64encode(to_encode).decode("utf-8")})
+
+        payload: dict[str, Any] = {"model": self.genai_config.model, "input": inputs}
+        for key in ("options", "keep_alive"):
+            if key in self.genai_config.provider_options:
+                payload[key] = self.genai_config.provider_options[key]
+
+        try:
+            # The ollama SDK's embed() validates input as strings only, so
+            # image items have to bypass it and post the JSON directly.
+            response = self.provider._request_raw("POST", "/api/embed", json=payload)
+            body = response.json()
+        except (
+            TimeoutException,
+            ResponseError,
+            RemoteProtocolError,
+            ConnectionError,
+            ValueError,
+        ) as e:
+            logger.warning("Ollama embeddings error: %s", str(e))
+            return []
+
+        vectors = body.get("embeddings") if isinstance(body, dict) else None
+        if not isinstance(vectors, list):
+            logger.warning("Ollama embeddings returned unexpected format")
+            return []
+
+        if len(vectors) != len(inputs):
+            logger.warning(
+                "Ollama returned %d embeddings for %d inputs",
+                len(vectors),
+                len(inputs),
+            )
+
+        return [np.asarray(v, dtype=np.float32).flatten() for v in vectors]
+
     def _build_request_params(
         self,
         messages: list[dict[str, Any]],

frigate/genai/utils.py

@@ -1,12 +1,42 @@
 """Shared helpers for GenAI providers and chat (OpenAI-style messages, tool call parsing)."""
 
+import io
 import json
 import logging
 from typing import Any
 
+from PIL import Image
+
 logger = logging.getLogger(__name__)
 
 
+def to_jpeg(img_bytes: bytes) -> bytes | None:
+    """Convert image bytes to JPEG.
+
+    Some provider image decoders (e.g. llama.cpp's STB) do not support WebP,
+    which is the format Frigate stores thumbnails in.
+    """
+    try:
+        img = Image.open(io.BytesIO(img_bytes))
+        if img.mode != "RGB":
+            img = img.convert("RGB")  # type: ignore[assignment]
+        buf = io.BytesIO()
+        img.save(buf, format="JPEG", quality=85)
+        return buf.getvalue()
+    except Exception as e:
+        logger.warning("Failed to convert image to JPEG: %s", e)
+        return None
+
+
+def synthetic_jpeg(width: int, height: int) -> bytes:
+    """A flat gray JPEG of the given dimensions, for measuring image token cost."""
+    buf = io.BytesIO()
+    Image.new("RGB", (width, height), (128, 128, 128)).save(
+        buf, format="JPEG", quality=60
+    )
+    return buf.getvalue()
+
+
 def interleave_images(
     prompt: str, images: list[bytes], captions: list[str] | None = None
 ) -> list[str | bytes]:

frigate/test/test_genai_providers.py

@@ -427,6 +427,148 @@ class TestOllamaProvider(unittest.TestCase):
         self.assertEqual(message["content"], "prompt\n[img]")
         self.assertEqual(message["images"], [b"a"])
 
+    def test_capabilities_drive_embeddings_and_thinking(self):
+        client = self._client()
+        client.provider = MagicMock()
+        client.provider.show.return_value = {"capabilities": ["embedding", "vision"]}
+
+        self.assertTrue(client.supports_embeddings)
+        self.assertFalse(client.supports_toggleable_thinking)
+        client.provider.show.assert_called_once()
+
+    def test_capability_lookup_failure_is_not_cached(self):
+        from ollama import ResponseError
+
+        client = self._client()
+        client.provider = MagicMock()
+        client.provider.show.side_effect = [
+            ResponseError("unavailable", 503),
+            {"capabilities": ["embedding"]},
+        ]
+
+        self.assertFalse(client.supports_embeddings)
+        self.assertTrue(client.supports_embeddings)
+
+    def test_thinking_rechecked_after_provider_recovers(self):
+        client = self._client()
+        client.provider = None
+
+        self.assertFalse(client.supports_toggleable_thinking)
+
+        client.provider = MagicMock()
+        client.provider.show.return_value = {"capabilities": ["thinking"]}
+
+        self.assertTrue(client.supports_toggleable_thinking)
+        params = client._build_request_params(
+            [{"role": "user", "content": "hi"}], None, None, enable_thinking=True
+        )
+        self.assertTrue(params["think"])
+
+    @staticmethod
+    def _webp_bytes():
+        import io
+
+        from PIL import Image
+
+        buf = io.BytesIO()
+        Image.new("RGB", (8, 8), (200, 10, 10)).save(buf, format="WEBP")
+        return buf.getvalue()
+
+    def test_embed_posts_text_and_image_items(self):
+        client = self._client()
+        client.provider = MagicMock()
+        client.provider._request_raw.return_value.json.return_value = {
+            "embeddings": [[0.1] * 768, [0.2] * 768]
+        }
+
+        result = client.embed(texts=["a person"], images=[self._webp_bytes()])
+
+        args = client.provider._request_raw.call_args
+        self.assertEqual(args.args, ("POST", "/api/embed"))
+        payload = args.kwargs["json"]
+        self.assertEqual(payload["model"], "llama3")
+        self.assertEqual(payload["input"][0], "a person")
+        self.assertEqual(list(payload["input"][1]), ["image"])
+        # WebP thumbnails are converted to JPEG before being sent
+        image = base64.b64decode(payload["input"][1]["image"])
+        self.assertEqual(image[:2], b"\xff\xd8")
+        self.assertEqual(len(result), 2)
+        self.assertAlmostEqual(float(result[1][0]), 0.2, places=5)
+
+    def test_embed_passes_configured_options(self):
+        client = _make_client(
+            "ollama",
+            model="embeddinggemma-2",
+            base_url="http://localhost:9999",
+            provider_options={"options": {"num_ctx": 2048}, "keep_alive": "10m"},
+        )
+        client.provider = MagicMock()
+        client.provider._request_raw.return_value.json.return_value = {
+            "embeddings": [[0.1] * 768]
+        }
+
+        client.embed(texts=["a"])
+
+        payload = client.provider._request_raw.call_args.kwargs["json"]
+        self.assertEqual(payload["options"], {"num_ctx": 2048})
+        self.assertEqual(payload["keep_alive"], "10m")
+
+    @staticmethod
+    def _chat_counting_prompt_tokens(**params):
+        """Fake chat that reports 10 prompt tokens plus 250 per image."""
+        images = params["messages"][0].get("images") or []
+        return {
+            "message": {"content": "."},
+            "prompt_eval_count": 10 + 250 * len(images),
+        }
+
+    def test_image_tokens_probed_with_one_token_requests(self):
+        client = _make_client(
+            "ollama",
+            model="qwen3-vl",
+            base_url="http://localhost:9999",
+            provider_options={"options": {"num_ctx": 16384}},
+        )
+        client.provider = MagicMock()
+        client.provider.chat.side_effect = self._chat_counting_prompt_tokens
+        client._supports_thinking_cache = False
+
+        self.assertEqual(client.estimate_image_tokens(320, 180), 250)
+
+        calls = client.provider.chat.call_args_list
+        self.assertEqual(len(calls), 2)
+        for call in calls:
+            self.assertEqual(
+                call.kwargs["options"], {"num_ctx": 16384, "num_predict": 1}
+            )
+        image = calls[1].kwargs["messages"][0]["images"][0]
+        self.assertEqual(image[:2], b"\xff\xd8")
+
+    def test_image_token_probe_error_uses_heuristic(self):
+        from ollama import ResponseError
+
+        client = self._client()
+        client.provider = MagicMock()
+        client.provider.chat.side_effect = [
+            {"message": {"content": "."}, "prompt_eval_count": 10},
+            ResponseError("model does not support images", 400),
+        ]
+        client._supports_thinking_cache = False
+
+        self.assertEqual(client.estimate_image_tokens(250, 100), 20)
+        self.assertEqual(client._image_token_cache, {})
+
+    def test_embed_server_error_returns_empty(self):
+        from ollama import ResponseError
+
+        client = self._client()
+        client.provider = MagicMock()
+        client.provider._request_raw.side_effect = ResponseError(
+            "model does not support media embeddings", 400
+        )
+
+        self.assertEqual(client.embed(images=[self._webp_bytes()]), [])
+
 
 # ---------------------------------------------------------------------------
 # llama.cpp
@@ -638,6 +780,46 @@ class TestLlamaCppProvider(unittest.TestCase):
         self.assertEqual([r.shape for r in result], [(768,), (768,)])
         self.assertEqual(float(result[1][-1]), 0.0)
 
+    @staticmethod
+    def _post_counting_prompt_tokens(url, json=None, timeout=None):
+        """Fake chat completion: 12 prompt tokens plus 300 per image part."""
+        content = json["messages"][0]["content"]
+        images = [p for p in content if p["type"] == "image_url"]
+        response = MagicMock()
+        response.json.return_value = {
+            "usage": {"prompt_tokens": 12 + 300 * len(images)}
+        }
+        return response
+
+    def test_image_tokens_probed_once_per_dimension(self):
+        client = self._client()
+
+        with patch.object(
+            client, "_post", side_effect=self._post_counting_prompt_tokens
+        ) as post:
+            self.assertEqual(client.estimate_image_tokens(320, 180), 300)
+            self.assertEqual(client.estimate_image_tokens(320, 180), 300)
+            self.assertEqual(client.estimate_image_tokens(640, 360), 300)
+
+        # one shared text baseline, then one image request per new dimension
+        self.assertEqual(post.call_count, 3)
+        payload = post.call_args_list[0].kwargs["json"]
+        self.assertEqual(payload["max_tokens"], 1)
+        self.assertEqual(
+            post.call_args_list[0].args[0], "http://localhost:9999/v1/chat/completions"
+        )
+
+    def test_image_token_probe_failure_is_not_cached(self):
+        client = self._client()
+
+        with patch.object(
+            client, "_post", side_effect=requests.exceptions.ConnectionError("down")
+        ):
+            self.assertEqual(client.estimate_image_tokens(250, 100), 20)
+
+        self.assertEqual(client._image_token_cache, {})
+        self.assertIsNone(client._text_baseline_tokens)
+
     def test_embed_request_error_returns_empty(self):
         client = self._client()
         response = MagicMock()
@@ -903,6 +1085,14 @@ class TestLlamaCppTranscribe(unittest.TestCase):
         post.assert_not_called()
 
 
+class TestImageTokenEstimate(unittest.TestCase):
+    def test_provider_without_token_counts_uses_heuristic(self):
+        client = _make_client("gemini", model="m", api_key="k")
+
+        self.assertEqual(client.estimate_image_tokens(250, 100), 20)
+        self.assertEqual(client._image_token_cache, {})
+
+
 class TestBaseClientTranscribe(unittest.TestCase):
     """Providers that don't implement the role must be inert, not broken."""