Ollama image improvements (#24605)

* Support embeddinggemma for ollama

* Refactor docs

* Allow Ollama to probe cost of image tokens

* Cleanup

frigate/genai/__init__.py

@@ -22,6 +22,7 @@ from frigate.genai.prompts import (
     build_review_description_response_format,
     build_review_summary_prompt,
 )
+from frigate.genai.utils import synthetic_jpeg
 from frigate.models import Event
 from frigate.util.builtin import has_non_finite_number
 
@@ -66,6 +67,8 @@ class GenAIClient:
         self.genai_config: GenAIConfig = genai_config
         self.timeout = timeout
         self.validate_model = validate_model
+        self._image_token_cache: dict[tuple[int, int], int] = {}
+        self._text_baseline_tokens: int | None = None
         self.provider = self._init_provider()
         self._last_init_attempt = time.monotonic()
 
@@ -372,10 +375,63 @@ class GenAIClient:
     def estimate_image_tokens(self, width: int, height: int) -> float:
         """Estimate prompt tokens consumed by a single image of the given dimensions.
 
-        Default heuristic: ~1 token per 1250 pixels. Providers that can measure or
-        know their model's exact image-token cost should override.
+        Providers that implement ``_count_prompt_tokens`` are probed for the
+        model's real cost: the same minimal prompt is counted with and without a
+        synthetic image, and the difference is cached per (width, height) since
+        image tokenization depends only on the dimensions and the loaded model.
+        Otherwise, or if probing fails, falls back to ~1 token per 1250 pixels.
         """
-        return (width * height) / 1250
+        heuristic = (width * height) / 1250
+
+        if self.provider is None:
+            return heuristic
+
+        cached = self._image_token_cache.get((width, height))
+
+        if cached is not None:
+            return cached
+
+        try:
+            if self._text_baseline_tokens is None:
+                self._text_baseline_tokens = self._count_prompt_tokens(None)
+
+            if self._text_baseline_tokens is None:
+                return heuristic
+
+            with_image = self._count_prompt_tokens(synthetic_jpeg(width, height))
+        except Exception as e:
+            logger.debug(
+                "%s image-token probe failed for %dx%d (%s); using heuristic",
+                self.__class__.__name__,
+                width,
+                height,
+                e,
+            )
+            return heuristic
+
+        if with_image is None:
+            return heuristic
+
+        tokens = max(1, with_image - self._text_baseline_tokens)
+        self._image_token_cache[(width, height)] = tokens
+        logger.debug(
+            "%s model '%s' uses ~%d tokens for %dx%d images",
+            self.__class__.__name__,
+            self.genai_config.model,
+            tokens,
+            width,
+            height,
+        )
+        return tokens
+
+    def _count_prompt_tokens(self, image: bytes | None) -> int | None:
+        """Prompt tokens the provider reports for a minimal "." request, with
+        ``image`` attached when given.
+
+        Return None when the provider cannot report prompt tokens; raise on
+        request failures. Used by estimate_image_tokens.
+        """
+        return None
 
     def embed(
         self,

frigate/genai/plugins/llama_cpp.py

@@ -1,7 +1,6 @@
 """llama.cpp Provider for Frigate AI."""
 
 import base64
-import io
 import json
 import logging
 from collections.abc import AsyncGenerator
@@ -10,11 +9,14 @@ from typing import Any, cast
 import httpx
 import numpy as np
 import requests
-from PIL import Image
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
-from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
+from frigate.genai.utils import (
+    interleave_images,
+    parse_tool_calls_from_message,
+    to_jpeg,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -76,20 +78,6 @@ def _parse_launch_arg(args: list[str], flag: str) -> str | None:
     return args[idx + 1]
 
 
-def _to_jpeg(img_bytes: bytes) -> bytes | None:
-    """Convert image bytes to JPEG. llama.cpp/STB does not support WebP."""
-    try:
-        img = Image.open(io.BytesIO(img_bytes))
-        if img.mode != "RGB":
-            img = img.convert("RGB")  # type: ignore[assignment]
-        buf = io.BytesIO()
-        img.save(buf, format="JPEG", quality=85)
-        return buf.getvalue()
-    except Exception as e:
-        logger.warning("Failed to convert image to JPEG: %s", e)
-        return None
-
-
 @register_genai_provider(GenAIProviderEnum.llamacpp)
 class LlamaCppClient(GenAIClient):
     """Generative AI client for Frigate using llama.cpp server."""
@@ -101,8 +89,6 @@ class LlamaCppClient(GenAIClient):
     _supports_audio: bool
     _supports_tools: bool
     _supports_reasoning: bool
-    _image_token_cache: dict[tuple[int, int], int]
-    _text_baseline_tokens: int | None
 
     @property
     def supports_embeddings(self) -> bool:
@@ -156,8 +142,6 @@ class LlamaCppClient(GenAIClient):
         self._supports_audio = False
         self._supports_tools = False
         self._supports_reasoning = False
-        self._image_token_cache = {}
-        self._text_baseline_tokens = None
 
         base_url = (
             self.genai_config.base_url.rstrip("/")
@@ -601,78 +585,23 @@ class LlamaCppClient(GenAIClient):
             return self._context_size
         return 4096
 
-    def estimate_image_tokens(self, width: int, height: int) -> float:
-        """Probe the llama.cpp server to learn the model's image-token cost at the
-        requested dimensions.
-
-        llama.cpp's image tokenization is a deterministic function of dimensions and
-        the loaded mmproj, so the result is cached per (width, height) for the
-        lifetime of the process. Falls back to the base pixel heuristic if the
-        server is unreachable or the response is malformed.
-        """
-        if self.provider is None:
-            return super().estimate_image_tokens(width, height)
-
-        cached = self._image_token_cache.get((width, height))
-
-        if cached is not None:
-            return cached
-
-        try:
-            baseline = self._probe_baseline_tokens()
-            with_image = self._probe_image_prompt_tokens(width, height)
-            tokens = max(1, with_image - baseline)
-        except Exception as e:
-            logger.debug(
-                "llama.cpp image-token probe failed for %dx%d (%s); using heuristic",
-                width,
-                height,
-                e,
-            )
-            return super().estimate_image_tokens(width, height)
-
-        self._image_token_cache[(width, height)] = tokens
-        logger.debug(
-            "llama.cpp model '%s' uses ~%d tokens for %dx%d images",
-            self.genai_config.model,
-            tokens,
-            width,
-            height,
-        )
-        return tokens
-
-    def _probe_baseline_tokens(self) -> int:
-        """Return prompt_tokens for a minimal text-only request. Cached after first call."""
-        if self._text_baseline_tokens is not None:
-            return self._text_baseline_tokens
-
-        self._text_baseline_tokens = self._probe_prompt_tokens(
-            [{"type": "text", "text": "."}]
-        )
-        return self._text_baseline_tokens
-
-    def _probe_image_prompt_tokens(self, width: int, height: int) -> int:
-        """Return prompt_tokens for a single synthetic image plus minimal text."""
-        img = Image.new("RGB", (width, height), (128, 128, 128))
-        buf = io.BytesIO()
-        img.save(buf, format="JPEG", quality=60)
-        encoded = base64.b64encode(buf.getvalue()).decode("utf-8")
-        return self._probe_prompt_tokens(
-            [
-                {"type": "text", "text": "."},
-                {
-                    "type": "image_url",
-                    "image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
-                },
-            ]
-        )
-
-    def _probe_prompt_tokens(self, content: list[dict[str, Any]]) -> int:
+    def _count_prompt_tokens(self, image: bytes | None) -> int | None:
         """POST a 1-token chat completion and return reported prompt_tokens.
 
         Uses a generous timeout to absorb a cold model load on the first probe
         when the server lazily loads models on demand (e.g. llama-swap).
         """
+        content: list[dict[str, Any]] = [{"type": "text", "text": "."}]
+
+        if image is not None:
+            encoded = base64.b64encode(image).decode("utf-8")
+            content.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{encoded}"},
+                }
+            )
+
         payload = {
             "model": self.genai_config.model,
             "messages": [{"role": "user", "content": content}],
@@ -807,7 +736,7 @@ class LlamaCppClient(GenAIClient):
 
         for img in images:
             # llama.cpp uses STB which does not support WebP; convert to JPEG
-            jpeg_bytes = _to_jpeg(img)
+            jpeg_bytes = to_jpeg(img)
             to_encode = jpeg_bytes if jpeg_bytes is not None else img
             encoded = base64.b64encode(to_encode).decode("utf-8")
             # The trailing newline keeps tokenization identical to the older

frigate/genai/plugins/ollama.py

@@ -7,6 +7,7 @@ import logging
 from collections.abc import AsyncGenerator
 from typing import Any
 
+import numpy as np
 from httpx import RemoteProtocolError, TimeoutException
 from ollama import AsyncClient as OllamaAsyncClient
 from ollama import Client as ApiClient
@@ -14,7 +15,11 @@ from ollama import ResponseError
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
-from frigate.genai.utils import interleave_images, parse_tool_calls_from_message
+from frigate.genai.utils import (
+    interleave_images,
+    parse_tool_calls_from_message,
+    to_jpeg,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -124,23 +129,39 @@ class OllamaClient(GenAIClient):
 
     provider: ApiClient | None
     provider_options: dict[str, Any]
+    _capabilities_cache: list[str] | None = None
     _supports_thinking_cache: bool | None = None
 
+    def _model_capabilities(self) -> list[str] | None:
+        """Capabilities Ollama reports for the configured model, or None when
+        they could not be fetched. Only successful lookups are cached."""
+        if self._capabilities_cache is not None:
+            return self._capabilities_cache
+        if self.provider is None:
+            return None
+        try:
+            response = self.provider.show(self.genai_config.model)
+        except Exception as e:
+            logger.debug("Failed to query Ollama model capabilities: %s", e)
+            return None
+        self._capabilities_cache = list(response.get("capabilities") or [])
+        return self._capabilities_cache
+
     @property
     def supports_toggleable_thinking(self) -> bool:
         if self._supports_thinking_cache is not None:
             return self._supports_thinking_cache
-        if self.provider is None:
+        capabilities = self._model_capabilities()
+        if capabilities is None:
             return False
-        try:
-            response = self.provider.show(self.genai_config.model)
-            capabilities = response.get("capabilities") or []
-            self._supports_thinking_cache = "thinking" in capabilities
-        except Exception as e:
-            logger.debug("Failed to query Ollama model capabilities: %s", e)
-            self._supports_thinking_cache = False
+        self._supports_thinking_cache = "thinking" in capabilities
         return self._supports_thinking_cache
 
+    @property
+    def supports_embeddings(self) -> bool:
+        """Whether Ollama reports the configured model as an embedding model."""
+        return "embedding" in (self._model_capabilities() or [])
+
     def _auth_headers(self) -> dict | None:
         if self.genai_config.api_key:
             return {"Authorization": "Bearer " + self.genai_config.api_key}
@@ -322,6 +343,93 @@ class OllamaClient(GenAIClient):
             self.genai_config.provider_options.get("options", {}).get("num_ctx", 4096)
         )
 
+    def _count_prompt_tokens(self, image: bytes | None) -> int | None:
+        """Send a 1-token chat request and return Ollama's prompt_eval_count.
+
+        Reuses the description request options so the probe runs with the same
+        num_ctx; a different value would make Ollama reload the model.
+        """
+        if self.provider is None:
+            return None
+
+        message: dict[str, Any] = {"role": "user", "content": "."}
+
+        if image is not None:
+            message["images"] = [image]
+
+        request_params = self._build_request_params(
+            [message], None, None, enable_thinking=False
+        )
+        request_params["options"] = {
+            **(request_params.get("options") or {}),
+            "num_predict": 1,
+        }
+        response = self.provider.chat(**request_params)
+        count = response.get("prompt_eval_count")
+        return int(count) if count is not None else None
+
+    def embed(
+        self,
+        texts: list[str] | None = None,
+        images: list[bytes] | None = None,
+    ) -> list[np.ndarray]:
+        """Generate embeddings via Ollama's /api/embed endpoint.
+
+        Each text is a plain string in `input` and each image is an
+        ``{"image": <base64>}`` item. Image input requires Ollama 0.40.1 or
+        newer and a model with a vision encoder (e.g. embeddinggemma-2:440m).
+        """
+        if self.provider is None:
+            logger.warning(
+                "Ollama provider has not been initialized. Check your Ollama configuration."
+            )
+            return []
+
+        texts = texts or []
+        images = images or []
+        if not texts and not images:
+            return []
+
+        inputs: list[str | dict[str, str]] = list(texts)
+        for img in images:
+            jpeg_bytes = to_jpeg(img)
+            to_encode = jpeg_bytes if jpeg_bytes is not None else img
+            inputs.append({"image": base64.b64encode(to_encode).decode("utf-8")})
+
+        payload: dict[str, Any] = {"model": self.genai_config.model, "input": inputs}
+        for key in ("options", "keep_alive"):
+            if key in self.genai_config.provider_options:
+                payload[key] = self.genai_config.provider_options[key]
+
+        try:
+            # The ollama SDK's embed() validates input as strings only, so
+            # image items have to bypass it and post the JSON directly.
+            response = self.provider._request_raw("POST", "/api/embed", json=payload)
+            body = response.json()
+        except (
+            TimeoutException,
+            ResponseError,
+            RemoteProtocolError,
+            ConnectionError,
+            ValueError,
+        ) as e:
+            logger.warning("Ollama embeddings error: %s", str(e))
+            return []
+
+        vectors = body.get("embeddings") if isinstance(body, dict) else None
+        if not isinstance(vectors, list):
+            logger.warning("Ollama embeddings returned unexpected format")
+            return []
+
+        if len(vectors) != len(inputs):
+            logger.warning(
+                "Ollama returned %d embeddings for %d inputs",
+                len(vectors),
+                len(inputs),
+            )
+
+        return [np.asarray(v, dtype=np.float32).flatten() for v in vectors]
+
     def _build_request_params(
         self,
         messages: list[dict[str, Any]],

frigate/genai/utils.py

@@ -1,12 +1,42 @@
 """Shared helpers for GenAI providers and chat (OpenAI-style messages, tool call parsing)."""
 
+import io
 import json
 import logging
 from typing import Any
 
+from PIL import Image
+
 logger = logging.getLogger(__name__)
 
 
+def to_jpeg(img_bytes: bytes) -> bytes | None:
+    """Convert image bytes to JPEG.
+
+    Some provider image decoders (e.g. llama.cpp's STB) do not support WebP,
+    which is the format Frigate stores thumbnails in.
+    """
+    try:
+        img = Image.open(io.BytesIO(img_bytes))
+        if img.mode != "RGB":
+            img = img.convert("RGB")  # type: ignore[assignment]
+        buf = io.BytesIO()
+        img.save(buf, format="JPEG", quality=85)
+        return buf.getvalue()
+    except Exception as e:
+        logger.warning("Failed to convert image to JPEG: %s", e)
+        return None
+
+
+def synthetic_jpeg(width: int, height: int) -> bytes:
+    """A flat gray JPEG of the given dimensions, for measuring image token cost."""
+    buf = io.BytesIO()
+    Image.new("RGB", (width, height), (128, 128, 128)).save(
+        buf, format="JPEG", quality=60
+    )
+    return buf.getvalue()
+
+
 def interleave_images(
     prompt: str, images: list[bytes], captions: list[str] | None = None
 ) -> list[str | bytes]: