Refactor genai (#22752)

* Switch to a feature-based roles so it is easier to choose models for different tasks

* Fallback and try llama-swap format

* List models supported by provider

* Cleanup

* Add frontend

* Improve model loading

* Make it possible to update genai without restarting

* Cleanup

* Cleanup

* Mypy

frigate/genai/__init__.py

@@ -320,6 +320,22 @@ Guidelines:
         """Submit a request to the provider."""
         return None
 
+    @property
+    def supports_vision(self) -> bool:
+        """Whether the model supports vision/image input.
+
+        Defaults to True for cloud providers. Providers that can detect
+        capability at runtime (e.g. llama.cpp) should override this.
+        """
+        return True
+
+    def list_models(self) -> list[str]:
+        """Return the list of model names available from this provider.
+
+        Providers should override this to query their backend.
+        """
+        return []
+
     def get_context_size(self) -> int:
         """Get the context window size for this provider in tokens."""
         return 4096

frigate/genai/azure-openai.py

@@ -82,6 +82,14 @@ class OpenAIClient(GenAIClient):
             return str(result.choices[0].message.content.strip())
         return None
 
+    def list_models(self) -> list[str]:
+        """Return available model IDs from Azure OpenAI."""
+        try:
+            return sorted(m.id for m in self.provider.models.list().data)
+        except Exception as e:
+            logger.warning("Failed to list Azure OpenAI models: %s", e)
+            return []
+
     def get_context_size(self) -> int:
         """Get the context window size for Azure OpenAI."""
         return 128000

frigate/genai/gemini.py

@@ -87,6 +87,14 @@ class GeminiClient(GenAIClient):
             return None
         return description
 
+    def list_models(self) -> list[str]:
+        """Return available model names from Gemini."""
+        try:
+            return sorted(m.name or "" for m in self.provider.models.list())
+        except Exception as e:
+            logger.warning("Failed to list Gemini models: %s", e)
+            return []
+
     def get_context_size(self) -> int:
         """Get the context window size for Gemini."""
         # Gemini Pro Vision has a 1M token context window

frigate/genai/llama_cpp.py

@@ -101,15 +101,26 @@ class LlamaCppClient(GenAIClient):
                 e,
             )
 
-        # Query /props for context size, modalities, and tool support
+        # Query /props for context size, modalities, and tool support.
+        # The standard /props?model=<name> endpoint works with llama-server.
+        # If it fails, try the llama-swap per-model passthrough endpoint which
+        # returns props for a specific model without requiring it to be loaded.
         try:
-            response = requests.get(
-                f"{base_url}/props",
-                params={"model": configured_model},
-                timeout=10,
-            )
-            response.raise_for_status()
-            props = response.json()
+            try:
+                response = requests.get(
+                    f"{base_url}/props",
+                    params={"model": configured_model},
+                    timeout=10,
+                )
+                response.raise_for_status()
+                props = response.json()
+            except Exception:
+                response = requests.get(
+                    f"{base_url}/upstream/{configured_model}/props",
+                    timeout=10,
+                )
+                response.raise_for_status()
+                props = response.json()
 
             # Context size from server runtime config
             default_settings = props.get("default_generation_settings", {})
@@ -126,7 +137,7 @@ class LlamaCppClient(GenAIClient):
             chat_caps = props.get("chat_template_caps", {})
             self._supports_tools = chat_caps.get("supports_tools", False)
 
-            logger.debug(
+            logger.info(
                 "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s",
                 configured_model,
                 self._context_size or "unknown",
@@ -225,6 +236,23 @@ class LlamaCppClient(GenAIClient):
         """Whether the loaded model supports tool/function calling."""
         return self._supports_tools
 
+    def list_models(self) -> list[str]:
+        """Return available model IDs from the llama.cpp server."""
+        if self.provider is None:
+            return []
+        try:
+            response = requests.get(f"{self.provider}/v1/models", timeout=10)
+            response.raise_for_status()
+            models = []
+            for m in response.json().get("data", []):
+                models.append(m.get("id", "unknown"))
+                for alias in m.get("aliases", []):
+                    models.append(alias)
+            return sorted(models)
+        except Exception as e:
+            logger.warning("Failed to list llama.cpp models: %s", e)
+            return []
+
     def get_context_size(self) -> int:
         """Get the context window size for llama.cpp.
 

frigate/genai/manager.py

@@ -1,15 +1,15 @@
 """GenAI client manager for Frigate.
 
-Manages GenAI provider clients from Frigate config. Configuration is read only
-in _update_config(); no other code should read config.genai. Exposes clients
-by role: tool_client, vision_client, embeddings_client.
+Manages GenAI provider clients from Frigate config. Clients are created lazily
+on first access so that providers whose roles are never used (e.g. chat when
+no chat feature is active) are never initialized.
 """
 
 import logging
 from typing import TYPE_CHECKING, Optional
 
 from frigate.config import FrigateConfig
-from frigate.config.camera.genai import GenAIRoleEnum
+from frigate.config.camera.genai import GenAIConfig, GenAIRoleEnum
 
 if TYPE_CHECKING:
     from frigate.genai import GenAIClient
@@ -21,68 +21,98 @@ class GenAIClientManager:
     """Manages GenAI provider clients from Frigate config."""
 
     def __init__(self, config: FrigateConfig) -> None:
-        self._tool_client: Optional[GenAIClient] = None
-        self._vision_client: Optional[GenAIClient] = None
-        self._embeddings_client: Optional[GenAIClient] = None
+        self._configs: dict[str, GenAIConfig] = {}
+        self._role_map: dict[GenAIRoleEnum, str] = {}
+        self._clients: dict[str, "GenAIClient"] = {}
         self.update_config(config)
 
     def update_config(self, config: FrigateConfig) -> None:
-        """Build role clients from current Frigate config.genai.
+        """Store provider configs and build the role→name mapping.
 
         Called from __init__ and can be called again when config is reloaded.
-        Each role (tools, vision, embeddings) gets the client for the provider
-        that has that role in its roles list.
+        Clients are not created here; they are instantiated lazily on first
+        access via a role property or list_models().
         """
         from frigate.genai import PROVIDERS, load_providers
 
-        self._tool_client = None
-        self._vision_client = None
-        self._embeddings_client = None
+        self._configs = {}
+        self._role_map = {}
+        self._clients = {}
 
         if not config.genai:
             return
 
         load_providers()
 
-        for _name, genai_cfg in config.genai.items():
+        for name, genai_cfg in config.genai.items():
             if not genai_cfg.provider:
                 continue
-            provider_cls = PROVIDERS.get(genai_cfg.provider)
-            if not provider_cls:
+            if genai_cfg.provider not in PROVIDERS:
                 logger.warning(
                     "Unknown GenAI provider %s in config, skipping.",
                     genai_cfg.provider,
                 )
                 continue
-            try:
-                client = provider_cls(genai_cfg)
-            except Exception as e:
-                logger.exception(
-                    "Failed to create GenAI client for provider %s: %s",
-                    genai_cfg.provider,
-                    e,
-                )
-                continue
+
+            self._configs[name] = genai_cfg
 
             for role in genai_cfg.roles:
-                if role == GenAIRoleEnum.tools:
-                    self._tool_client = client
-                elif role == GenAIRoleEnum.vision:
-                    self._vision_client = client
-                elif role == GenAIRoleEnum.embeddings:
-                    self._embeddings_client = client
+                self._role_map[role] = name
+
+    def _get_client(self, name: str) -> "Optional[GenAIClient]":
+        """Return the client for *name*, creating it on first access."""
+        if name in self._clients:
+            return self._clients[name]
+
+        from frigate.genai import PROVIDERS
+
+        genai_cfg = self._configs.get(name)
+        if not genai_cfg:
+            return None
+
+        if not genai_cfg.provider:
+            return None
+
+        provider_cls = PROVIDERS.get(genai_cfg.provider)
+        if not provider_cls:
+            return None
+
+        try:
+            client: "GenAIClient" = provider_cls(genai_cfg)
+        except Exception as e:
+            logger.exception(
+                "Failed to create GenAI client for provider %s: %s",
+                genai_cfg.provider,
+                e,
+            )
+            return None
+
+        self._clients[name] = client
+        return client
 
     @property
-    def tool_client(self) -> "Optional[GenAIClient]":
-        """Client configured for the tools role (e.g. chat with function calling)."""
-        return self._tool_client
+    def chat_client(self) -> "Optional[GenAIClient]":
+        """Client configured for the chat role (e.g. chat with function calling)."""
+        name = self._role_map.get(GenAIRoleEnum.chat)
+        return self._get_client(name) if name else None
 
     @property
-    def vision_client(self) -> "Optional[GenAIClient]":
-        """Client configured for the vision role (e.g. review descriptions, object descriptions)."""
-        return self._vision_client
+    def description_client(self) -> "Optional[GenAIClient]":
+        """Client configured for the descriptions role (e.g. review descriptions, object descriptions)."""
+        name = self._role_map.get(GenAIRoleEnum.descriptions)
+        return self._get_client(name) if name else None
 
     @property
     def embeddings_client(self) -> "Optional[GenAIClient]":
         """Client configured for the embeddings role."""
-        return self._embeddings_client
+        name = self._role_map.get(GenAIRoleEnum.embeddings)
+        return self._get_client(name) if name else None
+
+    def list_models(self) -> dict[str, list[str]]:
+        """Return available models keyed by config entry name."""
+        result: dict[str, list[str]] = {}
+        for name in self._configs:
+            client = self._get_client(name)
+            if client:
+                result[name] = client.list_models()
+        return result

frigate/genai/ollama.py

@@ -132,6 +132,19 @@ class OllamaClient(GenAIClient):
             logger.warning("Ollama returned an error: %s", str(e))
             return None
 
+    def list_models(self) -> list[str]:
+        """Return available model names from the Ollama server."""
+        if self.provider is None:
+            return []
+        try:
+            response = self.provider.list()
+            return sorted(
+                m.get("name", m.get("model", "")) for m in response.get("models", [])
+            )
+        except Exception as e:
+            logger.warning("Failed to list Ollama models: %s", e)
+            return []
+
     def get_context_size(self) -> int:
         """Get the context window size for Ollama."""
         return int(

frigate/genai/openai.py

@@ -86,6 +86,14 @@ class OpenAIClient(GenAIClient):
             logger.warning("OpenAI returned an error: %s", str(e))
             return None
 
+    def list_models(self) -> list[str]:
+        """Return available model IDs from the OpenAI-compatible API."""
+        try:
+            return sorted(m.id for m in self.provider.models.list().data)
+        except Exception as e:
+            logger.warning("Failed to list OpenAI models: %s", e)
+            return []
+
     def get_context_size(self) -> int:
         """Get the context window size for OpenAI."""
         if self.context_size is not None: