Add support for multiple GenAI Providers (#22144)

* GenAI client manager

* Add config migration

* Convert to roles list

* Support getting client via manager

* Cleanup

* Fix import issues

* Set model in llama.cpp config

* Clenaup

* Use config update

* Clenaup

* Add new title and desc

frigate/api/app.py

@@ -432,6 +432,7 @@ def config_set(request: Request, body: AppConfigSetBody):
     if body.requires_restart == 0 or body.update_topic:
         old_config: FrigateConfig = request.app.frigate_config
         request.app.frigate_config = config
+        request.app.genai_manager.update_config(config)
 
         if body.update_topic:
             if body.update_topic.startswith("config/cameras/"):

frigate/api/chat.py

@@ -23,7 +23,6 @@ from frigate.api.defs.response.chat_response import (
 )
 from frigate.api.defs.tags import Tags
 from frigate.api.event import events
-from frigate.genai import get_genai_client
 
 logger = logging.getLogger(__name__)
 
@@ -383,7 +382,7 @@ async def chat_completion(
     6. Repeats until final answer
     7. Returns response to user
     """
-    genai_client = get_genai_client(request.app.frigate_config)
+    genai_client = request.app.genai_manager.tool_client
     if not genai_client:
         return JSONResponse(
             content={

frigate/api/fastapi_app.py

@@ -33,6 +33,7 @@ from frigate.comms.event_metadata_updater import (
 from frigate.config import FrigateConfig
 from frigate.config.camera.updater import CameraConfigUpdatePublisher
 from frigate.embeddings import EmbeddingsContext
+from frigate.genai import GenAIClientManager
 from frigate.ptz.onvif import OnvifController
 from frigate.stats.emitter import StatsEmitter
 from frigate.storage import StorageMaintainer
@@ -134,6 +135,7 @@ def create_fastapi_app(
     app.include_router(record.router)
     # App Properties
     app.frigate_config = frigate_config
+    app.genai_manager = GenAIClientManager(frigate_config)
     app.embeddings = embeddings
     app.detected_frames_processor = detected_frames_processor
     app.storage_maintainer = storage_maintainer

frigate/api/review.py

@@ -33,7 +33,6 @@ from frigate.api.defs.response.review_response import (
     ReviewSummaryResponse,
 )
 from frigate.api.defs.tags import Tags
-from frigate.config import FrigateConfig
 from frigate.embeddings import EmbeddingsContext
 from frigate.models import Recordings, ReviewSegment, UserReviewStatus
 from frigate.review.types import SeverityEnum
@@ -747,9 +746,7 @@ async def set_not_reviewed(
     description="Use GenAI to summarize review items over a period of time.",
 )
 def generate_review_summary(request: Request, start_ts: float, end_ts: float):
-    config: FrigateConfig = request.app.frigate_config
+    if not request.app.genai_manager.vision_client:
-
-    if not config.genai.provider:
         return JSONResponse(
             content=(
                 {

frigate/config/camera/genai.py

@@ -6,7 +6,7 @@ from pydantic import Field
 from ..base import FrigateBaseModel
 from ..env import EnvString
 
-__all__ = ["GenAIConfig", "GenAIProviderEnum"]
+__all__ = ["GenAIConfig", "GenAIProviderEnum", "GenAIRoleEnum"]
 
 
 class GenAIProviderEnum(str, Enum):
@@ -17,15 +17,55 @@ class GenAIProviderEnum(str, Enum):
     llamacpp = "llamacpp"
 
 
+class GenAIRoleEnum(str, Enum):
+    tools = "tools"
+    vision = "vision"
+    embeddings = "embeddings"
+
+
 class GenAIConfig(FrigateBaseModel):
     """Primary GenAI Config to define GenAI Provider."""
 
-    api_key: Optional[EnvString] = Field(default=None, title="Provider API key.")
+    api_key: Optional[EnvString] = Field(
-    base_url: Optional[str] = Field(default=None, title="Provider base url.")
+        default=None,
-    model: str = Field(default="gpt-4o", title="GenAI model.")
+        title="API key",
-    provider: GenAIProviderEnum | None = Field(default=None, title="GenAI provider.")
+        description="API key required by some providers (can also be set via environment variables).",
+    )
+    base_url: Optional[str] = Field(
+        default=None,
+        title="Base URL",
+        description="Base URL for self-hosted or compatible providers (for example an Ollama instance).",
+    )
+    model: str = Field(
+        default="gpt-4o",
+        title="Model",
+        description="The model to use from the provider for generating descriptions or summaries.",
+    )
+    provider: GenAIProviderEnum | None = Field(
+        default=None,
+        title="Provider",
+        description="The GenAI provider to use (for example: ollama, gemini, openai).",
+    )
+    roles: list[GenAIRoleEnum] = Field(
+        default_factory=lambda: [
+            GenAIRoleEnum.embeddings,
+            GenAIRoleEnum.vision,
+            GenAIRoleEnum.tools,
+        ],
+        title="Roles",
+        description="GenAI roles (tools, vision, embeddings); one provider per role.",
+    )
     provider_options: dict[str, Any] = Field(
-        default={}, title="GenAI Provider extra options."
+        default={},
+        title="Provider options",
+        description="Additional provider-specific options to pass to the GenAI client.",
+        json_schema_extra={"additionalProperties": {"type": "string"}},
+    )
+    runtime_options: dict[str, Any] = Field(
+        default={},
+        title="Runtime options",
+        description="Runtime options passed to the provider for each inference call.",
+        json_schema_extra={"additionalProperties": {"type": "string"}},
     )
     runtime_options: dict[str, Any] = Field(
         default={}, title="Options to pass during inference calls."

frigate/config/config.py

@@ -45,7 +45,7 @@ from .camera.audio import AudioConfig
 from .camera.birdseye import BirdseyeConfig
 from .camera.detect import DetectConfig
 from .camera.ffmpeg import FfmpegConfig
-from .camera.genai import GenAIConfig
+from .camera.genai import GenAIConfig, GenAIRoleEnum
 from .camera.motion import MotionConfig
 from .camera.notification import NotificationConfig
 from .camera.objects import FilterConfig, ObjectConfig
@@ -347,9 +347,9 @@ class FrigateConfig(FrigateBaseModel):
         default_factory=ModelConfig, title="Detection model configuration."
     )
 
-    # GenAI config
+    # GenAI config (named provider configs: name -> GenAIConfig)
-    genai: GenAIConfig = Field(
+    genai: Dict[str, GenAIConfig] = Field(
-        default_factory=GenAIConfig, title="Generative AI configuration."
+        default_factory=dict, title="Generative AI configuration (named providers)."
     )
 
     # Camera config
@@ -431,6 +431,18 @@ class FrigateConfig(FrigateBaseModel):
         # set notifications state
         self.notifications.enabled_in_config = self.notifications.enabled
 
+        # validate genai: each role (tools, vision, embeddings) at most once
+        role_to_name: dict[GenAIRoleEnum, str] = {}
+        for name, genai_cfg in self.genai.items():
+            for role in genai_cfg.roles:
+                if role in role_to_name:
+                    raise ValueError(
+                        f"GenAI role '{role.value}' is assigned to both "
+                        f"'{role_to_name[role]}' and '{name}'; each role must have "
+                        "exactly one provider."
+                    )
+                role_to_name[role] = name
+
         # set default min_score for object attributes
         for attribute in self.model.all_attributes:
             if not self.objects.filters.get(attribute):

frigate/embeddings/maintainer.py

@@ -59,7 +59,7 @@ from frigate.data_processing.real_time.license_plate import (
 from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataEnum
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum
-from frigate.genai import get_genai_client
+from frigate.genai import GenAIClientManager
 from frigate.models import Event, Recordings, ReviewSegment, Trigger
 from frigate.util.builtin import serialize
 from frigate.util.file import get_event_thumbnail_bytes
@@ -144,7 +144,7 @@ class EmbeddingMaintainer(threading.Thread):
         self.frame_manager = SharedMemoryFrameManager()
 
         self.detected_license_plates: dict[str, dict[str, Any]] = {}
-        self.genai_client = get_genai_client(config)
+        self.genai_manager = GenAIClientManager(config)
 
         # model runners to share between realtime and post processors
         if self.config.lpr.enabled:
@@ -203,12 +203,15 @@ class EmbeddingMaintainer(threading.Thread):
         # post processors
         self.post_processors: list[PostProcessorApi] = []
 
-        if self.genai_client is not None and any(
+        if self.genai_manager.vision_client is not None and any(
             c.review.genai.enabled_in_config for c in self.config.cameras.values()
         ):
             self.post_processors.append(
                 ReviewDescriptionProcessor(
-                    self.config, self.requestor, self.metrics, self.genai_client
+                    self.config,
+                    self.requestor,
+                    self.metrics,
+                    self.genai_manager.vision_client,
                 )
             )
 
@@ -246,7 +249,7 @@ class EmbeddingMaintainer(threading.Thread):
             )
             self.post_processors.append(semantic_trigger_processor)
 
-        if self.genai_client is not None and any(
+        if self.genai_manager.vision_client is not None and any(
             c.objects.genai.enabled_in_config for c in self.config.cameras.values()
         ):
             self.post_processors.append(
@@ -255,7 +258,7 @@ class EmbeddingMaintainer(threading.Thread):
                     self.embeddings,
                     self.requestor,
                     self.metrics,
-                    self.genai_client,
+                    self.genai_manager.vision_client,
                     semantic_trigger_processor,
                 )
             )

frigate/genai/__init__.py

@@ -9,13 +9,24 @@ from typing import Any, Optional
 
 from playhouse.shortcuts import model_to_dict
 
-from frigate.config import CameraConfig, FrigateConfig, GenAIConfig, GenAIProviderEnum
+from frigate.config import CameraConfig, GenAIConfig, GenAIProviderEnum
 from frigate.const import CLIPS_DIR
 from frigate.data_processing.post.types import ReviewMetadata
+from frigate.genai.manager import GenAIClientManager
 from frigate.models import Event
 
 logger = logging.getLogger(__name__)
 
+__all__ = [
+    "GenAIClient",
+    "GenAIClientManager",
+    "GenAIConfig",
+    "GenAIProviderEnum",
+    "PROVIDERS",
+    "load_providers",
+    "register_genai_provider",
+]
+
 PROVIDERS = {}
 
 
@@ -352,19 +363,6 @@ Guidelines:
         }
 
 
-def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]:
-    """Get the GenAI client."""
-    if not config.genai.provider:
-        return None
-
-    load_providers()
-    provider = PROVIDERS.get(config.genai.provider)
-    if provider:
-        return provider(config.genai)
-
-    return None
-
-
 def load_providers():
     package_dir = os.path.dirname(__file__)
     for filename in os.listdir(package_dir):

frigate/genai/llama_cpp.py

@@ -67,6 +67,7 @@ class LlamaCppClient(GenAIClient):
 
             # Build request payload with llama.cpp native options
             payload = {
+                "model": self.genai_config.model,
                 "messages": [
                     {
                         "role": "user",
@@ -134,6 +135,7 @@ class LlamaCppClient(GenAIClient):
                     openai_tool_choice = "required"
 
             payload = {
+                "model": self.genai_config.model,
                 "messages": messages,
             }
 

frigate/genai/manager.py

@@ -0,0 +1,89 @@
+"""GenAI client manager for Frigate.
+
+Manages GenAI provider clients from Frigate config. Configuration is read only
+in _update_config(); no other code should read config.genai. Exposes clients
+by role: tool_client, vision_client, embeddings_client.
+"""
+
+import logging
+from typing import TYPE_CHECKING, Optional
+
+from frigate.config import FrigateConfig
+from frigate.config.camera.genai import GenAIRoleEnum
+
+if TYPE_CHECKING:
+    from frigate.genai import GenAIClient
+
+logger = logging.getLogger(__name__)
+
+
+class GenAIClientManager:
+    """Manages GenAI provider clients from Frigate config."""
+
+    def __init__(self, config: FrigateConfig) -> None:
+        self._config = config
+        self._tool_client: Optional[GenAIClient] = None
+        self._vision_client: Optional[GenAIClient] = None
+        self._embeddings_client: Optional[GenAIClient] = None
+        self._update_config()
+
+    def _update_config(self) -> None:
+        """Build role clients from current Frigate config.genai.
+
+        Called from __init__ and can be called again when config is reloaded.
+        Each role (tools, vision, embeddings) gets the client for the provider
+        that has that role in its roles list.
+        """
+        from frigate.genai import PROVIDERS, load_providers
+
+        self._tool_client = None
+        self._vision_client = None
+        self._embeddings_client = None
+
+        if not self._config.genai:
+            return
+
+        load_providers()
+
+        for _name, genai_cfg in self._config.genai.items():
+            if not genai_cfg.provider:
+                continue
+            provider_cls = PROVIDERS.get(genai_cfg.provider)
+            if not provider_cls:
+                logger.warning(
+                    "Unknown GenAI provider %s in config, skipping.",
+                    genai_cfg.provider,
+                )
+                continue
+            try:
+                client = provider_cls(genai_cfg)
+            except Exception as e:
+                logger.exception(
+                    "Failed to create GenAI client for provider %s: %s",
+                    genai_cfg.provider,
+                    e,
+                )
+                continue
+
+            for role in genai_cfg.roles:
+                if role == GenAIRoleEnum.tools:
+                    self._tool_client = client
+                elif role == GenAIRoleEnum.vision:
+                    self._vision_client = client
+                elif role == GenAIRoleEnum.embeddings:
+                    self._embeddings_client = client
+
+    @property
+    def tool_client(self) -> "Optional[GenAIClient]":
+        """Client configured for the tools role (e.g. chat with function calling)."""
+        return self._tool_client
+
+    @property
+    def vision_client(self) -> "Optional[GenAIClient]":
+        """Client configured for the vision role (e.g. review descriptions, object descriptions)."""
+        return self._vision_client
+
+    @property
+    def embeddings_client(self) -> "Optional[GenAIClient]":
+        """Client configured for the embeddings role."""
+        return self._embeddings_client

frigate/util/config.py

@@ -438,6 +438,13 @@ def migrate_018_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
     """Handle migrating frigate config to 0.18-0"""
     new_config = config.copy()
 
+    # Migrate GenAI to new format
+    genai = new_config.get("genai")
+
+    if genai and genai.get("provider"):
+        genai["roles"] = ["embeddings", "vision", "tools"]
+        new_config["genai"] = {"default": genai}
+
     # Remove deprecated sync_recordings from global record config
     if new_config.get("record", {}).get("sync_recordings") is not None:
         del new_config["record"]["sync_recordings"]