Add support for multiple GenAI Providers (#22144)

* GenAI client manager

* Add config migration

* Convert to roles list

* Support getting client via manager

* Cleanup

* Fix import issues

* Set model in llama.cpp config

* Clenaup

* Use config update

* Clenaup

* Add new title and desc

frigate/config/camera/genai.py

@@ -6,7 +6,7 @@ from pydantic import Field
 from ..base import FrigateBaseModel
 from ..env import EnvString
 
-__all__ = ["GenAIConfig", "GenAIProviderEnum"]
+__all__ = ["GenAIConfig", "GenAIProviderEnum", "GenAIRoleEnum"]
 
 
 class GenAIProviderEnum(str, Enum):
@@ -17,15 +17,55 @@ class GenAIProviderEnum(str, Enum):
     llamacpp = "llamacpp"
 
 
+class GenAIRoleEnum(str, Enum):
+    tools = "tools"
+    vision = "vision"
+    embeddings = "embeddings"
+
+
 class GenAIConfig(FrigateBaseModel):
     """Primary GenAI Config to define GenAI Provider."""
 
-    api_key: Optional[EnvString] = Field(default=None, title="Provider API key.")
-    base_url: Optional[str] = Field(default=None, title="Provider base url.")
-    model: str = Field(default="gpt-4o", title="GenAI model.")
-    provider: GenAIProviderEnum | None = Field(default=None, title="GenAI provider.")
+    api_key: Optional[EnvString] = Field(
+        default=None,
+        title="API key",
+        description="API key required by some providers (can also be set via environment variables).",
+    )
+    base_url: Optional[str] = Field(
+        default=None,
+        title="Base URL",
+        description="Base URL for self-hosted or compatible providers (for example an Ollama instance).",
+    )
+    model: str = Field(
+        default="gpt-4o",
+        title="Model",
+        description="The model to use from the provider for generating descriptions or summaries.",
+    )
+    provider: GenAIProviderEnum | None = Field(
+        default=None,
+        title="Provider",
+        description="The GenAI provider to use (for example: ollama, gemini, openai).",
+    )
+    roles: list[GenAIRoleEnum] = Field(
+        default_factory=lambda: [
+            GenAIRoleEnum.embeddings,
+            GenAIRoleEnum.vision,
+            GenAIRoleEnum.tools,
+        ],
+        title="Roles",
+        description="GenAI roles (tools, vision, embeddings); one provider per role.",
+    )
     provider_options: dict[str, Any] = Field(
-        default={}, title="GenAI Provider extra options."
+        default={},
+        title="Provider options",
+        description="Additional provider-specific options to pass to the GenAI client.",
+        json_schema_extra={"additionalProperties": {"type": "string"}},
+    )
+    runtime_options: dict[str, Any] = Field(
+        default={},
+        title="Runtime options",
+        description="Runtime options passed to the provider for each inference call.",
+        json_schema_extra={"additionalProperties": {"type": "string"}},
     )
     runtime_options: dict[str, Any] = Field(
         default={}, title="Options to pass during inference calls."

frigate/config/config.py

@@ -45,7 +45,7 @@ from .camera.audio import AudioConfig
 from .camera.birdseye import BirdseyeConfig
 from .camera.detect import DetectConfig
 from .camera.ffmpeg import FfmpegConfig
-from .camera.genai import GenAIConfig
+from .camera.genai import GenAIConfig, GenAIRoleEnum
 from .camera.motion import MotionConfig
 from .camera.notification import NotificationConfig
 from .camera.objects import FilterConfig, ObjectConfig
@@ -347,9 +347,9 @@ class FrigateConfig(FrigateBaseModel):
         default_factory=ModelConfig, title="Detection model configuration."
     )
 
-    # GenAI config
-    genai: GenAIConfig = Field(
-        default_factory=GenAIConfig, title="Generative AI configuration."
+    # GenAI config (named provider configs: name -> GenAIConfig)
+    genai: Dict[str, GenAIConfig] = Field(
+        default_factory=dict, title="Generative AI configuration (named providers)."
     )
 
     # Camera config
@@ -431,6 +431,18 @@ class FrigateConfig(FrigateBaseModel):
         # set notifications state
         self.notifications.enabled_in_config = self.notifications.enabled
 
+        # validate genai: each role (tools, vision, embeddings) at most once
+        role_to_name: dict[GenAIRoleEnum, str] = {}
+        for name, genai_cfg in self.genai.items():
+            for role in genai_cfg.roles:
+                if role in role_to_name:
+                    raise ValueError(
+                        f"GenAI role '{role.value}' is assigned to both "
+                        f"'{role_to_name[role]}' and '{name}'; each role must have "
+                        "exactly one provider."
+                    )
+                role_to_name[role] = name
+
         # set default min_score for object attributes
         for attribute in self.model.all_attributes:
             if not self.objects.filters.get(attribute):