diff --git a/frigate/api/app.py b/frigate/api/app.py index 126c613a7..9246095ca 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -432,6 +432,7 @@ def config_set(request: Request, body: AppConfigSetBody): if body.requires_restart == 0 or body.update_topic: old_config: FrigateConfig = request.app.frigate_config request.app.frigate_config = config + request.app.genai_manager.update_config(config) if body.update_topic: if body.update_topic.startswith("config/cameras/"): diff --git a/frigate/api/chat.py b/frigate/api/chat.py index 1f5cc2297..415f422da 100644 --- a/frigate/api/chat.py +++ b/frigate/api/chat.py @@ -23,7 +23,6 @@ from frigate.api.defs.response.chat_response import ( ) from frigate.api.defs.tags import Tags from frigate.api.event import events -from frigate.genai import get_genai_client logger = logging.getLogger(__name__) @@ -383,7 +382,7 @@ async def chat_completion( 6. Repeats until final answer 7. Returns response to user """ - genai_client = get_genai_client(request.app.frigate_config) + genai_client = request.app.genai_manager.tool_client if not genai_client: return JSONResponse( content={ diff --git a/frigate/api/fastapi_app.py b/frigate/api/fastapi_app.py index 496c8fada..3206c7b4a 100644 --- a/frigate/api/fastapi_app.py +++ b/frigate/api/fastapi_app.py @@ -33,6 +33,7 @@ from frigate.comms.event_metadata_updater import ( from frigate.config import FrigateConfig from frigate.config.camera.updater import CameraConfigUpdatePublisher from frigate.embeddings import EmbeddingsContext +from frigate.genai import GenAIClientManager from frigate.ptz.onvif import OnvifController from frigate.stats.emitter import StatsEmitter from frigate.storage import StorageMaintainer @@ -134,6 +135,7 @@ def create_fastapi_app( app.include_router(record.router) # App Properties app.frigate_config = frigate_config + app.genai_manager = GenAIClientManager(frigate_config) app.embeddings = embeddings app.detected_frames_processor = detected_frames_processor app.storage_maintainer = storage_maintainer diff --git a/frigate/api/review.py b/frigate/api/review.py index 76619dcb2..d2e8063d5 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -33,7 +33,6 @@ from frigate.api.defs.response.review_response import ( ReviewSummaryResponse, ) from frigate.api.defs.tags import Tags -from frigate.config import FrigateConfig from frigate.embeddings import EmbeddingsContext from frigate.models import Recordings, ReviewSegment, UserReviewStatus from frigate.review.types import SeverityEnum @@ -747,9 +746,7 @@ async def set_not_reviewed( description="Use GenAI to summarize review items over a period of time.", ) def generate_review_summary(request: Request, start_ts: float, end_ts: float): - config: FrigateConfig = request.app.frigate_config - - if not config.genai.provider: + if not request.app.genai_manager.vision_client: return JSONResponse( content=( { diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index 3dd596c3b..56d7322f5 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -6,7 +6,7 @@ from pydantic import Field from ..base import FrigateBaseModel from ..env import EnvString -__all__ = ["GenAIConfig", "GenAIProviderEnum"] +__all__ = ["GenAIConfig", "GenAIProviderEnum", "GenAIRoleEnum"] class GenAIProviderEnum(str, Enum): @@ -17,15 +17,55 @@ class GenAIProviderEnum(str, Enum): llamacpp = "llamacpp" +class GenAIRoleEnum(str, Enum): + tools = "tools" + vision = "vision" + embeddings = "embeddings" + + class GenAIConfig(FrigateBaseModel): """Primary GenAI Config to define GenAI Provider.""" - api_key: Optional[EnvString] = Field(default=None, title="Provider API key.") - base_url: Optional[str] = Field(default=None, title="Provider base url.") - model: str = Field(default="gpt-4o", title="GenAI model.") - provider: GenAIProviderEnum | None = Field(default=None, title="GenAI provider.") + api_key: Optional[EnvString] = Field( + default=None, + title="API key", + description="API key required by some providers (can also be set via environment variables).", + ) + base_url: Optional[str] = Field( + default=None, + title="Base URL", + description="Base URL for self-hosted or compatible providers (for example an Ollama instance).", + ) + model: str = Field( + default="gpt-4o", + title="Model", + description="The model to use from the provider for generating descriptions or summaries.", + ) + provider: GenAIProviderEnum | None = Field( + default=None, + title="Provider", + description="The GenAI provider to use (for example: ollama, gemini, openai).", + ) + roles: list[GenAIRoleEnum] = Field( + default_factory=lambda: [ + GenAIRoleEnum.embeddings, + GenAIRoleEnum.vision, + GenAIRoleEnum.tools, + ], + title="Roles", + description="GenAI roles (tools, vision, embeddings); one provider per role.", + ) provider_options: dict[str, Any] = Field( - default={}, title="GenAI Provider extra options." + default={}, + title="Provider options", + description="Additional provider-specific options to pass to the GenAI client.", + json_schema_extra={"additionalProperties": {"type": "string"}}, + ) + runtime_options: dict[str, Any] = Field( + default={}, + title="Runtime options", + description="Runtime options passed to the provider for each inference call.", + json_schema_extra={"additionalProperties": {"type": "string"}}, ) runtime_options: dict[str, Any] = Field( default={}, title="Options to pass during inference calls." diff --git a/frigate/config/config.py b/frigate/config/config.py index 370c89458..e31e3d8c8 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -45,7 +45,7 @@ from .camera.audio import AudioConfig from .camera.birdseye import BirdseyeConfig from .camera.detect import DetectConfig from .camera.ffmpeg import FfmpegConfig -from .camera.genai import GenAIConfig +from .camera.genai import GenAIConfig, GenAIRoleEnum from .camera.motion import MotionConfig from .camera.notification import NotificationConfig from .camera.objects import FilterConfig, ObjectConfig @@ -347,9 +347,9 @@ class FrigateConfig(FrigateBaseModel): default_factory=ModelConfig, title="Detection model configuration." ) - # GenAI config - genai: GenAIConfig = Field( - default_factory=GenAIConfig, title="Generative AI configuration." + # GenAI config (named provider configs: name -> GenAIConfig) + genai: Dict[str, GenAIConfig] = Field( + default_factory=dict, title="Generative AI configuration (named providers)." ) # Camera config @@ -431,6 +431,18 @@ class FrigateConfig(FrigateBaseModel): # set notifications state self.notifications.enabled_in_config = self.notifications.enabled + # validate genai: each role (tools, vision, embeddings) at most once + role_to_name: dict[GenAIRoleEnum, str] = {} + for name, genai_cfg in self.genai.items(): + for role in genai_cfg.roles: + if role in role_to_name: + raise ValueError( + f"GenAI role '{role.value}' is assigned to both " + f"'{role_to_name[role]}' and '{name}'; each role must have " + "exactly one provider." + ) + role_to_name[role] = name + # set default min_score for object attributes for attribute in self.model.all_attributes: if not self.objects.filters.get(attribute): diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index bd707de15..54831942a 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -59,7 +59,7 @@ from frigate.data_processing.real_time.license_plate import ( from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataEnum from frigate.db.sqlitevecq import SqliteVecQueueDatabase from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum -from frigate.genai import get_genai_client +from frigate.genai import GenAIClientManager from frigate.models import Event, Recordings, ReviewSegment, Trigger from frigate.util.builtin import serialize from frigate.util.file import get_event_thumbnail_bytes @@ -144,7 +144,7 @@ class EmbeddingMaintainer(threading.Thread): self.frame_manager = SharedMemoryFrameManager() self.detected_license_plates: dict[str, dict[str, Any]] = {} - self.genai_client = get_genai_client(config) + self.genai_manager = GenAIClientManager(config) # model runners to share between realtime and post processors if self.config.lpr.enabled: @@ -203,12 +203,15 @@ class EmbeddingMaintainer(threading.Thread): # post processors self.post_processors: list[PostProcessorApi] = [] - if self.genai_client is not None and any( + if self.genai_manager.vision_client is not None and any( c.review.genai.enabled_in_config for c in self.config.cameras.values() ): self.post_processors.append( ReviewDescriptionProcessor( - self.config, self.requestor, self.metrics, self.genai_client + self.config, + self.requestor, + self.metrics, + self.genai_manager.vision_client, ) ) @@ -246,7 +249,7 @@ class EmbeddingMaintainer(threading.Thread): ) self.post_processors.append(semantic_trigger_processor) - if self.genai_client is not None and any( + if self.genai_manager.vision_client is not None and any( c.objects.genai.enabled_in_config for c in self.config.cameras.values() ): self.post_processors.append( @@ -255,7 +258,7 @@ class EmbeddingMaintainer(threading.Thread): self.embeddings, self.requestor, self.metrics, - self.genai_client, + self.genai_manager.vision_client, semantic_trigger_processor, ) ) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 0ae664b9f..f52a19e45 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -9,13 +9,24 @@ from typing import Any, Optional from playhouse.shortcuts import model_to_dict -from frigate.config import CameraConfig, FrigateConfig, GenAIConfig, GenAIProviderEnum +from frigate.config import CameraConfig, GenAIConfig, GenAIProviderEnum from frigate.const import CLIPS_DIR from frigate.data_processing.post.types import ReviewMetadata +from frigate.genai.manager import GenAIClientManager from frigate.models import Event logger = logging.getLogger(__name__) +__all__ = [ + "GenAIClient", + "GenAIClientManager", + "GenAIConfig", + "GenAIProviderEnum", + "PROVIDERS", + "load_providers", + "register_genai_provider", +] + PROVIDERS = {} @@ -352,19 +363,6 @@ Guidelines: } -def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]: - """Get the GenAI client.""" - if not config.genai.provider: - return None - - load_providers() - provider = PROVIDERS.get(config.genai.provider) - if provider: - return provider(config.genai) - - return None - - def load_providers(): package_dir = os.path.dirname(__file__) for filename in os.listdir(package_dir): diff --git a/frigate/genai/llama_cpp.py b/frigate/genai/llama_cpp.py index fafef74ae..70a94eec5 100644 --- a/frigate/genai/llama_cpp.py +++ b/frigate/genai/llama_cpp.py @@ -67,6 +67,7 @@ class LlamaCppClient(GenAIClient): # Build request payload with llama.cpp native options payload = { + "model": self.genai_config.model, "messages": [ { "role": "user", @@ -134,6 +135,7 @@ class LlamaCppClient(GenAIClient): openai_tool_choice = "required" payload = { + "model": self.genai_config.model, "messages": messages, } diff --git a/frigate/genai/manager.py b/frigate/genai/manager.py new file mode 100644 index 000000000..e462a0c39 --- /dev/null +++ b/frigate/genai/manager.py @@ -0,0 +1,89 @@ +"""GenAI client manager for Frigate. + +Manages GenAI provider clients from Frigate config. Configuration is read only +in _update_config(); no other code should read config.genai. Exposes clients +by role: tool_client, vision_client, embeddings_client. +""" + +import logging +from typing import TYPE_CHECKING, Optional + +from frigate.config import FrigateConfig +from frigate.config.camera.genai import GenAIRoleEnum + +if TYPE_CHECKING: + from frigate.genai import GenAIClient + +logger = logging.getLogger(__name__) + + +class GenAIClientManager: + """Manages GenAI provider clients from Frigate config.""" + + def __init__(self, config: FrigateConfig) -> None: + self._config = config + self._tool_client: Optional[GenAIClient] = None + self._vision_client: Optional[GenAIClient] = None + self._embeddings_client: Optional[GenAIClient] = None + self._update_config() + + def _update_config(self) -> None: + """Build role clients from current Frigate config.genai. + + Called from __init__ and can be called again when config is reloaded. + Each role (tools, vision, embeddings) gets the client for the provider + that has that role in its roles list. + """ + from frigate.genai import PROVIDERS, load_providers + + self._tool_client = None + self._vision_client = None + self._embeddings_client = None + + if not self._config.genai: + return + + load_providers() + + for _name, genai_cfg in self._config.genai.items(): + if not genai_cfg.provider: + continue + provider_cls = PROVIDERS.get(genai_cfg.provider) + if not provider_cls: + logger.warning( + "Unknown GenAI provider %s in config, skipping.", + genai_cfg.provider, + ) + continue + try: + client = provider_cls(genai_cfg) + except Exception as e: + logger.exception( + "Failed to create GenAI client for provider %s: %s", + genai_cfg.provider, + e, + ) + continue + + for role in genai_cfg.roles: + if role == GenAIRoleEnum.tools: + self._tool_client = client + elif role == GenAIRoleEnum.vision: + self._vision_client = client + elif role == GenAIRoleEnum.embeddings: + self._embeddings_client = client + + @property + def tool_client(self) -> "Optional[GenAIClient]": + """Client configured for the tools role (e.g. chat with function calling).""" + return self._tool_client + + @property + def vision_client(self) -> "Optional[GenAIClient]": + """Client configured for the vision role (e.g. review descriptions, object descriptions).""" + return self._vision_client + + @property + def embeddings_client(self) -> "Optional[GenAIClient]": + """Client configured for the embeddings role.""" + return self._embeddings_client diff --git a/frigate/util/config.py b/frigate/util/config.py index 1af5c8e4e..62db3c42b 100644 --- a/frigate/util/config.py +++ b/frigate/util/config.py @@ -438,6 +438,13 @@ def migrate_018_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any] """Handle migrating frigate config to 0.18-0""" new_config = config.copy() + # Migrate GenAI to new format + genai = new_config.get("genai") + + if genai and genai.get("provider"): + genai["roles"] = ["embeddings", "vision", "tools"] + new_config["genai"] = {"default": genai} + # Remove deprecated sync_recordings from global record config if new_config.get("record", {}).get("sync_recordings") is not None: del new_config["record"]["sync_recordings"]