mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-02 11:50:31 +03:00
Mohit Varikuti
85e16eb31b
Adds an opt-in 'twelvelabs' GenAI provider implementing the embeddings role using TwelveLabs' Marengo multimodal model. Text and image inputs are embedded into a shared vector space, powering semantic search over event thumbnails. The provider only implements the embeddings role; descriptions/chat are left to other providers since Marengo is an embedding model. Marengo returns 512-dim vectors, which the existing GenAIEmbedding adapter pads to Frigate's 768-dim search schema. Includes no-network unit tests and a live test gated on TWELVELABS_API_KEY.
70 lines
2.1 KiB
Python
70 lines
2.1 KiB
Python
from enum import Enum
|
|
from typing import Any, Optional
|
|
|
|
from pydantic import Field
|
|
|
|
from ..base import FrigateBaseModel
|
|
from ..env import EnvString
|
|
|
|
__all__ = ["GenAIConfig", "GenAIProviderEnum", "GenAIRoleEnum"]
|
|
|
|
|
|
class GenAIProviderEnum(str, Enum):
|
|
openai = "openai"
|
|
azure_openai = "azure_openai"
|
|
gemini = "gemini"
|
|
ollama = "ollama"
|
|
llamacpp = "llamacpp"
|
|
twelvelabs = "twelvelabs"
|
|
|
|
|
|
class GenAIRoleEnum(str, Enum):
|
|
chat = "chat"
|
|
descriptions = "descriptions"
|
|
embeddings = "embeddings"
|
|
|
|
|
|
class GenAIConfig(FrigateBaseModel):
|
|
"""Primary GenAI Config to define GenAI Provider."""
|
|
|
|
api_key: Optional[EnvString] = Field(
|
|
default=None,
|
|
title="API key",
|
|
description="API key required by some providers (can also be set via environment variables).",
|
|
)
|
|
base_url: Optional[str] = Field(
|
|
default=None,
|
|
title="Base URL",
|
|
description="Base URL for self-hosted or compatible providers (for example an Ollama instance).",
|
|
)
|
|
model: str = Field(
|
|
default="",
|
|
title="Model",
|
|
description="The model to use from the provider for generating descriptions or summaries.",
|
|
)
|
|
provider: GenAIProviderEnum = Field(
|
|
title="Provider",
|
|
description="The GenAI provider to use (for example: ollama, gemini, openai).",
|
|
)
|
|
roles: list[GenAIRoleEnum] = Field(
|
|
default_factory=lambda: [
|
|
GenAIRoleEnum.embeddings,
|
|
GenAIRoleEnum.descriptions,
|
|
GenAIRoleEnum.chat,
|
|
],
|
|
title="Roles",
|
|
description="GenAI roles (chat, descriptions, embeddings); one provider per role.",
|
|
)
|
|
provider_options: dict[str, Any] = Field(
|
|
default={},
|
|
title="Provider options",
|
|
description="Additional provider-specific options to pass to the GenAI client.",
|
|
json_schema_extra={"additionalProperties": {}},
|
|
)
|
|
runtime_options: dict[str, Any] = Field(
|
|
default={},
|
|
title="Runtime options",
|
|
description="Runtime options passed to the provider for each inference call.",
|
|
json_schema_extra={"additionalProperties": {}},
|
|
)
|