diff --git a/frigate/genai/llama_cpp.py b/frigate/genai/llama_cpp.py index fbb1b45df8..73b0b7282b 100644 --- a/frigate/genai/llama_cpp.py +++ b/frigate/genai/llama_cpp.py @@ -38,18 +38,103 @@ class LlamaCppClient(GenAIClient): provider: str | None # base_url provider_options: dict[str, Any] + _context_size: int | None + _supports_vision: bool + _supports_audio: bool + _supports_tools: bool def _init_provider(self) -> str | None: - """Initialize the client.""" + """Initialize the client and query model metadata from the server.""" self.provider_options = { **self.genai_config.provider_options, } - return ( + self._context_size = None + self._supports_vision = False + self._supports_audio = False + self._supports_tools = False + + base_url = ( self.genai_config.base_url.rstrip("/") if self.genai_config.base_url else None ) + if base_url is None: + return None + + # Query /v1/models to validate the configured model exists + try: + response = requests.get( + f"{base_url}/v1/models", + timeout=10, + ) + response.raise_for_status() + models_data = response.json() + + model_found = False + for model in models_data.get("data", []): + if model.get("id") == self.genai_config.model: + model_found = True + # Extract context size from model metadata if available + meta = model.get("meta", {}) + if meta.get("n_ctx_train"): + self._context_size = int(meta["n_ctx_train"]) + break + + if not model_found: + available = [ + m.get("id", "unknown") for m in models_data.get("data", []) + ] + logger.error( + "Model '%s' not found on llama.cpp server. Available models: %s", + self.genai_config.model, + available, + ) + return None + except Exception as e: + logger.warning( + "Failed to query llama.cpp /v1/models endpoint: %s. " + "Model validation skipped.", + e, + ) + + # Query /props for server properties (context size, modalities) + try: + response = requests.get( + f"{base_url}/props", + timeout=10, + ) + response.raise_for_status() + props = response.json() + + # Context size from server runtime config + default_settings = props.get("default_generation_settings", {}) + if default_settings.get("n_ctx"): + self._context_size = int(default_settings["n_ctx"]) + + # Check supported modalities + modalities = props.get("modalities", {}) + self._supports_vision = modalities.get("vision", False) + self._supports_audio = modalities.get("audio", False) + self._supports_tools = modalities.get("tools", False) + + logger.info( + "llama.cpp model '%s' loaded — context: %s, vision: %s, audio: %s, tools: %s", + self.genai_config.model, + self._context_size or "unknown", + self._supports_vision, + self._supports_audio, + self._supports_tools, + ) + except Exception as e: + logger.warning( + "Failed to query llama.cpp /props endpoint: %s. " + "Using defaults for context size and modalities.", + e, + ) + + return base_url + def _send( self, prompt: str, @@ -117,9 +202,34 @@ class LlamaCppClient(GenAIClient): logger.warning("llama.cpp returned an error: %s", str(e)) return None + @property + def supports_vision(self) -> bool: + """Whether the loaded model supports vision/image input.""" + return self._supports_vision + + @property + def supports_audio(self) -> bool: + """Whether the loaded model supports audio input.""" + return self._supports_audio + + @property + def supports_tools(self) -> bool: + """Whether the loaded model supports tool/function calling.""" + return self._supports_tools + def get_context_size(self) -> int: - """Get the context window size for llama.cpp.""" - return int(self.provider_options.get("context_size", 4096)) + """Get the context window size for llama.cpp. + + Resolution order: + 1. provider_options["context_size"] (user override) + 2. Value queried from llama.cpp server at init + 3. Default fallback of 4096 + """ + if "context_size" in self.provider_options: + return int(self.provider_options["context_size"]) + if self._context_size is not None: + return self._context_size + return 4096 def _build_payload( self,