diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index 3c6baeb15..a4d9199af 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -26,3 +26,6 @@ class GenAIConfig(FrigateBaseModel): provider_options: dict[str, Any] = Field( default={}, title="GenAI Provider extra options." ) + runtime_options: dict[str, Any] = Field( + default={}, title="Options to pass during inference calls." + ) diff --git a/frigate/genai/azure-openai.py b/frigate/genai/azure-openai.py index eba8b47c0..eb08f7786 100644 --- a/frigate/genai/azure-openai.py +++ b/frigate/genai/azure-openai.py @@ -64,6 +64,7 @@ class OpenAIClient(GenAIClient): }, ], timeout=self.timeout, + **self.genai_config.runtime_options, ) except Exception as e: logger.warning("Azure OpenAI returned an error: %s", str(e)) diff --git a/frigate/genai/gemini.py b/frigate/genai/gemini.py index f94448d75..01e8ef758 100644 --- a/frigate/genai/gemini.py +++ b/frigate/genai/gemini.py @@ -35,10 +35,14 @@ class GeminiClient(GenAIClient): for img in images ] + [prompt] try: + # Merge runtime_options into generation_config if provided + generation_config_dict = {"candidate_count": 1} + generation_config_dict.update(self.genai_config.runtime_options) + response = self.provider.generate_content( data, generation_config=genai.types.GenerationConfig( - candidate_count=1, + **generation_config_dict ), request_options=genai.types.RequestOptions( timeout=self.timeout, diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py index a54141e4d..ab6d3c0b3 100644 --- a/frigate/genai/ollama.py +++ b/frigate/genai/ollama.py @@ -58,11 +58,15 @@ class OllamaClient(GenAIClient): ) return None try: + ollama_options = { + **self.provider_options, + **self.genai_config.runtime_options, + } result = self.provider.generate( self.genai_config.model, prompt, images=images if images else None, - **self.provider_options, + **ollama_options, ) logger.debug( f"Ollama tokens used: eval_count={result.get('eval_count')}, prompt_eval_count={result.get('prompt_eval_count')}" diff --git a/frigate/genai/openai.py b/frigate/genai/openai.py index 441e93a80..1fb0dd852 100644 --- a/frigate/genai/openai.py +++ b/frigate/genai/openai.py @@ -61,6 +61,7 @@ class OpenAIClient(GenAIClient): }, ], timeout=self.timeout, + **self.genai_config.runtime_options, ) if ( result is not None diff --git a/web/public/locales/en/config/genai.json b/web/public/locales/en/config/genai.json index 084b921c2..fed679d9e 100644 --- a/web/public/locales/en/config/genai.json +++ b/web/public/locales/en/config/genai.json @@ -15,6 +15,9 @@ }, "provider_options": { "label": "GenAI Provider extra options." + }, + "runtime_options": { + "label": "Options to pass during inference calls." } } -} \ No newline at end of file +}