Add option to set runtime options for LLM providers

frigate/config/camera/genai.py

@@ -26,3 +26,6 @@ class GenAIConfig(FrigateBaseModel):
     provider_options: dict[str, Any] = Field(
         default={}, title="GenAI Provider extra options."
     )
+    runtime_options: dict[str, Any] = Field(
+        default={}, title="Options to pass during inference calls."
+    )

frigate/genai/azure-openai.py

@@ -64,6 +64,7 @@ class OpenAIClient(GenAIClient):
                     },
                 ],
                 timeout=self.timeout,
+                **self.genai_config.runtime_options,
             )
         except Exception as e:
             logger.warning("Azure OpenAI returned an error: %s", str(e))

frigate/genai/gemini.py

@@ -35,10 +35,14 @@ class GeminiClient(GenAIClient):
             for img in images
         ] + [prompt]
         try:
+            # Merge runtime_options into generation_config if provided
+            generation_config_dict = {"candidate_count": 1}
+            generation_config_dict.update(self.genai_config.runtime_options)
+
             response = self.provider.generate_content(
                 data,
                 generation_config=genai.types.GenerationConfig(
-                    candidate_count=1,
+                    **generation_config_dict
                 ),
                 request_options=genai.types.RequestOptions(
                     timeout=self.timeout,

frigate/genai/ollama.py

@@ -58,11 +58,15 @@ class OllamaClient(GenAIClient):
             )
             return None
         try:
+            ollama_options = {
+                **self.provider_options,
+                **self.genai_config.runtime_options,
+            }
             result = self.provider.generate(
                 self.genai_config.model,
                 prompt,
                 images=images if images else None,
-                **self.provider_options,
+                **ollama_options,
             )
             logger.debug(
                 f"Ollama tokens used: eval_count={result.get('eval_count')}, prompt_eval_count={result.get('prompt_eval_count')}"

frigate/genai/openai.py

@@ -61,6 +61,7 @@ class OpenAIClient(GenAIClient):
                     },
                 ],
                 timeout=self.timeout,
+                **self.genai_config.runtime_options,
             )
             if (
                 result is not None

web/public/locales/en/config/genai.json

@@ -15,6 +15,9 @@
     },
     "provider_options": {
       "label": "GenAI Provider extra options."
+    },
+    "runtime_options": {
+      "label": "Options to pass during inference calls."
     }
   }
-}
+}