fix attribute area on detail stream hover

update gemini genai provider with new calls from SDK
provider_options specifies any http options
suppress unneeded info logging

docker/main/requirements-wheels.txt

@@ -47,7 +47,7 @@ onnxruntime == 1.22.*
 # Embeddings
 transformers == 4.45.*
 # Generative AI
-google-generativeai == 0.8.*
+google-genai == 1.58.*
 ollama == 0.6.*
 openai == 1.65.*
 # push notifications

frigate/embeddings/onnx/jina_v1_embedding.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import threading
 import warnings
 
 from transformers import AutoFeatureExtractor, AutoTokenizer
@@ -54,6 +55,7 @@ class JinaV1TextEmbedding(BaseEmbedding):
         self.tokenizer = None
         self.feature_extractor = None
         self.runner = None
+        self._lock = threading.Lock()
         files_names = list(self.download_urls.keys()) + [self.tokenizer_file]
 
         if not all(
@@ -134,17 +136,18 @@ class JinaV1TextEmbedding(BaseEmbedding):
             )
 
     def _preprocess_inputs(self, raw_inputs):
-        max_length = max(len(self.tokenizer.encode(text)) for text in raw_inputs)
+        with self._lock:
-        return [
+            max_length = max(len(self.tokenizer.encode(text)) for text in raw_inputs)
-            self.tokenizer(
+            return [
-                text,
+                self.tokenizer(
-                padding="max_length",
+                    text,
-                truncation=True,
+                    padding="max_length",
-                max_length=max_length,
+                    truncation=True,
-                return_tensors="np",
+                    max_length=max_length,
-            )
+                    return_tensors="np",
-            for text in raw_inputs
+                )
-        ]
+                for text in raw_inputs
+            ]
 
 
 class JinaV1ImageEmbedding(BaseEmbedding):
@@ -174,6 +177,7 @@ class JinaV1ImageEmbedding(BaseEmbedding):
         self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)
         self.feature_extractor = None
         self.runner: BaseModelRunner | None = None
+        self._lock = threading.Lock()
         files_names = list(self.download_urls.keys())
         if not all(
             os.path.exists(os.path.join(self.download_path, n)) for n in files_names
@@ -216,8 +220,9 @@ class JinaV1ImageEmbedding(BaseEmbedding):
             )
 
     def _preprocess_inputs(self, raw_inputs):
-        processed_images = [self._process_image(img) for img in raw_inputs]
+        with self._lock:
-        return [
+            processed_images = [self._process_image(img) for img in raw_inputs]
-            self.feature_extractor(images=image, return_tensors="np")
+            return [
-            for image in processed_images
+                self.feature_extractor(images=image, return_tensors="np")
-        ]
+                for image in processed_images
+            ]

frigate/genai/gemini.py

@@ -3,8 +3,8 @@
 import logging
 from typing import Optional
 
-import google.generativeai as genai
+from google import genai
-from google.api_core.exceptions import GoogleAPICallError
+from google.genai import errors, types
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
@@ -16,44 +16,51 @@ logger = logging.getLogger(__name__)
 class GeminiClient(GenAIClient):
     """Generative AI client for Frigate using Gemini."""
 
-    provider: genai.GenerativeModel
+    provider: genai.Client
 
     def _init_provider(self):
         """Initialize the client."""
-        genai.configure(api_key=self.genai_config.api_key)
+        # Merge provider_options into HttpOptions
-        return genai.GenerativeModel(
+        http_options_dict = {
-            self.genai_config.model, **self.genai_config.provider_options
+            "api_version": "v1",
+            "timeout": int(self.timeout * 1000),  # requires milliseconds
+        }
+
+        if isinstance(self.genai_config.provider_options, dict):
+            http_options_dict.update(self.genai_config.provider_options)
+
+        return genai.Client(
+            api_key=self.genai_config.api_key,
+            http_options=types.HttpOptions(**http_options_dict),
         )
 
     def _send(self, prompt: str, images: list[bytes]) -> Optional[str]:
         """Submit a request to Gemini."""
-        data = [
+        contents = [
-            {
+            types.Part.from_bytes(data=img, mime_type="image/jpeg") for img in images
-                "mime_type": "image/jpeg",
-                "data": img,
-            }
-            for img in images
         ] + [prompt]
         try:
             # Merge runtime_options into generation_config if provided
             generation_config_dict = {"candidate_count": 1}
             generation_config_dict.update(self.genai_config.runtime_options)
 
-            response = self.provider.generate_content(
+            response = self.provider.models.generate_content(
-                data,
+                model=self.genai_config.model,
-                generation_config=genai.types.GenerationConfig(
+                contents=contents,
-                    **generation_config_dict
+                config=types.GenerateContentConfig(
-                ),
+                    **generation_config_dict,
-                request_options=genai.types.RequestOptions(
-                    timeout=self.timeout,
                 ),
             )
-        except GoogleAPICallError as e:
+        except errors.APIError as e:
             logger.warning("Gemini returned an error: %s", str(e))
             return None
+        except Exception as e:
+            logger.warning("An unexpected error occurred with Gemini: %s", str(e))
+            return None
+
         try:
             description = response.text.strip()
-        except ValueError:
+        except (ValueError, AttributeError):
             # No description was generated
             return None
         return description

frigate/log.py

@@ -89,6 +89,7 @@ def apply_log_levels(default: str, log_levels: dict[str, LogLevel]) -> None:
         "ws4py": LogLevel.error,
         "PIL": LogLevel.warning,
         "numba": LogLevel.warning,
+        "google_genai.models": LogLevel.warning,
         **log_levels,
     }
 

web/src/components/timeline/DetailStream.tsx

@@ -887,7 +887,10 @@ function LifecycleItem({
                       </span>
                       <span className="font-medium text-foreground">
                         {attributeAreaPx}{" "}
-                        {t("information.pixels", { ns: "common" })}{" "}
+                        {t("information.pixels", {
+                          ns: "common",
+                          area: attributeAreaPx,
+                        })}{" "}
                         <span className="text-secondary-foreground">·</span>{" "}
                         {attributeAreaPct}%
                       </span>

web/src/views/classification/ModelTrainingView.tsx

@@ -75,6 +75,7 @@ import SearchDetailDialog, {
 } from "@/components/overlay/detail/SearchDetailDialog";
 import { SearchResult } from "@/types/search";
 import { HiSparkles } from "react-icons/hi";
+import { capitalizeFirstLetter } from "@/utils/stringUtil";
 
 type ModelTrainingViewProps = {
   model: CustomClassificationModelConfig;
@@ -88,7 +89,7 @@ export default function ModelTrainingView({ model }: ModelTrainingViewProps) {
   // title
 
   useEffect(() => {
-    document.title = `${model.name.toUpperCase()} - ${t("documentTitle")}`;
+    document.title = `${capitalizeFirstLetter(model.name)} - ${t("documentTitle")}`;
   }, [model.name, t]);
 
   // model state