Fixes (#18117)

* face library i18n fixes

* face library i18n fixes

* add ability to use ctrl/cmd S to save in the config editor

* Use datetime as ID

* Update metrics inference speed to start with 0 ms

* fix android formatted thumbnail

* ensure role is comma separated and stripped correctly

* improve face library deletion

- add a confirmation dialog
- add ability to select all / delete faces in collections

* Implement lazy loading for video previews

* Force GPU for large embedding model

* GPU is required

* settings i18n fixes

* Don't delete train tab

* webpush debugging logs

* Fix incorrectly copying zones

* copy path data

* Ensure that cache dir exists for Frigate+

* face docs update

* Add description to upload image step to clarify the image

* Clean up

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

frigate/api/auth.py

@@ -268,7 +268,9 @@ def auth(request: Request):
 
         # if comma-separated with "admin", use "admin", else use default role
         success_response.headers["remote-role"] = (
-            "admin" if role and "admin" in role else proxy_config.default_role
+            "admin"
+            if role and "admin" in [r.strip() for r in role.split(",")]
+            else proxy_config.default_role
         )
 
         return success_response

frigate/api/classification.py

@@ -1,10 +1,9 @@
 """Object classification APIs."""
 
+import datetime
 import logging
 import os
-import random
 import shutil
-import string
 
 import cv2
 from fastapi import APIRouter, Depends, Request, UploadFile
@@ -120,8 +119,7 @@ def train_face(request: Request, name: str, body: dict = None):
         )
 
     sanitized_name = sanitize_filename(name)
-    rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
-    new_name = f"{sanitized_name}-{rand_id}.webp"
+    new_name = f"{sanitized_name}-{datetime.datetime.now().timestamp()}.webp"
     new_file_folder = os.path.join(FACE_DIR, f"{sanitized_name}")
 
     if not os.path.exists(new_file_folder):

frigate/api/media.py

@@ -909,7 +909,7 @@ def event_thumbnail(
         elif extension == "webp":
             quality_params = [int(cv2.IMWRITE_WEBP_QUALITY), 60]
 
-        _, img = cv2.imencode(f".{img}", thumbnail, quality_params)
+        _, img = cv2.imencode(f".{extension}", thumbnail, quality_params)
         thumbnail_bytes = img.tobytes()
 
     return Response(

frigate/comms/webpush.py

@@ -303,6 +303,9 @@ class WebPushClient(Communicator):  # type: ignore[misc]
             and len(payload["before"]["data"]["zones"])
             == len(payload["after"]["data"]["zones"])
         ):
+            logger.debug(
+                f"Skipping notification for {camera} - message is an update and important fields don't have an update"
+            )
             return
 
         self.last_camera_notification_time[camera] = current_time
@@ -325,6 +328,8 @@ class WebPushClient(Communicator):  # type: ignore[misc]
         direct_url = f"/review?id={reviewId}" if state == "end" else f"/#{camera}"
         ttl = 3600 if state == "end" else 0
 
+        logger.debug(f"Sending push notification for {camera}, review ID {reviewId}")
+
         for user in self.web_pushers:
             self.send_push_notification(
                 user=user,

frigate/data_processing/common/license_plate/mixin.py

@@ -25,7 +25,7 @@ from frigate.comms.event_metadata_updater import (
 from frigate.const import CLIPS_DIR
 from frigate.embeddings.onnx.lpr_embedding import LPR_EMBEDDING_SIZE
 from frigate.types import TrackedObjectUpdateTypesEnum
-from frigate.util.builtin import EventsPerSecond
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
 from frigate.util.image import area
 
 logger = logging.getLogger(__name__)
@@ -36,8 +36,10 @@ WRITE_DEBUG_IMAGES = False
 class LicensePlateProcessingMixin:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.plate_rec_speed = InferenceSpeed(self.metrics.alpr_speed)
         self.plates_rec_second = EventsPerSecond()
         self.plates_rec_second.start()
+        self.plate_det_speed = InferenceSpeed(self.metrics.yolov9_lpr_speed)
         self.plates_det_second = EventsPerSecond()
         self.plates_det_second.start()
         self.event_metadata_publisher = EventMetadataPublisher()
@@ -1157,22 +1159,6 @@ class LicensePlateProcessingMixin:
         # 5. Return True if previous plate scores higher
         return prev_score > curr_score
 
-    def __update_yolov9_metrics(self, duration: float) -> None:
-        """
-        Update inference metrics.
-        """
-        self.metrics.yolov9_lpr_speed.value = (
-            self.metrics.yolov9_lpr_speed.value * 9 + duration
-        ) / 10
-
-    def __update_lpr_metrics(self, duration: float) -> None:
-        """
-        Update inference metrics.
-        """
-        self.metrics.alpr_speed.value = (
-            self.metrics.alpr_speed.value * 9 + duration
-        ) / 10
-
     def _generate_plate_event(self, camera: str, plate: str, plate_score: float) -> str:
         """Generate a unique ID for a plate event based on camera and text."""
         now = datetime.datetime.now().timestamp()
@@ -1228,7 +1214,7 @@ class LicensePlateProcessingMixin:
                 f"{camera}: YOLOv9 LPD inference time: {(datetime.datetime.now().timestamp() - yolov9_start) * 1000:.2f} ms"
             )
             self.plates_det_second.update()
-            self.__update_yolov9_metrics(
+            self.plate_det_speed.update(
                 datetime.datetime.now().timestamp() - yolov9_start
             )
 
@@ -1319,7 +1305,7 @@ class LicensePlateProcessingMixin:
                     f"{camera}: YOLOv9 LPD inference time: {(datetime.datetime.now().timestamp() - yolov9_start) * 1000:.2f} ms"
                 )
                 self.plates_det_second.update()
-                self.__update_yolov9_metrics(
+                self.plate_det_speed.update(
                     datetime.datetime.now().timestamp() - yolov9_start
                 )
 
@@ -1433,7 +1419,7 @@ class LicensePlateProcessingMixin:
             camera, id, license_plate_frame
         )
         self.plates_rec_second.update()
-        self.__update_lpr_metrics(datetime.datetime.now().timestamp() - start)
+        self.plate_rec_speed.update(datetime.datetime.now().timestamp() - start)
 
         if license_plates:
             for plate, confidence, text_area in zip(license_plates, confidences, areas):

frigate/data_processing/real_time/face.py

@@ -5,9 +5,7 @@ import datetime
 import json
 import logging
 import os
-import random
 import shutil
-import string
 from typing import Optional
 
 import cv2
@@ -27,7 +25,7 @@ from frigate.data_processing.common.face.model import (
     FaceRecognizer,
 )
 from frigate.types import TrackedObjectUpdateTypesEnum
-from frigate.util.builtin import EventsPerSecond
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
 from frigate.util.image import area
 
 from ..types import DataProcessorMetrics
@@ -58,6 +56,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
         self.person_face_history: dict[str, list[tuple[str, float, int]]] = {}
         self.recognizer: FaceRecognizer | None = None
         self.faces_per_second = EventsPerSecond()
+        self.inference_speed = InferenceSpeed(self.metrics.face_rec_speed)
 
         download_path = os.path.join(MODEL_CACHE_DIR, "facedet")
         self.model_files = {
@@ -155,9 +154,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
 
     def __update_metrics(self, duration: float) -> None:
         self.faces_per_second.update()
-        self.metrics.face_rec_speed.value = (
-            self.metrics.face_rec_speed.value * 9 + duration
-        ) / 10
+        self.inference_speed.update(duration)
 
     def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
         """Look for faces in image."""
@@ -343,11 +340,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
 
             return {"success": True, "score": score, "face_name": sub_label}
         elif topic == EmbeddingsRequestEnum.register_face.value:
-            rand_id = "".join(
-                random.choices(string.ascii_lowercase + string.digits, k=6)
-            )
             label = request_data["face_name"]
-            id = f"{label}-{rand_id}"
 
             if request_data.get("cropped"):
                 thumbnail = request_data["image"]
@@ -376,7 +369,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
 
             # write face to library
             folder = os.path.join(FACE_DIR, label)
-            file = os.path.join(folder, f"{id}.webp")
+            file = os.path.join(
+                folder, f"{label}_{datetime.datetime.now().timestamp()}.webp"
+            )
             os.makedirs(folder, exist_ok=True)
 
             # save face image

frigate/data_processing/types.py

@@ -7,7 +7,9 @@ from multiprocessing.sharedctypes import Synchronized
 
 class DataProcessorMetrics:
     image_embeddings_speed: Synchronized
+    image_embeddings_eps: Synchronized
     text_embeddings_speed: Synchronized
+    text_embeddings_eps: Synchronized
     face_rec_speed: Synchronized
     face_rec_fps: Synchronized
     alpr_speed: Synchronized
@@ -16,15 +18,15 @@ class DataProcessorMetrics:
     yolov9_lpr_pps: Synchronized
 
     def __init__(self):
-        self.image_embeddings_speed = mp.Value("d", 0.01)
+        self.image_embeddings_speed = mp.Value("d", 0.0)
         self.image_embeddings_eps = mp.Value("d", 0.0)
-        self.text_embeddings_speed = mp.Value("d", 0.01)
+        self.text_embeddings_speed = mp.Value("d", 0.0)
         self.text_embeddings_eps = mp.Value("d", 0.0)
-        self.face_rec_speed = mp.Value("d", 0.01)
+        self.face_rec_speed = mp.Value("d", 0.0)
         self.face_rec_fps = mp.Value("d", 0.0)
-        self.alpr_speed = mp.Value("d", 0.01)
+        self.alpr_speed = mp.Value("d", 0.0)
         self.alpr_pps = mp.Value("d", 0.0)
-        self.yolov9_lpr_speed = mp.Value("d", 0.01)
+        self.yolov9_lpr_speed = mp.Value("d", 0.0)
         self.yolov9_lpr_pps = mp.Value("d", 0.0)
 
 

frigate/detectors/detector_config.py

@@ -126,6 +126,9 @@ class ModelConfig(BaseModel):
         if not self.path or not self.path.startswith("plus://"):
             return
 
+        # ensure that model cache dir exists
+        os.makedirs(MODEL_CACHE_DIR, exist_ok=True)
+
         model_id = self.path[7:]
         self.path = os.path.join(MODEL_CACHE_DIR, model_id)
         model_info_path = f"{self.path}.json"

frigate/embeddings/__init__.py

@@ -235,7 +235,7 @@ class EmbeddingsContext:
             if os.path.isfile(file_path):
                 os.unlink(file_path)
 
-        if len(os.listdir(folder)) == 0:
+        if face != "train" and len(os.listdir(folder)) == 0:
             os.rmdir(folder)
 
         self.requestor.send_data(

frigate/embeddings/embeddings.py

@@ -21,7 +21,7 @@ from frigate.data_processing.types import DataProcessorMetrics
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.models import Event
 from frigate.types import ModelStatusTypesEnum
-from frigate.util.builtin import EventsPerSecond, serialize
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed, serialize
 from frigate.util.path import get_event_thumbnail_bytes
 
 from .onnx.jina_v1_embedding import JinaV1ImageEmbedding, JinaV1TextEmbedding
@@ -75,8 +75,10 @@ class Embeddings:
         self.metrics = metrics
         self.requestor = InterProcessRequestor()
 
+        self.image_inference_speed = InferenceSpeed(self.metrics.image_embeddings_speed)
         self.image_eps = EventsPerSecond()
         self.image_eps.start()
+        self.text_inference_speed = InferenceSpeed(self.metrics.text_embeddings_speed)
         self.text_eps = EventsPerSecond()
         self.text_eps.start()
 
@@ -183,10 +185,7 @@ class Embeddings:
                 (event_id, serialize(embedding)),
             )
 
-        duration = datetime.datetime.now().timestamp() - start
-        self.metrics.image_embeddings_speed.value = (
-            self.metrics.image_embeddings_speed.value * 9 + duration
-        ) / 10
+        self.image_inference_speed.update(datetime.datetime.now().timestamp() - start)
         self.image_eps.update()
 
         return embedding
@@ -220,9 +219,7 @@ class Embeddings:
             )
 
         duration = datetime.datetime.now().timestamp() - start
-        self.metrics.text_embeddings_speed.value = (
-            self.metrics.text_embeddings_speed.value * 9 + (duration / len(ids))
-        ) / 10
+        self.text_inference_speed.update(duration / len(ids))
 
         return embeddings
 
@@ -241,10 +238,7 @@ class Embeddings:
                 (event_id, serialize(embedding)),
             )
 
-        duration = datetime.datetime.now().timestamp() - start
-        self.metrics.text_embeddings_speed.value = (
-            self.metrics.text_embeddings_speed.value * 9 + duration
-        ) / 10
+        self.text_inference_speed.update(datetime.datetime.now().timestamp() - start)
         self.text_eps.update()
 
         return embedding
@@ -276,10 +270,7 @@ class Embeddings:
                 items,
             )
 
-        duration = datetime.datetime.now().timestamp() - start
-        self.metrics.text_embeddings_speed.value = (
-            self.metrics.text_embeddings_speed.value * 9 + (duration / len(ids))
-        ) / 10
+        self.text_inference_speed.update(datetime.datetime.now().timestamp() - start)
 
         return embeddings
 

frigate/embeddings/onnx/face_embedding.py

@@ -23,10 +23,7 @@ FACENET_INPUT_SIZE = 160
 
 
 class FaceNetEmbedding(BaseEmbedding):
-    def __init__(
-        self,
-        device: str = "AUTO",
-    ):
+    def __init__(self):
         super().__init__(
             model_name="facedet",
             model_file="facenet.tflite",
@@ -34,7 +31,6 @@ class FaceNetEmbedding(BaseEmbedding):
                 "facenet.tflite": "https://github.com/NickM-27/facenet-onnx/releases/download/v1.0/facenet.tflite",
             },
         )
-        self.device = device
         self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)
         self.tokenizer = None
         self.feature_extractor = None
@@ -113,10 +109,7 @@ class FaceNetEmbedding(BaseEmbedding):
 
 
 class ArcfaceEmbedding(BaseEmbedding):
-    def __init__(
-        self,
-        device: str = "AUTO",
-    ):
+    def __init__(self):
         super().__init__(
             model_name="facedet",
             model_file="arcface.onnx",
@@ -124,7 +117,6 @@ class ArcfaceEmbedding(BaseEmbedding):
                 "arcface.onnx": "https://github.com/NickM-27/facenet-onnx/releases/download/v1.0/arcface.onnx",
             },
         )
-        self.device = device
         self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)
         self.tokenizer = None
         self.feature_extractor = None
@@ -154,7 +146,7 @@ class ArcfaceEmbedding(BaseEmbedding):
 
             self.runner = ONNXModelRunner(
                 os.path.join(self.download_path, self.model_file),
-                self.device,
+                "GPU",
             )
 
     def _preprocess_inputs(self, raw_inputs):

frigate/review/maintainer.py

@@ -1,5 +1,6 @@
 """Maintain review segments in db."""
 
+import copy
 import json
 import logging
 import os
@@ -119,21 +120,23 @@ class PendingReviewSegment:
             )
 
     def get_data(self, ended: bool) -> dict:
-        return {
-            ReviewSegment.id.name: self.id,
-            ReviewSegment.camera.name: self.camera,
-            ReviewSegment.start_time.name: self.start_time,
-            ReviewSegment.end_time.name: self.last_update if ended else None,
-            ReviewSegment.severity.name: self.severity.value,
-            ReviewSegment.thumb_path.name: self.frame_path,
-            ReviewSegment.data.name: {
-                "detections": list(set(self.detections.keys())),
-                "objects": list(set(self.detections.values())),
-                "sub_labels": list(self.sub_labels.values()),
-                "zones": self.zones,
-                "audio": list(self.audio),
-            },
-        }.copy()
+        return copy.deepcopy(
+            {
+                ReviewSegment.id.name: self.id,
+                ReviewSegment.camera.name: self.camera,
+                ReviewSegment.start_time.name: self.start_time,
+                ReviewSegment.end_time.name: self.last_update if ended else None,
+                ReviewSegment.severity.name: self.severity.value,
+                ReviewSegment.thumb_path.name: self.frame_path,
+                ReviewSegment.data.name: {
+                    "detections": list(set(self.detections.keys())),
+                    "objects": list(set(self.detections.values())),
+                    "sub_labels": list(self.sub_labels.values()),
+                    "zones": self.zones,
+                    "audio": list(self.audio),
+                },
+            }
+        )
 
 
 class ReviewSegmentMaintainer(threading.Thread):

frigate/track/tracked_object.py

@@ -154,7 +154,7 @@ class TrackedObject:
                         "attributes": obj_data["attributes"],
                         "current_estimated_speed": self.current_estimated_speed,
                         "velocity_angle": self.velocity_angle,
-                        "path_data": self.path_data,
+                        "path_data": self.path_data.copy(),
                         "recognized_license_plate": obj_data.get(
                             "recognized_license_plate"
                         ),
@@ -378,7 +378,7 @@ class TrackedObject:
             "current_estimated_speed": self.current_estimated_speed,
             "average_estimated_speed": self.average_estimated_speed,
             "velocity_angle": self.velocity_angle,
-            "path_data": self.path_data,
+            "path_data": self.path_data.copy(),
             "recognized_license_plate": self.obj_data.get("recognized_license_plate"),
         }
 

frigate/util/builtin.py

@@ -11,6 +11,7 @@ import shlex
 import struct
 import urllib.parse
 from collections.abc import Mapping
+from multiprocessing.sharedctypes import Synchronized
 from pathlib import Path
 from typing import Any, Optional, Tuple, Union
 from zoneinfo import ZoneInfoNotFoundError
@@ -26,16 +27,16 @@ logger = logging.getLogger(__name__)
 
 
 class EventsPerSecond:
-    def __init__(self, max_events=1000, last_n_seconds=10):
+    def __init__(self, max_events=1000, last_n_seconds=10) -> None:
         self._start = None
         self._max_events = max_events
         self._last_n_seconds = last_n_seconds
         self._timestamps = []
 
-    def start(self):
+    def start(self) -> None:
         self._start = datetime.datetime.now().timestamp()
 
-    def update(self):
+    def update(self) -> None:
         now = datetime.datetime.now().timestamp()
         if self._start is None:
             self._start = now
@@ -45,7 +46,7 @@ class EventsPerSecond:
             self._timestamps = self._timestamps[(1 - self._max_events) :]
         self.expire_timestamps(now)
 
-    def eps(self):
+    def eps(self) -> float:
         now = datetime.datetime.now().timestamp()
         if self._start is None:
             self._start = now
@@ -58,12 +59,29 @@ class EventsPerSecond:
         return len(self._timestamps) / seconds
 
     # remove aged out timestamps
-    def expire_timestamps(self, now):
+    def expire_timestamps(self, now: float) -> None:
         threshold = now - self._last_n_seconds
         while self._timestamps and self._timestamps[0] < threshold:
             del self._timestamps[0]
 
 
+class InferenceSpeed:
+    def __init__(self, metric: Synchronized) -> None:
+        self.__metric = metric
+        self.__initialized = False
+
+    def update(self, inference_time: float) -> None:
+        if not self.__initialized:
+            self.__metric.value = inference_time
+            self.__initialized = True
+            return
+
+        self.__metric.value = (self.__metric.value * 9 + inference_time) / 10
+
+    def current(self) -> float:
+        return self.__metric.value
+
+
 def deep_merge(dct1: dict, dct2: dict, override=False, merge_lists=False) -> dict:
     """
     :param dct1: First dict to merge