From fad62b996ad9b66420128265f65ce1d035c25db0 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Mon, 17 Mar 2025 13:44:57 -0500
Subject: [PATCH 01/97] Add Frigate+ pane to Settings UI (#17208)
* add plus data to config api response
* add fields to frontend type
* add frigate+ page in settings
* add docs
* fix label in explore detail dialog
---
docs/docs/plus/faq.md | 10 +
frigate/api/app.py | 13 +
web/public/locales/en/views/settings.json | 37 ++-
.../overlay/detail/SearchDetailDialog.tsx | 2 +-
web/src/pages/Settings.tsx | 3 +
web/src/types/frigateConfig.ts | 6 +
.../settings/FrigatePlusSettingsView.tsx | 229 ++++++++++++++++++
7 files changed, 297 insertions(+), 3 deletions(-)
create mode 100644 web/src/views/settings/FrigatePlusSettingsView.tsx
diff --git a/docs/docs/plus/faq.md b/docs/docs/plus/faq.md
index fb0cd2512..151eb3f60 100644
--- a/docs/docs/plus/faq.md
+++ b/docs/docs/plus/faq.md
@@ -22,3 +22,13 @@ Yes. Models and metadata are stored in the `model_cache` directory within the co
### Can I keep using my Frigate+ models even if I do not renew my subscription?
Yes. Subscriptions to Frigate+ provide access to the infrastructure used to train the models. Models trained with your subscription are yours to keep and use forever. However, do note that the terms and conditions prohibit you from sharing, reselling, or creating derivative products from the models.
+
+### Why can't I submit images to Frigate+?
+
+If you've configured your API key and the Frigate+ Settings page in the UI shows that the key is active, you need to ensure that you've enabled both snapshots and `clean_copy` snapshots for the cameras you'd like to submit images for. Note that `clean_copy` is enabled by default when snapshots are enabled.
+
+```yaml
+snapshots:
+ enabled: true
+ clean_copy: true
+```
diff --git a/frigate/api/app.py b/frigate/api/app.py
index 05013ed12..9d7b3768f 100644
--- a/frigate/api/app.py
+++ b/frigate/api/app.py
@@ -9,6 +9,7 @@ import traceback
from datetime import datetime, timedelta
from functools import reduce
from io import StringIO
+from pathlib import Path as FilePath
from typing import Any, Optional
import aiofiles
@@ -174,6 +175,18 @@ def config(request: Request):
config["model"]["all_attributes"] = config_obj.model.all_attributes
config["model"]["non_logo_attributes"] = config_obj.model.non_logo_attributes
+ # Add model plus data if plus is enabled
+ if config["plus"]["enabled"]:
+ model_json_path = FilePath(config["model"]["path"]).with_suffix(".json")
+ try:
+ with open(model_json_path, "r") as f:
+ model_plus_data = json.load(f)
+ config["model"]["plus"] = model_plus_data
+ except FileNotFoundError:
+ config["model"]["plus"] = None
+ except json.JSONDecodeError:
+ config["model"]["plus"] = None
+
# use merged labelamp
for detector_config in config["detectors"].values():
detector_config["model"]["labelmap"] = (
diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json
index f19ac5ee6..3d25b92c1 100644
--- a/web/public/locales/en/views/settings.json
+++ b/web/public/locales/en/views/settings.json
@@ -7,7 +7,8 @@
"masksAndZones": "Mask and Zone Editor - Frigate",
"motionTuner": "Motion Tuner - Frigate",
"object": "Object Settings - Frigate",
- "general": "General Settings - Frigate"
+ "general": "General Settings - Frigate",
+ "frigatePlus": "Frigate+ Settings - Frigate"
},
"menu": {
"uiSettings": "UI Settings",
@@ -17,7 +18,8 @@
"motionTuner": "Motion Tuner",
"debug": "Debug",
"users": "Users",
- "notifications": "Notifications"
+ "notifications": "Notifications",
+ "frigateplus": "Frigate+"
},
"dialog": {
"unsavedChanges": {
@@ -515,5 +517,36 @@
"registerFailed": "Failed to save notification registration."
}
}
+ },
+ "frigatePlus": {
+ "title": "Frigate+ Settings",
+ "apiKey": {
+ "title": "Frigate+ API Key",
+ "validated": "Frigate+ API key is detected and validated",
+ "notValidated": "Frigate+ API key is not detected or not validated",
+ "desc": "The Frigate+ API key enables integration with the Frigate+ service.",
+ "plusLink": "Read more about Frigate+"
+ },
+ "snapshotConfig": {
+ "title": "Snapshot Configuration",
+ "desc": "Submitting to Frigate+ requires both snapshots and clean_copy snapshots to be enabled in your config.",
+ "documentation": "Read the documentation",
+ "cleanCopyWarning": "Some cameras have snapshots enabled but have the clean copy disabled. You need to enable clean_copy in your snapshot config to be able to submit images from these cameras to Frigate+.",
+ "table": {
+ "camera": "Camera",
+ "snapshots": "Snapshots",
+ "cleanCopySnapshots": "clean_copy Snapshots"
+ }
+ },
+ "modelInfo": {
+ "title": "Model Information",
+ "modelType": "Model Type",
+ "trainDate": "Train Date",
+ "baseModel": "Base Model",
+ "supportedDetectors": "Supported Detectors",
+ "cameras": "Cameras",
+ "loading": "Loading model information...",
+ "error": "Failed to load model information"
+ }
}
}
diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx
index d74efdf6d..891ce88b1 100644
--- a/web/src/components/overlay/detail/SearchDetailDialog.tsx
+++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx
@@ -563,7 +563,7 @@ function ObjectDetailsTab({
{t("details.label")}
{getIconForLabel(search.label, "size-4 text-primary")}
- {t("{search.label}", { ns: "objects" })}
+ {t(search.label, { ns: "objects" })}
{search.sub_label && ` (${search.sub_label})`}
diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx
index 353d0dbf8..6ccda34f3 100644
--- a/web/src/pages/Settings.tsx
+++ b/web/src/pages/Settings.tsx
@@ -37,6 +37,7 @@ import AuthenticationView from "@/views/settings/AuthenticationView";
import NotificationView from "@/views/settings/NotificationsSettingsView";
import ClassificationSettingsView from "@/views/settings/ClassificationSettingsView";
import UiSettingsView from "@/views/settings/UiSettingsView";
+import FrigatePlusSettingsView from "@/views/settings/FrigatePlusSettingsView";
import { useSearchEffect } from "@/hooks/use-overlay-state";
import { useSearchParams } from "react-router-dom";
import { useInitialCameraState } from "@/api/ws";
@@ -54,6 +55,7 @@ const allSettingsViews = [
"debug",
"users",
"notifications",
+ "frigateplus",
] as const;
type SettingsType = (typeof allSettingsViews)[number];
@@ -279,6 +281,7 @@ export default function Settings() {
{page == "notifications" && (
)}
+ {page == "frigateplus" && }
{confirmationDialogOpen && (
| null;
diff --git a/web/src/views/settings/FrigatePlusSettingsView.tsx b/web/src/views/settings/FrigatePlusSettingsView.tsx
new file mode 100644
index 000000000..965843f09
--- /dev/null
+++ b/web/src/views/settings/FrigatePlusSettingsView.tsx
@@ -0,0 +1,229 @@
+import Heading from "@/components/ui/heading";
+import { Label } from "@/components/ui/label";
+import { useEffect } from "react";
+import { Toaster } from "sonner";
+import { Separator } from "../../components/ui/separator";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { CheckCircle2, XCircle } from "lucide-react";
+import { Trans, useTranslation } from "react-i18next";
+import { IoIosWarning } from "react-icons/io";
+import { Link } from "react-router-dom";
+import { LuExternalLink } from "react-icons/lu";
+
+export default function FrigatePlusSettingsView() {
+ const { data: config } = useSWR("config");
+ const { t } = useTranslation("views/settings");
+
+ useEffect(() => {
+ document.title = t("documentTitle.frigatePlus");
+ }, [t]);
+
+ const needCleanSnapshots = () => {
+ if (!config) {
+ return false;
+ }
+ return Object.values(config.cameras).some(
+ (camera) => camera.snapshots.enabled && !camera.snapshots.clean_copy,
+ );
+ };
+
+ return (
+ <>
+
diff --git a/web/src/types/face.ts b/web/src/types/face.ts
new file mode 100644
index 000000000..e8f426a5b
--- /dev/null
+++ b/web/src/types/face.ts
@@ -0,0 +1,6 @@
+export type RecognizedFaceData = {
+ timestamp: number;
+ eventId: string;
+ name: string;
+ score: number;
+};
From 125c2665857c6ae38d63b584da42e6d869a4198c Mon Sep 17 00:00:00 2001
From: GuoQing Liu <842607283@qq.com>
Date: Wed, 19 Mar 2025 18:56:48 +0800
Subject: [PATCH 08/97] chore: use better translation (zh-CN) (#17239)
---
web/public/locales/zh-CN/views/faceLibrary.json | 7 ++++++-
web/public/locales/zh-CN/views/settings.json | 4 ++--
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/web/public/locales/zh-CN/views/faceLibrary.json b/web/public/locales/zh-CN/views/faceLibrary.json
index e8a86a446..76a98bf31 100644
--- a/web/public/locales/zh-CN/views/faceLibrary.json
+++ b/web/public/locales/zh-CN/views/faceLibrary.json
@@ -2,6 +2,12 @@
"description": {
"addFace": "我们将指导如何将新面孔添加到人脸库中。"
},
+ "details": {
+ "confidence": "置信度",
+ "face": "人脸详情",
+ "faceDesc": "人脸及相关对象的详细信息",
+ "timestamp": "时间戳"
+ },
"documentTitle": "人脸库 - Frigate",
"uploadFaceImage": {
"title": "上传人脸图片",
@@ -26,7 +32,6 @@
"readTheDocs": "阅读文档查看更多有关为人脸库优化图像的详细信息",
"trainFaceAs": "将人脸训练为：",
"trainFaceAsPerson": "将人脸训练为人物",
-
"toast": {
"success": {
"uploadedImage": "图片上传成功。",
diff --git a/web/public/locales/zh-CN/views/settings.json b/web/public/locales/zh-CN/views/settings.json
index 707e6c9f0..ccef8f151 100644
--- a/web/public/locales/zh-CN/views/settings.json
+++ b/web/public/locales/zh-CN/views/settings.json
@@ -26,8 +26,8 @@
"notifications": "通知"
},
"cameraSetting": {
- "camera": "相机",
- "noCamera": "没有相机"
+ "camera": "摄像头",
+ "noCamera": "没有摄像头"
},
"general": {
"title": "常规设置",
From e340c9aaba6b71136e142620d386c0ff9f24d9ff Mon Sep 17 00:00:00 2001
From: Jason Hunter
Date: Wed, 19 Mar 2025 08:23:24 -0400
Subject: [PATCH 09/97] Add openvino support for the DFINE model (#17238)
* add openvino support for the dfine model
* update docs to show DFINE support for openvino
* remove warning about OpenVINO for DFINE
---
docs/docs/configuration/object_detectors.md | 45 +++++++++++++++------
frigate/detectors/plugins/openvino.py | 18 ++++++++-
2 files changed, 49 insertions(+), 14 deletions(-)
diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md
index a4f4c7c20..8cc6b2f1e 100644
--- a/docs/docs/configuration/object_detectors.md
+++ b/docs/docs/configuration/object_detectors.md
@@ -129,8 +129,8 @@ detectors:
type: edgetpu
device: pci
```
----
+---
## Hailo-8
@@ -140,12 +140,13 @@ See the [installation docs](../frigate/installation.md#hailo-8l) for information
### Configuration
-When configuring the Hailo detector, you have two options to specify the model: a local **path** or a **URL**.
+When configuring the Hailo detector, you have two options to specify the model: a local **path** or a **URL**.
If both are provided, the detector will first check for the model at the given local path. If the file is not found, it will download the model from the specified URL. The model file is cached under `/config/model_cache/hailo`.
-#### YOLO
+#### YOLO
Use this configuration for YOLO-based models. When no custom model path or URL is provided, the detector automatically downloads the default model based on the detected hardware:
+
- **Hailo-8 hardware:** Uses **YOLOv6n** (default: `yolov6n.hef`)
- **Hailo-8L hardware:** Uses **YOLOv6n** (default: `yolov6n.hef`)
@@ -224,17 +225,16 @@ model:
# Alternatively, or as a fallback, provide a custom URL:
# path: https://custom-model-url.com/path/to/model.hef
```
+
For additional ready-to-use models, please visit: https://github.com/hailo-ai/hailo_model_zoo
-Hailo8 supports all models in the Hailo Model Zoo that include HailoRT post-processing. You're welcome to choose any of these pre-configured models for your implementation.
+Hailo8 supports all models in the Hailo Model Zoo that include HailoRT post-processing. You're welcome to choose any of these pre-configured models for your implementation.
-> **Note:**
+> **Note:**
> The config.path parameter can accept either a local file path or a URL ending with .hef. When provided, the detector will first check if the path is a local file path. If the file exists locally, it will use it directly. If the file is not found locally or if a URL was provided, it will attempt to download the model from the specified URL.
---
-
-
## OpenVINO Detector
The OpenVINO detector type runs an OpenVINO IR model on AMD and Intel CPUs, Intel GPUs and Intel VPU hardware. To configure an OpenVINO detector, set the `"type"` attribute to `"openvino"`.
@@ -340,6 +340,30 @@ model:
Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects.
+#### D-FINE
+
+[D-FINE](https://github.com/Peterande/D-FINE) is the [current state of the art](https://paperswithcode.com/sota/real-time-object-detection-on-coco?p=d-fine-redefine-regression-task-in-detrs-as) at the time of writing. The ONNX exported models are supported, but not included by default. See [the models section](#downloading-d-fine-model) for more information on downloading the D-FINE model for use in Frigate.
+
+After placing the downloaded onnx model in your config/model_cache folder, you can use the following configuration:
+
+```yaml
+detectors:
+ ov:
+ type: openvino
+ device: GPU
+
+model:
+ model_type: dfine
+ width: 640
+ height: 640
+ input_tensor: nchw
+ input_dtype: float
+ path: /config/model_cache/dfine_s_obj2coco.onnx
+ labelmap_path: /labelmap/coco-80.txt
+```
+
+Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects.
+
## NVidia TensorRT Detector
Nvidia GPUs may be used for object detection using the TensorRT libraries. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt`. This detector is designed to work with Yolo models for object detection.
@@ -529,6 +553,7 @@ $ docker exec -it frigate /bin/bash -c '(unset HSA_OVERRIDE_GFX_VERSION && /opt/
### Supported Models
See [ONNX supported models](#supported-models) for supported models, there are some caveats:
+
- D-FINE models are not supported
- YOLO-NAS models are known to not run well on integrated GPUs
@@ -626,12 +651,6 @@ Note that the labelmap uses a subset of the complete COCO label set that has onl
[D-FINE](https://github.com/Peterande/D-FINE) is the [current state of the art](https://paperswithcode.com/sota/real-time-object-detection-on-coco?p=d-fine-redefine-regression-task-in-detrs-as) at the time of writing. The ONNX exported models are supported, but not included by default. See [the models section](#downloading-d-fine-model) for more information on downloading the D-FINE model for use in Frigate.
-:::warning
-
-D-FINE is currently not supported on OpenVINO
-
-:::
-
After placing the downloaded onnx model in your config/model_cache folder, you can use the following configuration:
```yaml
diff --git a/frigate/detectors/plugins/openvino.py b/frigate/detectors/plugins/openvino.py
index 0f0b99a1f..75d956500 100644
--- a/frigate/detectors/plugins/openvino.py
+++ b/frigate/detectors/plugins/openvino.py
@@ -10,7 +10,7 @@ from typing_extensions import Literal
from frigate.const import MODEL_CACHE_DIR
from frigate.detectors.detection_api import DetectionApi
from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
-from frigate.util.model import post_process_yolov9
+from frigate.util.model import post_process_dfine, post_process_yolov9
logger = logging.getLogger(__name__)
@@ -29,6 +29,7 @@ class OvDetector(DetectionApi):
ModelTypeEnum.yolonas,
ModelTypeEnum.yolov9,
ModelTypeEnum.yolox,
+ ModelTypeEnum.dfine,
]
def __init__(self, detector_config: OvDetectorConfig):
@@ -163,6 +164,21 @@ class OvDetector(DetectionApi):
infer_request = self.interpreter.create_infer_request()
# TODO: see if we can use shared_memory=True
input_tensor = ov.Tensor(array=tensor_input)
+
+ if self.ov_model_type == ModelTypeEnum.dfine:
+ infer_request.set_tensor("images", input_tensor)
+ target_sizes_tensor = ov.Tensor(
+ np.array([[self.h, self.w]], dtype=np.int64)
+ )
+ infer_request.set_tensor("orig_target_sizes", target_sizes_tensor)
+ infer_request.infer()
+ tensor_output = (
+ infer_request.get_output_tensor(0).data,
+ infer_request.get_output_tensor(1).data,
+ infer_request.get_output_tensor(2).data,
+ )
+ return post_process_dfine(tensor_output, self.w, self.h)
+
infer_request.infer(input_tensor)
detections = np.zeros((20, 6), np.float32)
From ac9e24e2ed5bddd1cc8cac92e241abd11fc195f8 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Wed, 19 Mar 2025 12:23:44 +0000
Subject: [PATCH 10/97] Face Library: show name on details (#17245)
---
web/public/locales/en/views/faceLibrary.json | 1 +
web/src/pages/FaceLibrary.tsx | 4 ++++
2 files changed, 5 insertions(+)
diff --git a/web/public/locales/en/views/faceLibrary.json b/web/public/locales/en/views/faceLibrary.json
index 57d78b18a..fd5f50825 100644
--- a/web/public/locales/en/views/faceLibrary.json
+++ b/web/public/locales/en/views/faceLibrary.json
@@ -3,6 +3,7 @@
"addFace": "Walk through adding a new face to the Face Library."
},
"details": {
+ "person": "Person",
"confidence": "Confidence",
"face": "Face Details",
"faceDesc": "Details for the face and associated object",
diff --git a/web/src/pages/FaceLibrary.tsx b/web/src/pages/FaceLibrary.tsx
index a9b7dc230..73fd54c7a 100644
--- a/web/src/pages/FaceLibrary.tsx
+++ b/web/src/pages/FaceLibrary.tsx
@@ -368,6 +368,10 @@ function TrainingGrid({
{t("details.face")}{t("details.faceDesc")}
+
+
{t("details.person")}
+
{selectedEvent?.name}
+
{t("details.confidence")}
From dda7be99ebe65970d2143b1b87db24516c538b61 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Wed, 19 Mar 2025 12:44:07 +0000
Subject: [PATCH 11/97] Face rec: only consider webp files in /faces and
handle_request (#17244)
* Face rec: only consider webp files
* Allow png/jpg/jpeg as well as webp
---
frigate/api/classification.py | 5 ++++-
frigate/data_processing/real_time/face.py | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/frigate/api/classification.py b/frigate/api/classification.py
index df804f34a..b9d714ca3 100644
--- a/frigate/api/classification.py
+++ b/frigate/api/classification.py
@@ -39,7 +39,10 @@ def get_faces():
face_dict[name] = []
for file in sorted(
- os.listdir(face_dir),
+ filter(
+ lambda f: (f.lower().endswith((".webp", ".png", ".jpg", ".jpeg"))),
+ os.listdir(face_dir),
+ ),
key=lambda f: os.path.getctime(os.path.join(face_dir, f)),
reverse=True,
):
diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index acb891449..102913442 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -484,7 +484,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
shutil.move(current_file, new_file)
files = sorted(
- os.listdir(folder),
+ filter(lambda f: (f.endswith(".webp")), os.listdir(folder)),
key=lambda f: os.path.getctime(os.path.join(folder, f)),
reverse=True,
)
From 7f966df5a450b7f6244a24e1890bf10a6e4c7ff1 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Wed, 19 Mar 2025 07:50:36 -0500
Subject: [PATCH 12/97] Nginx fix: make locales public (#17248)
* make locales public
* remove cache
---
docker/main/rootfs/usr/local/nginx/conf/nginx.conf | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/docker/main/rootfs/usr/local/nginx/conf/nginx.conf b/docker/main/rootfs/usr/local/nginx/conf/nginx.conf
index 137d0ec3d..6c60019c7 100644
--- a/docker/main/rootfs/usr/local/nginx/conf/nginx.conf
+++ b/docker/main/rootfs/usr/local/nginx/conf/nginx.conf
@@ -300,6 +300,11 @@ http {
add_header Cache-Control "public";
}
+ location /locales/ {
+ access_log off;
+ add_header Cache-Control "public";
+ }
+
location ~ ^/.*-([A-Za-z0-9]+)\.webmanifest$ {
access_log off;
expires 1y;
From e33fa9659927d732d795632a217adaa033d4a29d Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Wed, 19 Mar 2025 09:02:25 -0600
Subject: [PATCH 13/97] Face recognize api (#17233)
* Add api to run face recognition on image
* Rework save attempts option
* Cleanup mobile object pane buttons
* Adjust api signature
* Remove param
* Cleanup
---
frigate/api/classification.py | 16 ++++++++++++
frigate/comms/embeddings_updater.py | 1 +
frigate/config/classification.py | 4 +--
frigate/data_processing/real_time/face.py | 25 +++++++++++++++++--
frigate/embeddings/__init__.py | 8 ++++++
.../overlay/detail/SearchDetailDialog.tsx | 4 ++-
6 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/frigate/api/classification.py b/frigate/api/classification.py
index b9d714ca3..498158ff2 100644
--- a/frigate/api/classification.py
+++ b/frigate/api/classification.py
@@ -201,6 +201,22 @@ async def register_face(request: Request, name: str, file: UploadFile):
)
+@router.post("/faces/recognize")
+async def recognize_face(request: Request, file: UploadFile):
+ if not request.app.frigate_config.face_recognition.enabled:
+ return JSONResponse(
+ status_code=400,
+ content={"message": "Face recognition is not enabled.", "success": False},
+ )
+
+ context: EmbeddingsContext = request.app.embeddings
+ result = context.recognize_face(await file.read())
+ return JSONResponse(
+ status_code=200 if result.get("success", True) else 400,
+ content=result,
+ )
+
+
@router.post("/faces/{name}/delete", dependencies=[Depends(require_role(["admin"]))])
def deregister_faces(request: Request, name: str, body: dict = None):
if not request.app.frigate_config.face_recognition.enabled:
diff --git a/frigate/comms/embeddings_updater.py b/frigate/comms/embeddings_updater.py
index 61c2331cf..fc35c4665 100644
--- a/frigate/comms/embeddings_updater.py
+++ b/frigate/comms/embeddings_updater.py
@@ -13,6 +13,7 @@ class EmbeddingsRequestEnum(Enum):
embed_description = "embed_description"
embed_thumbnail = "embed_thumbnail"
generate_search = "generate_search"
+ recognize_face = "recognize_face"
register_face = "register_face"
reprocess_face = "reprocess_face"
reprocess_plate = "reprocess_plate"
diff --git a/frigate/config/classification.py b/frigate/config/classification.py
index 30cd12b7c..cbe4880a1 100644
--- a/frigate/config/classification.py
+++ b/frigate/config/classification.py
@@ -70,8 +70,8 @@ class FaceRecognitionConfig(FrigateBaseModel):
min_area: int = Field(
default=500, title="Min area of face box to consider running face recognition."
)
- save_attempts: bool = Field(
- default=True, title="Save images of face detections for training."
+ save_attempts: int = Field(
+ default=100, ge=0, title="Number of face attempts to save in the train tab."
)
blur_confidence_filter: bool = Field(
default=True, title="Apply blur quality filter to face confidence."
diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index 102913442..ac6fa0f80 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -28,7 +28,6 @@ logger = logging.getLogger(__name__)
MAX_DETECTION_HEIGHT = 1080
-MAX_FACE_ATTEMPTS = 100
MIN_MATCHING_FACES = 2
@@ -407,6 +406,28 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
def handle_request(self, topic, request_data) -> dict[str, any] | None:
if topic == EmbeddingsRequestEnum.clear_face_classifier.value:
self.__clear_classifier()
+ elif topic == EmbeddingsRequestEnum.recognize_face.value:
+ img = cv2.imdecode(
+ np.frombuffer(base64.b64decode(request_data["image"]), dtype=np.uint8),
+ cv2.IMREAD_COLOR,
+ )
+
+ # detect faces with lower confidence since we expect the face
+ # to be visible in uploaded images
+ face_box = self.__detect_face(img, 0.5)
+
+ if not face_box:
+ return {"message": "No face was detected.", "success": False}
+
+ face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
+ res = self.__classify_face(face)
+
+ if not res:
+ return {"success": False, "message": "No face was recognized."}
+
+ sub_label, score = res
+
+ return {"success": True, "score": score, "face_name": sub_label}
elif topic == EmbeddingsRequestEnum.register_face.value:
rand_id = "".join(
random.choices(string.ascii_lowercase + string.digits, k=6)
@@ -490,7 +511,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
)
# delete oldest face image if maximum is reached
- if len(files) > MAX_FACE_ATTEMPTS:
+ if len(files) > self.config.face_recognition.save_attempts:
os.unlink(os.path.join(folder, files[-1]))
def expire_object(self, object_id: str):
diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py
index 0a0d7200a..c593a6c0d 100644
--- a/frigate/embeddings/__init__.py
+++ b/frigate/embeddings/__init__.py
@@ -197,6 +197,14 @@ class EmbeddingsContext:
},
)
+ def recognize_face(self, image_data: bytes) -> dict[str, any]:
+ return self.requestor.send_data(
+ EmbeddingsRequestEnum.recognize_face.value,
+ {
+ "image": base64.b64encode(image_data).decode("ASCII"),
+ },
+ )
+
def get_face_ids(self, name: str) -> list[str]:
sql_query = f"""
SELECT
diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx
index b22eb9a4c..afa428eda 100644
--- a/web/src/components/overlay/detail/SearchDetailDialog.tsx
+++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx
@@ -717,7 +717,9 @@ function ObjectDetailsTab({
draggable={false}
src={`${apiHost}api/events/${search.id}/thumbnail.webp`}
/>
-
From eabc316c7b5bfe2532282a2c8a1635329b89a868 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Mon, 24 Mar 2025 11:25:36 -0600
Subject: [PATCH 46/97] Various fixes (#17342)
* Remove imutils
* Ensure that state is maintained when setting search params
* Change script for version of setuptools
* Fix
* Fix
---
.cspell/frigate-dictionary.txt | 1 -
docker/main/Dockerfile | 5 ++++-
docker/main/requirements-wheels.txt | 1 -
docker/rocm/Dockerfile | 1 +
docker/tensorrt/Dockerfile.arm64 | 7 ++++---
frigate/motion/frigate_motion.py | 4 ++--
frigate/motion/improved_motion.py | 4 ++--
frigate/util/image.py | 13 +++++++++++++
web/src/hooks/use-overlay-state.tsx | 5 +++--
9 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/.cspell/frigate-dictionary.txt b/.cspell/frigate-dictionary.txt
index dbab9600e..77e4ede62 100644
--- a/.cspell/frigate-dictionary.txt
+++ b/.cspell/frigate-dictionary.txt
@@ -108,7 +108,6 @@ imagestream
imdecode
imencode
imread
-imutils
imwrite
interp
iostat
diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile
index fb23940f8..a71250813 100644
--- a/docker/main/Dockerfile
+++ b/docker/main/Dockerfile
@@ -78,8 +78,9 @@ COPY docker/main/requirements-ov.txt /requirements-ov.txt
RUN apt-get -qq update \
&& apt-get -qq install -y wget python3 python3-dev python3-distutils gcc pkg-config libhdf5-dev \
&& wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \
&& python3 get-pip.py "pip" \
- && pip install -r /requirements-ov.txt
+ && pip3 install -r /requirements-ov.txt
# Get OpenVino Model
RUN --mount=type=bind,source=docker/main/build_ov_model.py,target=/build_ov_model.py \
@@ -172,6 +173,7 @@ RUN apt-get -qq update \
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \
&& python3 get-pip.py "pip"
COPY docker/main/requirements.txt /requirements.txt
@@ -235,6 +237,7 @@ ENV DEFAULT_FFMPEG_VERSION="7.0"
ENV INCLUDED_FFMPEG_VERSIONS="${DEFAULT_FFMPEG_VERSION}:5.0"
RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \
&& python3 get-pip.py "pip"
RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \
diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt
index 9368cabcd..4ab7e03e6 100644
--- a/docker/main/requirements-wheels.txt
+++ b/docker/main/requirements-wheels.txt
@@ -7,7 +7,6 @@ starlette-context == 0.3.6
fastapi == 0.115.*
uvicorn == 0.30.*
slowapi == 0.1.*
-imutils == 0.5.*
joserfc == 1.0.*
pathvalidate == 3.2.*
markupsafe == 3.0.*
diff --git a/docker/rocm/Dockerfile b/docker/rocm/Dockerfile
index 78f91b96f..d04e93df3 100644
--- a/docker/rocm/Dockerfile
+++ b/docker/rocm/Dockerfile
@@ -39,6 +39,7 @@ WORKDIR /opt/frigate
COPY --from=rootfs / /
RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \
&& python3 get-pip.py "pip" --break-system-packages
RUN python3 -m pip config set global.break-system-packages true
diff --git a/docker/tensorrt/Dockerfile.arm64 b/docker/tensorrt/Dockerfile.arm64
index 5d5d5d939..6e2e0280f 100644
--- a/docker/tensorrt/Dockerfile.arm64
+++ b/docker/tensorrt/Dockerfile.arm64
@@ -9,9 +9,9 @@ ARG DEBIAN_FRONTEND
# Add deadsnakes PPA for python3.11
RUN apt-get -qq update && \
- apt-get -qq install -y --no-install-recommends \
- software-properties-common \
- && add-apt-repository ppa:deadsnakes/ppa
+ apt-get -qq install -y --no-install-recommends \
+ software-properties-common \
+ && add-apt-repository ppa:deadsnakes/ppa
# Use a separate container to build wheels to prevent build dependencies in final image
RUN apt-get -qq update \
@@ -24,6 +24,7 @@ RUN apt-get -qq update \
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
+ && sed -i 's/args.append("setuptools")/args.append("setuptools==77.0.3")/' get-pip.py \
&& python3 get-pip.py "pip"
FROM build-wheels AS trt-wheels
diff --git a/frigate/motion/frigate_motion.py b/frigate/motion/frigate_motion.py
index 72097667b..fd362de34 100644
--- a/frigate/motion/frigate_motion.py
+++ b/frigate/motion/frigate_motion.py
@@ -1,9 +1,9 @@
import cv2
-import imutils
import numpy as np
from frigate.config import MotionConfig
from frigate.motion import MotionDetector
+from frigate.util.image import grab_cv2_contours
class FrigateMotionDetector(MotionDetector):
@@ -103,7 +103,7 @@ class FrigateMotionDetector(MotionDetector):
contours = cv2.findContours(
thresh_dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
)
- contours = imutils.grab_contours(contours)
+ contours = grab_cv2_contours(contours)
# loop over the contours
for c in contours:
diff --git a/frigate/motion/improved_motion.py b/frigate/motion/improved_motion.py
index aae5167a4..69de6d015 100644
--- a/frigate/motion/improved_motion.py
+++ b/frigate/motion/improved_motion.py
@@ -1,7 +1,6 @@
import logging
import cv2
-import imutils
import numpy as np
from scipy.ndimage import gaussian_filter
@@ -9,6 +8,7 @@ from frigate.camera import PTZMetrics
from frigate.comms.config_updater import ConfigSubscriber
from frigate.config import MotionConfig
from frigate.motion import MotionDetector
+from frigate.util.image import grab_cv2_contours
logger = logging.getLogger(__name__)
@@ -147,7 +147,7 @@ class ImprovedMotionDetector(MotionDetector):
contours = cv2.findContours(
thresh_dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
)
- contours = imutils.grab_contours(contours)
+ contours = grab_cv2_contours(contours)
# loop over the contours
total_contour_area = 0
diff --git a/frigate/util/image.py b/frigate/util/image.py
index 0b80efe88..93827747c 100644
--- a/frigate/util/image.py
+++ b/frigate/util/image.py
@@ -265,6 +265,19 @@ def draw_box_with_label(
)
+def grab_cv2_contours(cnts):
+ # if the length the contours tuple returned by cv2.findContours
+ # is '2' then we are using either OpenCV v2.4, v4-beta, or
+ # v4-official
+ if len(cnts) == 2:
+ return cnts[0]
+
+ # if the length of the contours tuple is '3' then we are using
+ # either OpenCV v3, v4-pre, or v4-alpha
+ elif len(cnts) == 3:
+ return cnts[1]
+
+
def is_label_printable(label) -> bool:
"""Check if label is printable."""
return not bool(set(label) - set(printable))
diff --git a/web/src/hooks/use-overlay-state.tsx b/web/src/hooks/use-overlay-state.tsx
index 7a43383d4..5b41ca302 100644
--- a/web/src/hooks/use-overlay-state.tsx
+++ b/web/src/hooks/use-overlay-state.tsx
@@ -109,6 +109,7 @@ export function useSearchEffect(
key: string,
callback: (value: string) => boolean,
) {
+ const location = useLocation();
const [searchParams, setSearchParams] = useSearchParams();
const param = useMemo(() => {
@@ -129,7 +130,7 @@ export function useSearchEffect(
const remove = callback(param[1]);
if (remove) {
- setSearchParams(undefined, { replace: true });
+ setSearchParams(undefined, { state: location.state, replace: true });
}
- }, [param, callback, setSearchParams]);
+ }, [param, location.state, callback, setSearchParams]);
}
From 5c20cf456396e9b2122806211648e3c058053251 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Mon, 24 Mar 2025 17:56:27 +0000
Subject: [PATCH 47/97] Don't require /dev/bus/usb to be present (#17341)
---
docker-compose.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index c8eb765ab..4322dd23c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -23,8 +23,8 @@ services:
# capabilities: [gpu]
environment:
YOLO_MODELS: ""
- devices:
- - /dev/bus/usb:/dev/bus/usb
+ # devices:
+ # - /dev/bus/usb:/dev/bus/usb # Uncomment for Google Coral USB
# - /dev/dri:/dev/dri # for intel hwaccel, needs to be updated for your hardware
volumes:
- .:/workspace/frigate:cached
@@ -32,7 +32,7 @@ services:
- /etc/localtime:/etc/localtime:ro
- ./config:/config
- ./debug:/media/frigate
- - /dev/bus/usb:/dev/bus/usb
+ # - /dev/bus/usb:/dev/bus/usb # Uncomment for Google Coral USB
mqtt:
container_name: mqtt
image: eclipse-mosquitto:1.6
From 6ded9a1bc616aac97991bbc9a2c3f2051034c8df Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Mon, 24 Mar 2025 18:50:57 +0000
Subject: [PATCH 48/97] Update contributing.md (#17343)
---
docs/docs/development/contributing.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/docs/docs/development/contributing.md b/docs/docs/development/contributing.md
index a28320339..72b1af7ea 100644
--- a/docs/docs/development/contributing.md
+++ b/docs/docs/development/contributing.md
@@ -79,6 +79,7 @@ Create and place these files in a `debug` folder in the root of the repo. This i
VS Code will start the Docker Compose file for you and open a terminal window connected to `frigate-dev`.
+- Depending on what hardware you're developing on, you may need to amend `docker-compose.yml` in the project root to pass through a USB Coral or GPU for hardware acceleration.
- Run `python3 -m frigate` to start the backend.
- In a separate terminal window inside VS Code, change into the `web` directory and run `npm install && npm run dev` to start the frontend.
From 7e8b3c389d471fdf6b244339fe8226bdec341d91 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Mon, 24 Mar 2025 17:31:32 -0600
Subject: [PATCH 49/97] Update face_recognition.md (#17349)
---
docs/docs/configuration/face_recognition.md | 27 +++++++++++++++++----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md
index aac1be9b5..472cacada 100644
--- a/docs/docs/configuration/face_recognition.md
+++ b/docs/docs/configuration/face_recognition.md
@@ -74,19 +74,36 @@ The accuracy of face recognition is heavily dependent on the quality of data giv
When choosing images to include in the face training set it is recommended to always follow these recommendations:
- If it is difficult to make out details in a persons face it will not be helpful in training.
-- Avoid images with under/over-exposure.
+- Avoid images with extreme under/over-exposure.
- Avoid blurry / pixelated images.
-- Be careful when uploading images of people when they are wearing clothing that covers a lot of their face as this may confuse the training.
-- Do not upload too many images at the same time, it is recommended to train 4-6 images for each person each day so it is easier to know if the previously added images helped or hurt performance.
+- Be careful when uploading images of people when they are wearing clothing that covers a lot of their face as this may confuse the model.
+- Do not upload too many similar images at the same time, it is recommended to train no more than 4-6 similar images for each person to avoid overfitting.
:::
### Step 1 - Building a Strong Foundation
-When first enabling face recognition it is important to build a foundation of strong images. It is recommended to start by uploading 1-2 photos taken by a smartphone for each person. It is important that the person's face in the photo is straight-on and not turned which will ensure a good starting point.
+When first enabling face recognition it is important to build a foundation of strong images. It is recommended to start by uploading 1-5 "portrait" photos for each person. It is important that the person's face in the photo is straight-on and not turned which will ensure a good starting point.
-Then it is recommended to use the `Face Library` tab in Frigate to select and train images for each person as they are detected. When building a strong foundation it is strongly recommended to only train on images that are straight-on. Ignore images from cameras that recognize faces from an angle. Once a person starts to be consistently recognized correctly on images that are straight-on, it is time to move on to the next step.
+Then it is recommended to use the `Face Library` tab in Frigate to select and train images for each person as they are detected. When building a strong foundation it is strongly recommended to only train on images that are straight-on. Ignore images from cameras that recognize faces from an angle.
+
+Aim to strike a balance between the quality of images while also having a range of conditions (day / night, different weather conditions, different times of day, etc.) in order to have diversity in the images used for each person and not have overfitting.
+
+Once a person starts to be consistently recognized correctly on images that are straight-on, it is time to move on to the next step.
### Step 2 - Expanding The Dataset
Once straight-on images are performing well, start choosing slightly off-angle images to include for training. It is important to still choose images where enough face detail is visible to recognize someone.
+
+## FAQ
+
+### Why is every face tagged as a known face and not unknown?
+
+Any recognized face with a score >= `min_score` will show in the `Train` tab along with the recognition score. A low scoring face is effectively the same as `unknown`, but includes more information. This does not mean the recognition is not working well, and is part of the importance of choosing the correct `recognition_threshold`.
+
+### Why do unknown people score similarly to known people?
+
+This can happen for a few different reasons, but this is usually an indicator that the training set needs to be improved. This is often related to overfitting:
+- If you train with only a few images per person, especially if those images are very similar, the recognition model becomes overly specialized to those specific images.
+- When you provide images with different poses, lighting, and expressions, the algorithm extracts features that are consistent across those variations.
+- By training on a diverse set of images, the algorithm becomes less sensitive to minor variations and noise in the input image.
From 983dd87ffb1f4f73e8fa9e3b098b759698a8c396 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Tue, 25 Mar 2025 11:48:06 +0000
Subject: [PATCH 50/97] Revert go2rtc call checks for enabled camera state
(#17355)
* Revert "Fix camera enabled check (#17331)"
This reverts commit 1e45f63a7ce8768f712a166a5f6a6a95c8e16bfa.
* Revert "Fix webUI generating HTTP500s when camera disabled (#17305)"
This reverts commit 644faaf65b9effbd0b1c6071cfb46209b9a35022.
---
.../settings/CameraStreamingDialog.tsx | 9 +-------
web/src/views/live/LiveCameraView.tsx | 21 +++++++++----------
2 files changed, 11 insertions(+), 19 deletions(-)
diff --git a/web/src/components/settings/CameraStreamingDialog.tsx b/web/src/components/settings/CameraStreamingDialog.tsx
index cb16acaf2..b9bd8a4b6 100644
--- a/web/src/components/settings/CameraStreamingDialog.tsx
+++ b/web/src/components/settings/CameraStreamingDialog.tsx
@@ -1,5 +1,4 @@
import { useState, useCallback, useEffect, useMemo } from "react";
-import { useCameraActivity } from "@/hooks/use-camera-activity";
import { IoIosWarning } from "react-icons/io";
import { Button } from "@/components/ui/button";
import {
@@ -24,7 +23,6 @@ import { Checkbox } from "@/components/ui/checkbox";
import { Label } from "@/components/ui/label";
import {
FrigateConfig,
- CameraConfig,
GroupStreamingSettings,
StreamType,
} from "@/types/frigateConfig";
@@ -65,11 +63,6 @@ export function CameraStreamingDialog({
// metadata
- // camera enabled state
- const { enabled: isCameraEnabled } = useCameraActivity(
- config?.cameras[camera] ?? ({} as CameraConfig),
- );
-
const isRestreamed = useMemo(
() =>
config &&
@@ -78,7 +71,7 @@ export function CameraStreamingDialog({
);
const { data: cameraMetadata } = useSWR(
- isCameraEnabled && isRestreamed ? `go2rtc/streams/${streamName}` : null,
+ isRestreamed ? `go2rtc/streams/${streamName}` : null,
{
revalidateOnFocus: false,
},
diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx
index e067db157..34d61d684 100644
--- a/web/src/views/live/LiveCameraView.tsx
+++ b/web/src/views/live/LiveCameraView.tsx
@@ -112,7 +112,6 @@ import {
SelectTrigger,
} from "@/components/ui/select";
import { usePersistence } from "@/hooks/use-persistence";
-import { useCameraActivity } from "@/hooks/use-camera-activity";
import { Label } from "@/components/ui/label";
import { Switch } from "@/components/ui/switch";
import axios from "axios";
@@ -143,12 +142,8 @@ export default function LiveCameraView({
const [{ width: windowWidth, height: windowHeight }] =
useResizeObserver(window);
- // camera enabled state
- const { enabled: isCameraEnabled } = useCameraActivity(
- config?.cameras[camera.name] ?? ({} as CameraConfig),
- );
-
// supported features
+
const [streamName, setStreamName] = usePersistence(
`${camera.name}-stream`,
Object.values(camera.live.streams)[0],
@@ -162,7 +157,7 @@ export default function LiveCameraView({
);
const { data: cameraMetadata } = useSWR(
- isCameraEnabled && isRestreamed ? `go2rtc/streams/${streamName}` : null,
+ isRestreamed ? `go2rtc/streams/${streamName}` : null,
{
revalidateOnFocus: false,
},
@@ -197,6 +192,10 @@ export default function LiveCameraView({
);
}, [cameraMetadata]);
+ // camera enabled state
+ const { payload: enabledState } = useEnabledState(camera.name);
+ const cameraEnabled = enabledState === "ON";
+
// click overlay for ptzs
const [clickOverlay, setClickOverlay] = useState(false);
@@ -522,7 +521,7 @@ export default function LiveCameraView({
setPip(false);
}
}}
- disabled={!isCameraEnabled}
+ disabled={!cameraEnabled}
/>
)}
{supports2WayTalk && (
@@ -544,7 +543,7 @@ export default function LiveCameraView({
setAudio(true);
}
}}
- disabled={!isCameraEnabled}
+ disabled={!cameraEnabled}
/>
)}
{supportsAudioOutput && preferredLiveMode != "jsmpeg" && (
@@ -561,7 +560,7 @@ export default function LiveCameraView({
t("button.cameraAudio", { ns: "common" })
}
onClick={() => setAudio(!audio)}
- disabled={!isCameraEnabled}
+ disabled={!cameraEnabled}
/>
)}
From 9080305070ff789b6f39feb478354deba948bdcb Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Tue, 25 Mar 2025 08:16:56 -0500
Subject: [PATCH 51/97] Increase the initial stall timeout of the MSE player
(#17359)
---
web/src/components/player/MsePlayer.tsx | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/web/src/components/player/MsePlayer.tsx b/web/src/components/player/MsePlayer.tsx
index 554eb5af1..f3ef17a24 100644
--- a/web/src/components/player/MsePlayer.tsx
+++ b/web/src/components/player/MsePlayer.tsx
@@ -461,6 +461,7 @@ function MSEPlayer({
setBufferTimeout(undefined);
}
+ const timeoutDuration = bufferTime == 0 ? 5000 : 3000;
setBufferTimeout(
setTimeout(() => {
if (
@@ -471,7 +472,7 @@ function MSEPlayer({
onDisconnect();
onError("stalled");
}
- }, 3000),
+ }, timeoutDuration),
);
}
}, [
From e80caabee64d6398d58c86bd2759d9d5d8003b35 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Tue, 25 Mar 2025 11:46:24 -0500
Subject: [PATCH 52/97] Live and go2rtc docs updates (#17363)
---
docs/docs/configuration/live.md | 4 +++-
docs/docs/guides/configuring_go2rtc.md | 4 ++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/docs/docs/configuration/live.md b/docs/docs/configuration/live.md
index cee8b3dca..b5fa0cdd7 100644
--- a/docs/docs/configuration/live.md
+++ b/docs/docs/configuration/live.md
@@ -205,7 +205,9 @@ Note that disabling a camera through the config file (`enabled: False`) removes
When you have go2rtc configured, Live view initially attempts to load and play back your stream with a clearer, fluent stream technology (MSE). An initial timeout, a low bandwidth condition that would cause buffering of the stream, or decoding errors in the stream will cause Frigate to switch to the stream defined by the `detect` role, using the jsmpeg format. This is what the UI labels as "low bandwidth mode". On Live dashboards, the mode will automatically reset when smart streaming is configured and activity stops. Continuous streaming mode does not have an automatic reset mechanism, but you can use the _Reset_ option to force a reload of your stream.
- If you are still experiencing Frigate falling back to low bandwidth mode, you may need to adjust your camera's settings per the (recommendations above)[#camera_settings_recommendations].
+ If you are using continuous streaming or you are loading more than a few high resolution streams at once on the dashboard, your browser may struggle to begin playback of your streams before the timeout. Frigate always prioritizes showing a live stream as quickly as possible, even if it is a lower quality jsmpeg stream. You can use the "Reset" link/button to try loading your high resolution stream again.
+
+ If you are still experiencing Frigate falling back to low bandwidth mode, you may need to adjust your camera's settings per the [recommendations above](#camera_settings_recommendations).
3. **It doesn't seem like my cameras are streaming on the Live dashboard. Why?**
diff --git a/docs/docs/guides/configuring_go2rtc.md b/docs/docs/guides/configuring_go2rtc.md
index 3b8a8af1f..652aa3b26 100644
--- a/docs/docs/guides/configuring_go2rtc.md
+++ b/docs/docs/guides/configuring_go2rtc.md
@@ -115,3 +115,7 @@ section.
1. If the stream you added to go2rtc is also used by Frigate for the `record` or `detect` role, you can migrate your config to pull from the RTSP restream to reduce the number of connections to your camera as shown [here](/configuration/restream#reduce-connections-to-camera).
2. You may also prefer to [setup WebRTC](/configuration/live#webrtc-extra-configuration) for slightly lower latency than MSE. Note that WebRTC only supports h264 and specific audio formats and may require opening ports on your router.
+
+## Important considerations
+
+If you are configuring go2rtc to publish HomeKit camera streams, on pairing the configuration is written to the `/dev/shm/go2rtc.yaml` file inside the container. These changes must be manually copied across to the `go2rtc` section of your Frigate configuration in order to persist through restarts.
From 2c3ea5b74e3e512bce78b138b7f1eb93a3ad0426 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Tue, 25 Mar 2025 15:08:40 -0500
Subject: [PATCH 53/97] Make all object path points clickable (#17367)
---
.../overlay/detail/ObjectLifecycle.tsx | 17 ++++++++++++++---
.../components/overlay/detail/ObjectPath.tsx | 6 ++----
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/web/src/components/overlay/detail/ObjectLifecycle.tsx b/web/src/components/overlay/detail/ObjectLifecycle.tsx
index c359ee91d..0f77ecfbf 100644
--- a/web/src/components/overlay/detail/ObjectLifecycle.tsx
+++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx
@@ -285,9 +285,16 @@ export default function ObjectLifecycle({
useEffect(() => {
if (eventSequence && eventSequence.length > 0) {
- setTimeIndex(eventSequence?.[current].timestamp);
- handleSetBox(eventSequence?.[current].data.box ?? []);
- setLifecycleZones(eventSequence?.[current].data.zones);
+ if (current == -1) {
+ // normal path point
+ setBoxStyle(null);
+ setLifecycleZones([]);
+ } else {
+ // lifecycle point
+ setTimeIndex(eventSequence?.[current].timestamp);
+ handleSetBox(eventSequence?.[current].data.box ?? []);
+ setLifecycleZones(eventSequence?.[current].data.zones);
+ }
setSelectedZone("");
}
}, [current, imgLoaded, handleSetBox, eventSequence]);
@@ -322,6 +329,10 @@ export default function ObjectLifecycle({
mainApi.scrollTo(sequenceIndex);
thumbnailApi.scrollTo(sequenceIndex);
setCurrent(sequenceIndex);
+ } else {
+ // click on a normal path point, not a lifecycle point
+ setCurrent(-1);
+ setTimeIndex(pathPoints[index].timestamp);
}
},
[mainApi, thumbnailApi, eventSequence, pathPoints],
diff --git a/web/src/components/overlay/detail/ObjectPath.tsx b/web/src/components/overlay/detail/ObjectPath.tsx
index 80f454470..f8ee02080 100644
--- a/web/src/components/overlay/detail/ObjectPath.tsx
+++ b/web/src/components/overlay/detail/ObjectPath.tsx
@@ -95,10 +95,8 @@ export function ObjectPath({
fill={getPointColor(color, pos.lifecycle_item?.class_type)}
stroke="white"
strokeWidth={width / 2}
- onClick={() =>
- pos.lifecycle_item && onPointClick && onPointClick(index)
- }
- style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }}
+ onClick={() => onPointClick && onPointClick(index)}
+ style={{ cursor: "pointer" }}
/>
From b18d1fb9709a3d2585c3c3a7c47496bf37d29ca2 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Tue, 25 Mar 2025 18:59:03 -0600
Subject: [PATCH 54/97] Refactor face recognition (#17368)
* Refactor face recognition to allow for running lbph or embedding
* Cleanup
* Use weighted average for faces
* Set correct url
* Cleanup
* Update docs
* Update docs
* Use scipy trimmed mean
* Normalize
* Handle color and gray landmark detection
* Upgrade to new arcface model
* Implement sigmoid function
* Rename
* Rename to arcface
* Fix
* Add face recognition model size to ui config
* Update toast
---
docs/docs/configuration/face_recognition.md | 19 +-
frigate/config/classification.py | 3 +
frigate/data_processing/common/face/model.py | 308 ++++++++++++++++++
frigate/data_processing/real_time/face.py | 252 ++++----------
frigate/embeddings/onnx/base_embedding.py | 2 +
frigate/embeddings/onnx/facenet.py | 98 ++++++
web/public/locales/en/views/settings.json | 16 +-
web/src/types/frigateConfig.ts | 1 +
.../settings/ClassificationSettingsView.tsx | 62 +++-
9 files changed, 573 insertions(+), 188 deletions(-)
create mode 100644 frigate/data_processing/common/face/model.py
create mode 100644 frigate/embeddings/onnx/facenet.py
diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md
index 472cacada..a3cb4e308 100644
--- a/docs/docs/configuration/face_recognition.md
+++ b/docs/docs/configuration/face_recognition.md
@@ -7,7 +7,7 @@ Face recognition identifies known individuals by matching detected faces with pr
## Model Requirements
-Frigate has support for CV2 Local Binary Pattern Face Recognizer to recognize faces, which runs locally. A lightweight face landmark detection model is also used to align faces before running them through the face recognizer.
+### Face Detection
Users running a Frigate+ model (or any custom model that natively detects faces) should ensure that `face` is added to the [list of objects to track](../plus/#available-label-types) either globally or for a specific camera. This will allow face detection to run at the same time as object detection and be more efficient.
@@ -19,9 +19,19 @@ Frigate needs to first detect a `face` before it can recognize a face.
:::
+### Face Recognition
+
+Frigate has support for two face recognition model types:
+- **small**: Frigate will use CV2 Local Binary Pattern Face Recognizer to recognize faces, which runs locally on the CPU.
+- **large**: Frigate will run a face embedding model, this is only recommended to be run when an integrated or dedicated GPU is available.
+
+In both cases a lightweight face landmark detection model is also used to align faces before running them through the face recognizer.
+
## Minimum System Requirements
-Face recognition is lightweight and runs on the CPU, there are no significantly different system requirements than running Frigate itself.
+Face recognition is lightweight and runs on the CPU, there are no significantly different system requirements than running Frigate itself when using the `small` model.
+
+When using the `large` model an integrated or discrete GPU is recommended.
## Configuration
@@ -47,6 +57,7 @@ Fine-tune face recognition with these optional parameters:
### Recognition
+- `model_size`: Which model size to use, options are `small` or `large`
- `recognition_threshold`: Recognition confidence score required to add the face to the object as a sub label.
- Default: `0.9`.
- `blur_confidence_filter`: Enables a filter that calculates how blurry the face is and adjusts the confidence based on this.
@@ -107,3 +118,7 @@ This can happen for a few different reasons, but this is usually an indicator th
- If you train with only a few images per person, especially if those images are very similar, the recognition model becomes overly specialized to those specific images.
- When you provide images with different poses, lighting, and expressions, the algorithm extracts features that are consistent across those variations.
- By training on a diverse set of images, the algorithm becomes less sensitive to minor variations and noise in the input image.
+
+### I see scores above the threshold in the train tab, but a sub label wasn't assigned?
+
+The Frigate face recognizer collects face recognition scores from all of the frames across the person objects lifecycle. The scores are continually weighted based on the area of the face, and a sub label will only be assigned to person if there is a prominent person recognized. This avoids cases where a single high confidence recognition result would throw off the results.
diff --git a/frigate/config/classification.py b/frigate/config/classification.py
index 0070569a8..25c379546 100644
--- a/frigate/config/classification.py
+++ b/frigate/config/classification.py
@@ -51,6 +51,9 @@ class SemanticSearchConfig(FrigateBaseModel):
class FaceRecognitionConfig(FrigateBaseModel):
enabled: bool = Field(default=False, title="Enable face recognition.")
+ model_size: str = Field(
+ default="small", title="The size of the embeddings model used."
+ )
min_score: float = Field(
title="Minimum face distance score required to save the attempt.",
default=0.8,
diff --git a/frigate/data_processing/common/face/model.py b/frigate/data_processing/common/face/model.py
new file mode 100644
index 000000000..a006734ce
--- /dev/null
+++ b/frigate/data_processing/common/face/model.py
@@ -0,0 +1,308 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+
+import cv2
+import numpy as np
+from scipy import stats
+
+from frigate.config import FrigateConfig
+from frigate.const import MODEL_CACHE_DIR
+from frigate.embeddings.onnx.facenet import ArcfaceEmbedding
+
+logger = logging.getLogger(__name__)
+
+
+class FaceRecognizer(ABC):
+ """Face recognition runner."""
+
+ def __init__(self, config: FrigateConfig) -> None:
+ self.config = config
+ self.landmark_detector = cv2.face.createFacemarkLBF()
+ self.landmark_detector.loadModel(
+ os.path.join(MODEL_CACHE_DIR, "facedet/landmarkdet.yaml")
+ )
+
+ @abstractmethod
+ def build(self) -> None:
+ """Build face recognition model."""
+ pass
+
+ @abstractmethod
+ def clear(self) -> None:
+ """Clear current built model."""
+ pass
+
+ @abstractmethod
+ def classify(self, face_image: np.ndarray) -> tuple[str, float] | None:
+ pass
+
+ def align_face(
+ self,
+ image: np.ndarray,
+ output_width: int,
+ output_height: int,
+ ) -> np.ndarray:
+ # landmark is run on grayscale images
+
+ if image.ndim == 3:
+ land_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+ else:
+ land_image = image
+
+ _, lands = self.landmark_detector.fit(
+ land_image, np.array([(0, 0, land_image.shape[1], land_image.shape[0])])
+ )
+ landmarks: np.ndarray = lands[0][0]
+
+ # get landmarks for eyes
+ leftEyePts = landmarks[42:48]
+ rightEyePts = landmarks[36:42]
+
+ # compute the center of mass for each eye
+ leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
+ rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
+
+ # compute the angle between the eye centroids
+ dY = rightEyeCenter[1] - leftEyeCenter[1]
+ dX = rightEyeCenter[0] - leftEyeCenter[0]
+ angle = np.degrees(np.arctan2(dY, dX)) - 180
+
+ # compute the desired right eye x-coordinate based on the
+ # desired x-coordinate of the left eye
+ desiredRightEyeX = 1.0 - 0.35
+
+ # determine the scale of the new resulting image by taking
+ # the ratio of the distance between eyes in the *current*
+ # image to the ratio of distance between eyes in the
+ # *desired* image
+ dist = np.sqrt((dX**2) + (dY**2))
+ desiredDist = desiredRightEyeX - 0.35
+ desiredDist *= output_width
+ scale = desiredDist / dist
+
+ # compute center (x, y)-coordinates (i.e., the median point)
+ # between the two eyes in the input image
+ # grab the rotation matrix for rotating and scaling the face
+ eyesCenter = (
+ int((leftEyeCenter[0] + rightEyeCenter[0]) // 2),
+ int((leftEyeCenter[1] + rightEyeCenter[1]) // 2),
+ )
+ M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
+
+ # update the translation component of the matrix
+ tX = output_width * 0.5
+ tY = output_height * 0.35
+ M[0, 2] += tX - eyesCenter[0]
+ M[1, 2] += tY - eyesCenter[1]
+
+ # apply the affine transformation
+ return cv2.warpAffine(
+ image, M, (output_width, output_height), flags=cv2.INTER_CUBIC
+ )
+
+ def get_blur_factor(self, input: np.ndarray) -> float:
+ """Calculates the factor for the confidence based on the blur of the image."""
+ if not self.config.face_recognition.blur_confidence_filter:
+ return 1.0
+
+ variance = cv2.Laplacian(input, cv2.CV_64F).var()
+
+ if variance < 60: # image is very blurry
+ return 0.96
+ elif variance < 70: # image moderately blurry
+ return 0.98
+ elif variance < 80: # image is slightly blurry
+ return 0.99
+ else:
+ return 1.0
+
+
+class LBPHRecognizer(FaceRecognizer):
+ def __init__(self, config: FrigateConfig):
+ super().__init__(config)
+ self.label_map: dict[int, str] = {}
+ self.recognizer: cv2.face.LBPHFaceRecognizer | None = None
+
+ def clear(self) -> None:
+ self.face_recognizer = None
+ self.label_map = {}
+
+ def build(self):
+ if not self.landmark_detector:
+ return None
+
+ labels = []
+ faces = []
+ idx = 0
+
+ dir = "/media/frigate/clips/faces"
+ for name in os.listdir(dir):
+ if name == "train":
+ continue
+
+ face_folder = os.path.join(dir, name)
+
+ if not os.path.isdir(face_folder):
+ continue
+
+ self.label_map[idx] = name
+ for image in os.listdir(face_folder):
+ img = cv2.imread(os.path.join(face_folder, image))
+
+ if img is None:
+ continue
+
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+ img = self.align_face(img, img.shape[1], img.shape[0])
+ faces.append(img)
+ labels.append(idx)
+
+ idx += 1
+
+ if not faces:
+ return
+
+ self.recognizer: cv2.face.LBPHFaceRecognizer = (
+ cv2.face.LBPHFaceRecognizer_create(
+ radius=2, threshold=(1 - self.config.face_recognition.min_score) * 1000
+ )
+ )
+ self.recognizer.train(faces, np.array(labels))
+
+ def classify(self, face_image: np.ndarray) -> tuple[str, float] | None:
+ if not self.landmark_detector:
+ return None
+
+ if not self.label_map or not self.recognizer:
+ self.build()
+
+ if not self.recognizer:
+ return None
+
+ # face recognition is best run on grayscale images
+ img = cv2.cvtColor(face_image, cv2.COLOR_BGR2GRAY)
+
+ # get blur factor before aligning face
+ blur_factor = self.get_blur_factor(img)
+ logger.debug(f"face detected with bluriness {blur_factor}")
+
+ # align face and run recognition
+ img = self.align_face(img, img.shape[1], img.shape[0])
+ index, distance = self.recognizer.predict(img)
+
+ if index == -1:
+ return None
+
+ score = (1.0 - (distance / 1000)) * blur_factor
+ return self.label_map[index], round(score, 2)
+
+
+class ArcFaceRecognizer(FaceRecognizer):
+ def __init__(self, config: FrigateConfig):
+ super().__init__(config)
+ self.mean_embs: dict[int, np.ndarray] = {}
+ self.face_embedder: ArcfaceEmbedding = ArcfaceEmbedding()
+
+ def clear(self) -> None:
+ self.mean_embs = {}
+
+ def build(self):
+ if not self.landmark_detector:
+ return None
+
+ face_embeddings_map: dict[str, list[np.ndarray]] = {}
+ idx = 0
+
+ dir = "/media/frigate/clips/faces"
+ for name in os.listdir(dir):
+ if name == "train":
+ continue
+
+ face_folder = os.path.join(dir, name)
+
+ if not os.path.isdir(face_folder):
+ continue
+
+ face_embeddings_map[name] = []
+ for image in os.listdir(face_folder):
+ img = cv2.imread(os.path.join(face_folder, image))
+
+ if img is None:
+ continue
+
+ img = self.align_face(img, img.shape[1], img.shape[0])
+ emb = self.face_embedder([img])[0].squeeze()
+ face_embeddings_map[name].append(emb)
+
+ idx += 1
+
+ if not face_embeddings_map:
+ return
+
+ for name, embs in face_embeddings_map.items():
+ self.mean_embs[name] = stats.trim_mean(embs, 0.15)
+
+ def similarity_to_confidence(
+ self, cosine_similarity: float, median=0.3, range_width=0.6, slope_factor=12
+ ):
+ """
+ Default sigmoid function to map cosine similarity to confidence.
+
+ Args:
+ cosine_similarity (float): The input cosine similarity.
+ median (float): Assumed median of cosine similarity distribution.
+ range_width (float): Assumed range of cosine similarity distribution (90th percentile - 10th percentile).
+ slope_factor (float): Adjusts the steepness of the curve.
+
+ Returns:
+ float: The confidence score.
+ """
+
+ # Calculate slope and bias
+ slope = slope_factor / range_width
+ bias = median
+
+ # Calculate confidence
+ confidence = 1 / (1 + np.exp(-slope * (cosine_similarity - bias)))
+ return confidence
+
+ def classify(self, face_image):
+ if not self.landmark_detector:
+ return None
+
+ if not self.mean_embs:
+ self.build()
+
+ if not self.mean_embs:
+ return None
+
+ # face recognition is best run on grayscale images
+
+ # get blur factor before aligning face
+ blur_factor = self.get_blur_factor(face_image)
+ logger.debug(f"face detected with bluriness {blur_factor}")
+
+ # align face and run recognition
+ img = self.align_face(face_image, face_image.shape[1], face_image.shape[0])
+ embedding = self.face_embedder([img])[0].squeeze()
+
+ score = 0
+ label = ""
+
+ for name, mean_emb in self.mean_embs.items():
+ dot_product = np.dot(embedding, mean_emb)
+ magnitude_A = np.linalg.norm(embedding)
+ magnitude_B = np.linalg.norm(mean_emb)
+
+ cosine_similarity = dot_product / (magnitude_A * magnitude_B)
+ confidence = self.similarity_to_confidence(cosine_similarity)
+
+ if cosine_similarity > score:
+ score = confidence
+ label = name
+
+ if score < self.config.face_recognition.min_score:
+ return None
+
+ return label, round(score * blur_factor, 2)
diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index 7b49a2f47..e20dad633 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -19,6 +19,11 @@ from frigate.comms.event_metadata_updater import (
)
from frigate.config import FrigateConfig
from frigate.const import FACE_DIR, MODEL_CACHE_DIR
+from frigate.data_processing.common.face.model import (
+ ArcFaceRecognizer,
+ FaceRecognizer,
+ LBPHRecognizer,
+)
from frigate.util.image import area
from ..types import DataProcessorMetrics
@@ -31,6 +36,36 @@ MAX_DETECTION_HEIGHT = 1080
MIN_MATCHING_FACES = 2
+def weighted_average_by_area(results_list: list[tuple[str, float, int]]):
+ if len(results_list) < 3:
+ return "unknown", 0.0
+
+ score_count = {}
+ weighted_scores = {}
+ total_face_areas = {}
+
+ for name, score, face_area in results_list:
+ if name not in weighted_scores:
+ score_count[name] = 1
+ weighted_scores[name] = 0.0
+ total_face_areas[name] = 0.0
+ else:
+ score_count[name] += 1
+
+ weighted_scores[name] += score * face_area
+ total_face_areas[name] += face_area
+
+ prominent_name = max(score_count)
+
+ # if a single name is not prominent in the history then we are not confident
+ if score_count[prominent_name] / len(results_list) < 0.65:
+ return "unknown", 0.0
+
+ return prominent_name, weighted_scores[prominent_name] / total_face_areas[
+ prominent_name
+ ]
+
+
class FaceRealTimeProcessor(RealTimeProcessorApi):
def __init__(
self,
@@ -42,10 +77,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
self.face_config = config.face_recognition
self.sub_label_publisher = sub_label_publisher
self.face_detector: cv2.FaceDetectorYN = None
- self.landmark_detector: cv2.face.FacemarkLBF = None
- self.recognizer: cv2.face.LBPHFaceRecognizer = None
self.requires_face_detection = "face" not in self.config.objects.all_objects
- self.detected_faces: dict[str, float] = {}
+ self.person_face_history: dict[str, list[tuple[str, float, int]]] = {}
+ self.recognizer: FaceRecognizer | None = None
download_path = os.path.join(MODEL_CACHE_DIR, "facedet")
self.model_files = {
@@ -72,7 +106,13 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
self.__build_detector()
self.label_map: dict[int, str] = {}
- self.__build_classifier()
+
+ if self.face_config.model_size == "small":
+ self.recognizer = LBPHRecognizer(self.config)
+ else:
+ self.recognizer = ArcFaceRecognizer(self.config)
+
+ self.recognizer.build()
def __download_models(self, path: str) -> None:
try:
@@ -92,126 +132,6 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
score_threshold=0.5,
nms_threshold=0.3,
)
- self.landmark_detector = cv2.face.createFacemarkLBF()
- self.landmark_detector.loadModel(
- os.path.join(MODEL_CACHE_DIR, "facedet/landmarkdet.yaml")
- )
-
- def __build_classifier(self) -> None:
- if not self.landmark_detector:
- return None
-
- labels = []
- faces = []
-
- dir = "/media/frigate/clips/faces"
- for idx, name in enumerate(os.listdir(dir)):
- if name == "train":
- continue
-
- face_folder = os.path.join(dir, name)
-
- if not os.path.isdir(face_folder):
- continue
-
- self.label_map[idx] = name
- for image in os.listdir(face_folder):
- img = cv2.imread(os.path.join(face_folder, image))
-
- if img is None:
- continue
-
- img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
- img = self.__align_face(img, img.shape[1], img.shape[0])
- faces.append(img)
- labels.append(idx)
-
- if not faces:
- return
-
- self.recognizer: cv2.face.LBPHFaceRecognizer = (
- cv2.face.LBPHFaceRecognizer_create(
- radius=2, threshold=(1 - self.face_config.min_score) * 1000
- )
- )
- self.recognizer.train(faces, np.array(labels))
-
- def __align_face(
- self,
- image: np.ndarray,
- output_width: int,
- output_height: int,
- ) -> np.ndarray:
- _, lands = self.landmark_detector.fit(
- image, np.array([(0, 0, image.shape[1], image.shape[0])])
- )
- landmarks: np.ndarray = lands[0][0]
-
- # get landmarks for eyes
- leftEyePts = landmarks[42:48]
- rightEyePts = landmarks[36:42]
-
- # compute the center of mass for each eye
- leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
- rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
-
- # compute the angle between the eye centroids
- dY = rightEyeCenter[1] - leftEyeCenter[1]
- dX = rightEyeCenter[0] - leftEyeCenter[0]
- angle = np.degrees(np.arctan2(dY, dX)) - 180
-
- # compute the desired right eye x-coordinate based on the
- # desired x-coordinate of the left eye
- desiredRightEyeX = 1.0 - 0.35
-
- # determine the scale of the new resulting image by taking
- # the ratio of the distance between eyes in the *current*
- # image to the ratio of distance between eyes in the
- # *desired* image
- dist = np.sqrt((dX**2) + (dY**2))
- desiredDist = desiredRightEyeX - 0.35
- desiredDist *= output_width
- scale = desiredDist / dist
-
- # compute center (x, y)-coordinates (i.e., the median point)
- # between the two eyes in the input image
- # grab the rotation matrix for rotating and scaling the face
- eyesCenter = (
- int((leftEyeCenter[0] + rightEyeCenter[0]) // 2),
- int((leftEyeCenter[1] + rightEyeCenter[1]) // 2),
- )
- M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
-
- # update the translation component of the matrix
- tX = output_width * 0.5
- tY = output_height * 0.35
- M[0, 2] += tX - eyesCenter[0]
- M[1, 2] += tY - eyesCenter[1]
-
- # apply the affine transformation
- return cv2.warpAffine(
- image, M, (output_width, output_height), flags=cv2.INTER_CUBIC
- )
-
- def __get_blur_factor(self, input: np.ndarray) -> float:
- """Calculates the factor for the confidence based on the blur of the image."""
- if not self.face_config.blur_confidence_filter:
- return 1.0
-
- variance = cv2.Laplacian(input, cv2.CV_64F).var()
-
- if variance < 60: # image is very blurry
- return 0.96
- elif variance < 70: # image moderately blurry
- return 0.98
- elif variance < 80: # image is slightly blurry
- return 0.99
- else:
- return 1.0
-
- def __clear_classifier(self) -> None:
- self.face_recognizer = None
- self.label_map = {}
def __detect_face(
self, input: np.ndarray, threshold: float
@@ -254,33 +174,6 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
return face
- def __classify_face(self, face_image: np.ndarray) -> tuple[str, float] | None:
- if not self.landmark_detector:
- return None
-
- if not self.label_map or not self.recognizer:
- self.__build_classifier()
-
- if not self.recognizer:
- return None
-
- # face recognition is best run on grayscale images
- img = cv2.cvtColor(face_image, cv2.COLOR_BGR2GRAY)
-
- # get blur factor before aligning face
- blur_factor = self.__get_blur_factor(img)
- logger.debug(f"face detected with bluriness {blur_factor}")
-
- # align face and run recognition
- img = self.__align_face(img, img.shape[1], img.shape[0])
- index, distance = self.recognizer.predict(img)
-
- if index == -1:
- return None
-
- score = (1.0 - (distance / 1000)) * blur_factor
- return self.label_map[index], round(score, 2)
-
def __update_metrics(self, duration: float) -> None:
self.metrics.face_rec_fps.value = (
self.metrics.face_rec_fps.value * 9 + duration
@@ -301,7 +194,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
# don't overwrite sub label for objects that have a sub label
# that is not a face
- if obj_data.get("sub_label") and id not in self.detected_faces:
+ if obj_data.get("sub_label") and id not in self.person_face_history:
logger.debug(
f"Not processing face due to existing sub label: {obj_data.get('sub_label')}."
)
@@ -370,53 +263,46 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
]
- res = self.__classify_face(face_frame)
+ res = self.recognizer.classify(face_frame)
if not res:
+ self.__update_metrics(datetime.datetime.now().timestamp() - start)
return
sub_label, score = res
- # calculate the overall face score as the probability * area of face
- # this will help to reduce false positives from small side-angle faces
- # if a large front-on face image may have scored slightly lower but
- # is more likely to be accurate due to the larger face area
- face_score = round(score * face_frame.shape[0] * face_frame.shape[1], 2)
-
logger.debug(
- f"Detected best face for person as: {sub_label} with probability {score} and overall face score {face_score}"
+ f"Detected best face for person as: {sub_label} with probability {score}"
)
if self.config.face_recognition.save_attempts:
# write face to library
folder = os.path.join(FACE_DIR, "train")
- file = os.path.join(folder, f"{id}-{sub_label}-{score}-{face_score}.webp")
+ file = os.path.join(folder, f"{id}-{sub_label}-{score}-0.webp")
os.makedirs(folder, exist_ok=True)
cv2.imwrite(file, face_frame)
- if score < self.config.face_recognition.recognition_threshold:
- logger.debug(
- f"Recognized face distance {score} is less than threshold {self.config.face_recognition.recognition_threshold}"
- )
- self.__update_metrics(datetime.datetime.now().timestamp() - start)
- return
+ if id not in self.person_face_history:
+ self.person_face_history[id] = []
- if id in self.detected_faces and face_score <= self.detected_faces[id]:
- logger.debug(
- f"Recognized face distance {score} and overall score {face_score} is less than previous overall face score ({self.detected_faces.get(id)})."
- )
- self.__update_metrics(datetime.datetime.now().timestamp() - start)
- return
-
- self.sub_label_publisher.publish(
- EventMetadataTypeEnum.sub_label, (id, sub_label, score)
+ self.person_face_history[id].append(
+ (sub_label, score, face_frame.shape[0] * face_frame.shape[1])
)
- self.detected_faces[id] = face_score
+ (weighted_sub_label, weighted_score) = weighted_average_by_area(
+ self.person_face_history[id]
+ )
+
+ if weighted_score >= self.face_config.recognition_threshold:
+ self.sub_label_publisher.publish(
+ EventMetadataTypeEnum.sub_label,
+ (id, weighted_sub_label, weighted_score),
+ )
+
self.__update_metrics(datetime.datetime.now().timestamp() - start)
def handle_request(self, topic, request_data) -> dict[str, any] | None:
if topic == EmbeddingsRequestEnum.clear_face_classifier.value:
- self.__clear_classifier()
+ self.recognizer.clear()
elif topic == EmbeddingsRequestEnum.recognize_face.value:
img = cv2.imdecode(
np.frombuffer(base64.b64decode(request_data["image"]), dtype=np.uint8),
@@ -431,7 +317,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
return {"message": "No face was detected.", "success": False}
face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
- res = self.__classify_face(face)
+ res = self.recognizer.classify(face)
if not res:
return {"success": False, "message": "No face was recognized."}
@@ -480,7 +366,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
with open(file, "wb") as output:
output.write(thumbnail.tobytes())
- self.__clear_classifier()
+ self.recognizer.clear()
return {
"message": "Successfully registered face.",
"success": True,
@@ -500,7 +386,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
"success": False,
}
- res = self.__classify_face(img)
+ res = self.recognizer.classify(img)
if not res:
return
@@ -527,5 +413,5 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
os.unlink(os.path.join(folder, files[-1]))
def expire_object(self, object_id: str):
- if object_id in self.detected_faces:
- self.detected_faces.pop(object_id)
+ if object_id in self.person_face_history:
+ self.person_face_history.pop(object_id)
diff --git a/frigate/embeddings/onnx/base_embedding.py b/frigate/embeddings/onnx/base_embedding.py
index a2ea92674..7403f0ac1 100644
--- a/frigate/embeddings/onnx/base_embedding.py
+++ b/frigate/embeddings/onnx/base_embedding.py
@@ -69,6 +69,8 @@ class BaseEmbedding(ABC):
image = Image.open(BytesIO(response.content)).convert(output)
elif isinstance(image, bytes):
image = Image.open(BytesIO(image)).convert(output)
+ elif isinstance(image, np.ndarray):
+ image = Image.fromarray(image)
return image
diff --git a/frigate/embeddings/onnx/facenet.py b/frigate/embeddings/onnx/facenet.py
new file mode 100644
index 000000000..3439620a0
--- /dev/null
+++ b/frigate/embeddings/onnx/facenet.py
@@ -0,0 +1,98 @@
+"""Facenet Embeddings."""
+
+import logging
+import os
+
+import numpy as np
+
+from frigate.const import MODEL_CACHE_DIR
+from frigate.util.downloader import ModelDownloader
+
+from .base_embedding import BaseEmbedding
+from .runner import ONNXModelRunner
+
+logger = logging.getLogger(__name__)
+
+FACE_EMBEDDING_SIZE = 112
+
+
+class ArcfaceEmbedding(BaseEmbedding):
+ def __init__(
+ self,
+ device: str = "AUTO",
+ ):
+ super().__init__(
+ model_name="facedet",
+ model_file="arcface.onnx",
+ download_urls={
+ "arcface.onnx": "https://github.com/NickM-27/facenet-onnx/releases/download/v1.0/arcface.onnx",
+ },
+ )
+ self.device = device
+ self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)
+ self.tokenizer = None
+ self.feature_extractor = None
+ self.runner = None
+ files_names = list(self.download_urls.keys())
+
+ if not all(
+ os.path.exists(os.path.join(self.download_path, n)) for n in files_names
+ ):
+ logger.debug(f"starting model download for {self.model_name}")
+ self.downloader = ModelDownloader(
+ model_name=self.model_name,
+ download_path=self.download_path,
+ file_names=files_names,
+ download_func=self._download_model,
+ )
+ self.downloader.ensure_model_files()
+ else:
+ self.downloader = None
+ self._load_model_and_utils()
+ logger.debug(f"models are already downloaded for {self.model_name}")
+
+ def _load_model_and_utils(self):
+ if self.runner is None:
+ if self.downloader:
+ self.downloader.wait_for_download()
+
+ self.runner = ONNXModelRunner(
+ os.path.join(self.download_path, self.model_file),
+ self.device,
+ )
+
+ def _preprocess_inputs(self, raw_inputs):
+ pil = self._process_image(raw_inputs[0])
+
+ # handle images larger than input size
+ width, height = pil.size
+ if width != FACE_EMBEDDING_SIZE or height != FACE_EMBEDDING_SIZE:
+ if width > height:
+ new_height = int(((height / width) * FACE_EMBEDDING_SIZE) // 4 * 4)
+ pil = pil.resize((FACE_EMBEDDING_SIZE, new_height))
+ else:
+ new_width = int(((width / height) * FACE_EMBEDDING_SIZE) // 4 * 4)
+ pil = pil.resize((new_width, FACE_EMBEDDING_SIZE))
+
+ og = np.array(pil).astype(np.float32)
+
+ # Image must be FACE_EMBEDDING_SIZExFACE_EMBEDDING_SIZE
+ og_h, og_w, channels = og.shape
+ frame = np.zeros(
+ (FACE_EMBEDDING_SIZE, FACE_EMBEDDING_SIZE, channels), dtype=np.float32
+ )
+
+ # compute center offset
+ x_center = (FACE_EMBEDDING_SIZE - og_w) // 2
+ y_center = (FACE_EMBEDDING_SIZE - og_h) // 2
+
+ # copy img image into center of result image
+ frame[y_center : y_center + og_h, x_center : x_center + og_w] = og
+
+ # run arcface normalization
+ normalized_image = frame.astype(np.float32) / 255.0
+ frame = (normalized_image - 0.5) / 0.5
+
+ frame = np.transpose(frame, (2, 0, 1))
+ frame = np.expand_dims(frame, axis=0)
+ return [{"data": frame}]
diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json
index d642c12e6..ffb2434ff 100644
--- a/web/public/locales/en/views/settings.json
+++ b/web/public/locales/en/views/settings.json
@@ -107,7 +107,19 @@
"faceRecognition": {
"title": "Face Recognition",
"desc": "Face recognition allows people to be assigned names and when their face is recognized Frigate will assign the person's name as a sub label. This information is included in the UI, filters, as well as in notifications.",
- "readTheDocumentation": "Read the Documentation"
+ "readTheDocumentation": "Read the Documentation",
+ "modelSize": {
+ "label": "Model Size",
+ "desc": "The size of the model used for face recognition.",
+ "small": {
+ "title": "small",
+ "desc": "Using small employs a Local Binary Pattern Histogram model via OpenCV that runs efficiently on most CPUs."
+ },
+ "large": {
+ "title": "large",
+ "desc": "Using large employs an ArcFace Face embedding model and will automatically run on the GPU if applicable."
+ }
+ }
},
"licensePlateRecognition": {
"title": "License Plate Recognition",
@@ -115,7 +127,7 @@
"readTheDocumentation": "Read the Documentation"
},
"toast": {
- "success": "Classification settings have been saved.",
+ "success": "Classification settings have been saved. Restart Frigate to apply your changes.",
"error": "Failed to save config changes: {{errorMessage}}"
}
},
diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts
index 947c5cde9..5312bed8c 100644
--- a/web/src/types/frigateConfig.ts
+++ b/web/src/types/frigateConfig.ts
@@ -333,6 +333,7 @@ export interface FrigateConfig {
face_recognition: {
enabled: boolean;
+ model_size: SearchModelSize;
detection_threshold: number;
recognition_threshold: number;
};
diff --git a/web/src/views/settings/ClassificationSettingsView.tsx b/web/src/views/settings/ClassificationSettingsView.tsx
index 07926182b..2ce958f9e 100644
--- a/web/src/views/settings/ClassificationSettingsView.tsx
+++ b/web/src/views/settings/ClassificationSettingsView.tsx
@@ -30,6 +30,7 @@ type ClassificationSettings = {
};
face: {
enabled?: boolean;
+ model_size?: SearchModelSize;
};
lpr: {
enabled?: boolean;
@@ -59,6 +60,7 @@ export default function ClassificationSettingsView({
},
face: {
enabled: undefined,
+ model_size: undefined,
},
lpr: {
enabled: undefined,
@@ -74,6 +76,7 @@ export default function ClassificationSettingsView({
},
face: {
enabled: undefined,
+ model_size: undefined,
},
lpr: {
enabled: undefined,
@@ -91,6 +94,7 @@ export default function ClassificationSettingsView({
},
face: {
enabled: config.face_recognition.enabled,
+ model_size: config.face_recognition.model_size,
},
lpr: {
enabled: config.lpr.enabled,
@@ -106,6 +110,7 @@ export default function ClassificationSettingsView({
},
face: {
enabled: config.face_recognition.enabled,
+ model_size: config.face_recognition.model_size,
},
lpr: {
enabled: config.lpr.enabled,
@@ -136,7 +141,7 @@ export default function ClassificationSettingsView({
axios
.put(
- `config/set?semantic_search.enabled=${classificationSettings.search.enabled ? "True" : "False"}&semantic_search.reindex=${classificationSettings.search.reindex ? "True" : "False"}&semantic_search.model_size=${classificationSettings.search.model_size}&face_recognition.enabled=${classificationSettings.face.enabled ? "True" : "False"}&lpr.enabled=${classificationSettings.lpr.enabled ? "True" : "False"}`,
+ `config/set?semantic_search.enabled=${classificationSettings.search.enabled ? "True" : "False"}&semantic_search.reindex=${classificationSettings.search.reindex ? "True" : "False"}&semantic_search.model_size=${classificationSettings.search.model_size}&face_recognition.enabled=${classificationSettings.face.enabled ? "True" : "False"}&face_recognition.model_size=${classificationSettings.face.model_size}&lpr.enabled=${classificationSettings.lpr.enabled ? "True" : "False"}`,
{
requires_restart: 0,
},
@@ -384,6 +389,61 @@ export default function ClassificationSettingsView({
From bfee030d7b40efc559fba0b442152db6d19ceec2 Mon Sep 17 00:00:00 2001
From: GuoQing Liu <842607283@qq.com>
Date: Wed, 26 Mar 2025 18:57:56 +0800
Subject: [PATCH 55/97] add model chinese i18n keys (#17379)
---
web/public/locales/zh-CN/views/settings.json | 44 +++++++++++++++-----
1 file changed, 33 insertions(+), 11 deletions(-)
diff --git a/web/public/locales/zh-CN/views/settings.json b/web/public/locales/zh-CN/views/settings.json
index ccef8f151..4b39daf7c 100644
--- a/web/public/locales/zh-CN/views/settings.json
+++ b/web/public/locales/zh-CN/views/settings.json
@@ -7,13 +7,8 @@
"masksAndZones": "遮罩和区域编辑器 - Frigate",
"motionTuner": "运动调整器 - Frigate",
"object": "对象设置 - Frigate",
- "general": "常规设置 - Frigate"
- },
- "dialog": {
- "unsavedChanges": {
- "title": "你有未保存的更改。",
- "desc": "是否要在继续之前保存更改？"
- }
+ "general": "常规设置 - Frigate",
+ "frigatePlus": "Frigate+ 设置 - Frigate"
},
"menu": {
"uiSettings": "界面设置",
@@ -23,7 +18,14 @@
"motionTuner": "运动调整器",
"debug": "调试",
"users": "用户",
- "notifications": "通知"
+ "notifications": "通知",
+ "frigateplus": "Frigate+"
+ },
+ "dialog": {
+ "unsavedChanges": {
+ "title": "你有未保存的更改。",
+ "desc": "是否要在继续之前保存更改？"
+ }
},
"cameraSetting": {
"camera": "摄像头",
@@ -105,7 +107,19 @@
"faceRecognition": {
"title": "人脸识别",
"desc": "人脸识别功能允许为人物分配名称，当识别到他们的面孔时，Frigate 会将人物的名字作为子标签进行分配。这些信息会显示在界面、过滤器以及通知中。",
- "readTheDocumentation": "阅读文档（英文）"
+ "readTheDocumentation": "阅读文档（英文）",
+ "modelSize": {
+ "label": "模型大小",
+ "desc": "用于人脸识别的模型尺寸。",
+ "small": {
+ "title": "小模型",
+ "desc": "使用小模型将采用OpenCV的局部二值模式直方图(LBPH)算法，可在大多数CPU上高效运行。"
+ },
+ "large": {
+ "title": "大模型",
+ "desc": "使用大模型将采用ArcFace人脸嵌入模型，若适用将自动在GPU上运行。"
+ }
+ }
},
"licensePlateRecognition": {
"title": "车牌识别",
@@ -113,7 +127,7 @@
"readTheDocumentation": "阅读文档（英文）"
},
"toast": {
- "success": "分类设置已保存。",
+ "success": "分类设置已保存，请重启 Frigate 以应用更改。",
"error": "保存配置更改失败：{{errorMessage}}"
}
},
@@ -542,9 +556,17 @@
"trainDate": "训练日期",
"baseModel": "基础模型",
"supportedDetectors": "支持的检测器",
+ "dimensions": "大小",
"cameras": "摄像头",
"loading": "正在加载模型信息...",
- "error": "加载模型信息失败"
+ "error": "加载模型信息失败",
+ "availableModels": "可用模型",
+ "loadingAvailableModels": "正在加载可用模型...",
+ "modelSelect": "您可以在Frigate+上选择可用的模型。请注意，只能选择与当前探测器配置兼容的模型。"
+ },
+ "toast": {
+ "success": "Frigate+ 设置已保存。请重启 Frigate 以应用更改。",
+ "error": "配置更改保存失败：{{errorMessage}}"
}
}
}
From 6f707e8722e239827104bf97af0e51008eb2f99e Mon Sep 17 00:00:00 2001
From: Logan Garrett
Date: Wed, 26 Mar 2025 07:24:30 -0400
Subject: [PATCH 56/97] Add explanation of group_add for igpu (#17375)
Add explanation of group_add for igpu
---
docker-compose.yml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 4322dd23c..168e7fd10 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,7 +1,8 @@
services:
devcontainer:
container_name: frigate-devcontainer
- # add groups from host for render, plugdev, video
+ # Check host system's actual render/video/plugdev group IDs with 'getent group render', 'getent group video', and 'getent group plugdev'
+ # Must add these exact IDs in container's group_add section or OpenVINO GPU acceleration will fail
group_add:
- "109" # render
- "110" # render
@@ -37,4 +38,4 @@ services:
container_name: mqtt
image: eclipse-mosquitto:1.6
ports:
- - "1883:1883"
\ No newline at end of file
+ - "1883:1883"
From b30de965255f5ae87d3949c11c7d453cdabef265 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Wed, 26 Mar 2025 06:25:39 -0500
Subject: [PATCH 57/97] Reduce expansion of license_plate box for frigate+
models (#17373)
---
frigate/data_processing/common/license_plate/mixin.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py
index 9bf2119f9..1d80a4c02 100644
--- a/frigate/data_processing/common/license_plate/mixin.py
+++ b/frigate/data_processing/common/license_plate/mixin.py
@@ -1135,9 +1135,9 @@ class LicensePlateProcessingMixin:
license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
- # Expand the license_plate_box by 30%
+ # Expand the license_plate_box by 10%
box_array = np.array(license_plate_box)
- expansion = (box_array[2:] - box_array[:2]) * 0.30
+ expansion = (box_array[2:] - box_array[:2]) * 0.10
expanded_box = np.array(
[
license_plate_box[0] - expansion[0],
From 4ccf61a6d787298a32576e3e26f47b4e42f7aaf1 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Wed, 26 Mar 2025 12:16:12 +0000
Subject: [PATCH 58/97] Fix wrong value displayed in facerec settings (#17383)
---
web/src/views/settings/ClassificationSettingsView.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/web/src/views/settings/ClassificationSettingsView.tsx b/web/src/views/settings/ClassificationSettingsView.tsx
index 2ce958f9e..b40fde19f 100644
--- a/web/src/views/settings/ClassificationSettingsView.tsx
+++ b/web/src/views/settings/ClassificationSettingsView.tsx
@@ -424,7 +424,7 @@ export default function ClassificationSettingsView({
}
>
- {classificationSettings.search.model_size}
+ {classificationSettings.face.model_size}
From 4edf0d8cd36a84406a60de494ca5889f250e3935 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Wed, 26 Mar 2025 07:41:00 -0500
Subject: [PATCH 59/97] LPR bugfix (#17384)
* ensure image is numpy array
* clean up debugging
* clean up postprocessor
* process raw input as img
---
.../common/license_plate/mixin.py | 52 +++++++++----------
frigate/data_processing/post/license_plate.py | 4 +-
frigate/embeddings/onnx/lpr_embedding.py | 4 +-
3 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py
index 1d80a4c02..9cc988267 100644
--- a/frigate/data_processing/common/license_plate/mixin.py
+++ b/frigate/data_processing/common/license_plate/mixin.py
@@ -32,10 +32,6 @@ class LicensePlateProcessingMixin:
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.requires_license_plate_detection = (
- "license_plate" not in self.config.objects.all_objects
- )
-
self.event_metadata_publisher = EventMetadataPublisher()
self.ctc_decoder = CTCDecoder()
@@ -312,7 +308,6 @@ class LicensePlateProcessingMixin:
# get minimum bounding box (rotated rectangle) around the contour and the smallest side length.
points, min_side = self._get_min_boxes(contour)
- logger.debug(f"min side {index}, {min_side}")
if min_side < self.min_size:
continue
@@ -320,7 +315,6 @@ class LicensePlateProcessingMixin:
points = np.array(points)
score = self._box_score(output, contour)
- logger.debug(f"box score {index}, {score}")
if self.box_thresh > score:
continue
@@ -991,21 +985,21 @@ class LicensePlateProcessingMixin:
license_plate = self._detect_license_plate(rgb)
logger.debug(
- f"YOLOv9 LPD inference time: {(datetime.datetime.now().timestamp() - yolov9_start) * 1000:.2f} ms"
+ f"{camera}: YOLOv9 LPD inference time: {(datetime.datetime.now().timestamp() - yolov9_start) * 1000:.2f} ms"
)
self.__update_yolov9_metrics(
datetime.datetime.now().timestamp() - yolov9_start
)
if not license_plate:
- logger.debug("Detected no license plates in full frame.")
+ logger.debug(f"{camera}: Detected no license plates in full frame.")
return
license_plate_area = (license_plate[2] - license_plate[0]) * (
license_plate[3] - license_plate[1]
)
if license_plate_area < self.lpr_config.min_area:
- logger.debug("License plate area below minimum threshold.")
+ logger.debug(f"{camera}: License plate area below minimum threshold.")
return
license_plate_frame = rgb[
@@ -1027,13 +1021,15 @@ class LicensePlateProcessingMixin:
# don't run for non car objects
if obj_data.get("label") != "car":
- logger.debug("Not a processing license plate for non car object.")
+ logger.debug(
+ f"{camera}: Not a processing license plate for non car object."
+ )
return
# don't run for stationary car objects
if obj_data.get("stationary") == True:
logger.debug(
- "Not a processing license plate for a stationary car object."
+ f"{camera}: Not a processing license plate for a stationary car object."
)
return
@@ -1041,14 +1037,14 @@ class LicensePlateProcessingMixin:
# that is not a license plate
if obj_data.get("sub_label") and id not in self.detected_license_plates:
logger.debug(
- f"Not processing license plate due to existing sub label: {obj_data.get('sub_label')}."
+ f"{camera}: Not processing license plate due to existing sub label: {obj_data.get('sub_label')}."
)
return
license_plate: Optional[dict[str, any]] = None
- if self.requires_license_plate_detection:
- logger.debug("Running manual license_plate detection.")
+ if "license_plate" not in self.config.cameras[camera].objects.track:
+ logger.debug(f"{camera}: Running manual license_plate detection.")
car_box = obj_data.get("box")
@@ -1071,14 +1067,16 @@ class LicensePlateProcessingMixin:
yolov9_start = datetime.datetime.now().timestamp()
license_plate = self._detect_license_plate(car)
logger.debug(
- f"YOLOv9 LPD inference time: {(datetime.datetime.now().timestamp() - yolov9_start) * 1000:.2f} ms"
+ f"{camera}: YOLOv9 LPD inference time: {(datetime.datetime.now().timestamp() - yolov9_start) * 1000:.2f} ms"
)
self.__update_yolov9_metrics(
datetime.datetime.now().timestamp() - yolov9_start
)
if not license_plate:
- logger.debug("Detected no license plates for car object.")
+ logger.debug(
+ f"{camera}: Detected no license plates for car object."
+ )
return
license_plate_area = max(
@@ -1093,7 +1091,7 @@ class LicensePlateProcessingMixin:
license_plate_area
< self.config.cameras[obj_data["camera"]].lpr.min_area * 2
):
- logger.debug("License plate is less than min_area")
+ logger.debug(f"{camera}: License plate is less than min_area")
return
license_plate_frame = car[
@@ -1103,7 +1101,7 @@ class LicensePlateProcessingMixin:
else:
# don't run for object without attributes
if not obj_data.get("current_attributes"):
- logger.debug("No attributes to parse.")
+ logger.debug(f"{camera}: No attributes to parse.")
return
attributes: list[dict[str, any]] = obj_data.get(
@@ -1130,7 +1128,7 @@ class LicensePlateProcessingMixin:
or area(license_plate_box)
< self.config.cameras[obj_data["camera"]].lpr.min_area
):
- logger.debug(f"Invalid license plate box {license_plate}")
+ logger.debug(f"{camera}: Invalid license plate box {license_plate}")
return
license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
@@ -1184,7 +1182,7 @@ class LicensePlateProcessingMixin:
)
logger.debug(
- f"Detected text: {plate} (average confidence: {avg_confidence:.2f}, area: {text_area} pixels)"
+ f"{camera}: Detected text: {plate} (average confidence: {avg_confidence:.2f}, area: {text_area} pixels)"
)
else:
logger.debug("No text detected")
@@ -1204,7 +1202,7 @@ class LicensePlateProcessingMixin:
# Check against minimum confidence threshold
if avg_confidence < self.lpr_config.recognition_threshold:
logger.debug(
- f"Average confidence {avg_confidence} is less than threshold ({self.lpr_config.recognition_threshold})"
+ f"{camera}: Average confidence {avg_confidence} is less than threshold ({self.lpr_config.recognition_threshold})"
)
return
@@ -1223,7 +1221,7 @@ class LicensePlateProcessingMixin:
if similarity >= self.similarity_threshold:
plate_id = existing_id
logger.debug(
- f"Matched plate {top_plate} to {data['plate']} (similarity: {similarity:.3f})"
+ f"{camera}: Matched plate {top_plate} to {data['plate']} (similarity: {similarity:.3f})"
)
break
if plate_id is None:
@@ -1231,11 +1229,11 @@ class LicensePlateProcessingMixin:
obj_data, top_plate, avg_confidence
)
logger.debug(
- f"New plate event for dedicated LPR camera {plate_id}: {top_plate}"
+ f"{camera}: New plate event for dedicated LPR camera {plate_id}: {top_plate}"
)
else:
logger.debug(
- f"Matched existing plate event for dedicated LPR camera {plate_id}: {top_plate}"
+ f"{camera}: Matched existing plate event for dedicated LPR camera {plate_id}: {top_plate}"
)
self.detected_license_plates[plate_id]["last_seen"] = current_time
@@ -1246,7 +1244,7 @@ class LicensePlateProcessingMixin:
if self._should_keep_previous_plate(
id, top_plate, top_char_confidences, top_area, avg_confidence
):
- logger.debug("Keeping previous plate")
+ logger.debug(f"{camera}: Keeping previous plate")
return
# Determine subLabel based on known plates, use regex matching
@@ -1277,7 +1275,9 @@ class LicensePlateProcessingMixin:
if dedicated_lpr:
# save the best snapshot
- logger.debug(f"Writing snapshot for {id}, {top_plate}, {current_time}")
+ logger.debug(
+ f"{camera}: Writing snapshot for {id}, {top_plate}, {current_time}"
+ )
frame_bgr = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
self.sub_label_publisher.publish(
EventMetadataTypeEnum.save_lpr_snapshot,
diff --git a/frigate/data_processing/post/license_plate.py b/frigate/data_processing/post/license_plate.py
index e5c8a29a8..c78e56b9d 100644
--- a/frigate/data_processing/post/license_plate.py
+++ b/frigate/data_processing/post/license_plate.py
@@ -139,7 +139,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
scale_y = image.shape[0] / detect_height
# Determine which box to enlarge based on detection mode
- if self.requires_license_plate_detection:
+ if "license_plate" not in self.config.cameras[camera_name].objects.track:
# Scale and enlarge the car box
box = obj_data.get("box")
if not box:
@@ -189,7 +189,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
)
keyframe_obj_data = obj_data.copy()
- if self.requires_license_plate_detection:
+ if "license_plate" not in self.config.cameras[camera_name].objects.track:
# car box
keyframe_obj_data["box"] = [new_left, new_top, new_right, new_bottom]
else:
diff --git a/frigate/embeddings/onnx/lpr_embedding.py b/frigate/embeddings/onnx/lpr_embedding.py
index c3b9a8771..cfe8fb16f 100644
--- a/frigate/embeddings/onnx/lpr_embedding.py
+++ b/frigate/embeddings/onnx/lpr_embedding.py
@@ -261,8 +261,8 @@ class LicensePlateDetector(BaseEmbedding):
def _preprocess_inputs(self, raw_inputs):
if isinstance(raw_inputs, list):
raise ValueError("License plate embedding does not support batch inputs.")
- # Get image as numpy array
- img = self._process_image(raw_inputs)
+
+ img = raw_inputs
height, width, channels = img.shape
# Resize maintaining aspect ratio
From 395fc33ccce9f30c5b9130bd2c6e152e901d8f87 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Wed, 26 Mar 2025 07:14:36 -0600
Subject: [PATCH 60/97] Include all .so and .so.12 (#17388)
---
docker/tensorrt/Dockerfile.base | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docker/tensorrt/Dockerfile.base b/docker/tensorrt/Dockerfile.base
index 4305f1d74..bd8738792 100644
--- a/docker/tensorrt/Dockerfile.base
+++ b/docker/tensorrt/Dockerfile.base
@@ -20,8 +20,8 @@ RUN --mount=type=bind,source=docker/tensorrt/detector/tensorrt_libyolo.sh,target
# COPY required individual CUDA deps
RUN mkdir -p /usr/local/cuda-deps
RUN if [ "$TARGETARCH" = "amd64" ]; then \
- cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libcurand.so.* /usr/local/cuda-deps/ && \
- cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libnvrtc.so.* /usr/local/cuda-deps/ ; \
+ cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libcurand.s* /usr/local/cuda-deps/ && \
+ cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libnvrtc.s* /usr/local/cuda-deps/ ; \
fi
# Frigate w/ TensorRT Support as separate image
From e3d4b84803fce7e9f345592c797a8cdc62022dbb Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Wed, 26 Mar 2025 07:23:01 -0600
Subject: [PATCH 61/97] Face recognition improvements (#17387)
* Increase frequency of updates when internal face detection is used
* Adjust number of required faces based on detection type
* Adjust min_score config to unknown_score
* Only for person
* Improve typing
* Update face rec docs
* Cleanup ui colors
* Cleanup
---
docs/docs/configuration/face_recognition.md | 19 +++---
docs/docs/configuration/reference.md | 4 +-
frigate/camera/state.py | 20 ++++--
frigate/config/classification.py | 4 +-
frigate/data_processing/common/face/model.py | 9 +--
frigate/data_processing/real_time/face.py | 66 +++++++++++---------
web/src/pages/FaceLibrary.tsx | 25 ++++++--
web/src/types/frigateConfig.ts | 15 +++--
8 files changed, 97 insertions(+), 65 deletions(-)
diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md
index a3cb4e308..af6fd1eff 100644
--- a/docs/docs/configuration/face_recognition.md
+++ b/docs/docs/configuration/face_recognition.md
@@ -22,16 +22,16 @@ Frigate needs to first detect a `face` before it can recognize a face.
### Face Recognition
Frigate has support for two face recognition model types:
-- **small**: Frigate will use CV2 Local Binary Pattern Face Recognizer to recognize faces, which runs locally on the CPU.
-- **large**: Frigate will run a face embedding model, this is only recommended to be run when an integrated or dedicated GPU is available.
-In both cases a lightweight face landmark detection model is also used to align faces before running them through the face recognizer.
+- **small**: Frigate will use CV2 Local Binary Pattern Face Recognizer to recognize faces, which runs locally on the CPU. This model is optimized for efficiency and is not as accurate.
+- **large**: Frigate will run a face embedding model, this model is optimized for accuracy. It is only recommended to be run when an integrated or dedicated GPU is available.
+
+In both cases a lightweight face landmark detection model is also used to align faces before running the recognition model.
## Minimum System Requirements
-Face recognition is lightweight and runs on the CPU, there are no significantly different system requirements than running Frigate itself when using the `small` model.
-
-When using the `large` model an integrated or discrete GPU is recommended.
+The `small` model is optimized for efficiency and runs on the CPU, there are no significantly different system requirements.
+The `large` model is optimized for accuracy and an integrated or discrete GPU is highly recommended.
## Configuration
@@ -58,6 +58,8 @@ Fine-tune face recognition with these optional parameters:
### Recognition
- `model_size`: Which model size to use, options are `small` or `large`
+- `unknown_score`: Min score to mark a person as a potential match, matches below this will be marked as unknown.
+ - Default: `0.8`.
- `recognition_threshold`: Recognition confidence score required to add the face to the object as a sub label.
- Default: `0.9`.
- `blur_confidence_filter`: Enables a filter that calculates how blurry the face is and adjusts the confidence based on this.
@@ -108,13 +110,14 @@ Once straight-on images are performing well, start choosing slightly off-angle i
## FAQ
-### Why is every face tagged as a known face and not unknown?
+### Why can't I bulk upload photos?
-Any recognized face with a score >= `min_score` will show in the `Train` tab along with the recognition score. A low scoring face is effectively the same as `unknown`, but includes more information. This does not mean the recognition is not working well, and is part of the importance of choosing the correct `recognition_threshold`.
+It is important to methodically add photos to the library, bulk importing photos (especially from a general photo library) will lead to overfitting in that particular scenario and hurt recognition performance.
### Why do unknown people score similarly to known people?
This can happen for a few different reasons, but this is usually an indicator that the training set needs to be improved. This is often related to overfitting:
+
- If you train with only a few images per person, especially if those images are very similar, the recognition model becomes overly specialized to those specific images.
- When you provide images with different poses, lighting, and expressions, the algorithm extracts features that are consistent across those variations.
- By training on a diverse set of images, the algorithm becomes less sensitive to minor variations and noise in the input image.
diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md
index 8e24db0b7..e30ee1619 100644
--- a/docs/docs/configuration/reference.md
+++ b/docs/docs/configuration/reference.md
@@ -547,8 +547,8 @@ semantic_search:
face_recognition:
# Optional: Enable semantic search (default: shown below)
enabled: False
- # Optional: Minimum face distance score required to save the attempt (default: shown below)
- min_score: 0.8
+ # Optional: Minimum face distance score required to mark as a potential match (default: shown below)
+ unknown_score: 0.8
# Optional: Minimum face detection score required to detect a face (default: shown below)
# NOTE: This only applies when not running a Frigate+ model
detection_threshold: 0.7
diff --git a/frigate/camera/state.py b/frigate/camera/state.py
index 98f808bf6..65a3dcf5d 100644
--- a/frigate/camera/state.py
+++ b/frigate/camera/state.py
@@ -5,7 +5,7 @@ import logging
import os
import threading
from collections import defaultdict
-from typing import Callable
+from typing import Any, Callable
import cv2
import numpy as np
@@ -53,8 +53,19 @@ class CameraState:
self.callbacks = defaultdict(list)
self.ptz_autotracker_thread = ptz_autotracker_thread
self.prev_enabled = self.camera_config.enabled
+ self.requires_face_detection = (
+ self.config.face_recognition.enabled
+ and "face" not in self.config.objects.all_objects
+ )
- def get_current_frame(self, draw_options={}):
+ def get_max_update_frequency(self, obj: TrackedObject) -> int:
+ return (
+ 1
+ if self.requires_face_detection and obj.obj_data["label"] == "person"
+ else 5
+ )
+
+ def get_current_frame(self, draw_options: dict[str, Any] = {}):
with self.current_frame_lock:
frame_copy = np.copy(self._current_frame)
frame_time = self.current_frame_time
@@ -283,11 +294,12 @@ class CameraState:
updated_obj.last_updated = frame_time
- # if it has been more than 5 seconds since the last thumb update
+ # if it has been more than max_update_frequency seconds since the last thumb update
# and the last update is greater than the last publish or
# the object has changed significantly
if (
- frame_time - updated_obj.last_published > 5
+ frame_time - updated_obj.last_published
+ > self.get_max_update_frequency(updated_obj)
and updated_obj.last_updated > updated_obj.last_published
) or significant_update:
# call event handlers
diff --git a/frigate/config/classification.py b/frigate/config/classification.py
index 25c379546..aecbf6537 100644
--- a/frigate/config/classification.py
+++ b/frigate/config/classification.py
@@ -54,8 +54,8 @@ class FaceRecognitionConfig(FrigateBaseModel):
model_size: str = Field(
default="small", title="The size of the embeddings model used."
)
- min_score: float = Field(
- title="Minimum face distance score required to save the attempt.",
+ unknown_score: float = Field(
+ title="Minimum face distance score required to be marked as a potential match.",
default=0.8,
gt=0.0,
le=1.0,
diff --git a/frigate/data_processing/common/face/model.py b/frigate/data_processing/common/face/model.py
index a006734ce..5e15a2441 100644
--- a/frigate/data_processing/common/face/model.py
+++ b/frigate/data_processing/common/face/model.py
@@ -164,9 +164,7 @@ class LBPHRecognizer(FaceRecognizer):
return
self.recognizer: cv2.face.LBPHFaceRecognizer = (
- cv2.face.LBPHFaceRecognizer_create(
- radius=2, threshold=(1 - self.config.face_recognition.min_score) * 1000
- )
+ cv2.face.LBPHFaceRecognizer_create(radius=2, threshold=400)
)
self.recognizer.train(faces, np.array(labels))
@@ -243,6 +241,8 @@ class ArcFaceRecognizer(FaceRecognizer):
for name, embs in face_embeddings_map.items():
self.mean_embs[name] = stats.trim_mean(embs, 0.15)
+ logger.debug("Finished building ArcFace model")
+
def similarity_to_confidence(
self, cosine_similarity: float, median=0.3, range_width=0.6, slope_factor=12
):
@@ -302,7 +302,4 @@ class ArcFaceRecognizer(FaceRecognizer):
score = confidence
label = name
- if score < self.config.face_recognition.min_score:
- return None
-
return label, round(score * blur_factor, 2)
diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index e20dad633..9b479a527 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -36,36 +36,6 @@ MAX_DETECTION_HEIGHT = 1080
MIN_MATCHING_FACES = 2
-def weighted_average_by_area(results_list: list[tuple[str, float, int]]):
- if len(results_list) < 3:
- return "unknown", 0.0
-
- score_count = {}
- weighted_scores = {}
- total_face_areas = {}
-
- for name, score, face_area in results_list:
- if name not in weighted_scores:
- score_count[name] = 1
- weighted_scores[name] = 0.0
- total_face_areas[name] = 0.0
- else:
- score_count[name] += 1
-
- weighted_scores[name] += score * face_area
- total_face_areas[name] += face_area
-
- prominent_name = max(score_count)
-
- # if a single name is not prominent in the history then we are not confident
- if score_count[prominent_name] / len(results_list) < 0.65:
- return "unknown", 0.0
-
- return prominent_name, weighted_scores[prominent_name] / total_face_areas[
- prominent_name
- ]
-
-
class FaceRealTimeProcessor(RealTimeProcessorApi):
def __init__(
self,
@@ -271,6 +241,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
sub_label, score = res
+ if score < self.face_config.unknown_score:
+ sub_label = "unknown"
+
logger.debug(
f"Detected best face for person as: {sub_label} with probability {score}"
)
@@ -288,7 +261,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
self.person_face_history[id].append(
(sub_label, score, face_frame.shape[0] * face_frame.shape[1])
)
- (weighted_sub_label, weighted_score) = weighted_average_by_area(
+ (weighted_sub_label, weighted_score) = self.weighted_average_by_area(
self.person_face_history[id]
)
@@ -415,3 +388,34 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
def expire_object(self, object_id: str):
if object_id in self.person_face_history:
self.person_face_history.pop(object_id)
+
+ def weighted_average_by_area(self, results_list: list[tuple[str, float, int]]):
+ min_faces = 1 if self.requires_face_detection else 3
+
+ if len(results_list) < min_faces:
+ return "unknown", 0.0
+
+ score_count = {}
+ weighted_scores = {}
+ total_face_areas = {}
+
+ for name, score, face_area in results_list:
+ if name not in weighted_scores:
+ score_count[name] = 1
+ weighted_scores[name] = 0.0
+ total_face_areas[name] = 0.0
+ else:
+ score_count[name] += 1
+
+ weighted_scores[name] += score * face_area
+ total_face_areas[name] += face_area
+
+ prominent_name = max(score_count)
+
+ # if a single name is not prominent in the history then we are not confident
+ if score_count[prominent_name] / len(results_list) < 0.65:
+ return "unknown", 0.0
+
+ return prominent_name, weighted_scores[prominent_name] / total_face_areas[
+ prominent_name
+ ]
diff --git a/web/src/pages/FaceLibrary.tsx b/web/src/pages/FaceLibrary.tsx
index 0ac937283..696691997 100644
--- a/web/src/pages/FaceLibrary.tsx
+++ b/web/src/pages/FaceLibrary.tsx
@@ -33,7 +33,7 @@ import useKeyboardListener from "@/hooks/use-keyboard-listener";
import useOptimisticState from "@/hooks/use-optimistic-state";
import { cn } from "@/lib/utils";
import { FaceLibraryData, RecognizedFaceData } from "@/types/face";
-import { FrigateConfig } from "@/types/frigateConfig";
+import { FaceRecognitionConfig, FrigateConfig } from "@/types/frigateConfig";
import axios from "axios";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { isDesktop, isMobile } from "react-device-detect";
@@ -451,7 +451,7 @@ function TrainingGrid({
key={image}
image={image}
faceNames={faceNames}
- threshold={config.face_recognition.recognition_threshold}
+ recognitionConfig={config.face_recognition}
selected={selectedFaces.includes(image)}
onClick={(data, meta) => {
if (meta) {
@@ -471,7 +471,7 @@ function TrainingGrid({
type FaceAttemptProps = {
image: string;
faceNames: string[];
- threshold: number;
+ recognitionConfig: FaceRecognitionConfig;
selected: boolean;
onClick: (data: RecognizedFaceData, meta: boolean) => void;
onRefresh: () => void;
@@ -479,7 +479,7 @@ type FaceAttemptProps = {
function FaceAttempt({
image,
faceNames,
- threshold,
+ recognitionConfig,
selected,
onClick,
onRefresh,
@@ -496,6 +496,16 @@ function FaceAttempt({
};
}, [image]);
+ const scoreStatus = useMemo(() => {
+ if (data.score >= recognitionConfig.recognition_threshold) {
+ return "match";
+ } else if (data.score >= recognitionConfig.unknown_score) {
+ return "potential";
+ } else {
+ return "unknown";
+ }
+ }, [data, recognitionConfig]);
+
// interaction
const imgRef = useRef(null);
@@ -579,10 +589,13 @@ function FaceAttempt({
- t("exploreIsUnavailable.embeddingsReindexing.step.trackedObjectsProcessed")
+ {t(
+ "exploreIsUnavailable.embeddingsReindexing.step.trackedObjectsProcessed",
+ )}
{reindexState.processed_objects} /{" "}
{reindexState.total_objects}
From a35146ab616efc4affd8c81b9bf9f33cdbc5ec9c Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Thu, 27 Mar 2025 08:28:09 -0600
Subject: [PATCH 70/97] Various fixes (#17411)
* Remove initial requirement for history
* Clenaup conf
* Handle symlinks
---
docker/tensorrt/Dockerfile.base | 17 +++++++++++++++--
.../rootfs/etc/ld.so.conf.d/cuda_tensorrt.conf | 3 +--
frigate/data_processing/real_time/face.py | 5 -----
3 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/docker/tensorrt/Dockerfile.base b/docker/tensorrt/Dockerfile.base
index bd8738792..79a7f3c98 100644
--- a/docker/tensorrt/Dockerfile.base
+++ b/docker/tensorrt/Dockerfile.base
@@ -20,8 +20,21 @@ RUN --mount=type=bind,source=docker/tensorrt/detector/tensorrt_libyolo.sh,target
# COPY required individual CUDA deps
RUN mkdir -p /usr/local/cuda-deps
RUN if [ "$TARGETARCH" = "amd64" ]; then \
- cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libcurand.s* /usr/local/cuda-deps/ && \
- cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libnvrtc.s* /usr/local/cuda-deps/ ; \
+ cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libcurand.so.* /usr/local/cuda-deps/ && \
+ cp /usr/local/cuda-12.3/targets/x86_64-linux/lib/libnvrtc.so.* /usr/local/cuda-deps/ && \
+ cd /usr/local/cuda-deps/ && \
+ for lib in libnvrtc.so.*; do \
+ if [[ "$lib" =~ libnvrtc.so\.([0-9]+\.[0-9]+\.[0-9]+) ]]; then \
+ version="${BASH_REMATCH[1]}"; \
+ ln -sf "libnvrtc.so.$version" libnvrtc.so; \
+ fi; \
+ done && \
+ for lib in libcurand.so.*; do \
+ if [[ "$lib" =~ libcurand.so\.([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+) ]]; then \
+ version="${BASH_REMATCH[1]}"; \
+ ln -sf "libcurand.so.$version" libcurand.so; \
+ fi; \
+ done; \
fi
# Frigate w/ TensorRT Support as separate image
diff --git a/docker/tensorrt/detector/rootfs/etc/ld.so.conf.d/cuda_tensorrt.conf b/docker/tensorrt/detector/rootfs/etc/ld.so.conf.d/cuda_tensorrt.conf
index 72eec56e0..f66af7dc6 100644
--- a/docker/tensorrt/detector/rootfs/etc/ld.so.conf.d/cuda_tensorrt.conf
+++ b/docker/tensorrt/detector/rootfs/etc/ld.so.conf.d/cuda_tensorrt.conf
@@ -1,8 +1,7 @@
/usr/local/lib
/usr/local/cuda
+/usr/local/lib/python3.11/dist-packages/tensorrt
/usr/local/lib/python3.11/dist-packages/nvidia/cudnn/lib
/usr/local/lib/python3.11/dist-packages/nvidia/cuda_runtime/lib
/usr/local/lib/python3.11/dist-packages/nvidia/cublas/lib
-/usr/local/lib/python3.11/dist-packages/nvidia/cuda_nvrtc/lib
-/usr/local/lib/python3.11/dist-packages/tensorrt
/usr/local/lib/python3.11/dist-packages/nvidia/cufft/lib
\ No newline at end of file
diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index e3ebff079..d3fbc83df 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -390,11 +390,6 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
self.person_face_history.pop(object_id)
def weighted_average_by_area(self, results_list: list[tuple[str, float, int]]):
- min_faces = 1 if self.requires_face_detection else 3
-
- if len(results_list) < min_faces:
- return "unknown", 0.0
-
score_count = {}
weighted_scores = {}
total_face_areas = {}
From 6ec7d96ec9136b05cccd2c7e7404ac119cdc74ba Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Thu, 27 Mar 2025 09:49:10 -0500
Subject: [PATCH 71/97] remove LPR denoising (#17412)
---
.../common/license_plate/mixin.py | 26 +------------------
1 file changed, 1 insertion(+), 25 deletions(-)
diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py
index 48c191876..60066c48d 100644
--- a/frigate/data_processing/common/license_plate/mixin.py
+++ b/frigate/data_processing/common/license_plate/mixin.py
@@ -634,37 +634,13 @@ class LicensePlateProcessingMixin:
else:
gray = image
- # detect noise with Laplacian variance
- laplacian = cv2.Laplacian(gray, cv2.CV_64F)
- noise_variance = np.var(laplacian)
- brightness = cv2.mean(gray)[0]
- noise_threshold = 70
- brightness_threshold = 150
- is_noisy = (
- noise_variance > noise_threshold and brightness < brightness_threshold
- )
-
- # apply bilateral filter and sharpening only if noisy
- if is_noisy:
- logger.debug(
- f"Noise detected (variance: {noise_variance:.1f}, brightness: {brightness:.1f}) - denoising"
- )
- smoothed = cv2.bilateralFilter(gray, d=15, sigmaColor=100, sigmaSpace=100)
- sharpening_kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
- processed = cv2.filter2D(smoothed, -1, sharpening_kernel)
- else:
- logger.debug(
- f"No noise detected (variance: {noise_variance:.1f}, brightness: {brightness:.1f}) - skipping denoising and sharpening"
- )
- processed = gray
-
# apply CLAHE for contrast enhancement
grid_size = (
max(4, input_w // 40),
max(4, input_h // 40),
)
clahe = cv2.createCLAHE(clipLimit=1.5, tileGridSize=grid_size)
- enhanced = clahe.apply(processed)
+ enhanced = clahe.apply(gray)
# Convert back to 3-channel for model compatibility
image = cv2.cvtColor(enhanced, cv2.COLOR_GRAY2RGB)
From 8978d1ff743cb4b05d1f5cb7ddf9d4d8c143226b Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Thu, 27 Mar 2025 09:50:41 -0600
Subject: [PATCH 72/97] Tweak face recognition docs (#17413)
---
docs/docs/configuration/face_recognition.md | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md
index 278c592c0..9fb5c9a44 100644
--- a/docs/docs/configuration/face_recognition.md
+++ b/docs/docs/configuration/face_recognition.md
@@ -31,6 +31,7 @@ In both cases a lightweight face landmark detection model is also used to align
## Minimum System Requirements
The `small` model is optimized for efficiency and runs on the CPU, most CPUs should run the model efficiently.
+
The `large` model is optimized for accuracy, an integrated or discrete GPU is highly recommended.
## Configuration
@@ -65,7 +66,7 @@ Fine-tune face recognition with these optional parameters:
- `blur_confidence_filter`: Enables a filter that calculates how blurry the face is and adjusts the confidence based on this.
- Default: `True`.
-## Dataset
+## Creating a Robust Training Set
The number of images needed for a sufficient training set for face recognition varies depending on several factors:
@@ -74,11 +75,9 @@ The number of images needed for a sufficient training set for face recognition v
However, here are some general guidelines:
-- Minimum: For basic face recognition tasks, a minimum of 10-20 images per person is often recommended.
-- Recommended: For more robust and accurate systems, 30-50 images per person is a good starting point.
-- Ideal: For optimal performance, especially in challenging conditions, 100 or more images per person can be beneficial.
-
-## Creating a Robust Training Set
+- Minimum: For basic face recognition tasks, a minimum of 5-10 images per person is often recommended.
+- Recommended: For more robust and accurate systems, 20-30 images per person is a good starting point.
+- Ideal: For optimal performance, especially in challenging conditions, 50-100 images per person can be beneficial.
The accuracy of face recognition is heavily dependent on the quality of data given to it for training. It is recommended to build the face training library in phases.
@@ -89,7 +88,8 @@ When choosing images to include in the face training set it is recommended to al
- If it is difficult to make out details in a persons face it will not be helpful in training.
- Avoid images with extreme under/over-exposure.
- Avoid blurry / pixelated images.
-- Be careful when uploading images of people when they are wearing clothing that covers a lot of their face as this may confuse the model.
+- Avoid training on infrared (grayscale). The models are trained on color images and will be able to extract features from grayscale images.
+- Using images of people wearing hats / sunglasses may confuse the model.
- Do not upload too many similar images at the same time, it is recommended to train no more than 4-6 similar images for each person to avoid overfitting.
:::
@@ -124,4 +124,4 @@ This can happen for a few different reasons, but this is usually an indicator th
### I see scores above the threshold in the train tab, but a sub label wasn't assigned?
-The Frigate face recognizer collects face recognition scores from all of the frames across the person objects lifecycle. The scores are continually weighted based on the area of the face, and a sub label will only be assigned to person if there is a prominent person recognized. This avoids cases where a single high confidence recognition result would throw off the results.
+The Frigate considers the recognition scores across all recogntion attempts for each person object. The scores are continually weighted based on the area of the face, and a sub label will only be assigned to person if a person is confidently recognized consistently. This avoids cases where a single high confidence recognition would throw off the results.
From ccf20f456a21be15fd78414231c7ff02923830be Mon Sep 17 00:00:00 2001
From: aptalca <541623+aptalca@users.noreply.github.com>
Date: Thu, 27 Mar 2025 12:33:03 -0400
Subject: [PATCH 73/97] update YOLO_NAS notebook (#17414)
Google Colab updated to python 3.11
super-gradients v3.7.1 is not compatible with py3.11 and install fails
super-gradients committed a fix to master branch but did not cut a relase since (acquired by Nvidia in the meantime)
This commit installs super-gradients from master branch
---
notebooks/YOLO_NAS_Pretrained_Export.ipynb | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/notebooks/YOLO_NAS_Pretrained_Export.ipynb b/notebooks/YOLO_NAS_Pretrained_Export.ipynb
index e4e2222da..4fee19183 100644
--- a/notebooks/YOLO_NAS_Pretrained_Export.ipynb
+++ b/notebooks/YOLO_NAS_Pretrained_Export.ipynb
@@ -8,14 +8,14 @@
},
"outputs": [],
"source": [
- "! pip install -q super_gradients==3.7.1"
+ "! pip install -q git+https://github.com/Deci-AI/super-gradients.git"
]
},
{
"cell_type": "code",
"source": [
- "! sed -i 's/sghub.deci.ai/sg-hub-nv.s3.amazonaws.com/' /usr/local/lib/python3.10/dist-packages/super_gradients/training/pretrained_models.py\n",
- "! sed -i 's/sghub.deci.ai/sg-hub-nv.s3.amazonaws.com/' /usr/local/lib/python3.10/dist-packages/super_gradients/training/utils/checkpoint_utils.py"
+ "! sed -i 's/sghub.deci.ai/sg-hub-nv.s3.amazonaws.com/' /usr/local/lib/python3.11/dist-packages/super_gradients/training/pretrained_models.py\n",
+ "! sed -i 's/sghub.deci.ai/sg-hub-nv.s3.amazonaws.com/' /usr/local/lib/python3.11/dist-packages/super_gradients/training/utils/checkpoint_utils.py"
],
"metadata": {
"id": "NiRCt917KKcL"
@@ -84,4 +84,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
\ No newline at end of file
+}
From 67dd50a7f7940e6346aaa528d17f96522c7885a2 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Thu, 27 Mar 2025 16:33:49 +0000
Subject: [PATCH 74/97] Devcontainer: update Mosquitto from 1.6 to 2.0 (#17415)
---
docker-compose.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 168e7fd10..db63297d5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -36,6 +36,7 @@ services:
# - /dev/bus/usb:/dev/bus/usb # Uncomment for Google Coral USB
mqtt:
container_name: mqtt
- image: eclipse-mosquitto:1.6
+ image: eclipse-mosquitto:2.0
+ command: mosquitto -c /mosquitto-no-auth.conf # enable no-auth mode
ports:
- "1883:1883"
From 23c332387127bd03220c52f9da4a88ebc5ef7fba Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Thu, 27 Mar 2025 12:29:34 -0500
Subject: [PATCH 75/97] Dynamic embeddings reindexing (#17418)
* reindex with api endpoint and zmq
* threading
* frontend
* require admin role
---
frigate/api/classification.py | 46 +++++
frigate/comms/embeddings_updater.py | 1 +
frigate/embeddings/__init__.py | 3 +
frigate/embeddings/embeddings.py | 29 +++
frigate/embeddings/maintainer.py | 3 +
web/public/locales/en/views/settings.json | 12 +-
.../settings/ClassificationSettingsView.tsx | 172 +++++++++++-------
7 files changed, 193 insertions(+), 73 deletions(-)
diff --git a/frigate/api/classification.py b/frigate/api/classification.py
index 975a41c9d..4a6969cd3 100644
--- a/frigate/api/classification.py
+++ b/frigate/api/classification.py
@@ -298,3 +298,49 @@ def reprocess_license_plate(request: Request, event_id: str):
content=response,
status_code=200,
)
+
+
+@router.put("/reindex", dependencies=[Depends(require_role(["admin"]))])
+def reindex_embeddings(request: Request):
+ if not request.app.frigate_config.semantic_search.enabled:
+ message = (
+ "Cannot reindex tracked object embeddings, Semantic Search is not enabled."
+ )
+ logger.error(message)
+ return JSONResponse(
+ content=(
+ {
+ "success": False,
+ "message": message,
+ }
+ ),
+ status_code=400,
+ )
+
+ context: EmbeddingsContext = request.app.embeddings
+ response = context.reindex_embeddings()
+
+ if response == "started":
+ return JSONResponse(
+ content={
+ "success": True,
+ "message": "Embeddings reindexing has started.",
+ },
+ status_code=202, # 202 Accepted
+ )
+ elif response == "in_progress":
+ return JSONResponse(
+ content={
+ "success": False,
+ "message": "Embeddings reindexing is already in progress.",
+ },
+ status_code=409, # 409 Conflict
+ )
+ else:
+ return JSONResponse(
+ content={
+ "success": False,
+ "message": "Failed to start reindexing.",
+ },
+ status_code=500,
+ )
diff --git a/frigate/comms/embeddings_updater.py b/frigate/comms/embeddings_updater.py
index fc35c4665..6c26af3d1 100644
--- a/frigate/comms/embeddings_updater.py
+++ b/frigate/comms/embeddings_updater.py
@@ -17,6 +17,7 @@ class EmbeddingsRequestEnum(Enum):
register_face = "register_face"
reprocess_face = "reprocess_face"
reprocess_plate = "reprocess_plate"
+ reindex = "reindex"
class EmbeddingsResponder:
diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py
index e0673565b..c60465845 100644
--- a/frigate/embeddings/__init__.py
+++ b/frigate/embeddings/__init__.py
@@ -250,3 +250,6 @@ class EmbeddingsContext:
return self.requestor.send_data(
EmbeddingsRequestEnum.reprocess_plate.value, {"event": event}
)
+
+ def reindex_embeddings(self) -> dict[str, any]:
+ return self.requestor.send_data(EmbeddingsRequestEnum.reindex.value, {})
diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py
index 7e866d1fe..d2053f5ee 100644
--- a/frigate/embeddings/embeddings.py
+++ b/frigate/embeddings/embeddings.py
@@ -3,6 +3,7 @@
import datetime
import logging
import os
+import threading
import time
from numpy import ndarray
@@ -74,6 +75,10 @@ class Embeddings:
self.metrics = metrics
self.requestor = InterProcessRequestor()
+ self.reindex_lock = threading.Lock()
+ self.reindex_thread = None
+ self.reindex_running = False
+
# Create tables if they don't exist
self.db.create_embeddings_tables()
@@ -368,3 +373,27 @@ class Embeddings:
totals["status"] = "completed"
self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
+
+ def start_reindex(self) -> bool:
+ """Start reindexing in a separate thread if not already running."""
+ with self.reindex_lock:
+ if self.reindex_running:
+ logger.warning("Reindex embeddings is already running.")
+ return False
+
+ # Mark as running and start the thread
+ self.reindex_running = True
+ self.reindex_thread = threading.Thread(
+ target=self._reindex_wrapper, daemon=True
+ )
+ self.reindex_thread.start()
+ return True
+
+ def _reindex_wrapper(self) -> None:
+ """Wrapper to run reindex and reset running flag when done."""
+ try:
+ self.reindex()
+ finally:
+ with self.reindex_lock:
+ self.reindex_running = False
+ self.reindex_thread = None
diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index 9b90f6f2c..85b0e6d54 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -206,6 +206,9 @@ class EmbeddingMaintainer(threading.Thread):
self.embeddings.embed_description("", data, upsert=False),
pack=False,
)
+ elif topic == EmbeddingsRequestEnum.reindex.value:
+ response = self.embeddings.start_reindex()
+ return "started" if response else "in_progress"
processors = [self.realtime_processors, self.post_processors]
for processor_list in processors:
diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json
index 4a7693416..f6c5b2e99 100644
--- a/web/public/locales/en/views/settings.json
+++ b/web/public/locales/en/views/settings.json
@@ -87,9 +87,15 @@
"title": "Semantic Search",
"desc": "Semantic Search in Frigate allows you to find tracked objects within your review items using either the image itself, a user-defined text description, or an automatically generated one.",
"readTheDocumentation": "Read the Documentation",
- "reindexOnStartup": {
- "label": "Re-Index On Startup",
- "desc": "Re-indexing will reprocess all thumbnails and descriptions (if enabled) and apply the embeddings on each startup. Don't forget to disable the option after restarting!"
+ "reindexNow": {
+ "label": "Reindex Now",
+ "desc": "Reindexing will regenerate embeddings for all tracked object. This process runs in the background and may max out your CPU and take a fair amount of time depending on the number of tracked objects you have.",
+ "confirmTitle": "Confirm Reindexing",
+ "confirmDesc": "Are you sure you want to reindex all tracked object embeddings? This process will run in the background but it may max out your CPU and take a fair amount of time. You can watch the progress on the Explore page.",
+ "confirmButton": "Reindex",
+ "success": "Reindexing started successfully.",
+ "alreadyInProgress": "Reindexing is already in progress.",
+ "error": "Failed to start reindexing: {{errorMessage}}"
},
"modelSize": {
"label": "Model Size",
diff --git a/web/src/views/settings/ClassificationSettingsView.tsx b/web/src/views/settings/ClassificationSettingsView.tsx
index 24c3a9107..d12008f8f 100644
--- a/web/src/views/settings/ClassificationSettingsView.tsx
+++ b/web/src/views/settings/ClassificationSettingsView.tsx
@@ -21,11 +21,21 @@ import {
SelectTrigger,
} from "@/components/ui/select";
import { Trans, useTranslation } from "react-i18next";
+import {
+ AlertDialog,
+ AlertDialogAction,
+ AlertDialogCancel,
+ AlertDialogContent,
+ AlertDialogDescription,
+ AlertDialogFooter,
+ AlertDialogHeader,
+ AlertDialogTitle,
+} from "@/components/ui/alert-dialog";
+import { buttonVariants } from "@/components/ui/button";
type ClassificationSettings = {
search: {
enabled?: boolean;
- reindex?: boolean;
model_size?: SearchModelSize;
};
face: {
@@ -48,39 +58,22 @@ export default function ClassificationSettingsView({
useSWR("config");
const [changedValue, setChangedValue] = useState(false);
const [isLoading, setIsLoading] = useState(false);
+ const [isReindexDialogOpen, setIsReindexDialogOpen] = useState(false);
const { addMessage, removeMessage } = useContext(StatusBarMessagesContext)!;
const [classificationSettings, setClassificationSettings] =
useState({
- search: {
- enabled: undefined,
- reindex: undefined,
- model_size: undefined,
- },
- face: {
- enabled: undefined,
- model_size: undefined,
- },
- lpr: {
- enabled: undefined,
- },
+ search: { enabled: undefined, model_size: undefined },
+ face: { enabled: undefined, model_size: undefined },
+ lpr: { enabled: undefined },
});
const [origSearchSettings, setOrigSearchSettings] =
useState({
- search: {
- enabled: undefined,
- reindex: undefined,
- model_size: undefined,
- },
- face: {
- enabled: undefined,
- model_size: undefined,
- },
- lpr: {
- enabled: undefined,
- },
+ search: { enabled: undefined, model_size: undefined },
+ face: { enabled: undefined, model_size: undefined },
+ lpr: { enabled: undefined },
});
useEffect(() => {
@@ -89,32 +82,26 @@ export default function ClassificationSettingsView({
setClassificationSettings({
search: {
enabled: config.semantic_search.enabled,
- reindex: config.semantic_search.reindex,
model_size: config.semantic_search.model_size,
},
face: {
enabled: config.face_recognition.enabled,
model_size: config.face_recognition.model_size,
},
- lpr: {
- enabled: config.lpr.enabled,
- },
+ lpr: { enabled: config.lpr.enabled },
});
}
setOrigSearchSettings({
search: {
enabled: config.semantic_search.enabled,
- reindex: config.semantic_search.reindex,
model_size: config.semantic_search.model_size,
},
face: {
enabled: config.face_recognition.enabled,
model_size: config.face_recognition.model_size,
},
- lpr: {
- enabled: config.lpr.enabled,
- },
+ lpr: { enabled: config.lpr.enabled },
});
}
// we know that these deps are correct
@@ -125,10 +112,7 @@ export default function ClassificationSettingsView({
newConfig: Partial,
) => {
setClassificationSettings((prevConfig) => ({
- search: {
- ...prevConfig.search,
- ...newConfig.search,
- },
+ search: { ...prevConfig.search, ...newConfig.search },
face: { ...prevConfig.face, ...newConfig.face },
lpr: { ...prevConfig.lpr, ...newConfig.lpr },
}));
@@ -141,10 +125,8 @@ export default function ClassificationSettingsView({
axios
.put(
- `config/set?semantic_search.enabled=${classificationSettings.search.enabled ? "True" : "False"}&semantic_search.reindex=${classificationSettings.search.reindex ? "True" : "False"}&semantic_search.model_size=${classificationSettings.search.model_size}&face_recognition.enabled=${classificationSettings.face.enabled ? "True" : "False"}&face_recognition.model_size=${classificationSettings.face.model_size}&lpr.enabled=${classificationSettings.lpr.enabled ? "True" : "False"}`,
- {
- requires_restart: 0,
- },
+ `config/set?semantic_search.enabled=${classificationSettings.search.enabled ? "True" : "False"}&semantic_search.model_size=${classificationSettings.search.model_size}&face_recognition.enabled=${classificationSettings.face.enabled ? "True" : "False"}&face_recognition.model_size=${classificationSettings.face.model_size}&lpr.enabled=${classificationSettings.lpr.enabled ? "True" : "False"}`,
+ { requires_restart: 0 },
)
.then((res) => {
if (res.status === 200) {
@@ -156,9 +138,7 @@ export default function ClassificationSettingsView({
} else {
toast.error(
t("classification.toast.error", { errorMessage: res.statusText }),
- {
- position: "top-center",
- },
+ { position: "top-center" },
);
}
})
@@ -169,9 +149,7 @@ export default function ClassificationSettingsView({
"Unknown error";
toast.error(
t("toast.save.error.title", { errorMessage, ns: "common" }),
- {
- position: "top-center",
- },
+ { position: "top-center" },
);
})
.finally(() => {
@@ -191,6 +169,43 @@ export default function ClassificationSettingsView({
removeMessage("search_settings", "search_settings");
}, [origSearchSettings, removeMessage]);
+ const onReindex = useCallback(() => {
+ setIsLoading(true);
+
+ axios
+ .put("/reindex")
+ .then((res) => {
+ if (res.status === 202) {
+ toast.success(t("classification.semanticSearch.reindexNow.success"), {
+ position: "top-center",
+ });
+ } else {
+ toast.error(
+ t("classification.semanticSearch.reindexNow.error", {
+ errorMessage: res.statusText,
+ }),
+ { position: "top-center" },
+ );
+ }
+ })
+ .catch((error) => {
+ const errorMessage =
+ error.response?.data?.message ||
+ error.response?.data?.detail ||
+ "Unknown error";
+ toast.error(
+ t("classification.semanticSearch.reindexNow.error", {
+ errorMessage,
+ }),
+ { position: "top-center" },
+ );
+ })
+ .finally(() => {
+ setIsLoading(false);
+ setIsReindexDialogOpen(false);
+ });
+ }, [t]);
+
useEffect(() => {
if (changedValue) {
addMessage(
@@ -262,28 +277,18 @@ export default function ClassificationSettingsView({
From e1a40534262e7dfe6632ca4026d3129b563f64c2 Mon Sep 17 00:00:00 2001
From: leccelecce <24962424+leccelecce@users.noreply.github.com>
Date: Thu, 27 Mar 2025 18:48:43 +0000
Subject: [PATCH 76/97] Upgrade bundled nginx to 1.27.4 (#17419)
---
docker/main/build_nginx.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker/main/build_nginx.sh b/docker/main/build_nginx.sh
index 2591810e3..606682665 100755
--- a/docker/main/build_nginx.sh
+++ b/docker/main/build_nginx.sh
@@ -2,7 +2,7 @@
set -euxo pipefail
-NGINX_VERSION="1.25.3"
+NGINX_VERSION="1.27.4"
VOD_MODULE_VERSION="1.31"
SECURE_TOKEN_MODULE_VERSION="1.5"
SET_MISC_MODULE_VERSION="v0.33"
From 3f1b4438e4f42dbc8f63b21f78c39a07d98e357e Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Thu, 27 Mar 2025 13:41:22 -0600
Subject: [PATCH 77/97] Ensure landmark detector has a defualt value (#17420)
---
frigate/data_processing/common/face/model.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/frigate/data_processing/common/face/model.py b/frigate/data_processing/common/face/model.py
index eb27df68d..0aeb76792 100644
--- a/frigate/data_processing/common/face/model.py
+++ b/frigate/data_processing/common/face/model.py
@@ -20,6 +20,7 @@ class FaceRecognizer(ABC):
def __init__(self, config: FrigateConfig) -> None:
self.config = config
+ self.landmark_detector: cv2.face.FacemarkLBF = None
self.init_landmark_detector()
@abstractmethod
From 37e0b9b9049ce4bfc08ef0eb1872182a162aa254 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Fri, 28 Mar 2025 07:29:11 -0500
Subject: [PATCH 78/97] LPR tweaks (#17428)
* fix snapshot when using dedicated lpr
* enhancement and debugging config
* docs
---
.../license_plate_recognition.md | 19 ++++-
docs/docs/configuration/reference.md | 7 +-
frigate/config/classification.py | 16 ++++
.../common/license_plate/mixin.py | 77 +++++++++++++++----
4 files changed, 102 insertions(+), 17 deletions(-)
diff --git a/docs/docs/configuration/license_plate_recognition.md b/docs/docs/configuration/license_plate_recognition.md
index 13d45310e..f3581e598 100644
--- a/docs/docs/configuration/license_plate_recognition.md
+++ b/docs/docs/configuration/license_plate_recognition.md
@@ -87,6 +87,20 @@ Fine-tune the LPR feature using these optional parameters:
- For example, setting `match_distance: 1` allows a plate `ABCDE` to match `ABCBE` or `ABCD`.
- This parameter will _not_ operate on known plates that are defined as regular expressions. You should define the full string of your plate in `known_plates` in order to use `match_distance`.
+### Image Enhancement
+
+- **`enhancement`**: A value between **0 and 10** that adjusts the level of image enhancement applied to captured license plates before they are processed for recognition. This preprocessing step can sometimes improve accuracy but may also have the opposite effect.
+ - **Default:** `0` (no enhancement)
+ - Higher values increase contrast, sharpen details, and reduce noise, but excessive enhancement can blur or distort characters, actually making them much harder for Frigate to recognize.
+ - This setting is best adjusted **at the camera level** if running LPR on multiple cameras.
+ - If Frigate is already recognizing plates correctly, leave this setting at the default of `0`. However, if you're experiencing frequent character issues or incomplete plates and you can already easily read the plates yourself, try increasing the value gradually, starting at **5** and adjusting as needed. To preview how different enhancement levels affect your plates, use the `debug_save_plates` configuration option (see below).
+
+### Debugging
+
+- **`debug_save_plates`**: Set to `True` to save captured text on plates for debugging. These images are stored in `/media/frigate/clips/lpr`, organized into subdirectories by `/`, and named based on the capture timestamp.
+ - These saved images are not full plates but rather the specific areas of text detected on the plates. It is normal for the text detection model to sometimes find multiple areas of text on the plate. Use them to analyze what text Frigate recognized and how image enhancement affects detection.
+ - **Note:** Frigate does **not** automatically delete these debug images. Once LPR is functioning correctly, you should disable this option and manually remove the saved files to free up storage.
+
## Configuration Examples
These configuration parameters are available at the global level of your config. The only optional parameters that should be set at the camera level are `enabled` and `min_area`.
@@ -143,6 +157,7 @@ cameras:
lpr:
enabled: True
expire_time: 3 # optional, default
+ enhancement: 3 # optional, enhance the image before trying to recognize characters
ffmpeg: ...
detect:
enabled: False # optional, disable Frigate's standard object detection pipeline
@@ -151,7 +166,7 @@ cameras:
height: 1080
motion:
threshold: 30
- contour_area: 80 # use an increased value here to tune out small motion changes
+ contour_area: 60 # use an increased value here to tune out small motion changes
improve_contrast: false
mask: 0.704,0.007,0.709,0.052,0.989,0.055,0.993,0.001 # ensure your camera's timestamp is masked
record:
@@ -197,6 +212,7 @@ Ensure that:
- Your camera has a clear, human-readable, well-lit view of the plate. If you can't read the plate's characters, Frigate certainly won't be able to, even if the model is recognizing a `license_plate`. This may require changing video size, quality, or frame rate settings on your camera, depending on your scene and how fast the vehicles are traveling.
- The plate is large enough in the image (try adjusting `min_area`) or increasing the resolution of your camera's stream.
+- Your `enhancement` level (if you've changed it from the default of `0`) is not too high. Too much enhancement will run too much denoising and cause the plate characters to become blurry and unreadable.
If you are using a Frigate+ model or a custom model that detects license plates, ensure that `license_plate` is added to your list of objects to track.
If you are using the free model that ships with Frigate, you should _not_ add `license_plate` to the list of objects to track.
@@ -229,6 +245,7 @@ Use `match_distance` to allow small character mismatches. Alternatively, define
- If you are using a Frigate+ model or a model that detects license plates, watch the debug view (Settings --> Debug) to ensure that `license_plate` is being detected with a `car`.
- Watch the debug view to see plates recognized in real-time. For non-dedicated LPR cameras, the `car` label will change to the recognized plate when LPR is enabled and working.
- Adjust `detection_threshold` and `recognition_threshold` settings per the suggestions [above](#advanced-configuration).
+- Enable `debug_save_plates` to save images of detected text on plates to the clips directory (`/media/frigate/clips/lpr`).
- Enable debug logs for LPR by adding `frigate.data_processing.common.license_plate: debug` to your `logger` configuration. These logs are _very_ verbose, so only enable this when necessary.
```yaml
diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md
index e30ee1619..81ca1becc 100644
--- a/docs/docs/configuration/reference.md
+++ b/docs/docs/configuration/reference.md
@@ -562,7 +562,7 @@ face_recognition:
blur_confidence_filter: True
# Optional: Configuration for license plate recognition capability
-# NOTE: enabled and min_area can be overridden at the camera level
+# NOTE: enabled, min_area, and enhancement can be overridden at the camera level
lpr:
# Optional: Enable license plate recognition (default: shown below)
enabled: False
@@ -580,6 +580,11 @@ lpr:
match_distance: 1
# Optional: Known plates to track (strings or regular expressions) (default: shown below)
known_plates: {}
+ # Optional: Enhance the detected plate image with contrast adjustment and denoising (default: shown below)
+ # A value between 0 and 10. Higher values are not always better and may perform worse than lower values.
+ enhancement: 0
+ # Optional: Save plate images to /media/frigate/clips/lpr for debugging purposes (default: shown below)
+ debug_save_plates: False
# Optional: Configuration for AI generated tracked object descriptions
# WARNING: Depending on the provider, this will send thumbnails over the internet
diff --git a/frigate/config/classification.py b/frigate/config/classification.py
index aecbf6537..4026106b2 100644
--- a/frigate/config/classification.py
+++ b/frigate/config/classification.py
@@ -126,6 +126,16 @@ class LicensePlateRecognitionConfig(FrigateBaseModel):
known_plates: Optional[Dict[str, List[str]]] = Field(
default={}, title="Known plates to track (strings or regular expressions)."
)
+ enhancement: int = Field(
+ default=0,
+ title="Amount of contrast adjustment and denoising to apply to license plate images before recognition.",
+ ge=0,
+ le=10,
+ )
+ debug_save_plates: bool = Field(
+ default=False,
+ title="Save plates captured for LPR for debugging purposes.",
+ )
class CameraLicensePlateRecognitionConfig(FrigateBaseModel):
@@ -139,5 +149,11 @@ class CameraLicensePlateRecognitionConfig(FrigateBaseModel):
default=1000,
title="Minimum area of license plate to begin running recognition.",
)
+ enhancement: int = Field(
+ default=0,
+ title="Amount of contrast adjustment and denoising to apply to license plate images before recognition.",
+ ge=0,
+ le=10,
+ )
model_config = ConfigDict(extra="ignore", protected_namespaces=())
diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py
index 60066c48d..3d24d48d5 100644
--- a/frigate/data_processing/common/license_plate/mixin.py
+++ b/frigate/data_processing/common/license_plate/mixin.py
@@ -4,9 +4,11 @@ import base64
import datetime
import logging
import math
+import os
import random
import re
import string
+from pathlib import Path
from typing import List, Optional, Tuple
import cv2
@@ -20,6 +22,7 @@ from frigate.comms.event_metadata_updater import (
EventMetadataTypeEnum,
)
from frigate.config.camera.camera import CameraTypeEnum
+from frigate.const import CLIPS_DIR
from frigate.embeddings.onnx.lpr_embedding import LPR_EMBEDDING_SIZE
from frigate.util.image import area
@@ -107,7 +110,7 @@ class LicensePlateProcessingMixin:
return self._process_classification_output(images, outputs)
def _recognize(
- self, images: List[np.ndarray]
+ self, camera: string, images: List[np.ndarray]
) -> Tuple[List[str], List[List[float]]]:
"""
Recognize the characters on the detected license plates using the recognition model.
@@ -137,7 +140,7 @@ class LicensePlateProcessingMixin:
# preprocess the images based on the max aspect ratio
for i in range(index, min(num_images, index + self.batch_size)):
norm_image = self._preprocess_recognition_image(
- images[indices[i]], max_wh_ratio
+ camera, images[indices[i]], max_wh_ratio
)
norm_image = norm_image[np.newaxis, :]
norm_images.append(norm_image)
@@ -146,7 +149,7 @@ class LicensePlateProcessingMixin:
return self.ctc_decoder(outputs)
def _process_license_plate(
- self, image: np.ndarray
+ self, camera: string, id: string, image: np.ndarray
) -> Tuple[List[str], List[float], List[int]]:
"""
Complete pipeline for detecting, classifying, and recognizing license plates in the input image.
@@ -174,21 +177,37 @@ class LicensePlateProcessingMixin:
boxes = self._sort_boxes(list(boxes))
plate_images = [self._crop_license_plate(image, x) for x in boxes]
+ current_time = int(datetime.datetime.now().timestamp())
+
if WRITE_DEBUG_IMAGES:
- current_time = int(datetime.datetime.now().timestamp())
for i, img in enumerate(plate_images):
cv2.imwrite(
f"debug/frames/license_plate_cropped_{current_time}_{i + 1}.jpg",
img,
)
+ if self.config.lpr.debug_save_plates:
+ logger.debug(f"{camera}: Saving plates for event {id}")
+
+ Path(os.path.join(CLIPS_DIR, f"lpr/{camera}/{id}")).mkdir(
+ parents=True, exist_ok=True
+ )
+
+ for i, img in enumerate(plate_images):
+ cv2.imwrite(
+ os.path.join(
+ CLIPS_DIR, f"lpr/{camera}/{id}/{current_time}_{i + 1}.jpg"
+ ),
+ img,
+ )
+
# keep track of the index of each image for correct area calc later
sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in plate_images])
reverse_mapping = {
idx: original_idx for original_idx, idx in enumerate(sorted_indices)
}
- results, confidences = self._recognize(plate_images)
+ results, confidences = self._recognize(camera, plate_images)
if results:
license_plates = [""] * len(plate_images)
@@ -606,7 +625,7 @@ class LicensePlateProcessingMixin:
return images, results
def _preprocess_recognition_image(
- self, image: np.ndarray, max_wh_ratio: float
+ self, camera: string, image: np.ndarray, max_wh_ratio: float
) -> np.ndarray:
"""
Preprocess an image for recognition by dynamically adjusting its width.
@@ -634,13 +653,38 @@ class LicensePlateProcessingMixin:
else:
gray = image
- # apply CLAHE for contrast enhancement
- grid_size = (
- max(4, input_w // 40),
- max(4, input_h // 40),
- )
- clahe = cv2.createCLAHE(clipLimit=1.5, tileGridSize=grid_size)
- enhanced = clahe.apply(gray)
+ if self.config.cameras[camera].lpr.enhancement > 3:
+ # denoise using a configurable pixel neighborhood value
+ logger.debug(
+ f"{camera}: Denoising recognition image (level: {self.config.cameras[camera].lpr.enhancement})"
+ )
+ smoothed = cv2.bilateralFilter(
+ gray,
+ d=5 + self.config.cameras[camera].lpr.enhancement,
+ sigmaColor=10 * self.config.cameras[camera].lpr.enhancement,
+ sigmaSpace=10 * self.config.cameras[camera].lpr.enhancement,
+ )
+ sharpening_kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
+ processed = cv2.filter2D(smoothed, -1, sharpening_kernel)
+ else:
+ processed = gray
+
+ if self.config.cameras[camera].lpr.enhancement > 0:
+ # always apply the same CLAHE for contrast enhancement when enhancement level is above 3
+ logger.debug(
+ f"{camera}: Enhancing contrast for recognition image (level: {self.config.cameras[camera].lpr.enhancement})"
+ )
+ grid_size = (
+ max(4, input_w // 40),
+ max(4, input_h // 40),
+ )
+ clahe = cv2.createCLAHE(
+ clipLimit=2 if self.config.cameras[camera].lpr.enhancement > 5 else 1.5,
+ tileGridSize=grid_size,
+ )
+ enhanced = clahe.apply(processed)
+ else:
+ enhanced = processed
# Convert back to 3-channel for model compatibility
image = cv2.cvtColor(enhanced, cv2.COLOR_GRAY2RGB)
@@ -948,6 +992,8 @@ class LicensePlateProcessingMixin:
return
if dedicated_lpr:
+ id = "dedicated-lpr"
+
rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
# apply motion mask
@@ -1149,7 +1195,7 @@ class LicensePlateProcessingMixin:
# run detection, returns results sorted by confidence, best first
start = datetime.datetime.now().timestamp()
license_plates, confidences, areas = self._process_license_plate(
- license_plate_frame
+ camera, id, license_plate_frame
)
self.__update_lpr_metrics(datetime.datetime.now().timestamp() - start)
@@ -1257,9 +1303,10 @@ class LicensePlateProcessingMixin:
f"{camera}: Writing snapshot for {id}, {top_plate}, {current_time}"
)
frame_bgr = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+ _, encoded_img = cv2.imencode(".jpg", frame_bgr)
self.sub_label_publisher.publish(
EventMetadataTypeEnum.save_lpr_snapshot,
- (base64.b64encode(frame_bgr).decode("ASCII"), id, camera),
+ (base64.b64encode(encoded_img).decode("ASCII"), id, camera),
)
self.detected_license_plates[id] = {
From b14abffea31457b05cae1dabd9ff0ea11792a945 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Fri, 28 Mar 2025 12:52:12 -0600
Subject: [PATCH 79/97] Refactor face library page (#17424)
* Section faces by event id
* Make score keeping more robust
* layout improvements
* Cleanup dialog
* Fix clicking behavior
* Add view in explore option
* math.round
* Don't require events
* Cleanup
* Remove selection
* Don't require
* Change dialog size with snapshot
* Use filename as key
* fix key
* Rework layout for mobile
* Handle mobile landscape
* Fix train issue
* Match logic
* Move deletion logic
* Fix reprocessing
* Support creating a new face
* Translations
* Do sorting in frontend
* Adjust unknown
* Cleanup
* Set max limit to faces to recognize
* Fix sorting
* Fix
---
docs/docs/configuration/face_recognition.md | 2 +-
frigate/api/classification.py | 19 +-
frigate/data_processing/real_time/face.py | 97 +++--
web/public/locales/en/views/faceLibrary.json | 1 +
web/src/pages/FaceLibrary.tsx | 353 +++++++++++++------
web/src/types/face.ts | 1 +
6 files changed, 325 insertions(+), 148 deletions(-)
diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md
index 9fb5c9a44..d618dbba6 100644
--- a/docs/docs/configuration/face_recognition.md
+++ b/docs/docs/configuration/face_recognition.md
@@ -59,7 +59,7 @@ Fine-tune face recognition with these optional parameters:
### Recognition
- `model_size`: Which model size to use, options are `small` or `large`
-- `unknown_score`: Min score to mark a person as a potential match, matches below this will be marked as unknown.
+- `unknown_score`: Min score to mark a person as a potential match, matches at or below this will be marked as unknown.
- Default: `0.8`.
- `recognition_threshold`: Recognition confidence score required to add the face to the object as a sub label.
- Default: `0.9`.
diff --git a/frigate/api/classification.py b/frigate/api/classification.py
index 4a6969cd3..8c2d464d4 100644
--- a/frigate/api/classification.py
+++ b/frigate/api/classification.py
@@ -41,13 +41,9 @@ def get_faces():
face_dict[name] = []
- for file in sorted(
- filter(
- lambda f: (f.lower().endswith((".webp", ".png", ".jpg", ".jpeg"))),
- os.listdir(face_dir),
- ),
- key=lambda f: os.path.getctime(os.path.join(face_dir, f)),
- reverse=True,
+ for file in filter(
+ lambda f: (f.lower().endswith((".webp", ".png", ".jpg", ".jpeg"))),
+ os.listdir(face_dir),
):
face_dict[name].append(file)
@@ -125,10 +121,13 @@ def train_face(request: Request, name: str, body: dict = None):
sanitized_name = sanitize_filename(name)
rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
new_name = f"{sanitized_name}-{rand_id}.webp"
- new_file = os.path.join(FACE_DIR, f"{sanitized_name}/{new_name}")
+ new_file_folder = os.path.join(FACE_DIR, f"{sanitized_name}")
+
+ if not os.path.exists(new_file_folder):
+ os.mkdir(new_file_folder)
if training_file_name:
- shutil.move(training_file, new_file)
+ shutil.move(training_file, os.path.join(new_file_folder, new_name))
else:
try:
event: Event = Event.get(Event.id == event_id)
@@ -155,7 +154,7 @@ def train_face(request: Request, name: str, body: dict = None):
x2 = x1 + int(face_box[2] * detect_config.width) - 4
y2 = y1 + int(face_box[3] * detect_config.height) - 4
face = snapshot[y1:y2, x1:x2]
- cv2.imwrite(new_file, face)
+ cv2.imwrite(os.path.join(new_file_folder, new_name), face)
context: EmbeddingsContext = request.app.embeddings
context.clear_face_classifier()
diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index d3fbc83df..10479b92b 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -33,7 +33,8 @@ logger = logging.getLogger(__name__)
MAX_DETECTION_HEIGHT = 1080
-MIN_MATCHING_FACES = 2
+MAX_FACES_ATTEMPTS_AFTER_REC = 6
+MAX_FACE_ATTEMPTS = 12
class FaceRealTimeProcessor(RealTimeProcessorApi):
@@ -170,6 +171,23 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
)
return
+ # check if we have hit limits
+ if (
+ id in self.person_face_history
+ and len(self.person_face_history[id]) >= MAX_FACES_ATTEMPTS_AFTER_REC
+ ):
+ # if we are at max attempts after rec and we have a rec
+ if obj_data.get("sub_label"):
+ logger.debug(
+ "Not processing due to hitting max attempts after true recognition."
+ )
+ return
+
+ # if we don't have a rec and are at max attempts
+ if len(self.person_face_history[id]) >= MAX_FACE_ATTEMPTS:
+ logger.debug("Not processing due to hitting max rec attempts.")
+ return
+
face: Optional[dict[str, any]] = None
if self.requires_face_detection:
@@ -241,7 +259,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
sub_label, score = res
- if score < self.face_config.unknown_score:
+ if score <= self.face_config.unknown_score:
sub_label = "unknown"
logger.debug(
@@ -255,13 +273,23 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
os.makedirs(folder, exist_ok=True)
cv2.imwrite(file, face_frame)
+ files = sorted(
+ filter(lambda f: (f.endswith(".webp")), os.listdir(folder)),
+ key=lambda f: os.path.getctime(os.path.join(folder, f)),
+ reverse=True,
+ )
+
+ # delete oldest face image if maximum is reached
+ if len(files) > self.config.face_recognition.save_attempts:
+ os.unlink(os.path.join(folder, files[-1]))
+
if id not in self.person_face_history:
self.person_face_history[id] = []
self.person_face_history[id].append(
(sub_label, score, face_frame.shape[0] * face_frame.shape[1])
)
- (weighted_sub_label, weighted_score) = self.weighted_average_by_area(
+ (weighted_sub_label, weighted_score) = self.weighted_average(
self.person_face_history[id]
)
@@ -297,6 +325,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
sub_label, score = res
+ if score <= self.face_config.unknown_score:
+ sub_label = "unknown"
+
return {"success": True, "score": score, "face_name": sub_label}
elif topic == EmbeddingsRequestEnum.register_face.value:
rand_id = "".join(
@@ -366,6 +397,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
sub_label, score = res
+ if score <= self.face_config.unknown_score:
+ sub_label = "unknown"
+
if self.config.face_recognition.save_attempts:
# write face to library
folder = os.path.join(FACE_DIR, "train")
@@ -375,38 +409,49 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
)
shutil.move(current_file, new_file)
- files = sorted(
- filter(lambda f: (f.endswith(".webp")), os.listdir(folder)),
- key=lambda f: os.path.getctime(os.path.join(folder, f)),
- reverse=True,
- )
-
- # delete oldest face image if maximum is reached
- if len(files) > self.config.face_recognition.save_attempts:
- os.unlink(os.path.join(folder, files[-1]))
-
def expire_object(self, object_id: str):
if object_id in self.person_face_history:
self.person_face_history.pop(object_id)
- def weighted_average_by_area(self, results_list: list[tuple[str, float, int]]):
- score_count = {}
+ def weighted_average(
+ self, results_list: list[tuple[str, float, int]], max_weight: int = 4000
+ ):
+ """
+ Calculates a robust weighted average, capping the area weight and giving more weight to higher scores.
+
+ Args:
+ results_list: A list of tuples, where each tuple contains (name, score, face_area).
+ max_weight: The maximum weight to apply based on face area.
+
+ Returns:
+ A tuple containing the prominent name and its weighted average score, or (None, 0.0) if the list is empty.
+ """
+ if not results_list:
+ return None, 0.0
+
weighted_scores = {}
- total_face_areas = {}
+ total_weights = {}
for name, score, face_area in results_list:
+ if name == "unknown":
+ continue
+
if name not in weighted_scores:
- score_count[name] = 1
weighted_scores[name] = 0.0
- total_face_areas[name] = 0.0
- else:
- score_count[name] += 1
+ total_weights[name] = 0.0
- weighted_scores[name] += score * face_area
- total_face_areas[name] += face_area
+ # Capped weight based on face area
+ weight = min(face_area, max_weight)
- prominent_name = max(score_count)
+ # Score-based weighting (higher scores get more weight)
+ weight *= (score - self.face_config.unknown_score) * 10
+ weighted_scores[name] += score * weight
+ total_weights[name] += weight
- return prominent_name, weighted_scores[prominent_name] / total_face_areas[
- prominent_name
- ]
+ if not weighted_scores:
+ return None, 0.0
+
+ best_name = max(weighted_scores, key=weighted_scores.get)
+ weighted_average = weighted_scores[best_name] / total_weights[best_name]
+
+ return best_name, weighted_average
diff --git a/web/public/locales/en/views/faceLibrary.json b/web/public/locales/en/views/faceLibrary.json
index fd5f50825..baabed69a 100644
--- a/web/public/locales/en/views/faceLibrary.json
+++ b/web/public/locales/en/views/faceLibrary.json
@@ -17,6 +17,7 @@
"createFaceLibrary": {
"title": "Create Face Library",
"desc": "Create a new face library",
+ "new": "Create New Face",
"nextSteps": "It is recommended to use the Train tab to select and train images for each person as they are detected. When building a strong foundation it is strongly recommended to only train on images that are straight-on. Ignore images from cameras that recognize faces from an angle."
},
"train": {
diff --git a/web/src/pages/FaceLibrary.tsx b/web/src/pages/FaceLibrary.tsx
index 696691997..df57f729a 100644
--- a/web/src/pages/FaceLibrary.tsx
+++ b/web/src/pages/FaceLibrary.tsx
@@ -3,6 +3,7 @@ import TimeAgo from "@/components/dynamic/TimeAgo";
import AddFaceIcon from "@/components/icons/AddFaceIcon";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import CreateFaceWizardDialog from "@/components/overlay/detail/FaceCreateWizardDialog";
+import TextEntryDialog from "@/components/overlay/dialog/TextEntryDialog";
import UploadImageDialog from "@/components/overlay/dialog/UploadImageDialog";
import { Button } from "@/components/ui/button";
import {
@@ -32,13 +33,23 @@ import { useFormattedTimestamp } from "@/hooks/use-date-utils";
import useKeyboardListener from "@/hooks/use-keyboard-listener";
import useOptimisticState from "@/hooks/use-optimistic-state";
import { cn } from "@/lib/utils";
+import { Event } from "@/types/event";
import { FaceLibraryData, RecognizedFaceData } from "@/types/face";
import { FaceRecognitionConfig, FrigateConfig } from "@/types/frigateConfig";
+import { TooltipPortal } from "@radix-ui/react-tooltip";
import axios from "axios";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { isDesktop, isMobile } from "react-device-detect";
import { useTranslation } from "react-i18next";
-import { LuImagePlus, LuRefreshCw, LuScanFace, LuTrash2 } from "react-icons/lu";
+import {
+ LuImagePlus,
+ LuPlus,
+ LuRefreshCw,
+ LuScanFace,
+ LuSearch,
+ LuTrash2,
+} from "react-icons/lu";
+import { useNavigate } from "react-router-dom";
import { toast } from "sonner";
import useSWR from "swr";
@@ -391,14 +402,53 @@ function TrainingGrid({
onClickFace,
onRefresh,
}: TrainingGridProps) {
- const { t } = useTranslation(["views/faceLibrary"]);
+ const { t } = useTranslation(["views/faceLibrary", "views/explore"]);
+ const navigate = useNavigate();
// face data
- const [selectedEvent, setSelectedEvent] = useState();
+ const faceGroups = useMemo(() => {
+ const groups: { [eventId: string]: RecognizedFaceData[] } = {};
+
+ Array.from(new Set(attemptImages))
+ .sort()
+ .reverse()
+ .forEach((image) => {
+ const parts = image.split("-");
+ const data = {
+ filename: image,
+ timestamp: Number.parseFloat(parts[0]),
+ eventId: `${parts[0]}-${parts[1]}`,
+ name: parts[2],
+ score: Number.parseFloat(parts[3]),
+ };
+
+ if (groups[data.eventId]) {
+ groups[data.eventId].push(data);
+ } else {
+ groups[data.eventId] = [data];
+ }
+ });
+
+ return groups;
+ }, [attemptImages]);
+
+ const eventIdsQuery = useMemo(
+ () => Object.keys(faceGroups).join(","),
+ [faceGroups],
+ );
+
+ const { data: events } = useSWR([
+ "event_ids",
+ { ids: eventIdsQuery },
+ ]);
+
+ // selection
+
+ const [selectedEvent, setSelectedEvent] = useState();
const formattedDate = useFormattedTimestamp(
- selectedEvent?.timestamp ?? 0,
+ selectedEvent?.start_time ?? 0,
config?.ui.time_format == "24hour"
? t("time.formattedTimestampWithYear.24hour", { ns: "common" })
: t("time.formattedTimestampWithYear.12hour", { ns: "common" }),
@@ -415,23 +465,32 @@ function TrainingGrid({
}
}}
>
-
+ {t("details.face")}{t("details.faceDesc")}
![From bda7fcc7842defedd9457c55eadf19895b192318 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 29 Mar 2025 07:19:12 -0500 Subject: [PATCH 84/97] use ceil instead of round for recording snapshot (#17444) --- frigate/api/media.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frigate/api/media.py b/frigate/api/media.py index e9290e237..7dab6ae60 100644 --- a/frigate/api/media.py +++ b/frigate/api/media.py @@ -2,6 +2,7 @@ import glob import logging +import math import os import subprocess as sp import time @@ -262,7 +263,7 @@ def get_snapshot_from_recording( except DoesNotExist: # try again with a rounded frame time as it may be between # the rounded segment start time - frame_time = round(frame_time) + frame_time = math.ceil(frame_time) try: recording = ( Recordings.select( From 17912b4695b92f6aae105687ee551a1f30539f0d Mon Sep 17 00:00:00 2001 From: GuoQing Liu <842607283@qq.com> Date: Sun, 30 Mar 2025 18:55:26 +0800 Subject: [PATCH 85/97] update chinese i18n (#17450) --- web/public/locales/zh-CN/views/faceLibrary.json | 1 + web/public/locales/zh-CN/views/settings.json | 16 +++++++++++----- web/public/locales/zh-CN/views/system.json | 7 ++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/web/public/locales/zh-CN/views/faceLibrary.json b/web/public/locales/zh-CN/views/faceLibrary.json index 76a98bf31..f2dba6fc3 100644 --- a/web/public/locales/zh-CN/views/faceLibrary.json +++ b/web/public/locales/zh-CN/views/faceLibrary.json @@ -16,6 +16,7 @@ "createFaceLibrary": { "title": "创建人脸库", "desc": "创建一个新的人脸库", + "new": "新建人脸", "nextSteps": "建议使用“训练”选项卡为每个检测到的人选择并训练图像。在打好基础前，强烈建议训练仅使用正面图像。而不是从摄像机中识别到的角度拍摄的人脸图像。" }, "train": { diff --git a/web/public/locales/zh-CN/views/settings.json b/web/public/locales/zh-CN/views/settings.json index 0c1087916..779d4ca28 100644 --- a/web/public/locales/zh-CN/views/settings.json +++ b/web/public/locales/zh-CN/views/settings.json @@ -87,9 +87,15 @@ "title": "语义搜索", "desc": "Frigate的语义搜索能够让你使用自然语言根据图像本身、自定义的文本描述或自动生成的描述来搜索视频。", "readTheDocumentation": "阅读文档（英文）", - "reindexOnStartup": { - "label": "启动时重新索引", - "desc": "每次启动将重新索引并重新处理所有缩略图和描述。关闭该设置后不要忘记重启！" + "reindexNow": { + "label": "立即重建索引", + "desc": "重建索引将为所有跟踪对象重新生成特征向量。该过程将在后台运行，可能会使CPU满载，所需时间取决于跟踪对象的数量。", + "confirmTitle": "确认重建索引", + "confirmDesc": "确定要为所有跟踪对象重建特征向量索引吗？此过程将在后台运行，但可能会导致CPU满载并耗费较长时间。您可以在探索页面查看进度。", + "confirmButton": "重建索引", + "success": "重建索引已成功启动。", + "alreadyInProgress": "重建索引已在执行中。", + "error": "启动重建索引失败：{{errorMessage}}" }, "modelSize": { "label": "模型大小", @@ -113,11 +119,11 @@ "desc": "用于人脸识别的模型尺寸。", "small": { "title": "小模型", - "desc": "使用小模型将采用OpenCV的局部二值模式直方图(LBPH)算法，可在大多数CPU上高效运行。" + "desc": "使用小模型将采用FaceNet人脸特征提取模型，可在大多数CPU上高效运行。" }, "large": { "title": "大模型", - "desc": "使用大模型将采用ArcFace人脸嵌入模型，若适用将自动在GPU上运行。" + "desc": "使用大模型将采用ArcFace人脸特征提取模型，若条件允许将自动使用GPU运行。" } } }, diff --git a/web/public/locales/zh-CN/views/system.json b/web/public/locales/zh-CN/views/system.json index 01251f3c5..8e442162a 100644 --- a/web/public/locales/zh-CN/views/system.json +++ b/web/public/locales/zh-CN/views/system.json @@ -3,7 +3,7 @@ "cameras": "摄像头统计 - Frigate", "storage": "存储统计 - Frigate", "general": "常规统计 - Frigate", - "features": "功能统计 - Frigate", + "enrichments": "增强功能统计 - Frigate", "logs": { "frigate": "Frigate 日志 - Frigate", "go2rtc": "Go2RTC 日志 - Frigate", @@ -144,8 +144,9 @@ "healthy": "系统运行正常", "reindexingEmbeddings": "正在重新索引嵌入（已完成 {{processed}}%）" }, - "features": { - "title": "功能", + "enrichments": { + "title": "增强功能", + "infPerSecond": "每秒推理次数", "embeddings": { "image_embedding_speed": "图像特征提取速度", "face_embedding_speed": "人脸特征提取速度", From 2c1ded37a172c2b8e10a1b9bda5c2bb5faef7795 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 30 Mar 2025 07:17:25 -0500 Subject: [PATCH 86/97] Ensure we use the stream name from local storage on mobile (#17452) --- web/src/views/live/LiveDashboardView.tsx | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index 199146b0b..b00b1a2f5 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -472,11 +472,20 @@ export default function LiveDashboardView({ } else { grow = "aspect-video"; } - const streamName = - currentGroupStreamingSettings?.[camera.name]?.streamName || - camera?.live?.streams - ? Object?.values(camera?.live?.streams)?.[0] - : ""; + const availableStreams = camera.live.streams || {}; + const firstStreamEntry = Object.values(availableStreams)[0] || ""; + + const streamNameFromSettings = + currentGroupStreamingSettings?.[camera.name]?.streamName || ""; + const streamExists = + streamNameFromSettings && + Object.values(availableStreams).includes( + streamNameFromSettings, + ); + + const streamName = streamExists + ? streamNameFromSettings + : firstStreamEntry; const autoLive = currentGroupStreamingSettings?.[camera.name]?.streamType !== "no-streaming"; From 2920127ada801e39652aae796a26436d53590d48 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 30 Mar 2025 08:43:24 -0500 Subject: [PATCH 87/97] Dedicated LPR improvements (#17453) * remove license plate from attributes for dedicated lpr cameras * ensure we always have a color * use frigate+ models with dedicated lpr cameras * docs * docs clarity * docs enrichments * use license_plate as object type --- .../license_plate_recognition.md | 119 ++++++++++++++---- frigate/camera/state.py | 8 +- .../common/license_plate/mixin.py | 70 +++++++---- frigate/embeddings/maintainer.py | 6 +- frigate/track/tracked_object.py | 2 +- frigate/video.py | 37 ++++-- 6 files changed, 183 insertions(+), 59 deletions(-) diff --git a/docs/docs/configuration/license_plate_recognition.md b/docs/docs/configuration/license_plate_recognition.md index f3581e598..ccb9a16a6 100644 --- a/docs/docs/configuration/license_plate_recognition.md +++ b/docs/docs/configuration/license_plate_recognition.md @@ -62,7 +62,7 @@ Fine-tune the LPR feature using these optional parameters: - **`detection_threshold`**: License plate object detection confidence score required before recognition runs. - Default: `0.7` - - Note: This is field only applies to the standalone license plate detection model, `min_score` should be used to filter for models that have license plate detection built in. + - Note: This is field only applies to the standalone license plate detection model, `threshold` and `min_score` object filters should be used for models like Frigate+ that have license plate detection built in. - **`min_area`**: Defines the minimum area (in pixels) a license plate must be before recognition runs. - Default: `1000` pixels. Note: this is intentionally set very low as it is an _area_ measurement (length x width). For reference, 1000 pixels represents a ~32x32 pixel square in your camera image. - Depending on the resolution of your camera's `detect` stream, you can increase this value to ignore small or distant plates. @@ -137,17 +137,86 @@ lpr: - "MN D3163" ``` +:::note + +If you want to detect cars on cameras but don't want to use resources to run LPR on those cars, you should disable LPR for those specific cameras. + +```yaml +cameras: + side_yard: + lpr: + enabled: False + ... +``` + +::: + ## Dedicated LPR Cameras Dedicated LPR cameras are single-purpose cameras with powerful optical zoom to capture license plates on distant vehicles, often with fine-tuned settings to capture plates at night. -Users with a dedicated LPR camera can run Frigate's LPR by specifying a camera type of `lpr` in the camera configuration. An example config for a dedicated LPR camera might look like this: +Users can configure Frigate's LPR in two different ways depending on whether they are using a Frigate+ model: + +### Using a Frigate+ Model + +Users running a Frigate+ model (or any model that natively detects `license_plate`) can take advantage of `license_plate` detection. This allows license plates to be treated as standard objects in dedicated LPR mode, meaning that alerts, detections, snapshots, zones, and other Frigate features work as usual, and plates are detected efficiently through your configured object detector. + +An example configuration for a dedicated LPR camera using a Frigate+ model: + +```yaml +# LPR global configuration +lpr: + enabled: True + +# Dedicated LPR camera configuration +cameras: + dedicated_lpr_camera: + type: "lpr" # required to use dedicated LPR camera mode + detect: + enabled: True + fps: 5 # increase if vehicles move quickly + min_initialized: 2 # set at fps divided by 3 for very fast cars + width: 1920 + height: 1080 + objects: + track: + - license_plate + filters: + license_plate: + threshold: 0.7 + motion: + threshold: 30 + contour_area: 60 # use an increased value to tune out small motion changes + improve_contrast: false + mask: 0.704,0.007,0.709,0.052,0.989,0.055,0.993,0.001 # ensure your camera's timestamp is masked + record: + enabled: True # disable recording if you only want snapshots + snapshots: + enabled: True + review: + detections: + labels: + - license_plate +``` + +With this setup: + +- License plates are treated as normal objects in Frigate. +- Scores, alerts, detections, snapshots, zones, and object masks work as expected. +- Snapshots will have license plate bounding boxes on them. +- The `frigate/events` MQTT topic will publish tracked object updates. +- Debug view will display `license_plate` bounding boxes. + +### Using the Secondary LPR Pipeline (Without Frigate+) + +If you are not running a Frigate+ model, you can use Frigate’s built-in secondary dedicated LPR pipeline. In this mode, Frigate bypasses the standard object detection pipeline and runs a local license plate detector model on the full frame whenever motion activity occurs. + +An example configuration for a dedicated LPR camera using the secondary pipeline: ```yaml # LPR global configuration lpr: enabled: True - min_plate_length: 4 detection_threshold: 0.7 # change if necessary # Dedicated LPR camera configuration @@ -156,14 +225,15 @@ cameras: type: "lpr" # required to use dedicated LPR camera mode lpr: enabled: True - expire_time: 3 # optional, default enhancement: 3 # optional, enhance the image before trying to recognize characters ffmpeg: ... detect: - enabled: False # optional, disable Frigate's standard object detection pipeline - fps: 5 # keep this at 5, higher values are unnecessary for dedicated LPR mode and could overwhelm the detector + enabled: False # disable Frigate's standard object detection pipeline + fps: 5 # increase if necessary, though high values may slow down Frigate's enrichments pipeline and use considerable CPU width: 1920 height: 1080 + objects: + track: [] # required when not using a Frigate+ model for dedicated LPR mode motion: threshold: 30 contour_area: 60 # use an increased value here to tune out small motion changes @@ -178,31 +248,38 @@ cameras: default: 7 ``` -The camera-level `type` setting tells Frigate to treat your camera as a dedicated LPR camera. Setting this option bypasses Frigate's standard object detection pipeline so that a `car` does not need to be detected to run LPR. This dedicated LPR pipeline does not utilize defined zones or object masks, and the license plate detector is always run on the full frame whenever motion activity occurs. If a plate is found, a snapshot at the highest scoring moment is saved as a `car` object, visible in Explore and searchable by the recognized plate via Explore's More Filters. - -An optional config variable for dedicated LPR cameras only, `expire_time`, can be specified under the `lpr` configuration at the camera level to change the time it takes for Frigate to consider a previously tracked plate as expired. - -:::note - -When using `type: "lpr"` for a camera, a non-standard object detection pipeline is used. Any detected license plates on dedicated LPR cameras are treated similarly to manual events in Frigate. Note that for `car` objects with license plates: +With this setup: +- The standard object detection pipeline is bypassed. Any detected license plates on dedicated LPR cameras are treated similarly to manual events in Frigate. You must **not** specify `license_plate` as an object to track. +- The license plate detector runs on the full frame whenever motion is detected and processes frames according to your detect `fps` setting. - Review items will always be classified as a `detection`. - Snapshots will always be saved. -- Tracked objects are retained according to your retain settings for `record` and `snapshots`. -- Zones and object masks cannot be used. -- Debug view may not show `license_plate` bounding boxes, even if you are using a Frigate+ model for your standard object detection pipeline. -- The `frigate/events` MQTT topic will not publish tracked object updates, though `frigate/reviews` will if recordings are enabled. +- Zones and object masks are **not** used. +- The `frigate/events` MQTT topic will **not** publish tracked object updates, though `frigate/reviews` will if recordings are enabled. +- License plate snapshots are saved at the highest-scoring moment and appear in Explore. +- Debug view will not show `license_plate` bounding boxes. -::: +### Summary + +| Feature | Native `license_plate` detecting Model (like Frigate+) | Secondary Pipeline (without native model or Frigate+) | +| ----------------------- | ------------------------------------------------------ | --------------------------------------------------------------- | +| License Plate Detection | Uses `license_plate` as a tracked object | Runs a dedicated LPR pipeline | +| FPS Setting | 5 (increase for fast-moving cars) | 5 (increase for fast-moving cars, but it may use much more CPU) | +| Object Detection | Standard Frigate+ detection applies | Bypasses standard object detection | +| Zones & Object Masks | Supported | Not supported | +| Debug View | May show `license_plate` bounding boxes | May **not** show `license_plate` bounding boxes | +| MQTT `frigate/events` | Publishes tracked object updates | Does **not** publish tracked object updates | +| Explore | Recognized plates available in More Filters | Recognized plates available in More Filters | + +By selecting the appropriate configuration, users can optimize their dedicated LPR cameras based on whether they are using a Frigate+ model or the secondary LPR pipeline. ### Best practices for using Dedicated LPR camera mode - Tune your motion detection and increase the `contour_area` until you see only larger motion boxes being created as cars pass through the frame (likely somewhere between 50-90 for a 1920x1080 detect stream). Increasing the `contour_area` filters out small areas of motion and will prevent excessive resource use from looking for license plates in frames that don't even have a car passing through it. - Disable the `improve_contrast` motion setting, especially if you are running LPR at night and the frame is mostly dark. This will prevent small pixel changes and smaller areas of motion from triggering license plate detection. - Ensure your camera's timestamp is covered with a motion mask so that it's not incorrectly detected as a license plate. -- While not strictly required, it may be beneficial to disable standard object detection on your dedicated LPR camera (`detect` --> `enabled: False`). If you've set the camera type to `"lpr"`, license plate detection will still be performed on the entire frame when motion occurs. -- If multiple tracked objects are being produced for the same license plate, you can tweak the `expire_time` to prevent plates from being expired from the view as quickly. -- You may need to change your camera settings for a clearer image or decrease your global `recognition_threshold` config if your plates are not being accurately recognized at night. +- For non-Frigate+ users, you may need to change your camera settings for a clearer image or decrease your global `recognition_threshold` config if your plates are not being accurately recognized at night. +- The secondary pipeline mode runs a local AI model on your CPU to detect plates. Increasing detect `fps` will increase CPU usage proportionally. ## FAQ diff --git a/frigate/camera/state.py b/frigate/camera/state.py index 65a3dcf5d..267c7d457 100644 --- a/frigate/camera/state.py +++ b/frigate/camera/state.py @@ -88,7 +88,9 @@ class CameraState: thickness = 1 else: thickness = 2 - color = self.config.model.colormap[obj["label"]] + color = self.config.model.colormap.get( + obj["label"], (255, 255, 255) + ) else: thickness = 1 color = (255, 0, 0) @@ -110,7 +112,9 @@ class CameraState: and obj["frame_time"] == frame_time ): thickness = 5 - color = self.config.model.colormap[obj["label"]] + color = self.config.model.colormap.get( + obj["label"], (255, 255, 255) + ) # debug autotracking zooming - show the zoom factor box if ( diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py index 63285ac79..7f731704f 100644 --- a/frigate/data_processing/common/license_plate/mixin.py +++ b/frigate/data_processing/common/license_plate/mixin.py @@ -21,7 +21,6 @@ from frigate.comms.event_metadata_updater import ( EventMetadataPublisher, EventMetadataTypeEnum, ) -from frigate.config.camera.camera import CameraTypeEnum from frigate.const import CLIPS_DIR from frigate.embeddings.onnx.lpr_embedding import LPR_EMBEDDING_SIZE from frigate.util.builtin import EventsPerSecond @@ -972,7 +971,7 @@ class LicensePlateProcessingMixin: ( now, camera, - "car", + "license_plate", event_id, True, plate_score, @@ -994,9 +993,7 @@ class LicensePlateProcessingMixin: if not self.config.cameras[camera].lpr.enabled: return - if not dedicated_lpr and self.config.cameras[camera].type == CameraTypeEnum.lpr: - return - + # dedicated LPR cam without frigate+ if dedicated_lpr: id = "dedicated-lpr" @@ -1050,8 +1047,11 @@ class LicensePlateProcessingMixin: else: id = obj_data["id"] - # don't run for non car objects - if obj_data.get("label") != "car": + # don't run for non car or non license plate (dedicated lpr with frigate+) objects + if ( + obj_data.get("label") != "car" + and obj_data.get("label") != "license_plate" + ): logger.debug( f"{camera}: Not a processing license plate for non car object." ) @@ -1131,26 +1131,34 @@ class LicensePlateProcessingMixin: license_plate[0] : license_plate[2], ] else: - # don't run for object without attributes - if not obj_data.get("current_attributes"): + # don't run for object without attributes if this isn't dedicated lpr with frigate+ + if ( + not obj_data.get("current_attributes") + and obj_data.get("label") != "license_plate" + ): logger.debug(f"{camera}: No attributes to parse.") return - attributes: list[dict[str, any]] = obj_data.get( - "current_attributes", [] - ) - for attr in attributes: - if attr.get("label") != "license_plate": - continue + if obj_data.get("label") == "car": + attributes: list[dict[str, any]] = obj_data.get( + "current_attributes", [] + ) + for attr in attributes: + if attr.get("label") != "license_plate": + continue - if license_plate is None or attr.get( - "score", 0.0 - ) > license_plate.get("score", 0.0): - license_plate = attr + if license_plate is None or attr.get( + "score", 0.0 + ) > license_plate.get("score", 0.0): + license_plate = attr - # no license plates detected in this frame - if not license_plate: - return + # no license plates detected in this frame + if not license_plate: + return + + # we are using dedicated lpr with frigate+ + if obj_data.get("label") == "license_plate": + license_plate = obj_data license_plate_box = license_plate.get("box") @@ -1160,7 +1168,9 @@ class LicensePlateProcessingMixin: or area(license_plate_box) < self.config.cameras[obj_data["camera"]].lpr.min_area ): - logger.debug(f"{camera}: Invalid license plate box {license_plate}") + logger.debug( + f"{camera}: Area for license plate box {area(license_plate_box)} is less than min_area {self.config.cameras[obj_data['camera']].lpr.min_area}" + ) return license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420) @@ -1239,8 +1249,11 @@ class LicensePlateProcessingMixin: ) return - # For LPR cameras, match or assign plate ID using Jaro-Winkler distance - if dedicated_lpr: + # For dedicated LPR cameras, match or assign plate ID using Jaro-Winkler distance + if ( + dedicated_lpr + and "license_plate" not in self.config.cameras[camera].objects.track + ): plate_id = None for existing_id, data in self.detected_license_plates.items(): @@ -1306,8 +1319,11 @@ class LicensePlateProcessingMixin: (id, top_plate, avg_confidence), ) - if dedicated_lpr: - # save the best snapshot + # save the best snapshot for dedicated lpr cams not using frigate+ + if ( + dedicated_lpr + and "license_plate" not in self.config.cameras[camera].objects.track + ): logger.debug( f"{camera}: Writing snapshot for {id}, {top_plate}, {current_time}" ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index eee8e0e96..be7843c9c 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -457,7 +457,11 @@ class EmbeddingMaintainer(threading.Thread): camera_config = self.config.cameras[camera] - if not camera_config.type == CameraTypeEnum.lpr: + if ( + camera_config.type != CameraTypeEnum.lpr + or "license_plate" in camera_config.objects.track + ): + # we're not a dedicated lpr camera or we are one but we're using frigate+ return try: diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index 7a4829c2a..978671512 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -442,7 +442,7 @@ class TrackedObject: if bounding_box: thickness = 2 - color = self.colormap[self.obj_data["label"]] + color = self.colormap.get(self.obj_data["label"], (255, 255, 255)) # draw the bounding boxes on the frame box = self.thumbnail_data["box"] diff --git a/frigate/video.py b/frigate/video.py index 91e92fee1..b14f8567c 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -15,6 +15,7 @@ from frigate.camera import CameraMetrics, PTZMetrics from frigate.comms.config_updater import ConfigSubscriber from frigate.comms.inter_process import InterProcessRequestor from frigate.config import CameraConfig, DetectConfig, ModelConfig +from frigate.config.camera.camera import CameraTypeEnum from frigate.const import ( CACHE_DIR, CACHE_SEGMENT_FORMAT, @@ -519,6 +520,7 @@ def track_camera( frame_queue, frame_shape, model_config, + config, config.detect, frame_manager, motion_detector, @@ -585,6 +587,7 @@ def process_frames( frame_queue: mp.Queue, frame_shape, model_config: ModelConfig, + camera_config: CameraConfig, detect_config: DetectConfig, frame_manager: FrameManager, motion_detector: MotionDetector, @@ -612,6 +615,29 @@ def process_frames( region_min_size = get_min_region_size(model_config) + attributes_map = model_config.attributes_map + all_attributes = model_config.all_attributes + + # remove license_plate from attributes if this camera is a dedicated LPR cam + if camera_config.type == CameraTypeEnum.lpr: + modified_attributes_map = model_config.attributes_map.copy() + + if ( + "car" in modified_attributes_map + and "license_plate" in modified_attributes_map["car"] + ): + modified_attributes_map["car"] = [ + attr + for attr in modified_attributes_map["car"] + if attr != "license_plate" + ] + + attributes_map = modified_attributes_map + + all_attributes = [ + attr for attr in model_config.all_attributes if attr != "license_plate" + ] + while not stop_event.is_set(): _, updated_enabled_config = enabled_config_subscriber.check_for_update() @@ -805,9 +831,7 @@ def process_frames( # if detection was run on this frame, consolidate if len(regions) > 0: tracked_detections = [ - d - for d in consolidated_detections - if d[0] not in model_config.all_attributes + d for d in consolidated_detections if d[0] not in all_attributes ] # now that we have refined our detections, we need to track objects object_tracker.match_and_update( @@ -819,7 +843,7 @@ def process_frames( # group the attribute detections based on what label they apply to attribute_detections: dict[str, list[TrackedObjectAttribute]] = {} - for label, attribute_labels in model_config.attributes_map.items(): + for label, attribute_labels in attributes_map.items(): attribute_detections[label] = [ TrackedObjectAttribute(d) for d in consolidated_detections @@ -836,8 +860,7 @@ def process_frames( for attributes in attribute_detections.values(): for attribute in attributes: filtered_objects = filter( - lambda o: attribute.label - in model_config.attributes_map.get(o["label"], []), + lambda o: attribute.label in attributes_map.get(o["label"], []), all_objects, ) selected_object_id = attribute.find_best_object(filtered_objects) @@ -885,7 +908,7 @@ def process_frames( for obj in object_tracker.tracked_objects.values(): if obj["frame_time"] == frame_time: thickness = 2 - color = model_config.colormap[obj["label"]] + color = model_config.colormap.get(obj["label"], (255, 255, 255)) else: thickness = 1 color = (255, 0, 0) From 7f3f62e46dcb96dfee7560108ee03e97cb8d04ff Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sun, 30 Mar 2025 08:33:52 -0600 Subject: [PATCH 88/97] Clean up and clarify face docs (#17454) --- docs/docs/configuration/face_recognition.md | 24 ++++++++------------- docs/docs/configuration/reference.md | 4 ++-- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md index d618dbba6..b894133fb 100644 --- a/docs/docs/configuration/face_recognition.md +++ b/docs/docs/configuration/face_recognition.md @@ -9,15 +9,9 @@ Face recognition identifies known individuals by matching detected faces with pr ### Face Detection -Users running a Frigate+ model (or any custom model that natively detects faces) should ensure that `face` is added to the [list of objects to track](../plus/#available-label-types) either globally or for a specific camera. This will allow face detection to run at the same time as object detection and be more efficient. +When running a Frigate+ model (or any custom model that natively detects faces) should ensure that `face` is added to the [list of objects to track](../plus/#available-label-types) either globally or for a specific camera. This will allow face detection to run at the same time as object detection and be more efficient. -Users without a model that detects faces can still run face recognition. Frigate uses a lightweight DNN face detection model that runs on the CPU. In this case, you should _not_ define `face` in your list of objects to track. - -:::note - -Frigate needs to first detect a `face` before it can recognize a face. - -::: +When running a default COCO model or another model that does not include `face` as a detectable label, face detection will run via CV2 using a lightweight DNN model that runs on the CPU. In this case, you should _not_ define `face` in your list of objects to track. ### Face Recognition @@ -26,7 +20,7 @@ Frigate has support for two face recognition model types: - **small**: Frigate will run a FaceNet embedding model to recognize faces, which runs locally on the CPU. This model is optimized for efficiency and is not as accurate. - **large**: Frigate will run a large ArcFace embedding model that is optimized for accuracy. It is only recommended to be run when an integrated or dedicated GPU is available. -In both cases a lightweight face landmark detection model is also used to align faces before running the recognition model. +In both cases, a lightweight face landmark detection model is also used to align faces before running recognition. ## Minimum System Requirements @@ -88,9 +82,9 @@ When choosing images to include in the face training set it is recommended to al - If it is difficult to make out details in a persons face it will not be helpful in training. - Avoid images with extreme under/over-exposure. - Avoid blurry / pixelated images. -- Avoid training on infrared (grayscale). The models are trained on color images and will be able to extract features from grayscale images. +- Avoid training on infrared (gray-scale). The models are trained on color images and will be able to extract features from gray-scale images. - Using images of people wearing hats / sunglasses may confuse the model. -- Do not upload too many similar images at the same time, it is recommended to train no more than 4-6 similar images for each person to avoid overfitting. +- Do not upload too many similar images at the same time, it is recommended to train no more than 4-6 similar images for each person to avoid over-fitting. ::: @@ -100,7 +94,7 @@ When first enabling face recognition it is important to build a foundation of st Then it is recommended to use the `Face Library` tab in Frigate to select and train images for each person as they are detected. When building a strong foundation it is strongly recommended to only train on images that are straight-on. Ignore images from cameras that recognize faces from an angle. -Aim to strike a balance between the quality of images while also having a range of conditions (day / night, different weather conditions, different times of day, etc.) in order to have diversity in the images used for each person and not have overfitting. +Aim to strike a balance between the quality of images while also having a range of conditions (day / night, different weather conditions, different times of day, etc.) in order to have diversity in the images used for each person and not have over-fitting. Once a person starts to be consistently recognized correctly on images that are straight-on, it is time to move on to the next step. @@ -112,11 +106,11 @@ Once straight-on images are performing well, start choosing slightly off-angle i ### Why can't I bulk upload photos? -It is important to methodically add photos to the library, bulk importing photos (especially from a general photo library) will lead to overfitting in that particular scenario and hurt recognition performance. +It is important to methodically add photos to the library, bulk importing photos (especially from a general photo library) will lead to over-fitting in that particular scenario and hurt recognition performance. ### Why do unknown people score similarly to known people? -This can happen for a few different reasons, but this is usually an indicator that the training set needs to be improved. This is often related to overfitting: +This can happen for a few different reasons, but this is usually an indicator that the training set needs to be improved. This is often related to over-fitting: - If you train with only a few images per person, especially if those images are very similar, the recognition model becomes overly specialized to those specific images. - When you provide images with different poses, lighting, and expressions, the algorithm extracts features that are consistent across those variations. @@ -124,4 +118,4 @@ This can happen for a few different reasons, but this is usually an indicator th ### I see scores above the threshold in the train tab, but a sub label wasn't assigned? -The Frigate considers the recognition scores across all recogntion attempts for each person object. The scores are continually weighted based on the area of the face, and a sub label will only be assigned to person if a person is confidently recognized consistently. This avoids cases where a single high confidence recognition would throw off the results. +The Frigate considers the recognition scores across all recognition attempts for each person object. The scores are continually weighted based on the area of the face, and a sub label will only be assigned to person if a person is confidently recognized consistently. This avoids cases where a single high confidence recognition would throw off the results. diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 81ca1becc..26e51b616 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -556,8 +556,8 @@ face_recognition: recognition_threshold: 0.9 # Optional: Min area of detected face box to consider running face recognition (default: shown below) min_area: 500 - # Optional: Save images of recognized faces for training (default: shown below) - save_attempts: True + # Optional: Number of images of recognized faces to save for training (default: shown below) + save_attempts: 100 # Optional: Apply a blur quality filter to adjust confidence based on the blur level of the image (default: shown below) blur_confidence_filter: True From 1dd5007fa83ef1fb39fc5837295709aaa3ea8f57 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 31 Mar 2025 12:47:33 -0600 Subject: [PATCH 89/97] Update nvidia inference time docs (#17469) --- docs/docs/frigate/hardware.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 62152996d..72d340682 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -135,6 +135,7 @@ Inference speeds will vary greatly depending on the GPU and the model used. | GTX 1660 SUPER | ~ 4 ms | | | | RTX 3050 | 5 - 7 ms | 320: ~ 10 ms 640: ~ 16 ms | 336: ~ 16 ms 560: ~ 40 ms | | RTX 3070 Mobile | ~ 5 ms | | | +| RTX 3070 | 4 - 6 ms | 320: ~ 6 ms 640: ~ 12 ms | 336: ~ 14 ms 560: ~ 36 ms | | Quadro P400 2GB | 20 - 25 ms | | | | Quadro P2000 | ~ 12 ms | | | From 207d1d280698a08d92a916fd2167d9c0170de246 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 31 Mar 2025 15:49:56 -0600 Subject: [PATCH 90/97] Face UI cleanup (#17472) * Add note * Sort by event id * Fix reprocess causing shift * Move event group to separate comp * Handle selecting events * implement event selection * Implement selected handler * handle right click * Toggle ctrl + a * Stop propogation * Fix --- docs/docs/configuration/face_recognition.md | 4 + frigate/data_processing/real_time/face.py | 52 ++-- web/src/hooks/use-contextmenu.ts | 1 + web/src/pages/FaceLibrary.tsx | 319 +++++++++++++------- 4 files changed, 255 insertions(+), 121 deletions(-) diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md index b894133fb..fe6ed1f99 100644 --- a/docs/docs/configuration/face_recognition.md +++ b/docs/docs/configuration/face_recognition.md @@ -108,6 +108,10 @@ Once straight-on images are performing well, start choosing slightly off-angle i It is important to methodically add photos to the library, bulk importing photos (especially from a general photo library) will lead to over-fitting in that particular scenario and hurt recognition performance. +### Why can't I bulk reprocess faces? + +Face embedding models work by breaking apart faces into different features. This means that when reprocessing an image, only images from a similar angle will have its score affected. + ### Why do unknown people score similarly to known people? This can happen for a few different reasons, but this is usually an indicator that the training set needs to be improved. This is often related to over-fitting: diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py index 5b20a6303..dd18aeffc 100644 --- a/frigate/data_processing/real_time/face.py +++ b/frigate/data_processing/real_time/face.py @@ -272,22 +272,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): f"Detected best face for person as: {sub_label} with probability {score}" ) - if self.config.face_recognition.save_attempts: - # write face to library - folder = os.path.join(FACE_DIR, "train") - file = os.path.join(folder, f"{id}-{sub_label}-{score}-0.webp") - os.makedirs(folder, exist_ok=True) - cv2.imwrite(file, face_frame) - - files = sorted( - filter(lambda f: (f.endswith(".webp")), os.listdir(folder)), - key=lambda f: os.path.getctime(os.path.join(folder, f)), - reverse=True, - ) - - # delete oldest face image if maximum is reached - if len(files) > self.config.face_recognition.save_attempts: - os.unlink(os.path.join(folder, files[-1])) + self.write_face_attempt( + face_frame, id, datetime.datetime.now().timestamp(), sub_label, score + ) if id not in self.person_face_history: self.person_face_history[id] = [] @@ -383,9 +370,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): } elif topic == EmbeddingsRequestEnum.reprocess_face.value: current_file: str = request_data["image_file"] - id = current_file[0 : current_file.index("-", current_file.index("-") + 1)] - face_score = current_file[current_file.rfind("-") : current_file.rfind(".")] + (id_time, id_rand, timestamp, _, _) = current_file.split("-") img = None + id = f"{id_time}-{id_rand}" if current_file: img = cv2.imread(current_file) @@ -411,7 +398,7 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): folder = os.path.join(FACE_DIR, "train") os.makedirs(folder, exist_ok=True) new_file = os.path.join( - folder, f"{id}-{sub_label}-{score}-{face_score}.webp" + folder, f"{id}-{timestamp}-{sub_label}-{score}.webp" ) shutil.move(current_file, new_file) @@ -461,3 +448,30 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): weighted_average = weighted_scores[best_name] / total_weights[best_name] return best_name, weighted_average + + def write_face_attempt( + self, + frame: np.ndarray, + event_id: str, + timestamp: float, + sub_label: str, + score: float, + ) -> None: + if self.config.face_recognition.save_attempts: + # write face to library + folder = os.path.join(FACE_DIR, "train") + file = os.path.join( + folder, f"{event_id}-{timestamp}-{sub_label}-{score}.webp" + ) + os.makedirs(folder, exist_ok=True) + cv2.imwrite(file, frame) + + files = sorted( + filter(lambda f: (f.endswith(".webp")), os.listdir(folder)), + key=lambda f: os.path.getctime(os.path.join(folder, f)), + reverse=True, + ) + + # delete oldest face image if maximum is reached + if len(files) > self.config.face_recognition.save_attempts: + os.unlink(os.path.join(folder, files[-1])) diff --git a/web/src/hooks/use-contextmenu.ts b/web/src/hooks/use-contextmenu.ts index f121846ae..21c03e353 100644 --- a/web/src/hooks/use-contextmenu.ts +++ b/web/src/hooks/use-contextmenu.ts @@ -33,6 +33,7 @@ export default function useContextMenu( }; } else { const context = (e: MouseEvent) => { + e.stopPropagation(); e.preventDefault(); callback(); }; diff --git a/web/src/pages/FaceLibrary.tsx b/web/src/pages/FaceLibrary.tsx index df57f729a..0b18f3d39 100644 --- a/web/src/pages/FaceLibrary.tsx +++ b/web/src/pages/FaceLibrary.tsx @@ -142,29 +142,33 @@ export default function FaceLibrary() { const [selectedFaces, setSelectedFaces] = useState([]); - const onClickFace = useCallback( - (imageId: string, ctrl: boolean) => { + const onClickFaces = useCallback( + (images: string[], ctrl: boolean) => { if (selectedFaces.length == 0 && !ctrl) { return; } - const index = selectedFaces.indexOf(imageId); + let newSelectedFaces = [...selectedFaces]; - if (index != -1) { - if (selectedFaces.length == 1) { - setSelectedFaces([]); + images.forEach((imageId) => { + const index = newSelectedFaces.indexOf(imageId); + + if (index != -1) { + if (selectedFaces.length == 1) { + newSelectedFaces = []; + } else { + const copy = [ + ...newSelectedFaces.slice(0, index), + ...newSelectedFaces.slice(index + 1), + ]; + newSelectedFaces = copy; + } } else { - const copy = [ - ...selectedFaces.slice(0, index), - ...selectedFaces.slice(index + 1), - ]; - setSelectedFaces(copy); + newSelectedFaces.push(imageId); } - } else { - const copy = [...selectedFaces]; - copy.push(imageId); - setSelectedFaces(copy); - } + }); + + setSelectedFaces(newSelectedFaces); }, [selectedFaces, setSelectedFaces], ); @@ -212,7 +216,11 @@ export default function FaceLibrary() { switch (key) { case "a": if (modifiers.ctrl) { - setSelectedFaces([...trainImages]); + if (selectedFaces.length) { + setSelectedFaces([]); + } else { + setSelectedFaces([...trainImages]); + } } break; case "Escape": @@ -253,6 +261,16 @@ export default function FaceLibrary() { /> {selectedFaces?.length > 0 ? ( + + {`${selectedFaces.length} selected`} + {"|"} + setSelectedFaces([])} + > + {t("button.unselect", { ns: "common" })} + +]({`${baseUrl}api/events/${search?.id}/snapshot.jpg`}){kind=link}