From bf22d89f67d68f1b10dab94c472b62509ff5fede Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 17 Mar 2025 15:57:46 -0600 Subject: [PATCH] Improve Face Library Management (#17213) * Set maximum number of face images to be kept * Fix vertical camera scaling * adjust wording * Add attributes to search data * Add button to train face from event * Handle event id saving in API --- frigate/api/classification.py | 57 ++++++++-- frigate/api/event.py | 1 + frigate/data_processing/real_time/face.py | 11 ++ frigate/util/path.py | 8 ++ web/public/locales/en/views/faceLibrary.json | 2 +- .../overlay/detail/SearchDetailDialog.tsx | 107 +++++++++++++++--- web/src/pages/FaceLibrary.tsx | 2 +- web/src/types/search.ts | 1 + web/src/views/settings/MotionTunerView.tsx | 2 +- web/src/views/settings/ObjectSettingsView.tsx | 2 +- 10 files changed, 167 insertions(+), 26 deletions(-) diff --git a/frigate/api/classification.py b/frigate/api/classification.py index 85b604379..df804f34a 100644 --- a/frigate/api/classification.py +++ b/frigate/api/classification.py @@ -6,6 +6,7 @@ import random import shutil import string +import cv2 from fastapi import APIRouter, Depends, Request, UploadFile from fastapi.responses import JSONResponse from pathvalidate import sanitize_filename @@ -14,9 +15,11 @@ from playhouse.shortcuts import model_to_dict from frigate.api.auth import require_role from frigate.api.defs.tags import Tags +from frigate.config.camera import DetectConfig from frigate.const import FACE_DIR from frigate.embeddings import EmbeddingsContext from frigate.models import Event +from frigate.util.path import get_event_snapshot logger = logging.getLogger(__name__) @@ -87,16 +90,27 @@ def train_face(request: Request, name: str, body: dict = None): ) json: dict[str, any] = body or {} - training_file = os.path.join( - FACE_DIR, f"train/{sanitize_filename(json.get('training_file', ''))}" - ) + training_file_name = sanitize_filename(json.get("training_file", "")) + training_file = os.path.join(FACE_DIR, f"train/{training_file_name}") + event_id = json.get("event_id") - if not training_file or not os.path.isfile(training_file): + if not training_file_name and not event_id: return JSONResponse( content=( { "success": False, - "message": f"Invalid filename or no file exists: {training_file}", + "message": "A training file or event_id must be passed.", + } + ), + status_code=400, + ) + + if training_file_name and not os.path.isfile(training_file): + return JSONResponse( + content=( + { + "success": False, + "message": f"Invalid filename or no file exists: {training_file_name}", } ), status_code=404, @@ -106,7 +120,36 @@ def train_face(request: Request, name: str, body: dict = None): rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) new_name = f"{sanitized_name}-{rand_id}.webp" new_file = os.path.join(FACE_DIR, f"{sanitized_name}/{new_name}") - shutil.move(training_file, new_file) + + if training_file_name: + shutil.move(training_file, new_file) + else: + try: + event: Event = Event.get(Event.id == event_id) + except DoesNotExist: + return JSONResponse( + content=( + { + "success": False, + "message": f"Invalid event_id or no event exists: {event_id}", + } + ), + status_code=404, + ) + + snapshot = get_event_snapshot(event) + face_box = event.data["attributes"][0]["box"] + detect_config: DetectConfig = request.app.frigate_config.cameras[ + event.camera + ].detect + + # crop onto the face box minus the bounding box itself + x1 = int(face_box[0] * detect_config.width) + 2 + y1 = int(face_box[1] * detect_config.height) + 2 + x2 = x1 + int(face_box[2] * detect_config.width) - 4 + y2 = y1 + int(face_box[3] * detect_config.height) - 4 + face = snapshot[y1:y2, x1:x2] + cv2.imwrite(new_file, face) context: EmbeddingsContext = request.app.embeddings context.clear_face_classifier() @@ -115,7 +158,7 @@ def train_face(request: Request, name: str, body: dict = None): content=( { "success": True, - "message": f"Successfully saved {training_file} as {new_name}.", + "message": f"Successfully saved {training_file_name} as {new_name}.", } ), status_code=200, diff --git a/frigate/api/event.py b/frigate/api/event.py index 88a865318..c4c763bf7 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -701,6 +701,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) for k, v in event["data"].items() if k in [ + "attributes", "type", "score", "top_score", diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py index b51b7a20f..acb891449 100644 --- a/frigate/data_processing/real_time/face.py +++ b/frigate/data_processing/real_time/face.py @@ -28,6 +28,7 @@ logger = logging.getLogger(__name__) MAX_DETECTION_HEIGHT = 1080 +MAX_FACE_ATTEMPTS = 100 MIN_MATCHING_FACES = 2 @@ -482,6 +483,16 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): ) shutil.move(current_file, new_file) + files = sorted( + os.listdir(folder), + key=lambda f: os.path.getctime(os.path.join(folder, f)), + reverse=True, + ) + + # delete oldest face image if maximum is reached + if len(files) > MAX_FACE_ATTEMPTS: + os.unlink(os.path.join(folder, files[-1])) + def expire_object(self, object_id: str): if object_id in self.detected_faces: self.detected_faces.pop(object_id) diff --git a/frigate/util/path.py b/frigate/util/path.py index dbe51abe5..565f5a357 100644 --- a/frigate/util/path.py +++ b/frigate/util/path.py @@ -4,6 +4,9 @@ import base64 import os from pathlib import Path +import cv2 +from numpy import ndarray + from frigate.const import CLIPS_DIR, THUMB_DIR from frigate.models import Event @@ -21,6 +24,11 @@ def get_event_thumbnail_bytes(event: Event) -> bytes | None: return None +def get_event_snapshot(event: Event) -> ndarray: + media_name = f"{event.camera}-{event.id}" + return cv2.imread(f"{os.path.join(CLIPS_DIR, media_name)}.jpg") + + ### Deletion diff --git a/web/public/locales/en/views/faceLibrary.json b/web/public/locales/en/views/faceLibrary.json index 4028690e3..46842b7ea 100644 --- a/web/public/locales/en/views/faceLibrary.json +++ b/web/public/locales/en/views/faceLibrary.json @@ -25,7 +25,7 @@ }, "readTheDocs": "Read the documentation to view more details on refining images for the Face Library", "trainFaceAs": "Train Face as:", - "trainFaceAsPerson": "Train Face as Person", + "trainFace": "Train Face", "toast": { "success": { "uploadedImage": "Successfully uploaded image.", diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 891ce88b1..b22eb9a4c 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -57,6 +57,7 @@ import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, + DropdownMenuLabel, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch"; @@ -69,11 +70,12 @@ import { PopoverContent, PopoverTrigger, } from "@/components/ui/popover"; -import { LuInfo } from "react-icons/lu"; +import { LuInfo, LuSearch } from "react-icons/lu"; import { TooltipPortal } from "@radix-ui/react-tooltip"; import { FaPencilAlt } from "react-icons/fa"; import TextEntryDialog from "@/components/overlay/dialog/TextEntryDialog"; import { useTranslation } from "react-i18next"; +import { TbFaceId } from "react-icons/tb"; const SEARCH_TABS = [ "details", @@ -99,7 +101,7 @@ export default function SearchDetailDialog({ setSimilarity, setInputFocused, }: SearchDetailDialogProps) { - const { t } = useTranslation(["views/explore"]); + const { t } = useTranslation(["views/explore", "views/faceLibrary"]); const { data: config } = useSWR("config", { revalidateOnFocus: false, }); @@ -555,6 +557,48 @@ function ObjectDetailsTab({ [search, apiHost, mutate, setSearch, t], ); + // face training + + const hasFace = useMemo(() => { + if (!config?.face_recognition.enabled || !search) { + return false; + } + + return search.data.attributes?.find((attr) => attr.label == "face"); + }, [config, search]); + + const { data: faceData } = useSWR(hasFace ? "faces" : null); + + const faceNames = useMemo( + () => + faceData ? Object.keys(faceData).filter((face) => face != "train") : [], + [faceData], + ); + + const onTrainFace = useCallback( + (trainName: string) => { + axios + .post(`/faces/train/${trainName}/classify`, { event_id: search.id }) + .then((resp) => { + if (resp.status == 200) { + toast.success(t("toast.success.trainedFace"), { + position: "top-center", + }); + } + }) + .catch((error) => { + const errorMessage = + error.response?.data?.message || + error.response?.data?.detail || + "Unknown error"; + toast.error(t("toast.error.trainFailed", { errorMessage }), { + position: "top-center", + }); + }); + }, + [search, t], + ); + return (
@@ -673,20 +717,53 @@ function ObjectDetailsTab({ draggable={false} src={`${apiHost}api/events/${search.id}/thumbnail.webp`} /> - {config?.semantic_search.enabled && search.data.type == "object" && ( - - )} + if (setSimilarity) { + setSimilarity(); + } + }} + > +
+ + {t("itemMenu.findSimilar.label")} +
+ + )} + {hasFace && ( + + + + + + + {t("trainFaceAs", { ns: "views/faceLibrary" })} + + {faceNames.map((faceName) => ( + onTrainFace(faceName)} + > + {faceName} + + ))} + + + )} +
diff --git a/web/src/pages/FaceLibrary.tsx b/web/src/pages/FaceLibrary.tsx index afa196f35..94a7f6947 100644 --- a/web/src/pages/FaceLibrary.tsx +++ b/web/src/pages/FaceLibrary.tsx @@ -472,7 +472,7 @@ function FaceAttempt({ ))} - {t("trainFaceAsPerson")} + {t("trainFace")} diff --git a/web/src/types/search.ts b/web/src/types/search.ts index 5dca11973..2a57385f7 100644 --- a/web/src/types/search.ts +++ b/web/src/types/search.ts @@ -50,6 +50,7 @@ export type SearchResult = { score: number; sub_label_score?: number; region: number[]; + attributes?: [{ box: number[]; label: string; score: number }]; box: number[]; area: number; ratio: number; diff --git a/web/src/views/settings/MotionTunerView.tsx b/web/src/views/settings/MotionTunerView.tsx index d1027a14d..98169b4f8 100644 --- a/web/src/views/settings/MotionTunerView.tsx +++ b/web/src/views/settings/MotionTunerView.tsx @@ -323,7 +323,7 @@ export default function MotionTunerView({
{cameraConfig ? ( -
+
{cameraConfig ? ( -
+