Generalize postprocessing (#15931)

* Actually send result to face registration

* Define postprocessing api and move face processing to fit

* Standardize request handling

* Standardize handling of processors

* Rename processing metrics

* Cleanup

* Standardize object end

* Update to newer formatting

* One more

* One more
This commit is contained in:
Nicolas Mowen
2025-02-08 12:47:01 -06:00
committed by Blake Blackshear
parent 3f1d85e189
commit 88686c44fe
12 changed files with 504 additions and 416 deletions
+4 -4
View File
@@ -20,14 +20,14 @@ from frigate.models import Event
from frigate.util.builtin import serialize
from frigate.util.services import listen
from ..postprocessing.types import PostProcessingMetrics
from .maintainer import EmbeddingMaintainer
from .types import EmbeddingsMetrics
from .util import ZScoreNormalization
logger = logging.getLogger(__name__)
def manage_embeddings(config: FrigateConfig, metrics: EmbeddingsMetrics) -> None:
def manage_embeddings(config: FrigateConfig, metrics: PostProcessingMetrics) -> None:
# Only initialize embeddings if semantic search is enabled
if not config.semantic_search.enabled:
return
@@ -192,8 +192,8 @@ class EmbeddingsContext:
return results
def register_face(self, face_name: str, image_data: bytes) -> None:
self.requestor.send_data(
def register_face(self, face_name: str, image_data: bytes) -> dict[str, any]:
return self.requestor.send_data(
EmbeddingsRequestEnum.register_face.value,
{
"face_name": face_name,
+2 -2
View File
@@ -21,8 +21,8 @@ from frigate.models import Event
from frigate.types import ModelStatusTypesEnum
from frigate.util.builtin import serialize
from ..postprocessing.types import PostProcessingMetrics
from .functions.onnx import GenericONNXEmbedding, ModelTypeEnum
from .types import EmbeddingsMetrics
logger = logging.getLogger(__name__)
@@ -65,7 +65,7 @@ class Embeddings:
self,
config: FrigateConfig,
db: SqliteVecQueueDatabase,
metrics: EmbeddingsMetrics,
metrics: PostProcessingMetrics,
) -> None:
self.config = config
self.db = db
+17 -210
View File
@@ -4,9 +4,7 @@ import base64
import datetime
import logging
import os
import random
import re
import string
import threading
from multiprocessing.synchronize import Event as MpEvent
from pathlib import Path
@@ -28,7 +26,6 @@ from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.const import (
CLIPS_DIR,
FACE_DIR,
FRIGATE_LOCALHOST,
UPDATE_EVENT_DESCRIPTION,
)
@@ -36,13 +33,14 @@ from frigate.embeddings.lpr.lpr import LicensePlateRecognition
from frigate.events.types import EventTypeEnum
from frigate.genai import get_genai_client
from frigate.models import Event
from frigate.postprocessing.face_processor import FaceProcessor
from frigate.postprocessing.processor_api import ProcessorApi
from frigate.types import TrackedObjectUpdateTypesEnum
from frigate.util.builtin import serialize
from frigate.util.image import SharedMemoryFrameManager, area, calculate_region
from frigate.util.model import FaceClassificationModel
from ..postprocessing.types import PostProcessingMetrics
from .embeddings import Embeddings
from .types import EmbeddingsMetrics
logger = logging.getLogger(__name__)
@@ -56,7 +54,7 @@ class EmbeddingMaintainer(threading.Thread):
self,
db: SqliteQueueDatabase,
config: FrigateConfig,
metrics: EmbeddingsMetrics,
metrics: PostProcessingMetrics,
stop_event: MpEvent,
) -> None:
super().__init__(name="embeddings_maintainer")
@@ -75,16 +73,10 @@ class EmbeddingMaintainer(threading.Thread):
)
self.embeddings_responder = EmbeddingsResponder()
self.frame_manager = SharedMemoryFrameManager()
self.processors: list[ProcessorApi] = []
# set face recognition conditions
self.face_recognition_enabled = self.config.face_recognition.enabled
self.requires_face_detection = "face" not in self.config.objects.all_objects
self.detected_faces: dict[str, float] = {}
self.face_classifier = (
FaceClassificationModel(self.config.face_recognition, db)
if self.face_recognition_enabled
else None
)
if self.config.face_recognition.enabled:
self.processors.append(FaceProcessor(self.config, metrics))
# create communication for updating event descriptions
self.requestor = InterProcessRequestor()
@@ -142,46 +134,12 @@ class EmbeddingMaintainer(threading.Thread):
self.embeddings.embed_description("", data, upsert=False),
pack=False,
)
elif topic == EmbeddingsRequestEnum.register_face.value:
if not self.face_recognition_enabled:
return False
else:
for processor in self.processors:
resp = processor.handle_request(data)
rand_id = "".join(
random.choices(string.ascii_lowercase + string.digits, k=6)
)
label = data["face_name"]
id = f"{label}-{rand_id}"
if data.get("cropped"):
pass
else:
img = cv2.imdecode(
np.frombuffer(
base64.b64decode(data["image"]), dtype=np.uint8
),
cv2.IMREAD_COLOR,
)
face_box = self._detect_face(img)
if not face_box:
return False
face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
ret, thumbnail = cv2.imencode(
".webp", face, [int(cv2.IMWRITE_WEBP_QUALITY), 100]
)
# write face to library
folder = os.path.join(FACE_DIR, label)
file = os.path.join(folder, f"{id}.webp")
os.makedirs(folder, exist_ok=True)
# save face image
with open(file, "wb") as output:
output.write(thumbnail.tobytes())
self.face_classifier.clear_classifier()
return True
if resp is not None:
return resp
except Exception as e:
logger.error(f"Unable to handle embeddings request {e}")
@@ -204,8 +162,8 @@ class EmbeddingMaintainer(threading.Thread):
# no need to process updated objects if face recognition, lpr, genai are disabled
if (
not camera_config.genai.enabled
and not self.face_recognition_enabled
and not self.lpr_config.enabled
and len(self.processors) == 0
):
return
@@ -223,15 +181,8 @@ class EmbeddingMaintainer(threading.Thread):
)
return
if self.face_recognition_enabled:
start = datetime.datetime.now().timestamp()
processed = self._process_face(data, yuv_frame)
if processed:
duration = datetime.datetime.now().timestamp() - start
self.metrics.face_rec_fps.value = (
self.metrics.face_rec_fps.value * 9 + duration
) / 10
for processor in self.processors:
processor.process_frame(data, yuv_frame)
if self.lpr_config.enabled:
start = datetime.datetime.now().timestamp()
@@ -271,8 +222,8 @@ class EmbeddingMaintainer(threading.Thread):
event_id, camera, updated_db = ended
camera_config = self.config.cameras[camera]
if event_id in self.detected_faces:
self.detected_faces.pop(event_id)
for processor in self.processors:
processor.expire_object(event_id)
if event_id in self.detected_license_plates:
self.detected_license_plates.pop(event_id)
@@ -399,150 +350,6 @@ class EmbeddingMaintainer(threading.Thread):
if event_id:
self.handle_regenerate_description(event_id, source)
def _detect_face(self, input: np.ndarray) -> tuple[int, int, int, int]:
"""Detect faces in input image."""
faces = self.face_classifier.detect_faces(input)
if faces is None or faces[1] is None:
return None
face = None
for _, potential_face in enumerate(faces[1]):
raw_bbox = potential_face[0:4].astype(np.uint16)
x: int = max(raw_bbox[0], 0)
y: int = max(raw_bbox[1], 0)
w: int = raw_bbox[2]
h: int = raw_bbox[3]
bbox = (x, y, x + w, y + h)
if face is None or area(bbox) > area(face):
face = bbox
return face
def _process_face(self, obj_data: dict[str, any], frame: np.ndarray) -> bool:
"""Look for faces in image."""
id = obj_data["id"]
# don't run for non person objects
if obj_data.get("label") != "person":
logger.debug("Not a processing face for non person object.")
return False
# don't overwrite sub label for objects that have a sub label
# that is not a face
if obj_data.get("sub_label") and id not in self.detected_faces:
logger.debug(
f"Not processing face due to existing sub label: {obj_data.get('sub_label')}."
)
return False
face: Optional[dict[str, any]] = None
if self.requires_face_detection:
logger.debug("Running manual face detection.")
person_box = obj_data.get("box")
if not person_box:
return False
rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
left, top, right, bottom = person_box
person = rgb[top:bottom, left:right]
face_box = self._detect_face(person)
if not face_box:
logger.debug("Detected no faces for person object.")
return False
face_frame = person[
max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
]
face_frame = cv2.cvtColor(face_frame, cv2.COLOR_RGB2BGR)
else:
# don't run for object without attributes
if not obj_data.get("current_attributes"):
logger.debug("No attributes to parse.")
return False
attributes: list[dict[str, any]] = obj_data.get("current_attributes", [])
for attr in attributes:
if attr.get("label") != "face":
continue
if face is None or attr.get("score", 0.0) > face.get("score", 0.0):
face = attr
# no faces detected in this frame
if not face:
return False
face_box = face.get("box")
# check that face is valid
if not face_box or area(face_box) < self.config.face_recognition.min_area:
logger.debug(f"Invalid face box {face}")
return False
face_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
face_frame = face_frame[
max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
]
res = self.face_classifier.classify_face(face_frame)
if not res:
return False
sub_label, score = res
# calculate the overall face score as the probability * area of face
# this will help to reduce false positives from small side-angle faces
# if a large front-on face image may have scored slightly lower but
# is more likely to be accurate due to the larger face area
face_score = round(score * face_frame.shape[0] * face_frame.shape[1], 2)
logger.debug(
f"Detected best face for person as: {sub_label} with probability {score} and overall face score {face_score}"
)
if self.config.face_recognition.save_attempts:
# write face to library
folder = os.path.join(FACE_DIR, "train")
file = os.path.join(folder, f"{id}-{sub_label}-{score}-{face_score}.webp")
os.makedirs(folder, exist_ok=True)
cv2.imwrite(file, face_frame)
if score < self.config.face_recognition.threshold:
logger.debug(
f"Recognized face distance {score} is less than threshold {self.config.face_recognition.threshold}"
)
return True
if id in self.detected_faces and face_score <= self.detected_faces[id]:
logger.debug(
f"Recognized face distance {score} and overall score {face_score} is less than previous overall face score ({self.detected_faces.get(id)})."
)
return True
resp = requests.post(
f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
json={
"camera": obj_data.get("camera"),
"subLabel": sub_label,
"subLabelScore": score,
},
)
if resp.status_code == 200:
self.detected_faces[id] = face_score
return True
def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
"""Return the dimensions of the input image as [x, y, width, height]."""
height, width = input.shape[:2]
-17
View File
@@ -1,17 +0,0 @@
"""Embeddings types."""
import multiprocessing as mp
from multiprocessing.sharedctypes import Synchronized
class EmbeddingsMetrics:
image_embeddings_fps: Synchronized
text_embeddings_sps: Synchronized
face_rec_fps: Synchronized
alpr_pps: Synchronized
def __init__(self):
self.image_embeddings_fps = mp.Value("d", 0.01)
self.text_embeddings_sps = mp.Value("d", 0.01)
self.face_rec_fps = mp.Value("d", 0.01)
self.alpr_pps = mp.Value("d", 0.01)