mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-22 05:23:01 +03:00
Generalize postprocessing (#15931)
* Actually send result to face registration * Define postprocessing api and move face processing to fit * Standardize request handling * Standardize handling of processors * Rename processing metrics * Cleanup * Standardize object end * Update to newer formatting * One more * One more
This commit is contained in:
committed by
Blake Blackshear
parent
3f1d85e189
commit
88686c44fe
@@ -20,14 +20,14 @@ from frigate.models import Event
|
||||
from frigate.util.builtin import serialize
|
||||
from frigate.util.services import listen
|
||||
|
||||
from ..postprocessing.types import PostProcessingMetrics
|
||||
from .maintainer import EmbeddingMaintainer
|
||||
from .types import EmbeddingsMetrics
|
||||
from .util import ZScoreNormalization
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def manage_embeddings(config: FrigateConfig, metrics: EmbeddingsMetrics) -> None:
|
||||
def manage_embeddings(config: FrigateConfig, metrics: PostProcessingMetrics) -> None:
|
||||
# Only initialize embeddings if semantic search is enabled
|
||||
if not config.semantic_search.enabled:
|
||||
return
|
||||
@@ -192,8 +192,8 @@ class EmbeddingsContext:
|
||||
|
||||
return results
|
||||
|
||||
def register_face(self, face_name: str, image_data: bytes) -> None:
|
||||
self.requestor.send_data(
|
||||
def register_face(self, face_name: str, image_data: bytes) -> dict[str, any]:
|
||||
return self.requestor.send_data(
|
||||
EmbeddingsRequestEnum.register_face.value,
|
||||
{
|
||||
"face_name": face_name,
|
||||
|
||||
@@ -21,8 +21,8 @@ from frigate.models import Event
|
||||
from frigate.types import ModelStatusTypesEnum
|
||||
from frigate.util.builtin import serialize
|
||||
|
||||
from ..postprocessing.types import PostProcessingMetrics
|
||||
from .functions.onnx import GenericONNXEmbedding, ModelTypeEnum
|
||||
from .types import EmbeddingsMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -65,7 +65,7 @@ class Embeddings:
|
||||
self,
|
||||
config: FrigateConfig,
|
||||
db: SqliteVecQueueDatabase,
|
||||
metrics: EmbeddingsMetrics,
|
||||
metrics: PostProcessingMetrics,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.db = db
|
||||
|
||||
@@ -4,9 +4,7 @@ import base64
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import threading
|
||||
from multiprocessing.synchronize import Event as MpEvent
|
||||
from pathlib import Path
|
||||
@@ -28,7 +26,6 @@ from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.const import (
|
||||
CLIPS_DIR,
|
||||
FACE_DIR,
|
||||
FRIGATE_LOCALHOST,
|
||||
UPDATE_EVENT_DESCRIPTION,
|
||||
)
|
||||
@@ -36,13 +33,14 @@ from frigate.embeddings.lpr.lpr import LicensePlateRecognition
|
||||
from frigate.events.types import EventTypeEnum
|
||||
from frigate.genai import get_genai_client
|
||||
from frigate.models import Event
|
||||
from frigate.postprocessing.face_processor import FaceProcessor
|
||||
from frigate.postprocessing.processor_api import ProcessorApi
|
||||
from frigate.types import TrackedObjectUpdateTypesEnum
|
||||
from frigate.util.builtin import serialize
|
||||
from frigate.util.image import SharedMemoryFrameManager, area, calculate_region
|
||||
from frigate.util.model import FaceClassificationModel
|
||||
|
||||
from ..postprocessing.types import PostProcessingMetrics
|
||||
from .embeddings import Embeddings
|
||||
from .types import EmbeddingsMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -56,7 +54,7 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
self,
|
||||
db: SqliteQueueDatabase,
|
||||
config: FrigateConfig,
|
||||
metrics: EmbeddingsMetrics,
|
||||
metrics: PostProcessingMetrics,
|
||||
stop_event: MpEvent,
|
||||
) -> None:
|
||||
super().__init__(name="embeddings_maintainer")
|
||||
@@ -75,16 +73,10 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
)
|
||||
self.embeddings_responder = EmbeddingsResponder()
|
||||
self.frame_manager = SharedMemoryFrameManager()
|
||||
self.processors: list[ProcessorApi] = []
|
||||
|
||||
# set face recognition conditions
|
||||
self.face_recognition_enabled = self.config.face_recognition.enabled
|
||||
self.requires_face_detection = "face" not in self.config.objects.all_objects
|
||||
self.detected_faces: dict[str, float] = {}
|
||||
self.face_classifier = (
|
||||
FaceClassificationModel(self.config.face_recognition, db)
|
||||
if self.face_recognition_enabled
|
||||
else None
|
||||
)
|
||||
if self.config.face_recognition.enabled:
|
||||
self.processors.append(FaceProcessor(self.config, metrics))
|
||||
|
||||
# create communication for updating event descriptions
|
||||
self.requestor = InterProcessRequestor()
|
||||
@@ -142,46 +134,12 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
self.embeddings.embed_description("", data, upsert=False),
|
||||
pack=False,
|
||||
)
|
||||
elif topic == EmbeddingsRequestEnum.register_face.value:
|
||||
if not self.face_recognition_enabled:
|
||||
return False
|
||||
else:
|
||||
for processor in self.processors:
|
||||
resp = processor.handle_request(data)
|
||||
|
||||
rand_id = "".join(
|
||||
random.choices(string.ascii_lowercase + string.digits, k=6)
|
||||
)
|
||||
label = data["face_name"]
|
||||
id = f"{label}-{rand_id}"
|
||||
|
||||
if data.get("cropped"):
|
||||
pass
|
||||
else:
|
||||
img = cv2.imdecode(
|
||||
np.frombuffer(
|
||||
base64.b64decode(data["image"]), dtype=np.uint8
|
||||
),
|
||||
cv2.IMREAD_COLOR,
|
||||
)
|
||||
face_box = self._detect_face(img)
|
||||
|
||||
if not face_box:
|
||||
return False
|
||||
|
||||
face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
|
||||
ret, thumbnail = cv2.imencode(
|
||||
".webp", face, [int(cv2.IMWRITE_WEBP_QUALITY), 100]
|
||||
)
|
||||
|
||||
# write face to library
|
||||
folder = os.path.join(FACE_DIR, label)
|
||||
file = os.path.join(folder, f"{id}.webp")
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
|
||||
# save face image
|
||||
with open(file, "wb") as output:
|
||||
output.write(thumbnail.tobytes())
|
||||
|
||||
self.face_classifier.clear_classifier()
|
||||
return True
|
||||
if resp is not None:
|
||||
return resp
|
||||
except Exception as e:
|
||||
logger.error(f"Unable to handle embeddings request {e}")
|
||||
|
||||
@@ -204,8 +162,8 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
# no need to process updated objects if face recognition, lpr, genai are disabled
|
||||
if (
|
||||
not camera_config.genai.enabled
|
||||
and not self.face_recognition_enabled
|
||||
and not self.lpr_config.enabled
|
||||
and len(self.processors) == 0
|
||||
):
|
||||
return
|
||||
|
||||
@@ -223,15 +181,8 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
)
|
||||
return
|
||||
|
||||
if self.face_recognition_enabled:
|
||||
start = datetime.datetime.now().timestamp()
|
||||
processed = self._process_face(data, yuv_frame)
|
||||
|
||||
if processed:
|
||||
duration = datetime.datetime.now().timestamp() - start
|
||||
self.metrics.face_rec_fps.value = (
|
||||
self.metrics.face_rec_fps.value * 9 + duration
|
||||
) / 10
|
||||
for processor in self.processors:
|
||||
processor.process_frame(data, yuv_frame)
|
||||
|
||||
if self.lpr_config.enabled:
|
||||
start = datetime.datetime.now().timestamp()
|
||||
@@ -271,8 +222,8 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
event_id, camera, updated_db = ended
|
||||
camera_config = self.config.cameras[camera]
|
||||
|
||||
if event_id in self.detected_faces:
|
||||
self.detected_faces.pop(event_id)
|
||||
for processor in self.processors:
|
||||
processor.expire_object(event_id)
|
||||
|
||||
if event_id in self.detected_license_plates:
|
||||
self.detected_license_plates.pop(event_id)
|
||||
@@ -399,150 +350,6 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
if event_id:
|
||||
self.handle_regenerate_description(event_id, source)
|
||||
|
||||
def _detect_face(self, input: np.ndarray) -> tuple[int, int, int, int]:
|
||||
"""Detect faces in input image."""
|
||||
faces = self.face_classifier.detect_faces(input)
|
||||
|
||||
if faces is None or faces[1] is None:
|
||||
return None
|
||||
|
||||
face = None
|
||||
|
||||
for _, potential_face in enumerate(faces[1]):
|
||||
raw_bbox = potential_face[0:4].astype(np.uint16)
|
||||
x: int = max(raw_bbox[0], 0)
|
||||
y: int = max(raw_bbox[1], 0)
|
||||
w: int = raw_bbox[2]
|
||||
h: int = raw_bbox[3]
|
||||
bbox = (x, y, x + w, y + h)
|
||||
|
||||
if face is None or area(bbox) > area(face):
|
||||
face = bbox
|
||||
|
||||
return face
|
||||
|
||||
def _process_face(self, obj_data: dict[str, any], frame: np.ndarray) -> bool:
|
||||
"""Look for faces in image."""
|
||||
id = obj_data["id"]
|
||||
|
||||
# don't run for non person objects
|
||||
if obj_data.get("label") != "person":
|
||||
logger.debug("Not a processing face for non person object.")
|
||||
return False
|
||||
|
||||
# don't overwrite sub label for objects that have a sub label
|
||||
# that is not a face
|
||||
if obj_data.get("sub_label") and id not in self.detected_faces:
|
||||
logger.debug(
|
||||
f"Not processing face due to existing sub label: {obj_data.get('sub_label')}."
|
||||
)
|
||||
return False
|
||||
|
||||
face: Optional[dict[str, any]] = None
|
||||
|
||||
if self.requires_face_detection:
|
||||
logger.debug("Running manual face detection.")
|
||||
person_box = obj_data.get("box")
|
||||
|
||||
if not person_box:
|
||||
return False
|
||||
|
||||
rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
|
||||
left, top, right, bottom = person_box
|
||||
person = rgb[top:bottom, left:right]
|
||||
face_box = self._detect_face(person)
|
||||
|
||||
if not face_box:
|
||||
logger.debug("Detected no faces for person object.")
|
||||
return False
|
||||
|
||||
face_frame = person[
|
||||
max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
|
||||
max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
|
||||
]
|
||||
face_frame = cv2.cvtColor(face_frame, cv2.COLOR_RGB2BGR)
|
||||
else:
|
||||
# don't run for object without attributes
|
||||
if not obj_data.get("current_attributes"):
|
||||
logger.debug("No attributes to parse.")
|
||||
return False
|
||||
|
||||
attributes: list[dict[str, any]] = obj_data.get("current_attributes", [])
|
||||
for attr in attributes:
|
||||
if attr.get("label") != "face":
|
||||
continue
|
||||
|
||||
if face is None or attr.get("score", 0.0) > face.get("score", 0.0):
|
||||
face = attr
|
||||
|
||||
# no faces detected in this frame
|
||||
if not face:
|
||||
return False
|
||||
|
||||
face_box = face.get("box")
|
||||
|
||||
# check that face is valid
|
||||
if not face_box or area(face_box) < self.config.face_recognition.min_area:
|
||||
logger.debug(f"Invalid face box {face}")
|
||||
return False
|
||||
|
||||
face_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
|
||||
|
||||
face_frame = face_frame[
|
||||
max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
|
||||
max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
|
||||
]
|
||||
|
||||
res = self.face_classifier.classify_face(face_frame)
|
||||
|
||||
if not res:
|
||||
return False
|
||||
|
||||
sub_label, score = res
|
||||
|
||||
# calculate the overall face score as the probability * area of face
|
||||
# this will help to reduce false positives from small side-angle faces
|
||||
# if a large front-on face image may have scored slightly lower but
|
||||
# is more likely to be accurate due to the larger face area
|
||||
face_score = round(score * face_frame.shape[0] * face_frame.shape[1], 2)
|
||||
|
||||
logger.debug(
|
||||
f"Detected best face for person as: {sub_label} with probability {score} and overall face score {face_score}"
|
||||
)
|
||||
|
||||
if self.config.face_recognition.save_attempts:
|
||||
# write face to library
|
||||
folder = os.path.join(FACE_DIR, "train")
|
||||
file = os.path.join(folder, f"{id}-{sub_label}-{score}-{face_score}.webp")
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
cv2.imwrite(file, face_frame)
|
||||
|
||||
if score < self.config.face_recognition.threshold:
|
||||
logger.debug(
|
||||
f"Recognized face distance {score} is less than threshold {self.config.face_recognition.threshold}"
|
||||
)
|
||||
return True
|
||||
|
||||
if id in self.detected_faces and face_score <= self.detected_faces[id]:
|
||||
logger.debug(
|
||||
f"Recognized face distance {score} and overall score {face_score} is less than previous overall face score ({self.detected_faces.get(id)})."
|
||||
)
|
||||
return True
|
||||
|
||||
resp = requests.post(
|
||||
f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
|
||||
json={
|
||||
"camera": obj_data.get("camera"),
|
||||
"subLabel": sub_label,
|
||||
"subLabelScore": score,
|
||||
},
|
||||
)
|
||||
|
||||
if resp.status_code == 200:
|
||||
self.detected_faces[id] = face_score
|
||||
|
||||
return True
|
||||
|
||||
def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
|
||||
"""Return the dimensions of the input image as [x, y, width, height]."""
|
||||
height, width = input.shape[:2]
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
"""Embeddings types."""
|
||||
|
||||
import multiprocessing as mp
|
||||
from multiprocessing.sharedctypes import Synchronized
|
||||
|
||||
|
||||
class EmbeddingsMetrics:
|
||||
image_embeddings_fps: Synchronized
|
||||
text_embeddings_sps: Synchronized
|
||||
face_rec_fps: Synchronized
|
||||
alpr_pps: Synchronized
|
||||
|
||||
def __init__(self):
|
||||
self.image_embeddings_fps = mp.Value("d", 0.01)
|
||||
self.text_embeddings_sps = mp.Value("d", 0.01)
|
||||
self.face_rec_fps = mp.Value("d", 0.01)
|
||||
self.alpr_pps = mp.Value("d", 0.01)
|
||||
Reference in New Issue
Block a user