From 60b34bcfcafc8ced4843f530a541a911a156c8f7 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 21 Feb 2025 07:51:37 -0600 Subject: [PATCH 01/10] Refactor processors and add LPR postprocessing (#16722) * recordings data pub/sub * function to process recording stream frames * model runner * lpr model runner * refactor to mixin class and use model runner * separate out realtime and post processors * move model and mixin folders * basic postprocessor * clean up * docs * postprocessing logic * clean up * return none if recordings are disabled * run postprocessor handle_requests too * tweak expansion * add put endpoint * postprocessor tweaks with endpoint --- .../license_plate_recognition.md | 8 +- frigate/api/classification.py | 36 +++ frigate/comms/embeddings_updater.py | 1 + frigate/comms/recordings_updater.py | 36 +++ .../license_plate/mixin.py} | 95 ++----- .../common/license_plate/model.py | 31 +++ frigate/data_processing/post/api.py | 10 +- frigate/data_processing/post/license_plate.py | 231 ++++++++++++++++++ frigate/data_processing/real_time/api.py | 10 +- .../real_time/{bird_processor.py => bird.py} | 2 +- .../real_time/{face_processor.py => face.py} | 2 +- .../real_time/license_plate.py | 53 ++++ frigate/data_processing/types.py | 7 + frigate/embeddings/__init__.py | 9 +- frigate/embeddings/maintainer.py | 123 ++++++++-- frigate/record/maintainer.py | 18 ++ 16 files changed, 568 insertions(+), 104 deletions(-) create mode 100644 frigate/comms/recordings_updater.py rename frigate/data_processing/{real_time/license_plate_processor.py => common/license_plate/mixin.py} (93%) create mode 100644 frigate/data_processing/common/license_plate/model.py create mode 100644 frigate/data_processing/post/license_plate.py rename frigate/data_processing/real_time/{bird_processor.py => bird.py} (99%) rename frigate/data_processing/real_time/{face_processor.py => face.py} (99%) create mode 100644 frigate/data_processing/real_time/license_plate.py diff --git a/docs/docs/configuration/license_plate_recognition.md b/docs/docs/configuration/license_plate_recognition.md index 4fd7aa568..103c3bf14 100644 --- a/docs/docs/configuration/license_plate_recognition.md +++ b/docs/docs/configuration/license_plate_recognition.md @@ -41,6 +41,8 @@ lpr: Ensure that your camera is configured to detect objects of type `car`, and that a car is actually being detected by Frigate. Otherwise, LPR will not run. +Like the other real-time processors in Frigate, license plate recognition runs on the camera stream defined by the `detect` role in your config. To ensure optimal performance, select a suitable resolution for this stream in your camera's firmware that fits your specific scene and requirements. + ## Advanced Configuration Fine-tune the LPR feature using these optional parameters: @@ -52,7 +54,7 @@ Fine-tune the LPR feature using these optional parameters: - Note: If you are using a Frigate+ model and you set the `threshold` in your objects config for `license_plate` higher than this value, recognition will never run. It's best to ensure these values match, or this `detection_threshold` is lower than your object config `threshold`. - **`min_area`**: Defines the minimum size (in pixels) a license plate must be before recognition runs. - Default: `1000` pixels. - - Depending on the resolution of your cameras, you can increase this value to ignore small or distant plates. + - Depending on the resolution of your camera's `detect` stream, you can increase this value to ignore small or distant plates. ### Recognition @@ -114,7 +116,7 @@ lpr: Ensure that: - Your camera has a clear, well-lit view of the plate. -- The plate is large enough in the image (try adjusting `min_area`). +- The plate is large enough in the image (try adjusting `min_area`) or increasing the resolution of your camera's stream. - A `car` is detected first, as LPR only runs on recognized vehicles. If you are using a Frigate+ model or a custom model that detects license plates, ensure that `license_plate` is added to your list of objects to track. @@ -143,7 +145,7 @@ Use `match_distance` to allow small character mismatches. Alternatively, define - View MQTT messages for `frigate/events` to verify detected plates. - Adjust `detection_threshold` and `recognition_threshold` settings. - If you are using a Frigate+ model or a model that detects license plates, watch the debug view (Settings --> Debug) to ensure that `license_plate` is being detected with a `car`. -- Enable debug logs for LPR by adding `frigate.data_processing.real_time.license_plate_processor: debug` to your `logger` configuration. These logs are _very_ verbose, so only enable this when necessary. +- Enable debug logs for LPR by adding `frigate.data_processing.common.license_plate: debug` to your `logger` configuration. These logs are _very_ verbose, so only enable this when necessary. ### Will LPR slow down my system? diff --git a/frigate/api/classification.py b/frigate/api/classification.py index 7cd127d07..bd395737a 100644 --- a/frigate/api/classification.py +++ b/frigate/api/classification.py @@ -9,10 +9,13 @@ import string from fastapi import APIRouter, Request, UploadFile from fastapi.responses import JSONResponse from pathvalidate import sanitize_filename +from peewee import DoesNotExist +from playhouse.shortcuts import model_to_dict from frigate.api.defs.tags import Tags from frigate.const import FACE_DIR from frigate.embeddings import EmbeddingsContext +from frigate.models import Event logger = logging.getLogger(__name__) @@ -176,3 +179,36 @@ def deregister_faces(request: Request, name: str, body: dict = None): content=({"success": True, "message": "Successfully deleted faces."}), status_code=200, ) + + +@router.put("/lpr/reprocess") +def reprocess_license_plate(request: Request, event_id: str): + if not request.app.frigate_config.lpr.enabled: + message = "License plate recognition is not enabled." + logger.error(message) + return JSONResponse( + content=( + { + "success": False, + "message": message, + } + ), + status_code=400, + ) + + try: + event = Event.get(Event.id == event_id) + except DoesNotExist: + message = f"Event {event_id} not found" + logger.error(message) + return JSONResponse( + content=({"success": False, "message": message}), status_code=404 + ) + + context: EmbeddingsContext = request.app.embeddings + response = context.reprocess_plate(model_to_dict(event)) + + return JSONResponse( + content=response, + status_code=200, + ) diff --git a/frigate/comms/embeddings_updater.py b/frigate/comms/embeddings_updater.py index 58f012e7d..61c2331cf 100644 --- a/frigate/comms/embeddings_updater.py +++ b/frigate/comms/embeddings_updater.py @@ -15,6 +15,7 @@ class EmbeddingsRequestEnum(Enum): generate_search = "generate_search" register_face = "register_face" reprocess_face = "reprocess_face" + reprocess_plate = "reprocess_plate" class EmbeddingsResponder: diff --git a/frigate/comms/recordings_updater.py b/frigate/comms/recordings_updater.py new file mode 100644 index 000000000..862ec1041 --- /dev/null +++ b/frigate/comms/recordings_updater.py @@ -0,0 +1,36 @@ +"""Facilitates communication between processes.""" + +import logging +from enum import Enum + +from .zmq_proxy import Publisher, Subscriber + +logger = logging.getLogger(__name__) + + +class RecordingsDataTypeEnum(str, Enum): + all = "" + recordings_available_through = "recordings_available_through" + + +class RecordingsDataPublisher(Publisher): + """Publishes latest recording data.""" + + topic_base = "recordings/" + + def __init__(self, topic: RecordingsDataTypeEnum) -> None: + topic = topic.value + super().__init__(topic) + + def publish(self, payload: tuple[str, float]) -> None: + super().publish(payload) + + +class RecordingsDataSubscriber(Subscriber): + """Receives latest recording data.""" + + topic_base = "recordings/" + + def __init__(self, topic: RecordingsDataTypeEnum) -> None: + topic = topic.value + super().__init__(topic) diff --git a/frigate/data_processing/real_time/license_plate_processor.py b/frigate/data_processing/common/license_plate/mixin.py similarity index 93% rename from frigate/data_processing/real_time/license_plate_processor.py rename to frigate/data_processing/common/license_plate/mixin.py index bd7441928..1723d213e 100644 --- a/frigate/data_processing/real_time/license_plate_processor.py +++ b/frigate/data_processing/common/license_plate/mixin.py @@ -13,34 +13,21 @@ from Levenshtein import distance from pyclipper import ET_CLOSEDPOLYGON, JT_ROUND, PyclipperOffset from shapely.geometry import Polygon -from frigate.comms.inter_process import InterProcessRequestor -from frigate.config import FrigateConfig from frigate.const import FRIGATE_LOCALHOST -from frigate.embeddings.onnx.lpr_embedding import ( - LicensePlateDetector, - PaddleOCRClassification, - PaddleOCRDetection, - PaddleOCRRecognition, -) from frigate.util.image import area -from ..types import DataProcessorMetrics -from .api import RealTimeProcessorApi - logger = logging.getLogger(__name__) WRITE_DEBUG_IMAGES = False -class LicensePlateProcessor(RealTimeProcessorApi): - def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics): - super().__init__(config, metrics) - self.requestor = InterProcessRequestor() - self.lpr_config = config.lpr +class LicensePlateProcessingMixin: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.requires_license_plate_detection = ( "license_plate" not in self.config.objects.all_objects ) - self.detected_license_plates: dict[str, dict[str, any]] = {} self.ctc_decoder = CTCDecoder() @@ -52,42 +39,6 @@ class LicensePlateProcessor(RealTimeProcessorApi): self.box_thresh = 0.8 self.mask_thresh = 0.8 - self.lpr_detection_model = None - self.lpr_classification_model = None - self.lpr_recognition_model = None - - if self.config.lpr.enabled: - self.detection_model = PaddleOCRDetection( - model_size="large", - requestor=self.requestor, - device="CPU", - ) - - self.classification_model = PaddleOCRClassification( - model_size="large", - requestor=self.requestor, - device="CPU", - ) - - self.recognition_model = PaddleOCRRecognition( - model_size="large", - requestor=self.requestor, - device="CPU", - ) - - self.yolov9_detection_model = LicensePlateDetector( - model_size="large", - requestor=self.requestor, - device="CPU", - ) - - if self.lpr_config.enabled: - # all models need to be loaded to run LPR - self.detection_model._load_model_and_utils() - self.classification_model._load_model_and_utils() - self.recognition_model._load_model_and_utils() - self.yolov9_detection_model._load_model_and_utils() - def _detect(self, image: np.ndarray) -> List[np.ndarray]: """ Detect possible license plates in the input image by first resizing and normalizing it, @@ -114,7 +65,7 @@ class LicensePlateProcessor(RealTimeProcessorApi): resized_image, ) - outputs = self.detection_model([normalized_image])[0] + outputs = self.model_runner.detection_model([normalized_image])[0] outputs = outputs[0, :, :] boxes, _ = self._boxes_from_bitmap(outputs, outputs > self.mask_thresh, w, h) @@ -143,7 +94,7 @@ class LicensePlateProcessor(RealTimeProcessorApi): norm_img = norm_img[np.newaxis, :] norm_images.append(norm_img) - outputs = self.classification_model(norm_images) + outputs = self.model_runner.classification_model(norm_images) return self._process_classification_output(images, outputs) @@ -183,7 +134,7 @@ class LicensePlateProcessor(RealTimeProcessorApi): norm_image = norm_image[np.newaxis, :] norm_images.append(norm_image) - outputs = self.recognition_model(norm_images) + outputs = self.model_runner.recognition_model(norm_images) return self.ctc_decoder(outputs) def _process_license_plate( @@ -199,9 +150,9 @@ class LicensePlateProcessor(RealTimeProcessorApi): Tuple[List[str], List[float], List[int]]: Detected license plate texts, confidence scores, and areas of the plates. """ if ( - self.detection_model.runner is None - or self.classification_model.runner is None - or self.recognition_model.runner is None + self.model_runner.detection_model.runner is None + or self.model_runner.classification_model.runner is None + or self.model_runner.recognition_model.runner is None ): # we might still be downloading the models logger.debug("Model runners not loaded") @@ -665,7 +616,9 @@ class LicensePlateProcessor(RealTimeProcessorApi): input_w = int(input_h * max_wh_ratio) # check for model-specific input width - model_input_w = self.recognition_model.runner.ort.get_inputs()[0].shape[3] + model_input_w = self.model_runner.recognition_model.runner.ort.get_inputs()[ + 0 + ].shape[3] if isinstance(model_input_w, int) and model_input_w > 0: input_w = model_input_w @@ -732,19 +685,13 @@ class LicensePlateProcessor(RealTimeProcessorApi): image = np.rot90(image, k=3) return image - def __update_metrics(self, duration: float) -> None: - """ - Update inference metrics. - """ - self.metrics.alpr_pps.value = (self.metrics.alpr_pps.value * 9 + duration) / 10 - def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]: """ Use a lightweight YOLOv9 model to detect license plates for users without Frigate+ Return the dimensions of the detected plate as [x1, y1, x2, y2]. """ - predictions = self.yolov9_detection_model(input) + predictions = self.model_runner.yolov9_detection_model(input) confidence_threshold = self.lpr_config.detection_threshold @@ -770,8 +717,8 @@ class LicensePlateProcessor(RealTimeProcessorApi): # Return the top scoring bounding box if found if top_box is not None: - # expand box by 15% to help with OCR - expansion = (top_box[2:] - top_box[:2]) * 0.1 + # expand box by 30% to help with OCR + expansion = (top_box[2:] - top_box[:2]) * 0.30 # Expand box expanded_box = np.array( @@ -869,9 +816,8 @@ class LicensePlateProcessor(RealTimeProcessorApi): # 5. Return True if we should keep the previous plate (i.e., if it scores higher) return prev_score > curr_score - def process_frame(self, obj_data: dict[str, any], frame: np.ndarray): + def lpr_process(self, obj_data: dict[str, any], frame: np.ndarray): """Look for license plates in image.""" - start = datetime.datetime.now().timestamp() id = obj_data["id"] @@ -934,7 +880,7 @@ class LicensePlateProcessor(RealTimeProcessorApi): # check that license plate is valid # double the value because we've doubled the size of the car - if license_plate_area < self.config.lpr.min_area * 2: + if license_plate_area < self.lpr_config.min_area * 2: logger.debug("License plate is less than min_area") return @@ -972,7 +918,7 @@ class LicensePlateProcessor(RealTimeProcessorApi): # check that license plate is valid if ( not license_plate_box - or area(license_plate_box) < self.config.lpr.min_area + or area(license_plate_box) < self.lpr_config.min_area ): logger.debug(f"Invalid license plate box {license_plate}") return @@ -1078,10 +1024,9 @@ class LicensePlateProcessor(RealTimeProcessorApi): "plate": top_plate, "char_confidences": top_char_confidences, "area": top_area, + "obj_data": obj_data, } - self.__update_metrics(datetime.datetime.now().timestamp() - start) - def handle_request(self, topic, request_data) -> dict[str, any] | None: return diff --git a/frigate/data_processing/common/license_plate/model.py b/frigate/data_processing/common/license_plate/model.py new file mode 100644 index 000000000..25e7b2caf --- /dev/null +++ b/frigate/data_processing/common/license_plate/model.py @@ -0,0 +1,31 @@ +from frigate.embeddings.onnx.lpr_embedding import ( + LicensePlateDetector, + PaddleOCRClassification, + PaddleOCRDetection, + PaddleOCRRecognition, +) + +from ...types import DataProcessorModelRunner + + +class LicensePlateModelRunner(DataProcessorModelRunner): + def __init__(self, requestor, device: str = "CPU", model_size: str = "large"): + super().__init__(requestor, device, model_size) + self.detection_model = PaddleOCRDetection( + model_size=model_size, requestor=requestor, device=device + ) + self.classification_model = PaddleOCRClassification( + model_size=model_size, requestor=requestor, device=device + ) + self.recognition_model = PaddleOCRRecognition( + model_size=model_size, requestor=requestor, device=device + ) + self.yolov9_detection_model = LicensePlateDetector( + model_size=model_size, requestor=requestor, device=device + ) + + # Load all models once + self.detection_model._load_model_and_utils() + self.classification_model._load_model_and_utils() + self.recognition_model._load_model_and_utils() + self.yolov9_detection_model._load_model_and_utils() diff --git a/frigate/data_processing/post/api.py b/frigate/data_processing/post/api.py index 5c88221c2..c40caef71 100644 --- a/frigate/data_processing/post/api.py +++ b/frigate/data_processing/post/api.py @@ -5,16 +5,22 @@ from abc import ABC, abstractmethod from frigate.config import FrigateConfig -from ..types import DataProcessorMetrics, PostProcessDataEnum +from ..types import DataProcessorMetrics, DataProcessorModelRunner, PostProcessDataEnum logger = logging.getLogger(__name__) class PostProcessorApi(ABC): @abstractmethod - def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics) -> None: + def __init__( + self, + config: FrigateConfig, + metrics: DataProcessorMetrics, + model_runner: DataProcessorModelRunner, + ) -> None: self.config = config self.metrics = metrics + self.model_runner = model_runner pass @abstractmethod diff --git a/frigate/data_processing/post/license_plate.py b/frigate/data_processing/post/license_plate.py new file mode 100644 index 000000000..9a9974bc7 --- /dev/null +++ b/frigate/data_processing/post/license_plate.py @@ -0,0 +1,231 @@ +"""Handle post processing for license plate recognition.""" + +import datetime +import logging + +import cv2 +import numpy as np +from peewee import DoesNotExist + +from frigate.comms.embeddings_updater import EmbeddingsRequestEnum +from frigate.config import FrigateConfig +from frigate.data_processing.common.license_plate.mixin import ( + WRITE_DEBUG_IMAGES, + LicensePlateProcessingMixin, +) +from frigate.data_processing.common.license_plate.model import ( + LicensePlateModelRunner, +) +from frigate.data_processing.types import PostProcessDataEnum +from frigate.models import Recordings +from frigate.util.image import get_image_from_recording + +from ..types import DataProcessorMetrics +from .api import PostProcessorApi + +logger = logging.getLogger(__name__) + + +class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi): + def __init__( + self, + config: FrigateConfig, + metrics: DataProcessorMetrics, + model_runner: LicensePlateModelRunner, + detected_license_plates: dict[str, dict[str, any]], + ): + self.detected_license_plates = detected_license_plates + self.model_runner = model_runner + self.lpr_config = config.lpr + self.config = config + super().__init__(config, metrics, model_runner) + + def __update_metrics(self, duration: float) -> None: + """ + Update inference metrics. + """ + self.metrics.alpr_pps.value = (self.metrics.alpr_pps.value * 9 + duration) / 10 + + def process_data( + self, data: dict[str, any], data_type: PostProcessDataEnum + ) -> None: + """Look for license plates in recording stream image + Args: + data (dict): containing data about the input. + data_type (enum): Describing the data that is being processed. + + Returns: + None. + """ + start = datetime.datetime.now().timestamp() + + event_id = data["event_id"] + camera_name = data["camera"] + + if data_type == PostProcessDataEnum.recording: + obj_data = data["obj_data"] + frame_time = obj_data["frame_time"] + recordings_available_through = data["recordings_available"] + + if frame_time > recordings_available_through: + logger.debug( + f"LPR post processing: No recordings available for this frame time {frame_time}, available through {recordings_available_through}" + ) + + elif data_type == PostProcessDataEnum.tracked_object: + # non-functional, need to think about snapshot time + obj_data = data["event"]["data"] + obj_data["id"] = data["event"]["id"] + obj_data["camera"] = data["event"]["camera"] + # TODO: snapshot time? + frame_time = data["event"]["start_time"] + + else: + logger.error("No data type passed to LPR postprocessing") + return + + recording_query = ( + Recordings.select( + Recordings.path, + Recordings.start_time, + ) + .where( + ( + (frame_time >= Recordings.start_time) + & (frame_time <= Recordings.end_time) + ) + ) + .where(Recordings.camera == camera_name) + .order_by(Recordings.start_time.desc()) + .limit(1) + ) + + try: + recording: Recordings = recording_query.get() + time_in_segment = frame_time - recording.start_time + codec = "mjpeg" + + image_data = get_image_from_recording( + self.config.ffmpeg, recording.path, time_in_segment, codec, None + ) + + if not image_data: + logger.debug( + "LPR post processing: Unable to fetch license plate from recording" + ) + + # Convert bytes to numpy array + image_array = np.frombuffer(image_data, dtype=np.uint8) + + if len(image_array) == 0: + logger.debug("LPR post processing: No image") + return + + image = cv2.imdecode(image_array, cv2.IMREAD_COLOR) + + except DoesNotExist: + logger.debug("Error fetching license plate for postprocessing") + return + + if WRITE_DEBUG_IMAGES: + cv2.imwrite(f"debug/frames/lpr_post_{start}.jpg", image) + + # convert to yuv for processing + frame = cv2.cvtColor(image, cv2.COLOR_BGR2YUV_I420) + + detect_width = self.config.cameras[camera_name].detect.width + detect_height = self.config.cameras[camera_name].detect.height + + # Scale the boxes based on detect dimensions + scale_x = image.shape[1] / detect_width + scale_y = image.shape[0] / detect_height + + # Determine which box to enlarge based on detection mode + if self.requires_license_plate_detection: + # Scale and enlarge the car box + box = obj_data.get("box") + if not box: + return + + # Scale original car box to detection dimensions + left = int(box[0] * scale_x) + top = int(box[1] * scale_y) + right = int(box[2] * scale_x) + bottom = int(box[3] * scale_y) + box = [left, top, right, bottom] + else: + # Get the license plate box from attributes + if not obj_data.get("current_attributes"): + return + + license_plate = None + for attr in obj_data["current_attributes"]: + if attr.get("label") != "license_plate": + continue + if license_plate is None or attr.get("score", 0.0) > license_plate.get( + "score", 0.0 + ): + license_plate = attr + + if not license_plate or not license_plate.get("box"): + return + + # Scale license plate box to detection dimensions + orig_box = license_plate["box"] + left = int(orig_box[0] * scale_x) + top = int(orig_box[1] * scale_y) + right = int(orig_box[2] * scale_x) + bottom = int(orig_box[3] * scale_y) + box = [left, top, right, bottom] + + width_box = right - left + height_box = bottom - top + + # Enlarge box slightly to account for drift in detect vs recording stream + enlarge_factor = 0.3 + new_left = max(0, int(left - (width_box * enlarge_factor / 2))) + new_top = max(0, int(top - (height_box * enlarge_factor / 2))) + new_right = min(image.shape[1], int(right + (width_box * enlarge_factor / 2))) + new_bottom = min( + image.shape[0], int(bottom + (height_box * enlarge_factor / 2)) + ) + + keyframe_obj_data = obj_data.copy() + if self.requires_license_plate_detection: + # car box + keyframe_obj_data["box"] = [new_left, new_top, new_right, new_bottom] + else: + # Update the license plate box in the attributes + new_attributes = [] + for attr in obj_data["current_attributes"]: + if attr.get("label") == "license_plate": + new_attr = attr.copy() + new_attr["box"] = [new_left, new_top, new_right, new_bottom] + new_attributes.append(new_attr) + else: + new_attributes.append(attr) + keyframe_obj_data["current_attributes"] = new_attributes + + # run the frame through lpr processing + logger.debug(f"Post processing plate: {event_id}, {frame_time}") + self.lpr_process(keyframe_obj_data, frame) + + self.__update_metrics(datetime.datetime.now().timestamp() - start) + + def handle_request(self, topic, request_data) -> dict[str, any] | None: + if topic == EmbeddingsRequestEnum.reprocess_plate.value: + event = request_data["event"] + + self.process_data( + { + "event_id": event["id"], + "camera": event["camera"], + "event": event, + }, + PostProcessDataEnum.tracked_object, + ) + + return { + "message": "Successfully requested reprocessing of license plate.", + "success": True, + } diff --git a/frigate/data_processing/real_time/api.py b/frigate/data_processing/real_time/api.py index 205431a36..cd8f3e493 100644 --- a/frigate/data_processing/real_time/api.py +++ b/frigate/data_processing/real_time/api.py @@ -7,16 +7,22 @@ import numpy as np from frigate.config import FrigateConfig -from ..types import DataProcessorMetrics +from ..types import DataProcessorMetrics, DataProcessorModelRunner logger = logging.getLogger(__name__) class RealTimeProcessorApi(ABC): @abstractmethod - def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics) -> None: + def __init__( + self, + config: FrigateConfig, + metrics: DataProcessorMetrics, + model_runner: DataProcessorModelRunner, + ) -> None: self.config = config self.metrics = metrics + self.model_runner = model_runner pass @abstractmethod diff --git a/frigate/data_processing/real_time/bird_processor.py b/frigate/data_processing/real_time/bird.py similarity index 99% rename from frigate/data_processing/real_time/bird_processor.py rename to frigate/data_processing/real_time/bird.py index 1199f6124..01490d895 100644 --- a/frigate/data_processing/real_time/bird_processor.py +++ b/frigate/data_processing/real_time/bird.py @@ -22,7 +22,7 @@ except ModuleNotFoundError: logger = logging.getLogger(__name__) -class BirdProcessor(RealTimeProcessorApi): +class BirdRealTimeProcessor(RealTimeProcessorApi): def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics): super().__init__(config, metrics) self.interpreter: Interpreter = None diff --git a/frigate/data_processing/real_time/face_processor.py b/frigate/data_processing/real_time/face.py similarity index 99% rename from frigate/data_processing/real_time/face_processor.py rename to frigate/data_processing/real_time/face.py index 086c59658..d2b677653 100644 --- a/frigate/data_processing/real_time/face_processor.py +++ b/frigate/data_processing/real_time/face.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) MIN_MATCHING_FACES = 2 -class FaceProcessor(RealTimeProcessorApi): +class FaceRealTimeProcessor(RealTimeProcessorApi): def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics): super().__init__(config, metrics) self.face_config = config.face_recognition diff --git a/frigate/data_processing/real_time/license_plate.py b/frigate/data_processing/real_time/license_plate.py new file mode 100644 index 000000000..a5a1577fe --- /dev/null +++ b/frigate/data_processing/real_time/license_plate.py @@ -0,0 +1,53 @@ +"""Handle processing images for face detection and recognition.""" + +import datetime +import logging + +import numpy as np + +from frigate.config import FrigateConfig +from frigate.data_processing.common.license_plate.mixin import ( + LicensePlateProcessingMixin, +) +from frigate.data_processing.common.license_plate.model import ( + LicensePlateModelRunner, +) + +from ..types import DataProcessorMetrics +from .api import RealTimeProcessorApi + +logger = logging.getLogger(__name__) + + +class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcessorApi): + def __init__( + self, + config: FrigateConfig, + metrics: DataProcessorMetrics, + model_runner: LicensePlateModelRunner, + detected_license_plates: dict[str, dict[str, any]], + ): + self.detected_license_plates = detected_license_plates + self.model_runner = model_runner + self.lpr_config = config.lpr + self.config = config + super().__init__(config, metrics, model_runner) + + def __update_metrics(self, duration: float) -> None: + """ + Update inference metrics. + """ + self.metrics.alpr_pps.value = (self.metrics.alpr_pps.value * 9 + duration) / 10 + + def process_frame(self, obj_data: dict[str, any], frame: np.ndarray): + """Look for license plates in image.""" + start = datetime.datetime.now().timestamp() + self.lpr_process(obj_data, frame) + self.__update_metrics(datetime.datetime.now().timestamp() - start) + + def handle_request(self, topic, request_data) -> dict[str, any] | None: + return + + def expire_object(self, object_id: str): + if object_id in self.detected_license_plates: + self.detected_license_plates.pop(object_id) diff --git a/frigate/data_processing/types.py b/frigate/data_processing/types.py index 39f355667..6f87f77f9 100644 --- a/frigate/data_processing/types.py +++ b/frigate/data_processing/types.py @@ -18,6 +18,13 @@ class DataProcessorMetrics: self.alpr_pps = mp.Value("d", 0.01) +class DataProcessorModelRunner: + def __init__(self, requestor, device: str = "CPU", model_size: str = "large"): + self.requestor = requestor + self.device = device + self.model_size = model_size + + class PostProcessDataEnum(str, Enum): recording = "recording" review = "review" diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 185d5436b..18673c4e9 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -17,7 +17,7 @@ from frigate.config import FrigateConfig from frigate.const import CONFIG_DIR, FACE_DIR from frigate.data_processing.types import DataProcessorMetrics from frigate.db.sqlitevecq import SqliteVecQueueDatabase -from frigate.models import Event +from frigate.models import Event, Recordings from frigate.util.builtin import serialize from frigate.util.services import listen @@ -55,7 +55,7 @@ def manage_embeddings(config: FrigateConfig, metrics: DataProcessorMetrics) -> N timeout=max(60, 10 * len([c for c in config.cameras.values() if c.enabled])), load_vec_extension=True, ) - models = [Event] + models = [Event, Recordings] db.bind(models) maintainer = EmbeddingMaintainer( @@ -234,3 +234,8 @@ class EmbeddingsContext: EmbeddingsRequestEnum.embed_description.value, {"id": event_id, "description": description}, ) + + def reprocess_plate(self, event: dict[str, any]) -> dict[str, any]: + return self.requestor.send_data( + EmbeddingsRequestEnum.reprocess_plate.value, {"event": event} + ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 7925345b2..a18ca7a7f 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -20,18 +20,29 @@ from frigate.comms.event_metadata_updater import ( ) from frigate.comms.events_updater import EventEndSubscriber, EventUpdateSubscriber from frigate.comms.inter_process import InterProcessRequestor +from frigate.comms.recordings_updater import ( + RecordingsDataSubscriber, + RecordingsDataTypeEnum, +) from frigate.config import FrigateConfig from frigate.const import ( CLIPS_DIR, UPDATE_EVENT_DESCRIPTION, ) -from frigate.data_processing.real_time.api import RealTimeProcessorApi -from frigate.data_processing.real_time.bird_processor import BirdProcessor -from frigate.data_processing.real_time.face_processor import FaceProcessor -from frigate.data_processing.real_time.license_plate_processor import ( - LicensePlateProcessor, +from frigate.data_processing.common.license_plate.model import ( + LicensePlateModelRunner, ) -from frigate.data_processing.types import DataProcessorMetrics +from frigate.data_processing.post.api import PostProcessorApi +from frigate.data_processing.post.license_plate import ( + LicensePlatePostProcessor, +) +from frigate.data_processing.real_time.api import RealTimeProcessorApi +from frigate.data_processing.real_time.bird import BirdRealTimeProcessor +from frigate.data_processing.real_time.face import FaceRealTimeProcessor +from frigate.data_processing.real_time.license_plate import ( + LicensePlateRealTimeProcessor, +) +from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataEnum from frigate.events.types import EventTypeEnum from frigate.genai import get_genai_client from frigate.models import Event @@ -66,40 +77,71 @@ class EmbeddingMaintainer(threading.Thread): if config.semantic_search.reindex: self.embeddings.reindex() + # create communication for updating event descriptions + self.requestor = InterProcessRequestor() + self.event_subscriber = EventUpdateSubscriber() self.event_end_subscriber = EventEndSubscriber() self.event_metadata_subscriber = EventMetadataSubscriber( EventMetadataTypeEnum.regenerate_description ) + self.recordings_subscriber = RecordingsDataSubscriber( + RecordingsDataTypeEnum.recordings_available_through + ) self.embeddings_responder = EmbeddingsResponder() self.frame_manager = SharedMemoryFrameManager() - self.processors: list[RealTimeProcessorApi] = [] + + self.detected_license_plates: dict[str, dict[str, any]] = {} + + # model runners to share between realtime and post processors + if self.config.lpr.enabled: + lpr_model_runner = LicensePlateModelRunner(self.requestor) + + # realtime processors + self.realtime_processors: list[RealTimeProcessorApi] = [] if self.config.face_recognition.enabled: - self.processors.append(FaceProcessor(self.config, metrics)) + self.realtime_processors.append(FaceRealTimeProcessor(self.config, metrics)) if self.config.classification.bird.enabled: - self.processors.append(BirdProcessor(self.config, metrics)) + self.realtime_processors.append(BirdRealTimeProcessor(self.config, metrics)) if self.config.lpr.enabled: - self.processors.append(LicensePlateProcessor(self.config, metrics)) + self.realtime_processors.append( + LicensePlateRealTimeProcessor( + self.config, metrics, lpr_model_runner, self.detected_license_plates + ) + ) + + # post processors + self.post_processors: list[PostProcessorApi] = [] + + if self.config.lpr.enabled: + self.post_processors.append( + LicensePlatePostProcessor( + self.config, metrics, lpr_model_runner, self.detected_license_plates + ) + ) - # create communication for updating event descriptions - self.requestor = InterProcessRequestor() self.stop_event = stop_event self.tracked_events: dict[str, list[any]] = {} self.genai_client = get_genai_client(config) + # recordings data + self.recordings_available_through: dict[str, float] = {} + def run(self) -> None: """Maintain a SQLite-vec database for semantic search.""" while not self.stop_event.is_set(): self._process_requests() self._process_updates() + self._process_recordings_updates() self._process_finalized() self._process_event_metadata() self.event_subscriber.stop() self.event_end_subscriber.stop() + self.recordings_subscriber.stop() self.event_metadata_subscriber.stop() self.embeddings_responder.stop() self.requestor.stop() @@ -129,13 +171,15 @@ class EmbeddingMaintainer(threading.Thread): pack=False, ) else: - for processor in self.processors: - resp = processor.handle_request(topic, data) + processors = [self.realtime_processors, self.post_processors] + for processor_list in processors: + for processor in processor_list: + resp = processor.handle_request(topic, data) if resp is not None: return resp except Exception as e: - logger.error(f"Unable to handle embeddings request {e}") + logger.error(f"Unable to handle embeddings request {e}", exc_info=True) self.embeddings_responder.check_for_request(_handle_request) @@ -154,7 +198,7 @@ class EmbeddingMaintainer(threading.Thread): camera_config = self.config.cameras[camera] # no need to process updated objects if face recognition, lpr, genai are disabled - if not camera_config.genai.enabled and len(self.processors) == 0: + if not camera_config.genai.enabled and len(self.realtime_processors) == 0: return # Create our own thumbnail based on the bounding box and the frame time @@ -171,7 +215,7 @@ class EmbeddingMaintainer(threading.Thread): ) return - for processor in self.processors: + for processor in self.realtime_processors: processor.process_frame(data, yuv_frame) # no need to save our own thumbnails if genai is not enabled @@ -202,7 +246,32 @@ class EmbeddingMaintainer(threading.Thread): event_id, camera, updated_db = ended camera_config = self.config.cameras[camera] - for processor in self.processors: + # call any defined post processors + for processor in self.post_processors: + if isinstance(processor, LicensePlatePostProcessor): + recordings_available = self.recordings_available_through.get(camera) + if ( + recordings_available is not None + and event_id in self.detected_license_plates + ): + processor.process_data( + { + "event_id": event_id, + "camera": camera, + "recordings_available": self.recordings_available_through[ + camera + ], + "obj_data": self.detected_license_plates[event_id][ + "obj_data" + ], + }, + PostProcessDataEnum.recording, + ) + else: + processor.process_data(event_id, PostProcessDataEnum.event_id) + + # expire in realtime processors + for processor in self.realtime_processors: processor.expire_object(event_id) if updated_db: @@ -315,6 +384,24 @@ class EmbeddingMaintainer(threading.Thread): if event_id in self.tracked_events: del self.tracked_events[event_id] + def _process_recordings_updates(self) -> None: + """Process recordings updates.""" + while True: + recordings_data = self.recordings_subscriber.check_for_update(timeout=0.01) + + if recordings_data == None: + break + + camera, recordings_available_through_timestamp = recordings_data + + self.recordings_available_through[camera] = ( + recordings_available_through_timestamp + ) + + logger.debug( + f"{camera} now has recordings available through {recordings_available_through_timestamp}" + ) + def _process_event_metadata(self): # Check for regenerate description requests (topic, event_id, source) = self.event_metadata_subscriber.check_for_update( diff --git a/frigate/record/maintainer.py b/frigate/record/maintainer.py index a4c23763d..faa41f75f 100644 --- a/frigate/record/maintainer.py +++ b/frigate/record/maintainer.py @@ -19,6 +19,10 @@ import psutil from frigate.comms.config_updater import ConfigSubscriber from frigate.comms.detections_updater import DetectionSubscriber, DetectionTypeEnum from frigate.comms.inter_process import InterProcessRequestor +from frigate.comms.recordings_updater import ( + RecordingsDataPublisher, + RecordingsDataTypeEnum, +) from frigate.config import FrigateConfig, RetainModeEnum from frigate.const import ( CACHE_DIR, @@ -70,6 +74,9 @@ class RecordingMaintainer(threading.Thread): self.requestor = InterProcessRequestor() self.config_subscriber = ConfigSubscriber("config/record/") self.detection_subscriber = DetectionSubscriber(DetectionTypeEnum.all) + self.recordings_publisher = RecordingsDataPublisher( + RecordingsDataTypeEnum.recordings_available_through + ) self.stop_event = stop_event self.object_recordings_info: dict[str, list] = defaultdict(list) @@ -213,6 +220,16 @@ class RecordingMaintainer(threading.Thread): [self.validate_and_move_segment(camera, reviews, r) for r in recordings] ) + # publish most recently available recording time and None if disabled + self.recordings_publisher.publish( + ( + camera, + recordings[0]["start_time"].timestamp() + if self.config.cameras[camera].record.enabled + else None, + ) + ) + recordings_to_insert: list[Optional[Recordings]] = await asyncio.gather(*tasks) # fire and forget recordings entries @@ -582,4 +599,5 @@ class RecordingMaintainer(threading.Thread): self.requestor.stop() self.config_subscriber.stop() self.detection_subscriber.stop() + self.recordings_publisher.stop() logger.info("Exiting recording maintenance...") From b3c1b21f80c26c976470c6a7eb48727efa9e4d4b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 21 Feb 2025 14:26:03 -0600 Subject: [PATCH 02/10] Don't require model_runner for realtime processors (#16728) --- frigate/data_processing/real_time/api.py | 4 +--- frigate/data_processing/real_time/license_plate.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/frigate/data_processing/real_time/api.py b/frigate/data_processing/real_time/api.py index cd8f3e493..1ba01d5da 100644 --- a/frigate/data_processing/real_time/api.py +++ b/frigate/data_processing/real_time/api.py @@ -7,7 +7,7 @@ import numpy as np from frigate.config import FrigateConfig -from ..types import DataProcessorMetrics, DataProcessorModelRunner +from ..types import DataProcessorMetrics logger = logging.getLogger(__name__) @@ -18,11 +18,9 @@ class RealTimeProcessorApi(ABC): self, config: FrigateConfig, metrics: DataProcessorMetrics, - model_runner: DataProcessorModelRunner, ) -> None: self.config = config self.metrics = metrics - self.model_runner = model_runner pass @abstractmethod diff --git a/frigate/data_processing/real_time/license_plate.py b/frigate/data_processing/real_time/license_plate.py index a5a1577fe..2809e861f 100644 --- a/frigate/data_processing/real_time/license_plate.py +++ b/frigate/data_processing/real_time/license_plate.py @@ -31,7 +31,7 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess self.model_runner = model_runner self.lpr_config = config.lpr self.config = config - super().__init__(config, metrics, model_runner) + super().__init__(config, metrics) def __update_metrics(self, duration: float) -> None: """ From 844ee089d838a2df6ffce0b0095bf283199e497a Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sat, 22 Feb 2025 08:04:09 -0700 Subject: [PATCH 03/10] Fix preview fetch (#16741) --- frigate/record/export.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frigate/record/export.py b/frigate/record/export.py index f05fae6aa..0e64021b4 100644 --- a/frigate/record/export.py +++ b/frigate/record/export.py @@ -363,10 +363,13 @@ class RecordingExporter(threading.Thread): } ).execute() - if self.playback_source == PlaybackSourceEnum.recordings: - ffmpeg_cmd, playlist_lines = self.get_record_export_command(video_path) - else: - ffmpeg_cmd, playlist_lines = self.get_preview_export_command(video_path) + try: + if self.playback_source == PlaybackSourceEnum.recordings: + ffmpeg_cmd, playlist_lines = self.get_record_export_command(video_path) + else: + ffmpeg_cmd, playlist_lines = self.get_preview_export_command(video_path) + except DoesNotExist: + return p = sp.run( ffmpeg_cmd, From 71f1ea86d28c6887423efa731cc99b7d71440d86 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 22 Feb 2025 09:19:37 -0600 Subject: [PATCH 04/10] Add note for notifications on iOS devices (#16744) --- docs/docs/configuration/notifications.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/configuration/notifications.md b/docs/docs/configuration/notifications.md index 8ae2f6d47..b5e1600e4 100644 --- a/docs/docs/configuration/notifications.md +++ b/docs/docs/configuration/notifications.md @@ -14,6 +14,7 @@ In order to use notifications the following requirements must be met: - Frigate must be accessed via a secure `https` connection ([see the authorization docs](/configuration/authentication)). - A supported browser must be used. Currently Chrome, Firefox, and Safari are known to be supported. - In order for notifications to be usable externally, Frigate must be accessible externally. +- For iOS devices, some users have also indicated that the Notifications switch needs to be enabled in iOS Settings --> Apps --> Safari --> Advanced --> Features. ### Configuration From 22cbf74dc893efe29671f45405b8c25824d850a0 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 23 Feb 2025 07:25:50 -0600 Subject: [PATCH 05/10] Fix frigate log deduplication (#16759) --- frigate/util/services.py | 49 +++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/frigate/util/services.py b/frigate/util/services.py index d7966bd00..c9c1b61a2 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -659,25 +659,42 @@ def process_logs( if " " not in clean_line: clean_line = f"{datetime.now()} {clean_line}" - # Find the position of the first double space to extract timestamp and message - date_end = clean_line.index(" ") - timestamp = clean_line[:date_end] - message_part = clean_line[date_end:].strip() + try: + # Find the position of the first double space to extract timestamp and message + date_end = clean_line.index(" ") + timestamp = clean_line[:date_end] + full_message = clean_line[date_end:].strip() - if message_part == last_message: - repeat_count += 1 - continue - else: - if repeat_count > 0: - # Insert a deduplication message formatted the same way as logs - dedup_message = f"{last_timestamp} [LOGGING] Last message repeated {repeat_count} times" - log_lines.append(dedup_message) - repeat_count = 0 + # For frigate, remove the date part from message comparison + if service == "frigate": + # Skip the date at the start of the message if it exists + date_parts = full_message.split("]", 1) + if len(date_parts) > 1: + message_part = date_parts[1].strip() + else: + message_part = full_message + else: + message_part = full_message + if message_part == last_message: + repeat_count += 1 + continue + else: + if repeat_count > 0: + # Insert a deduplication message formatted the same way as logs + dedup_message = f"{last_timestamp} [LOGGING] Last message repeated {repeat_count} times" + log_lines.append(dedup_message) + repeat_count = 0 + + log_lines.append(clean_line) + last_timestamp = timestamp + + last_message = message_part + + except ValueError: + # If we can't parse the line properly, just add it as is log_lines.append(clean_line) - last_timestamp = timestamp - - last_message = message_part + continue # If there were repeated messages at the end, log the count if repeat_count > 0: From 202b9d1c7994233f019161fb150fa9b3ffd6ac3d Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 23 Feb 2025 12:11:18 -0600 Subject: [PATCH 06/10] Check websocket correctly when no cameras are enabled/defined (#16762) --- web/src/api/ws.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index a8cedf953..7ca9ae69d 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -46,7 +46,7 @@ function useValue(): useValueReturn { const cameraActivity: { [key: string]: object } = JSON.parse(activityValue); - if (!cameraActivity) { + if (Object.keys(cameraActivity).length === 0) { return; } From 04a718dda8717ca5e18b8ee6faf3c52921526307 Mon Sep 17 00:00:00 2001 From: Tibladar <52620063+Tibladar@users.noreply.github.com> Date: Sun, 23 Feb 2025 18:44:41 +0000 Subject: [PATCH 07/10] Docs: Fix broken shm calculation (#16755) * Docs: Fix broken shm calculation * Docs: Change wording of shm template --- docs/docs/frigate/installation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index a1c8550b9..9b3920ebc 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -80,12 +80,12 @@ The Frigate container also stores logs in shm, which can take up to **40MB**, so You can calculate the **minimum** shm size for each camera with the following formula using the resolution specified for detect: ```console -# Replace and +# Template for one camera without logs, replace and $ python -c 'print("{:.2f}MB".format(( * * 1.5 * 20 + 270480) / 1048576))' # Example for 1280x720, including logs -$ python -c 'print("{:.2f}MB".format((1280 * 720 * 1.5 * 20 + 270480) / 1048576)) + 40' -46.63MB +$ python -c 'print("{:.2f}MB".format((1280 * 720 * 1.5 * 20 + 270480) / 1048576 + 40))' +66.63MB # Example for eight cameras detecting at 1280x720, including logs $ python -c 'print("{:.2f}MB".format(((1280 * 720 * 1.5 * 20 + 270480) / 1048576) * 8 + 40))' From 9414e001f32bd61c9217529be45751de020a751a Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 23 Feb 2025 17:56:48 -0600 Subject: [PATCH 08/10] Edit sub labels from the UI (#16764) * Add ability to edit sub labels from tracked object detail dialog * add allowEmpty prop * use TextEntryDialog * clean up * text consistency --- .../overlay/detail/SearchDetailDialog.tsx | 95 +++++++++++++++++++ .../overlay/dialog/TextEntryDialog.tsx | 19 +++- 2 files changed, 110 insertions(+), 4 deletions(-) diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 03054d811..9d3610e49 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -71,6 +71,8 @@ import { } from "@/components/ui/popover"; import { LuInfo } from "react-icons/lu"; import { TooltipPortal } from "@radix-ui/react-tooltip"; +import { FaPencilAlt } from "react-icons/fa"; +import TextEntryDialog from "@/components/overlay/dialog/TextEntryDialog"; const SEARCH_TABS = [ "details", @@ -288,6 +290,7 @@ function ObjectDetailsTab({ // data const [desc, setDesc] = useState(search?.data.description); + const [isSubLabelDialogOpen, setIsSubLabelDialogOpen] = useState(false); const handleDescriptionFocus = useCallback(() => { setInputFocused(true); @@ -430,6 +433,74 @@ function ObjectDetailsTab({ [search, config], ); + const handleSubLabelSave = useCallback( + (text: string) => { + if (!search) return; + + // set score to 1.0 if we're manually entering a sub label + const subLabelScore = + text === "" ? undefined : search.data?.sub_label_score || 1.0; + + axios + .post(`${apiHost}api/events/${search.id}/sub_label`, { + camera: search.camera, + subLabel: text, + subLabelScore: subLabelScore, + }) + .then((response) => { + if (response.status === 200) { + toast.success("Successfully updated sub label.", { + position: "top-center", + }); + + mutate( + (key) => + typeof key === "string" && + (key.includes("events") || + key.includes("events/search") || + key.includes("events/explore")), + (currentData: SearchResult[][] | SearchResult[] | undefined) => { + if (!currentData) return currentData; + return currentData.flat().map((event) => + event.id === search.id + ? { + ...event, + sub_label: text, + data: { + ...event.data, + sub_label_score: subLabelScore, + }, + } + : event, + ); + }, + { + optimisticData: true, + rollbackOnError: true, + revalidate: false, + }, + ); + + setSearch({ + ...search, + sub_label: text, + data: { + ...search.data, + sub_label_score: subLabelScore, + }, + }); + setIsSubLabelDialogOpen(false); + } + }) + .catch(() => { + toast.error("Failed to update sub label.", { + position: "top-center", + }); + }); + }, + [search, apiHost, mutate, setSearch], + ); + return (
@@ -440,6 +511,21 @@ function ObjectDetailsTab({ {getIconForLabel(search.label, "size-4 text-primary")} {search.label} {search.sub_label && ` (${search.sub_label})`} + + + + { + setIsSubLabelDialogOpen(true); + }} + /> + + + + Edit sub label + +
@@ -616,6 +702,15 @@ function ObjectDetailsTab({ Save )} +
diff --git a/web/src/components/overlay/dialog/TextEntryDialog.tsx b/web/src/components/overlay/dialog/TextEntryDialog.tsx index 1b0655078..d7b90aabb 100644 --- a/web/src/components/overlay/dialog/TextEntryDialog.tsx +++ b/web/src/components/overlay/dialog/TextEntryDialog.tsx @@ -10,7 +10,7 @@ import { import { Form, FormControl, FormField, FormItem } from "@/components/ui/form"; import { Input } from "@/components/ui/input"; import { zodResolver } from "@hookform/resolvers/zod"; -import { useCallback } from "react"; +import { useCallback, useEffect } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; @@ -20,13 +20,18 @@ type TextEntryDialogProps = { description?: string; setOpen: (open: boolean) => void; onSave: (text: string) => void; + defaultValue?: string; + allowEmpty?: boolean; }; + export default function TextEntryDialog({ open, title, description, setOpen, onSave, + defaultValue = "", + allowEmpty = false, }: TextEntryDialogProps) { const formSchema = z.object({ text: z.string(), @@ -34,6 +39,7 @@ export default function TextEntryDialog({ const form = useForm>({ resolver: zodResolver(formSchema), + defaultValues: { text: defaultValue }, }); const fileRef = form.register("text"); @@ -41,15 +47,20 @@ export default function TextEntryDialog({ const onSubmit = useCallback( (data: z.infer) => { - if (!data["text"]) { + if (!allowEmpty && !data["text"]) { return; } - onSave(data["text"]); }, - [onSave], + [onSave, allowEmpty], ); + useEffect(() => { + if (open) { + form.reset({ text: defaultValue }); + } + }, [open, defaultValue, form]); + return ( From 1d8f1bd7ae7b7c379ccf4a2566cc22ad7ba081db Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 24 Feb 2025 08:02:36 -0600 Subject: [PATCH 09/10] Ensure sub label is null when submitting an empty string (#16779) * null sub_label when submitting an empty string * prevent cancel from submitting form * fix test --- frigate/api/event.py | 24 ++++++++++--------- frigate/test/test_http.py | 2 +- .../overlay/dialog/TextEntryDialog.tsx | 4 +++- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/frigate/api/event.py b/frigate/api/event.py index 2df32471e..bb1bf7395 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -991,6 +991,10 @@ def set_sub_label( new_sub_label = body.subLabel new_score = body.subLabelScore + if new_sub_label == "": + new_sub_label = None + new_score = None + if tracked_obj: tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score) @@ -1001,21 +1005,19 @@ def set_sub_label( if event: event.sub_label = new_sub_label - - if new_score: - data = event.data + data = event.data + if new_sub_label is None: + data["sub_label_score"] = None + elif new_score is not None: data["sub_label_score"] = new_score - event.data = data - + event.data = data event.save() return JSONResponse( - content=( - { - "success": True, - "message": "Event " + event_id + " sub label set to " + new_sub_label, - } - ), + content={ + "success": True, + "message": f"Event {event_id} sub label set to {new_sub_label if new_sub_label is not None else 'None'}", + }, status_code=200, ) diff --git a/frigate/test/test_http.py b/frigate/test/test_http.py index 8c89e0433..46de1307f 100644 --- a/frigate/test/test_http.py +++ b/frigate/test/test_http.py @@ -275,7 +275,7 @@ class TestHttp(unittest.TestCase): event = client.get(f"/events/{id}").json() assert event assert event["id"] == id - assert event["sub_label"] == "" + assert event["sub_label"] == None def test_sub_label_list(self): app = create_fastapi_app( diff --git a/web/src/components/overlay/dialog/TextEntryDialog.tsx b/web/src/components/overlay/dialog/TextEntryDialog.tsx index d7b90aabb..c11a84ae7 100644 --- a/web/src/components/overlay/dialog/TextEntryDialog.tsx +++ b/web/src/components/overlay/dialog/TextEntryDialog.tsx @@ -86,7 +86,9 @@ export default function TextEntryDialog({ )} /> - + From 0de928703facc46fd2ed947dbb2be486b41ae4dc Mon Sep 17 00:00:00 2001 From: Jason Hunter Date: Mon, 24 Feb 2025 10:56:01 -0500 Subject: [PATCH 10/10] Initial implementation of D-FINE model via ONNX (#16772) * initial implementation of D-FINE model * revert docker-compose * add docs for D-FINE * remove weird auto-format issue --- .devcontainer/devcontainer.json | 47 +++++++++++++--- .../onnxruntime-gpu/devcontainer-feature.json | 22 ++++++++ .../features/onnxruntime-gpu/install.sh | 15 +++++ docs/docs/configuration/object_detectors.md | 55 ++++++++++++++++++- frigate/detectors/detector_config.py | 1 + frigate/detectors/plugins/onnx.py | 17 +++++- frigate/util/model.py | 27 +++++++++ 7 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 .devcontainer/features/onnxruntime-gpu/devcontainer-feature.json create mode 100644 .devcontainer/features/onnxruntime-gpu/install.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 63adae73d..c782fb32f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,9 +8,25 @@ "overrideCommand": false, "remoteUser": "vscode", "features": { - "ghcr.io/devcontainers/features/common-utils:1": {} + "ghcr.io/devcontainers/features/common-utils:2": {} + // Uncomment the following lines to use ONNX Runtime with CUDA support + // "ghcr.io/devcontainers/features/nvidia-cuda:1": { + // "installCudnn": true, + // "installNvtx": true, + // "installToolkit": true, + // "cudaVersion": "12.5", + // "cudnnVersion": "9.4.0.58" + // }, + // "./features/onnxruntime-gpu": {} }, - "forwardPorts": [8971, 5000, 5001, 5173, 8554, 8555], + "forwardPorts": [ + 8971, + 5000, + 5001, + 5173, + 8554, + 8555 + ], "portsAttributes": { "8971": { "label": "External NGINX", @@ -64,10 +80,18 @@ "editor.formatOnType": true, "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": true, - "python.testing.unittestArgs": ["-v", "-s", "./frigate/test"], + "python.testing.unittestArgs": [ + "-v", + "-s", + "./frigate/test" + ], "files.trimTrailingWhitespace": true, - "eslint.workingDirectories": ["./web"], - "isort.args": ["--settings-path=./pyproject.toml"], + "eslint.workingDirectories": [ + "./web" + ], + "isort.args": [ + "--settings-path=./pyproject.toml" + ], "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, @@ -86,9 +110,16 @@ ], "editor.tabSize": 2 }, - "cSpell.ignoreWords": ["rtmp"], - "cSpell.words": ["preact", "astype", "hwaccel", "mqtt"] + "cSpell.ignoreWords": [ + "rtmp" + ], + "cSpell.words": [ + "preact", + "astype", + "hwaccel", + "mqtt" + ] } } } -} +} \ No newline at end of file diff --git a/.devcontainer/features/onnxruntime-gpu/devcontainer-feature.json b/.devcontainer/features/onnxruntime-gpu/devcontainer-feature.json new file mode 100644 index 000000000..30514442b --- /dev/null +++ b/.devcontainer/features/onnxruntime-gpu/devcontainer-feature.json @@ -0,0 +1,22 @@ +{ + "id": "onnxruntime-gpu", + "version": "0.0.1", + "name": "ONNX Runtime GPU (Nvidia)", + "description": "Installs ONNX Runtime for Nvidia GPUs.", + "documentationURL": "", + "options": { + "version": { + "type": "string", + "proposals": [ + "latest", + "1.20.1", + "1.20.0" + ], + "default": "latest", + "description": "Version of ONNX Runtime to install" + } + }, + "installsAfter": [ + "ghcr.io/devcontainers/features/nvidia-cuda" + ] +} \ No newline at end of file diff --git a/.devcontainer/features/onnxruntime-gpu/install.sh b/.devcontainer/features/onnxruntime-gpu/install.sh new file mode 100644 index 000000000..0c090beec --- /dev/null +++ b/.devcontainer/features/onnxruntime-gpu/install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +VERSION=${VERSION} + +python3 -m pip config set global.break-system-packages true +# if VERSION == "latest" or VERSION is empty, install the latest version +if [ "$VERSION" == "latest" ] || [ -z "$VERSION" ]; then + python3 -m pip install onnxruntime-gpu +else + python3 -m pip install onnxruntime-gpu==$VERSION +fi + +echo "Done!" \ No newline at end of file diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 21ba46c2d..bc76779cb 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -10,25 +10,31 @@ title: Object Detectors Frigate supports multiple different detectors that work on different types of hardware: **Most Hardware** + - [Coral EdgeTPU](#edge-tpu-detector): The Google Coral EdgeTPU is available in USB and m.2 format allowing for a wide range of compatibility with devices. - [Hailo](#hailo-8l): The Hailo8 AI Acceleration module is available in m.2 format with a HAT for RPi devices, offering a wide range of compatibility with devices. **AMD** + - [ROCm](#amdrocm-gpu-detector): ROCm can run on AMD Discrete GPUs to provide efficient object detection. - [ONNX](#onnx): ROCm will automatically be detected and used as a detector in the `-rocm` Frigate image when a supported ONNX model is configured. **Intel** + - [OpenVino](#openvino-detector): OpenVino can run on Intel Arc GPUs, Intel integrated GPUs, and Intel CPUs to provide efficient object detection. - [ONNX](#onnx): OpenVINO will automatically be detected and used as a detector in the default Frigate image when a supported ONNX model is configured. **Nvidia** + - [TensortRT](#nvidia-tensorrt-detector): TensorRT can run on Nvidia GPUs and Jetson devices, using one of many default models. - [ONNX](#onnx): TensorRT will automatically be detected and used as a detector in the `-tensorrt` or `-tensorrt-jp(4/5)` Frigate images when a supported ONNX model is configured. **Rockchip** + - [RKNN](#rockchip-platform): RKNN models can run on Rockchip devices with included NPUs. **For Testing** + - [CPU Detector (not recommended for actual use](#cpu-detector-not-recommended): Use a CPU to run tflite model, this is not recommended and in most cases OpenVINO can be used in CPU mode with better results. ::: @@ -147,7 +153,6 @@ model: path: /config/model_cache/h8l_cache/ssd_mobilenet_v1.hef ``` - ## OpenVINO Detector The OpenVINO detector type runs an OpenVINO IR model on AMD and Intel CPUs, Intel GPUs and Intel VPU hardware. To configure an OpenVINO detector, set the `"type"` attribute to `"openvino"`. @@ -412,7 +417,7 @@ When using docker compose: ```yaml services: frigate: -... + environment: HSA_OVERRIDE_GFX_VERSION: "9.0.0" ``` @@ -555,6 +560,50 @@ model: Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. +#### D-FINE + +[D-FINE](https://github.com/Peterande/D-FINE) is the [current state of the art](https://paperswithcode.com/sota/real-time-object-detection-on-coco?p=d-fine-redefine-regression-task-in-detrs-as) at the time of writing. The ONNX exported models are supported, but not included by default. + +To export as ONNX: + +1. Clone: https://github.com/Peterande/D-FINE and install all dependencies. +2. Select and download a checkpoint from the [readme](https://github.com/Peterande/D-FINE). +3. Modify line 58 of `tools/deployment/export_onnx.py` and change batch size to 1: `data = torch.rand(1, 3, 640, 640)` +4. Run the export, making sure you select the right config, for your checkpoint. + +Example: + +``` +python3 tools/deployment/export_onnx.py -c configs/dfine/objects365/dfine_hgnetv2_m_obj2coco.yml -r output/dfine_m_obj2coco.pth +``` + +:::tip + +Model export has only been tested on Linux (or WSL2). Not all dependencies are in `requirements.txt`. Some live in the deployment folder, and some are still missing entirely and must be installed manually. + +Make sure you change the batch size to 1 before exporting. + +::: + +After placing the downloaded onnx model in your config folder, you can use the following configuration: + +```yaml +detectors: + onnx: + type: onnx + +model: + model_type: dfine + width: 640 + height: 640 + input_tensor: nchw + input_dtype: float + path: /config/model_cache/dfine_m_obj2coco.onnx + labelmap_path: /labelmap/coco-80.txt +``` + +Note that the labelmap uses a subset of the complete COCO label set that has only 80 objects. + ## CPU Detector (not recommended) The CPU detector type runs a TensorFlow Lite model utilizing the CPU without hardware acceleration. It is recommended to use a hardware accelerated detector type instead for better performance. To configure a CPU based detector, set the `"type"` attribute to `"cpu"`. @@ -704,7 +753,7 @@ To convert a onnx model to the rknn format using the [rknn-toolkit2](https://git This is an example configuration file that you need to adjust to your specific onnx model: ```yaml -soc: ["rk3562","rk3566", "rk3568", "rk3576", "rk3588"] +soc: ["rk3562", "rk3566", "rk3568", "rk3576", "rk3588"] quantization: false output_name: "{input_basename}" diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py index c8aea0a1d..16599b141 100644 --- a/frigate/detectors/detector_config.py +++ b/frigate/detectors/detector_config.py @@ -37,6 +37,7 @@ class ModelTypeEnum(str, Enum): yolox = "yolox" yolov9 = "yolov9" yolonas = "yolonas" + dfine = "dfine" class ModelConfig(BaseModel): diff --git a/frigate/detectors/plugins/onnx.py b/frigate/detectors/plugins/onnx.py index c8589145a..13a948de9 100644 --- a/frigate/detectors/plugins/onnx.py +++ b/frigate/detectors/plugins/onnx.py @@ -9,7 +9,11 @@ from frigate.detectors.detector_config import ( BaseDetectorConfig, ModelTypeEnum, ) -from frigate.util.model import get_ort_providers, post_process_yolov9 +from frigate.util.model import ( + get_ort_providers, + post_process_dfine, + post_process_yolov9, +) logger = logging.getLogger(__name__) @@ -41,6 +45,7 @@ class ONNXDetector(DetectionApi): providers, options = get_ort_providers( detector_config.device == "CPU", detector_config.device ) + self.model = ort.InferenceSession( path, providers=providers, provider_options=options ) @@ -55,6 +60,16 @@ class ONNXDetector(DetectionApi): logger.info(f"ONNX: {path} loaded") def detect_raw(self, tensor_input: np.ndarray): + if self.onnx_model_type == ModelTypeEnum.dfine: + tensor_output = self.model.run( + None, + { + "images": tensor_input, + "orig_target_sizes": np.array([[self.h, self.w]], dtype=np.int64), + }, + ) + return post_process_dfine(tensor_output, self.w, self.h) + model_input_name = self.model.get_inputs()[0].name tensor_output = self.model.run(None, {model_input_name: tensor_input}) diff --git a/frigate/util/model.py b/frigate/util/model.py index da7b1a50a..0428a42ff 100644 --- a/frigate/util/model.py +++ b/frigate/util/model.py @@ -9,7 +9,34 @@ import onnxruntime as ort logger = logging.getLogger(__name__) + ### Post Processing +def post_process_dfine(tensor_output: np.ndarray, width, height) -> np.ndarray: + class_ids = tensor_output[0][tensor_output[2] > 0.4] + boxes = tensor_output[1][tensor_output[2] > 0.4] + scores = tensor_output[2][tensor_output[2] > 0.4] + + input_shape = np.array([height, width, height, width]) + boxes = np.divide(boxes, input_shape, dtype=np.float32) + indices = cv2.dnn.NMSBoxes(boxes, scores, score_threshold=0.4, nms_threshold=0.4) + detections = np.zeros((20, 6), np.float32) + + for i, (bbox, confidence, class_id) in enumerate( + zip(boxes[indices], scores[indices], class_ids[indices]) + ): + if i == 20: + break + + detections[i] = [ + class_id, + confidence, + bbox[1], + bbox[0], + bbox[3], + bbox[2], + ] + + return detections def post_process_yolov9(predictions: np.ndarray, width, height) -> np.ndarray: