From 9e34581faf4e330befbe422009123475d5afa0cb Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 28 Mar 2025 07:05:46 -0500 Subject: [PATCH] enhancement and debugging config --- frigate/config/classification.py | 16 +++++ .../common/license_plate/mixin.py | 71 ++++++++++++++----- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/frigate/config/classification.py b/frigate/config/classification.py index aecbf6537..4026106b2 100644 --- a/frigate/config/classification.py +++ b/frigate/config/classification.py @@ -126,6 +126,16 @@ class LicensePlateRecognitionConfig(FrigateBaseModel): known_plates: Optional[Dict[str, List[str]]] = Field( default={}, title="Known plates to track (strings or regular expressions)." ) + enhancement: int = Field( + default=0, + title="Amount of contrast adjustment and denoising to apply to license plate images before recognition.", + ge=0, + le=10, + ) + debug_save_plates: bool = Field( + default=False, + title="Save plates captured for LPR for debugging purposes.", + ) class CameraLicensePlateRecognitionConfig(FrigateBaseModel): @@ -139,5 +149,11 @@ class CameraLicensePlateRecognitionConfig(FrigateBaseModel): default=1000, title="Minimum area of license plate to begin running recognition.", ) + enhancement: int = Field( + default=0, + title="Amount of contrast adjustment and denoising to apply to license plate images before recognition.", + ge=0, + le=10, + ) model_config = ConfigDict(extra="ignore", protected_namespaces=()) diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py index a2528380b..3d24d48d5 100644 --- a/frigate/data_processing/common/license_plate/mixin.py +++ b/frigate/data_processing/common/license_plate/mixin.py @@ -4,9 +4,11 @@ import base64 import datetime import logging import math +import os import random import re import string +from pathlib import Path from typing import List, Optional, Tuple import cv2 @@ -20,6 +22,7 @@ from frigate.comms.event_metadata_updater import ( EventMetadataTypeEnum, ) from frigate.config.camera.camera import CameraTypeEnum +from frigate.const import CLIPS_DIR from frigate.embeddings.onnx.lpr_embedding import LPR_EMBEDDING_SIZE from frigate.util.image import area @@ -107,7 +110,7 @@ class LicensePlateProcessingMixin: return self._process_classification_output(images, outputs) def _recognize( - self, images: List[np.ndarray] + self, camera: string, images: List[np.ndarray] ) -> Tuple[List[str], List[List[float]]]: """ Recognize the characters on the detected license plates using the recognition model. @@ -137,7 +140,7 @@ class LicensePlateProcessingMixin: # preprocess the images based on the max aspect ratio for i in range(index, min(num_images, index + self.batch_size)): norm_image = self._preprocess_recognition_image( - images[indices[i]], max_wh_ratio + camera, images[indices[i]], max_wh_ratio ) norm_image = norm_image[np.newaxis, :] norm_images.append(norm_image) @@ -146,7 +149,7 @@ class LicensePlateProcessingMixin: return self.ctc_decoder(outputs) def _process_license_plate( - self, image: np.ndarray + self, camera: string, id: string, image: np.ndarray ) -> Tuple[List[str], List[float], List[int]]: """ Complete pipeline for detecting, classifying, and recognizing license plates in the input image. @@ -174,21 +177,37 @@ class LicensePlateProcessingMixin: boxes = self._sort_boxes(list(boxes)) plate_images = [self._crop_license_plate(image, x) for x in boxes] + current_time = int(datetime.datetime.now().timestamp()) + if WRITE_DEBUG_IMAGES: - current_time = int(datetime.datetime.now().timestamp()) for i, img in enumerate(plate_images): cv2.imwrite( f"debug/frames/license_plate_cropped_{current_time}_{i + 1}.jpg", img, ) + if self.config.lpr.debug_save_plates: + logger.debug(f"{camera}: Saving plates for event {id}") + + Path(os.path.join(CLIPS_DIR, f"lpr/{camera}/{id}")).mkdir( + parents=True, exist_ok=True + ) + + for i, img in enumerate(plate_images): + cv2.imwrite( + os.path.join( + CLIPS_DIR, f"lpr/{camera}/{id}/{current_time}_{i + 1}.jpg" + ), + img, + ) + # keep track of the index of each image for correct area calc later sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in plate_images]) reverse_mapping = { idx: original_idx for original_idx, idx in enumerate(sorted_indices) } - results, confidences = self._recognize(plate_images) + results, confidences = self._recognize(camera, plate_images) if results: license_plates = [""] * len(plate_images) @@ -606,7 +625,7 @@ class LicensePlateProcessingMixin: return images, results def _preprocess_recognition_image( - self, image: np.ndarray, max_wh_ratio: float + self, camera: string, image: np.ndarray, max_wh_ratio: float ) -> np.ndarray: """ Preprocess an image for recognition by dynamically adjusting its width. @@ -634,20 +653,38 @@ class LicensePlateProcessingMixin: else: gray = image - if False: - smoothed = cv2.bilateralFilter(gray, d=3, sigmaColor=50, sigmaSpace=50) + if self.config.cameras[camera].lpr.enhancement > 3: + # denoise using a configurable pixel neighborhood value + logger.debug( + f"{camera}: Denoising recognition image (level: {self.config.cameras[camera].lpr.enhancement})" + ) + smoothed = cv2.bilateralFilter( + gray, + d=5 + self.config.cameras[camera].lpr.enhancement, + sigmaColor=10 * self.config.cameras[camera].lpr.enhancement, + sigmaSpace=10 * self.config.cameras[camera].lpr.enhancement, + ) sharpening_kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]]) processed = cv2.filter2D(smoothed, -1, sharpening_kernel) else: processed = gray - # apply CLAHE for contrast enhancement - grid_size = ( - max(4, input_w // 40), - max(4, input_h // 40), - ) - clahe = cv2.createCLAHE(clipLimit=1.5, tileGridSize=grid_size) - enhanced = clahe.apply(processed) + if self.config.cameras[camera].lpr.enhancement > 0: + # always apply the same CLAHE for contrast enhancement when enhancement level is above 3 + logger.debug( + f"{camera}: Enhancing contrast for recognition image (level: {self.config.cameras[camera].lpr.enhancement})" + ) + grid_size = ( + max(4, input_w // 40), + max(4, input_h // 40), + ) + clahe = cv2.createCLAHE( + clipLimit=2 if self.config.cameras[camera].lpr.enhancement > 5 else 1.5, + tileGridSize=grid_size, + ) + enhanced = clahe.apply(processed) + else: + enhanced = processed # Convert back to 3-channel for model compatibility image = cv2.cvtColor(enhanced, cv2.COLOR_GRAY2RGB) @@ -955,6 +992,8 @@ class LicensePlateProcessingMixin: return if dedicated_lpr: + id = "dedicated-lpr" + rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420) # apply motion mask @@ -1156,7 +1195,7 @@ class LicensePlateProcessingMixin: # run detection, returns results sorted by confidence, best first start = datetime.datetime.now().timestamp() license_plates, confidences, areas = self._process_license_plate( - license_plate_frame + camera, id, license_plate_frame ) self.__update_lpr_metrics(datetime.datetime.now().timestamp() - start)