diff --git a/frigate/data_processing/real_time/custom_classification.py b/frigate/data_processing/real_time/custom_classification.py index 94b7ebeb68..7b5ab74dbc 100644 --- a/frigate/data_processing/real_time/custom_classification.py +++ b/frigate/data_processing/real_time/custom_classification.py @@ -17,6 +17,7 @@ from frigate.config.classification import CustomClassificationConfig from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR from frigate.log import suppress_stderr_during from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels +from frigate.util.classification import resize_classification_image from frigate.util.file import trim_oldest_files from frigate.util.image import calculate_region from frigate.util.object import box_overlaps @@ -275,7 +276,7 @@ class CustomStateClassificationProcessor(DeferredRealtimeProcessorApi): cropped_frame = rgb[y1:y2, x1:x2] try: - resized_frame = cv2.resize(cropped_frame, (224, 224)) + resized_frame = resize_classification_image(cropped_frame) except Exception: logger.warning("Failed to resize image for state classification") return @@ -583,7 +584,7 @@ class CustomObjectClassificationProcessor(DeferredRealtimeProcessorApi): crop = rgb[y:y2, x:x2] try: - resized_crop = cv2.resize(crop, (224, 224)) + resized_crop = resize_classification_image(crop) except Exception: logger.warning("Failed to resize image for object classification") return diff --git a/frigate/test/test_classification_input_parity.py b/frigate/test/test_classification_input_parity.py new file mode 100644 index 0000000000..452b732172 --- /dev/null +++ b/frigate/test/test_classification_input_parity.py @@ -0,0 +1,198 @@ +"""Training and inference must give a classification model the same input. + +The realtime processor saves the crop at its own size for the user to +categorise, and resizes it to the model input for inference. Training later +loads the saved crop from the dataset. If the two resize it differently, the +model learns pixels it never sees live. +""" + +import math +import os +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +import cv2 +import numpy as np + +# Mock TFLite before importing the classification processor +_MOCK_MODULES = [ + "tflite_runtime", + "tflite_runtime.interpreter", + "ai_edge_litert", + "ai_edge_litert.interpreter", +] +for mod in _MOCK_MODULES: + if mod not in sys.modules: + sys.modules[mod] = MagicMock() + +import frigate.util.classification as classification # noqa: E402 +from frigate.data_processing.real_time.custom_classification import ( # noqa: E402 + CustomStateClassificationProcessor, +) + +CAMERA = "cam" +MODEL_NAME = "parity" +FRAME_WIDTH = 1280 +FRAME_HEIGHT = 720 +# Below, near and above the 224 px model input, so crops are both upscaled +# and downscaled. 79 px is the crop size this was found on. +CROP_SIZES = (64, 79, 224, 600) + + +class _TrainingStopped(Exception): + pass + + +def _frame() -> np.ndarray: + """A fixed random I420 frame.""" + return np.random.default_rng(0).integers( + 0, 256, (FRAME_HEIGHT * 3 // 2, FRAME_WIDTH), dtype=np.uint8 + ) + + +def _square_crop(size: int, x: int = 100, y: int = 60) -> list[float]: + """Relative crop coordinates for a size x size px crop at (x, y). + + The half pixel keeps the processor's int() truncation on the intended + pixel. + """ + return [ + (x + 0.5) / FRAME_WIDTH, + (y + 0.5) / FRAME_HEIGHT, + (x + size + 0.5) / FRAME_WIDTH, + (y + size + 0.5) / FRAME_HEIGHT, + ] + + +def _inference_input_and_saved_crop( + frame: np.ndarray, crop: list[float] +) -> tuple[np.ndarray, np.ndarray]: + """Run the state processor on a frame. + + Returns the image it classifies and the crop it saves for training. + """ + height = frame.shape[0] * 2 // 3 + width = frame.shape[1] + + config = MagicMock() + config.cameras[CAMERA].detect.width = width + config.cameras[CAMERA].detect.height = height + model_config = MagicMock() + model_config.name = MODEL_NAME + model_config.state_config.cameras = {CAMERA: MagicMock(crop=crop)} + model_config.state_config.interval = 1 + + with tempfile.TemporaryDirectory() as model_cache: + with patch( + "frigate.data_processing.real_time.custom_classification.MODEL_CACHE_DIR", + model_cache, + ): + processor = CustomStateClassificationProcessor( + config, model_config, MagicMock(), None + ) + + try: + tasks = [] + processor._enqueue_task = tasks.append + processor.last_run = -math.inf + processor.process_frame({"camera": CAMERA, "motion": []}, frame) + finally: + processor.shutdown() + + assert len(tasks) == 1, "processor did not classify the frame" + _, _, _, resized_frame, crop_bgr = tasks[0] + return resized_frame, crop_bgr + + +def _training_batch(dataset_crop_bgr: np.ndarray) -> np.ndarray: + """Run the training process up to the first batch it would train on.""" + with tempfile.TemporaryDirectory() as root: + clips_dir = os.path.join(root, "clips") + model_cache = os.path.join(root, "model_cache") + + # Two classes are required. Both get the same image, so every image + # in the batch must match the inference input regardless of shuffling. + for class_name in ("a", "b"): + class_dir = os.path.join(clips_dir, MODEL_NAME, "dataset", class_name) + os.makedirs(class_dir) + cv2.imwrite(os.path.join(class_dir, "crop.png"), dataset_crop_bgr) + + import tensorflow as tf + + captured = [] + + def fit(train_gen, *args, **kwargs): + # read the batch now, while the training images still exist + captured.append(train_gen[0][0]) + raise _TrainingStopped() + + model = MagicMock() + model.fit.side_effect = fit + + process = classification.ClassificationTrainingProcess(MODEL_NAME) + process.BASE_WEIGHT_URL = "" + + with ( + patch.object(classification, "CLIPS_DIR", clips_dir), + patch.object(classification, "MODEL_CACHE_DIR", model_cache), + patch.object(tf.keras.applications, "MobileNetV2"), + patch.object(tf.keras.models, "Sequential", return_value=model), + ): + process._ClassificationTrainingProcess__train_classification_model() + + assert captured, "training did not reach model.fit" + return captured[0] + + +class TestTrainingMatchesInference(unittest.TestCase): + def test_training_input_matches_inference_input(self): + frame = _frame() + + for size in CROP_SIZES: + with self.subTest(crop_size=size): + inference_input, saved_crop = _inference_input_and_saved_crop( + frame, _square_crop(size) + ) + + # Saved training crops are webp, which cv2 writes losslessly + # by default, and categorising converts them to png. Writing + # png here gives the same dataset pixels. + batch = _training_batch(saved_crop) + + # training rescales by 1/255; the quantised model takes uint8 + # input + training_input = np.rint(batch * 255.0).astype(np.uint8) + + self.assertEqual(training_input.shape[1:], inference_input.shape) + for image in training_input: + np.testing.assert_array_equal(image, inference_input) + + +class TestPrepareTrainingDataset(unittest.TestCase): + def test_skips_files_that_are_not_images(self): + with ( + tempfile.TemporaryDirectory() as dataset_dir, + tempfile.TemporaryDirectory() as output_dir, + ): + os.makedirs(os.path.join(dataset_dir, "open")) + cv2.imwrite( + os.path.join(dataset_dir, "open", "a.png"), + np.zeros((79, 79, 3), dtype=np.uint8), + ) + + with open(os.path.join(dataset_dir, "open", "broken.png"), "w") as f: + f.write("not an image") + + with open(os.path.join(dataset_dir, ".training_metadata.json"), "w") as f: + f.write("{}") + + classification.prepare_training_dataset(dataset_dir, output_dir) + + self.assertEqual(os.listdir(output_dir), ["open"]) + self.assertEqual(os.listdir(os.path.join(output_dir, "open")), ["a.png"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/frigate/util/classification.py b/frigate/util/classification.py index b750451209..6fa2036d3d 100644 --- a/frigate/util/classification.py +++ b/frigate/util/classification.py @@ -6,6 +6,7 @@ import logging import os import random import shutil +import tempfile from collections import defaultdict import cv2 @@ -38,10 +39,48 @@ BATCH_SIZE = 16 EPOCHS = 50 LEARNING_RATE = 0.001 TRAINING_METADATA_FILE = ".training_metadata.json" +CLASSIFICATION_INPUT_SIZE = (224, 224) logger = logging.getLogger(__name__) +def resize_classification_image(image: np.ndarray) -> np.ndarray: + """Resize an image to the classification model input size. + + Training and inference both need to use this, otherwise the model is + trained on different pixels than the ones it is later run on. + """ + return cv2.resize(image, CLASSIFICATION_INPUT_SIZE) + + +def prepare_training_dataset(dataset_dir: str, output_dir: str) -> None: + """Write a copy of the dataset that is resized the same way as inference. + + The dataset holds crops at their original size. Keras would resize them + with nearest neighbor while inference uses cv2, so the images are resized + here and Keras is given images that are already the model input size. + """ + for class_name in os.listdir(dataset_dir): + class_dir = os.path.join(dataset_dir, class_name) + + if not os.path.isdir(class_dir): + continue + + output_class_dir = os.path.join(output_dir, class_name) + os.makedirs(output_class_dir, exist_ok=True) + + for file in os.listdir(class_dir): + img = cv2.imread(os.path.join(class_dir, file)) + + if img is None: + continue + + output_path = os.path.join(output_class_dir, file) + + if not cv2.imwrite(output_path, resize_classification_image(img)): + raise OSError(f"Failed to write training image {output_path}") + + def write_training_metadata(model_name: str, image_count: int) -> None: """ Write training metadata to a hidden file in the model's clips directory. @@ -160,7 +199,7 @@ class ClassificationTrainingProcess(FrigateProcess): for path in image_paths[:300]: img = cv2.imread(path) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - img = cv2.resize(img, (224, 224)) + img = resize_classification_image(img) img_array = np.array(img, dtype=np.float32) / 255.0 img_array = img_array[None, ...] yield [img_array] @@ -232,32 +271,35 @@ class ClassificationTrainingProcess(FrigateProcess): metrics=["accuracy"], ) - # create training set - datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2) - train_gen = datagen.flow_from_directory( - dataset_dir, - target_size=(224, 224), - batch_size=BATCH_SIZE, - class_mode="categorical", - subset="training", - ) + # create training set, resized the same way as inference + with tempfile.TemporaryDirectory() as training_dir: + prepare_training_dataset(dataset_dir, training_dir) + datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2) + train_gen = datagen.flow_from_directory( + training_dir, + target_size=CLASSIFICATION_INPUT_SIZE, + batch_size=BATCH_SIZE, + class_mode="categorical", + subset="training", + ) - total_images = train_gen.samples - logger.debug( - f"Training {self.model_name}: {total_images} images across {num_classes} classes" - ) + total_images = train_gen.samples + logger.debug( + f"Training {self.model_name}: {total_images} images across {num_classes} classes" + ) - # write labelmap - class_indices = train_gen.class_indices - index_to_class = {v: k for k, v in class_indices.items()} - sorted_classes = [index_to_class[i] for i in range(len(index_to_class))] - with open(os.path.join(model_dir, "labelmap.txt"), "w") as f: - for class_name in sorted_classes: - f.write(f"{class_name}\n") + # write labelmap + class_indices = train_gen.class_indices + index_to_class = {v: k for k, v in class_indices.items()} + sorted_classes = [index_to_class[i] for i in range(len(index_to_class))] + with open(os.path.join(model_dir, "labelmap.txt"), "w") as f: + for class_name in sorted_classes: + f.write(f"{class_name}\n") + + # train the model + logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...") + model.fit(train_gen, epochs=EPOCHS, verbose=0) - # train the model - logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...") - model.fit(train_gen, epochs=EPOCHS, verbose=0) logger.debug(f"Converting {self.model_name} to TFLite...") # convert model to tflite @@ -602,7 +644,7 @@ def _extract_keyframes( return idx, None cropped = img[y1_clipped:y2_clipped, x1_clipped:x2_clipped] - resized = cv2.resize(cropped, (224, 224)) + resized = resize_classification_image(cropped) output_path = os.path.join(output_dir, f"frame_{idx:04d}.jpg") cv2.imwrite(output_path, resized) @@ -880,7 +922,7 @@ def _extract_event_thumbnails( if img is None: continue - resized = cv2.resize(img, (224, 224)) + resized = resize_classification_image(img) output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg") cv2.imwrite(output_path, resized) image_paths.append(output_path)