Resize classification training images the same way as inference (#24988)
CI / AMD64 Smoke Test (push) Blocked by required conditions
CI / ARM Build (push) Waiting to run
CI / Jetson Jetpack 6 (push) Waiting to run
CI / AMD64 Build (push) Waiting to run
CI / AMD64 Extra Build (push) Blocked by required conditions
CI / ARM Extra Build (push) Blocked by required conditions
CI / Synaptics Build (push) Blocked by required conditions
CI / Assemble and push default build (push) Blocked by required conditions

* Resize classification training images the same way as inference

Training let Keras resize dataset images to 224x224, which uses nearest
neighbor by default. Inference resizes the crop with cv2.resize
(bilinear). For crops much smaller than 224 px the two results look very
different, so a model could fit its training images perfectly and still
report the same state for every live frame.

Add resize_classification_image() and use it everywhere an image is
resized for a classification model. Before training, the dataset is
copied to a temporary directory through that same function, so Keras is
given images that are already the model input size and does no resizing
of its own.

The new test runs a frame through the state processor and the crop it
saves through the training process (stopped at model.fit), and checks
the model receives the same pixels from both, for random frame and crop
sizes. Adds hypothesis as a dev dependency.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Fail training when a prepared image cannot be written

cv2.imwrite returns False instead of raising, so a full temporary
directory would have dropped images from training without any error.
Raise with the file path instead; the training process already logs
the exception and fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Trim classification parity tests and drop hypothesis

Per review: the parity test is now a plain loop over fixed crop sizes
(64, 79, 224 and 600 px) on a fixed 1280x720 frame, which covers both
upscaled and downscaled crops. The 224x224 check and the failed-write
test are removed; the parity test already covers the prepared image
size. hypothesis is no longer a dev dependency.

The 64, 79 and 600 px cases still fail on dev and pass with the fix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Victor Bajanov <victorb@spunmonkey.org>
Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Victor Bajanov
2026-11-16 16:01:52 -07:00
committed by GitHub
co-authored by Victor Bajanov Claude Fable 5.2
parent f22278bcfa
commit f2a264c15e
3 changed files with 269 additions and 28 deletions
@@ -17,6 +17,7 @@ from frigate.config.classification import CustomClassificationConfig
from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR
from frigate.log import suppress_stderr_during from frigate.log import suppress_stderr_during
from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels
from frigate.util.classification import resize_classification_image
from frigate.util.file import trim_oldest_files from frigate.util.file import trim_oldest_files
from frigate.util.image import calculate_region from frigate.util.image import calculate_region
from frigate.util.object import box_overlaps from frigate.util.object import box_overlaps
@@ -275,7 +276,7 @@ class CustomStateClassificationProcessor(DeferredRealtimeProcessorApi):
cropped_frame = rgb[y1:y2, x1:x2] cropped_frame = rgb[y1:y2, x1:x2]
try: try:
resized_frame = cv2.resize(cropped_frame, (224, 224)) resized_frame = resize_classification_image(cropped_frame)
except Exception: except Exception:
logger.warning("Failed to resize image for state classification") logger.warning("Failed to resize image for state classification")
return return
@@ -583,7 +584,7 @@ class CustomObjectClassificationProcessor(DeferredRealtimeProcessorApi):
crop = rgb[y:y2, x:x2] crop = rgb[y:y2, x:x2]
try: try:
resized_crop = cv2.resize(crop, (224, 224)) resized_crop = resize_classification_image(crop)
except Exception: except Exception:
logger.warning("Failed to resize image for object classification") logger.warning("Failed to resize image for object classification")
return return
@@ -0,0 +1,198 @@
"""Training and inference must give a classification model the same input.
The realtime processor saves the crop at its own size for the user to
categorise, and resizes it to the model input for inference. Training later
loads the saved crop from the dataset. If the two resize it differently, the
model learns pixels it never sees live.
"""
import math
import os
import sys
import tempfile
import unittest
from unittest.mock import MagicMock, patch
import cv2
import numpy as np
# Mock TFLite before importing the classification processor
_MOCK_MODULES = [
"tflite_runtime",
"tflite_runtime.interpreter",
"ai_edge_litert",
"ai_edge_litert.interpreter",
]
for mod in _MOCK_MODULES:
if mod not in sys.modules:
sys.modules[mod] = MagicMock()
import frigate.util.classification as classification # noqa: E402
from frigate.data_processing.real_time.custom_classification import ( # noqa: E402
CustomStateClassificationProcessor,
)
CAMERA = "cam"
MODEL_NAME = "parity"
FRAME_WIDTH = 1280
FRAME_HEIGHT = 720
# Below, near and above the 224 px model input, so crops are both upscaled
# and downscaled. 79 px is the crop size this was found on.
CROP_SIZES = (64, 79, 224, 600)
class _TrainingStopped(Exception):
pass
def _frame() -> np.ndarray:
"""A fixed random I420 frame."""
return np.random.default_rng(0).integers(
0, 256, (FRAME_HEIGHT * 3 // 2, FRAME_WIDTH), dtype=np.uint8
)
def _square_crop(size: int, x: int = 100, y: int = 60) -> list[float]:
"""Relative crop coordinates for a size x size px crop at (x, y).
The half pixel keeps the processor's int() truncation on the intended
pixel.
"""
return [
(x + 0.5) / FRAME_WIDTH,
(y + 0.5) / FRAME_HEIGHT,
(x + size + 0.5) / FRAME_WIDTH,
(y + size + 0.5) / FRAME_HEIGHT,
]
def _inference_input_and_saved_crop(
frame: np.ndarray, crop: list[float]
) -> tuple[np.ndarray, np.ndarray]:
"""Run the state processor on a frame.
Returns the image it classifies and the crop it saves for training.
"""
height = frame.shape[0] * 2 // 3
width = frame.shape[1]
config = MagicMock()
config.cameras[CAMERA].detect.width = width
config.cameras[CAMERA].detect.height = height
model_config = MagicMock()
model_config.name = MODEL_NAME
model_config.state_config.cameras = {CAMERA: MagicMock(crop=crop)}
model_config.state_config.interval = 1
with tempfile.TemporaryDirectory() as model_cache:
with patch(
"frigate.data_processing.real_time.custom_classification.MODEL_CACHE_DIR",
model_cache,
):
processor = CustomStateClassificationProcessor(
config, model_config, MagicMock(), None
)
try:
tasks = []
processor._enqueue_task = tasks.append
processor.last_run = -math.inf
processor.process_frame({"camera": CAMERA, "motion": []}, frame)
finally:
processor.shutdown()
assert len(tasks) == 1, "processor did not classify the frame"
_, _, _, resized_frame, crop_bgr = tasks[0]
return resized_frame, crop_bgr
def _training_batch(dataset_crop_bgr: np.ndarray) -> np.ndarray:
"""Run the training process up to the first batch it would train on."""
with tempfile.TemporaryDirectory() as root:
clips_dir = os.path.join(root, "clips")
model_cache = os.path.join(root, "model_cache")
# Two classes are required. Both get the same image, so every image
# in the batch must match the inference input regardless of shuffling.
for class_name in ("a", "b"):
class_dir = os.path.join(clips_dir, MODEL_NAME, "dataset", class_name)
os.makedirs(class_dir)
cv2.imwrite(os.path.join(class_dir, "crop.png"), dataset_crop_bgr)
import tensorflow as tf
captured = []
def fit(train_gen, *args, **kwargs):
# read the batch now, while the training images still exist
captured.append(train_gen[0][0])
raise _TrainingStopped()
model = MagicMock()
model.fit.side_effect = fit
process = classification.ClassificationTrainingProcess(MODEL_NAME)
process.BASE_WEIGHT_URL = ""
with (
patch.object(classification, "CLIPS_DIR", clips_dir),
patch.object(classification, "MODEL_CACHE_DIR", model_cache),
patch.object(tf.keras.applications, "MobileNetV2"),
patch.object(tf.keras.models, "Sequential", return_value=model),
):
process._ClassificationTrainingProcess__train_classification_model()
assert captured, "training did not reach model.fit"
return captured[0]
class TestTrainingMatchesInference(unittest.TestCase):
def test_training_input_matches_inference_input(self):
frame = _frame()
for size in CROP_SIZES:
with self.subTest(crop_size=size):
inference_input, saved_crop = _inference_input_and_saved_crop(
frame, _square_crop(size)
)
# Saved training crops are webp, which cv2 writes losslessly
# by default, and categorising converts them to png. Writing
# png here gives the same dataset pixels.
batch = _training_batch(saved_crop)
# training rescales by 1/255; the quantised model takes uint8
# input
training_input = np.rint(batch * 255.0).astype(np.uint8)
self.assertEqual(training_input.shape[1:], inference_input.shape)
for image in training_input:
np.testing.assert_array_equal(image, inference_input)
class TestPrepareTrainingDataset(unittest.TestCase):
def test_skips_files_that_are_not_images(self):
with (
tempfile.TemporaryDirectory() as dataset_dir,
tempfile.TemporaryDirectory() as output_dir,
):
os.makedirs(os.path.join(dataset_dir, "open"))
cv2.imwrite(
os.path.join(dataset_dir, "open", "a.png"),
np.zeros((79, 79, 3), dtype=np.uint8),
)
with open(os.path.join(dataset_dir, "open", "broken.png"), "w") as f:
f.write("not an image")
with open(os.path.join(dataset_dir, ".training_metadata.json"), "w") as f:
f.write("{}")
classification.prepare_training_dataset(dataset_dir, output_dir)
self.assertEqual(os.listdir(output_dir), ["open"])
self.assertEqual(os.listdir(os.path.join(output_dir, "open")), ["a.png"])
if __name__ == "__main__":
unittest.main()
+48 -6
View File
@@ -6,6 +6,7 @@ import logging
import os import os
import random import random
import shutil import shutil
import tempfile
from collections import defaultdict from collections import defaultdict
import cv2 import cv2
@@ -38,10 +39,48 @@ BATCH_SIZE = 16
EPOCHS = 50 EPOCHS = 50
LEARNING_RATE = 0.001 LEARNING_RATE = 0.001
TRAINING_METADATA_FILE = ".training_metadata.json" TRAINING_METADATA_FILE = ".training_metadata.json"
CLASSIFICATION_INPUT_SIZE = (224, 224)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def resize_classification_image(image: np.ndarray) -> np.ndarray:
"""Resize an image to the classification model input size.
Training and inference both need to use this, otherwise the model is
trained on different pixels than the ones it is later run on.
"""
return cv2.resize(image, CLASSIFICATION_INPUT_SIZE)
def prepare_training_dataset(dataset_dir: str, output_dir: str) -> None:
"""Write a copy of the dataset that is resized the same way as inference.
The dataset holds crops at their original size. Keras would resize them
with nearest neighbor while inference uses cv2, so the images are resized
here and Keras is given images that are already the model input size.
"""
for class_name in os.listdir(dataset_dir):
class_dir = os.path.join(dataset_dir, class_name)
if not os.path.isdir(class_dir):
continue
output_class_dir = os.path.join(output_dir, class_name)
os.makedirs(output_class_dir, exist_ok=True)
for file in os.listdir(class_dir):
img = cv2.imread(os.path.join(class_dir, file))
if img is None:
continue
output_path = os.path.join(output_class_dir, file)
if not cv2.imwrite(output_path, resize_classification_image(img)):
raise OSError(f"Failed to write training image {output_path}")
def write_training_metadata(model_name: str, image_count: int) -> None: def write_training_metadata(model_name: str, image_count: int) -> None:
""" """
Write training metadata to a hidden file in the model's clips directory. Write training metadata to a hidden file in the model's clips directory.
@@ -160,7 +199,7 @@ class ClassificationTrainingProcess(FrigateProcess):
for path in image_paths[:300]: for path in image_paths[:300]:
img = cv2.imread(path) img = cv2.imread(path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (224, 224)) img = resize_classification_image(img)
img_array = np.array(img, dtype=np.float32) / 255.0 img_array = np.array(img, dtype=np.float32) / 255.0
img_array = img_array[None, ...] img_array = img_array[None, ...]
yield [img_array] yield [img_array]
@@ -232,11 +271,13 @@ class ClassificationTrainingProcess(FrigateProcess):
metrics=["accuracy"], metrics=["accuracy"],
) )
# create training set # create training set, resized the same way as inference
with tempfile.TemporaryDirectory() as training_dir:
prepare_training_dataset(dataset_dir, training_dir)
datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2) datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2)
train_gen = datagen.flow_from_directory( train_gen = datagen.flow_from_directory(
dataset_dir, training_dir,
target_size=(224, 224), target_size=CLASSIFICATION_INPUT_SIZE,
batch_size=BATCH_SIZE, batch_size=BATCH_SIZE,
class_mode="categorical", class_mode="categorical",
subset="training", subset="training",
@@ -258,6 +299,7 @@ class ClassificationTrainingProcess(FrigateProcess):
# train the model # train the model
logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...") logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...")
model.fit(train_gen, epochs=EPOCHS, verbose=0) model.fit(train_gen, epochs=EPOCHS, verbose=0)
logger.debug(f"Converting {self.model_name} to TFLite...") logger.debug(f"Converting {self.model_name} to TFLite...")
# convert model to tflite # convert model to tflite
@@ -602,7 +644,7 @@ def _extract_keyframes(
return idx, None return idx, None
cropped = img[y1_clipped:y2_clipped, x1_clipped:x2_clipped] cropped = img[y1_clipped:y2_clipped, x1_clipped:x2_clipped]
resized = cv2.resize(cropped, (224, 224)) resized = resize_classification_image(cropped)
output_path = os.path.join(output_dir, f"frame_{idx:04d}.jpg") output_path = os.path.join(output_dir, f"frame_{idx:04d}.jpg")
cv2.imwrite(output_path, resized) cv2.imwrite(output_path, resized)
@@ -880,7 +922,7 @@ def _extract_event_thumbnails(
if img is None: if img is None:
continue continue
resized = cv2.resize(img, (224, 224)) resized = resize_classification_image(img)
output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg") output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg")
cv2.imwrite(output_path, resized) cv2.imwrite(output_path, resized)
image_paths.append(output_path) image_paths.append(output_path)