Resize classification training images the same way as inference (#24988)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s

* Resize classification training images the same way as inference

Training let Keras resize dataset images to 224x224, which uses nearest
neighbor by default. Inference resizes the crop with cv2.resize
(bilinear). For crops much smaller than 224 px the two results look very
different, so a model could fit its training images perfectly and still
report the same state for every live frame.

Add resize_classification_image() and use it everywhere an image is
resized for a classification model. Before training, the dataset is
copied to a temporary directory through that same function, so Keras is
given images that are already the model input size and does no resizing
of its own.

The new test runs a frame through the state processor and the crop it
saves through the training process (stopped at model.fit), and checks
the model receives the same pixels from both, for random frame and crop
sizes. Adds hypothesis as a dev dependency.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Fail training when a prepared image cannot be written

cv2.imwrite returns False instead of raising, so a full temporary
directory would have dropped images from training without any error.
Raise with the file path instead; the training process already logs
the exception and fails.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Trim classification parity tests and drop hypothesis

Per review: the parity test is now a plain loop over fixed crop sizes
(64, 79, 224 and 600 px) on a fixed 1280x720 frame, which covers both
upscaled and downscaled crops. The 224x224 check and the failed-write
test are removed; the parity test already covers the prepared image
size. hypothesis is no longer a dev dependency.

The 64, 79 and 600 px cases still fail on dev and pass with the fix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Victor Bajanov <victorb@spunmonkey.org>
Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Victor Bajanov
2026-11-16 16:01:52 -07:00
committed by GitHub
co-authored by Victor Bajanov Claude Fable 5.2
parent f22278bcfa
commit f2a264c15e
3 changed files with 269 additions and 28 deletions
+68 -26
View File
@@ -6,6 +6,7 @@ import logging
import os
import random
import shutil
import tempfile
from collections import defaultdict
import cv2
@@ -38,10 +39,48 @@ BATCH_SIZE = 16
EPOCHS = 50
LEARNING_RATE = 0.001
TRAINING_METADATA_FILE = ".training_metadata.json"
CLASSIFICATION_INPUT_SIZE = (224, 224)
logger = logging.getLogger(__name__)
def resize_classification_image(image: np.ndarray) -> np.ndarray:
"""Resize an image to the classification model input size.
Training and inference both need to use this, otherwise the model is
trained on different pixels than the ones it is later run on.
"""
return cv2.resize(image, CLASSIFICATION_INPUT_SIZE)
def prepare_training_dataset(dataset_dir: str, output_dir: str) -> None:
"""Write a copy of the dataset that is resized the same way as inference.
The dataset holds crops at their original size. Keras would resize them
with nearest neighbor while inference uses cv2, so the images are resized
here and Keras is given images that are already the model input size.
"""
for class_name in os.listdir(dataset_dir):
class_dir = os.path.join(dataset_dir, class_name)
if not os.path.isdir(class_dir):
continue
output_class_dir = os.path.join(output_dir, class_name)
os.makedirs(output_class_dir, exist_ok=True)
for file in os.listdir(class_dir):
img = cv2.imread(os.path.join(class_dir, file))
if img is None:
continue
output_path = os.path.join(output_class_dir, file)
if not cv2.imwrite(output_path, resize_classification_image(img)):
raise OSError(f"Failed to write training image {output_path}")
def write_training_metadata(model_name: str, image_count: int) -> None:
"""
Write training metadata to a hidden file in the model's clips directory.
@@ -160,7 +199,7 @@ class ClassificationTrainingProcess(FrigateProcess):
for path in image_paths[:300]:
img = cv2.imread(path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (224, 224))
img = resize_classification_image(img)
img_array = np.array(img, dtype=np.float32) / 255.0
img_array = img_array[None, ...]
yield [img_array]
@@ -232,32 +271,35 @@ class ClassificationTrainingProcess(FrigateProcess):
metrics=["accuracy"],
)
# create training set
datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2)
train_gen = datagen.flow_from_directory(
dataset_dir,
target_size=(224, 224),
batch_size=BATCH_SIZE,
class_mode="categorical",
subset="training",
)
# create training set, resized the same way as inference
with tempfile.TemporaryDirectory() as training_dir:
prepare_training_dataset(dataset_dir, training_dir)
datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2)
train_gen = datagen.flow_from_directory(
training_dir,
target_size=CLASSIFICATION_INPUT_SIZE,
batch_size=BATCH_SIZE,
class_mode="categorical",
subset="training",
)
total_images = train_gen.samples
logger.debug(
f"Training {self.model_name}: {total_images} images across {num_classes} classes"
)
total_images = train_gen.samples
logger.debug(
f"Training {self.model_name}: {total_images} images across {num_classes} classes"
)
# write labelmap
class_indices = train_gen.class_indices
index_to_class = {v: k for k, v in class_indices.items()}
sorted_classes = [index_to_class[i] for i in range(len(index_to_class))]
with open(os.path.join(model_dir, "labelmap.txt"), "w") as f:
for class_name in sorted_classes:
f.write(f"{class_name}\n")
# write labelmap
class_indices = train_gen.class_indices
index_to_class = {v: k for k, v in class_indices.items()}
sorted_classes = [index_to_class[i] for i in range(len(index_to_class))]
with open(os.path.join(model_dir, "labelmap.txt"), "w") as f:
for class_name in sorted_classes:
f.write(f"{class_name}\n")
# train the model
logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...")
model.fit(train_gen, epochs=EPOCHS, verbose=0)
# train the model
logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...")
model.fit(train_gen, epochs=EPOCHS, verbose=0)
logger.debug(f"Converting {self.model_name} to TFLite...")
# convert model to tflite
@@ -602,7 +644,7 @@ def _extract_keyframes(
return idx, None
cropped = img[y1_clipped:y2_clipped, x1_clipped:x2_clipped]
resized = cv2.resize(cropped, (224, 224))
resized = resize_classification_image(cropped)
output_path = os.path.join(output_dir, f"frame_{idx:04d}.jpg")
cv2.imwrite(output_path, resized)
@@ -880,7 +922,7 @@ def _extract_event_thumbnails(
if img is None:
continue
resized = cv2.resize(img, (224, 224))
resized = resize_classification_image(img)
output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg")
cv2.imwrite(output_path, resized)
image_paths.append(output_path)