mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-22 05:23:01 +03:00
Resize classification training images the same way as inference (#24988)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
* Resize classification training images the same way as inference Training let Keras resize dataset images to 224x224, which uses nearest neighbor by default. Inference resizes the crop with cv2.resize (bilinear). For crops much smaller than 224 px the two results look very different, so a model could fit its training images perfectly and still report the same state for every live frame. Add resize_classification_image() and use it everywhere an image is resized for a classification model. Before training, the dataset is copied to a temporary directory through that same function, so Keras is given images that are already the model input size and does no resizing of its own. The new test runs a frame through the state processor and the crop it saves through the training process (stopped at model.fit), and checks the model receives the same pixels from both, for random frame and crop sizes. Adds hypothesis as a dev dependency. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Fail training when a prepared image cannot be written cv2.imwrite returns False instead of raising, so a full temporary directory would have dropped images from training without any error. Raise with the file path instead; the training process already logs the exception and fails. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Trim classification parity tests and drop hypothesis Per review: the parity test is now a plain loop over fixed crop sizes (64, 79, 224 and 600 px) on a fixed 1280x720 frame, which covers both upscaled and downscaled crops. The 224x224 check and the failed-write test are removed; the parity test already covers the prepared image size. hypothesis is no longer a dev dependency. The 64, 79 and 600 px cases still fail on dev and pass with the fix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Victor Bajanov <victorb@spunmonkey.org> Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Victor Bajanov
Claude Fable 5.2
parent
f22278bcfa
commit
f2a264c15e
@@ -6,6 +6,7 @@ import logging
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
|
||||
import cv2
|
||||
@@ -38,10 +39,48 @@ BATCH_SIZE = 16
|
||||
EPOCHS = 50
|
||||
LEARNING_RATE = 0.001
|
||||
TRAINING_METADATA_FILE = ".training_metadata.json"
|
||||
CLASSIFICATION_INPUT_SIZE = (224, 224)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resize_classification_image(image: np.ndarray) -> np.ndarray:
|
||||
"""Resize an image to the classification model input size.
|
||||
|
||||
Training and inference both need to use this, otherwise the model is
|
||||
trained on different pixels than the ones it is later run on.
|
||||
"""
|
||||
return cv2.resize(image, CLASSIFICATION_INPUT_SIZE)
|
||||
|
||||
|
||||
def prepare_training_dataset(dataset_dir: str, output_dir: str) -> None:
|
||||
"""Write a copy of the dataset that is resized the same way as inference.
|
||||
|
||||
The dataset holds crops at their original size. Keras would resize them
|
||||
with nearest neighbor while inference uses cv2, so the images are resized
|
||||
here and Keras is given images that are already the model input size.
|
||||
"""
|
||||
for class_name in os.listdir(dataset_dir):
|
||||
class_dir = os.path.join(dataset_dir, class_name)
|
||||
|
||||
if not os.path.isdir(class_dir):
|
||||
continue
|
||||
|
||||
output_class_dir = os.path.join(output_dir, class_name)
|
||||
os.makedirs(output_class_dir, exist_ok=True)
|
||||
|
||||
for file in os.listdir(class_dir):
|
||||
img = cv2.imread(os.path.join(class_dir, file))
|
||||
|
||||
if img is None:
|
||||
continue
|
||||
|
||||
output_path = os.path.join(output_class_dir, file)
|
||||
|
||||
if not cv2.imwrite(output_path, resize_classification_image(img)):
|
||||
raise OSError(f"Failed to write training image {output_path}")
|
||||
|
||||
|
||||
def write_training_metadata(model_name: str, image_count: int) -> None:
|
||||
"""
|
||||
Write training metadata to a hidden file in the model's clips directory.
|
||||
@@ -160,7 +199,7 @@ class ClassificationTrainingProcess(FrigateProcess):
|
||||
for path in image_paths[:300]:
|
||||
img = cv2.imread(path)
|
||||
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
||||
img = cv2.resize(img, (224, 224))
|
||||
img = resize_classification_image(img)
|
||||
img_array = np.array(img, dtype=np.float32) / 255.0
|
||||
img_array = img_array[None, ...]
|
||||
yield [img_array]
|
||||
@@ -232,32 +271,35 @@ class ClassificationTrainingProcess(FrigateProcess):
|
||||
metrics=["accuracy"],
|
||||
)
|
||||
|
||||
# create training set
|
||||
datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2)
|
||||
train_gen = datagen.flow_from_directory(
|
||||
dataset_dir,
|
||||
target_size=(224, 224),
|
||||
batch_size=BATCH_SIZE,
|
||||
class_mode="categorical",
|
||||
subset="training",
|
||||
)
|
||||
# create training set, resized the same way as inference
|
||||
with tempfile.TemporaryDirectory() as training_dir:
|
||||
prepare_training_dataset(dataset_dir, training_dir)
|
||||
datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2)
|
||||
train_gen = datagen.flow_from_directory(
|
||||
training_dir,
|
||||
target_size=CLASSIFICATION_INPUT_SIZE,
|
||||
batch_size=BATCH_SIZE,
|
||||
class_mode="categorical",
|
||||
subset="training",
|
||||
)
|
||||
|
||||
total_images = train_gen.samples
|
||||
logger.debug(
|
||||
f"Training {self.model_name}: {total_images} images across {num_classes} classes"
|
||||
)
|
||||
total_images = train_gen.samples
|
||||
logger.debug(
|
||||
f"Training {self.model_name}: {total_images} images across {num_classes} classes"
|
||||
)
|
||||
|
||||
# write labelmap
|
||||
class_indices = train_gen.class_indices
|
||||
index_to_class = {v: k for k, v in class_indices.items()}
|
||||
sorted_classes = [index_to_class[i] for i in range(len(index_to_class))]
|
||||
with open(os.path.join(model_dir, "labelmap.txt"), "w") as f:
|
||||
for class_name in sorted_classes:
|
||||
f.write(f"{class_name}\n")
|
||||
# write labelmap
|
||||
class_indices = train_gen.class_indices
|
||||
index_to_class = {v: k for k, v in class_indices.items()}
|
||||
sorted_classes = [index_to_class[i] for i in range(len(index_to_class))]
|
||||
with open(os.path.join(model_dir, "labelmap.txt"), "w") as f:
|
||||
for class_name in sorted_classes:
|
||||
f.write(f"{class_name}\n")
|
||||
|
||||
# train the model
|
||||
logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...")
|
||||
model.fit(train_gen, epochs=EPOCHS, verbose=0)
|
||||
|
||||
# train the model
|
||||
logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...")
|
||||
model.fit(train_gen, epochs=EPOCHS, verbose=0)
|
||||
logger.debug(f"Converting {self.model_name} to TFLite...")
|
||||
|
||||
# convert model to tflite
|
||||
@@ -602,7 +644,7 @@ def _extract_keyframes(
|
||||
return idx, None
|
||||
|
||||
cropped = img[y1_clipped:y2_clipped, x1_clipped:x2_clipped]
|
||||
resized = cv2.resize(cropped, (224, 224))
|
||||
resized = resize_classification_image(cropped)
|
||||
|
||||
output_path = os.path.join(output_dir, f"frame_{idx:04d}.jpg")
|
||||
cv2.imwrite(output_path, resized)
|
||||
@@ -880,7 +922,7 @@ def _extract_event_thumbnails(
|
||||
if img is None:
|
||||
continue
|
||||
|
||||
resized = cv2.resize(img, (224, 224))
|
||||
resized = resize_classification_image(img)
|
||||
output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg")
|
||||
cv2.imwrite(output_path, resized)
|
||||
image_paths.append(output_path)
|
||||
|
||||
Reference in New Issue
Block a user