mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-17 02:53:01 +03:00
Resize classification training images the same way as inference (#24988)
CI / AMD64 Smoke Test (push) Blocked by required conditions
CI / ARM Build (push) Waiting to run
CI / Jetson Jetpack 6 (push) Waiting to run
CI / AMD64 Build (push) Waiting to run
CI / AMD64 Extra Build (push) Blocked by required conditions
CI / ARM Extra Build (push) Blocked by required conditions
CI / Synaptics Build (push) Blocked by required conditions
CI / Assemble and push default build (push) Blocked by required conditions
CI / AMD64 Smoke Test (push) Blocked by required conditions
CI / ARM Build (push) Waiting to run
CI / Jetson Jetpack 6 (push) Waiting to run
CI / AMD64 Build (push) Waiting to run
CI / AMD64 Extra Build (push) Blocked by required conditions
CI / ARM Extra Build (push) Blocked by required conditions
CI / Synaptics Build (push) Blocked by required conditions
CI / Assemble and push default build (push) Blocked by required conditions
* Resize classification training images the same way as inference Training let Keras resize dataset images to 224x224, which uses nearest neighbor by default. Inference resizes the crop with cv2.resize (bilinear). For crops much smaller than 224 px the two results look very different, so a model could fit its training images perfectly and still report the same state for every live frame. Add resize_classification_image() and use it everywhere an image is resized for a classification model. Before training, the dataset is copied to a temporary directory through that same function, so Keras is given images that are already the model input size and does no resizing of its own. The new test runs a frame through the state processor and the crop it saves through the training process (stopped at model.fit), and checks the model receives the same pixels from both, for random frame and crop sizes. Adds hypothesis as a dev dependency. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Fail training when a prepared image cannot be written cv2.imwrite returns False instead of raising, so a full temporary directory would have dropped images from training without any error. Raise with the file path instead; the training process already logs the exception and fails. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Trim classification parity tests and drop hypothesis Per review: the parity test is now a plain loop over fixed crop sizes (64, 79, 224 and 600 px) on a fixed 1280x720 frame, which covers both upscaled and downscaled crops. The 224x224 check and the failed-write test are removed; the parity test already covers the prepared image size. hypothesis is no longer a dev dependency. The 64, 79 and 600 px cases still fail on dev and pass with the fix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Victor Bajanov <victorb@spunmonkey.org> Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Victor Bajanov
Claude Fable 5.2
parent
f22278bcfa
commit
f2a264c15e
@@ -17,6 +17,7 @@ from frigate.config.classification import CustomClassificationConfig
|
|||||||
from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR
|
from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR
|
||||||
from frigate.log import suppress_stderr_during
|
from frigate.log import suppress_stderr_during
|
||||||
from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels
|
from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels
|
||||||
|
from frigate.util.classification import resize_classification_image
|
||||||
from frigate.util.file import trim_oldest_files
|
from frigate.util.file import trim_oldest_files
|
||||||
from frigate.util.image import calculate_region
|
from frigate.util.image import calculate_region
|
||||||
from frigate.util.object import box_overlaps
|
from frigate.util.object import box_overlaps
|
||||||
@@ -275,7 +276,7 @@ class CustomStateClassificationProcessor(DeferredRealtimeProcessorApi):
|
|||||||
cropped_frame = rgb[y1:y2, x1:x2]
|
cropped_frame = rgb[y1:y2, x1:x2]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
resized_frame = cv2.resize(cropped_frame, (224, 224))
|
resized_frame = resize_classification_image(cropped_frame)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Failed to resize image for state classification")
|
logger.warning("Failed to resize image for state classification")
|
||||||
return
|
return
|
||||||
@@ -583,7 +584,7 @@ class CustomObjectClassificationProcessor(DeferredRealtimeProcessorApi):
|
|||||||
crop = rgb[y:y2, x:x2]
|
crop = rgb[y:y2, x:x2]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
resized_crop = cv2.resize(crop, (224, 224))
|
resized_crop = resize_classification_image(crop)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Failed to resize image for object classification")
|
logger.warning("Failed to resize image for object classification")
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""Training and inference must give a classification model the same input.
|
||||||
|
|
||||||
|
The realtime processor saves the crop at its own size for the user to
|
||||||
|
categorise, and resizes it to the model input for inference. Training later
|
||||||
|
loads the saved crop from the dataset. If the two resize it differently, the
|
||||||
|
model learns pixels it never sees live.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
# Mock TFLite before importing the classification processor
|
||||||
|
_MOCK_MODULES = [
|
||||||
|
"tflite_runtime",
|
||||||
|
"tflite_runtime.interpreter",
|
||||||
|
"ai_edge_litert",
|
||||||
|
"ai_edge_litert.interpreter",
|
||||||
|
]
|
||||||
|
for mod in _MOCK_MODULES:
|
||||||
|
if mod not in sys.modules:
|
||||||
|
sys.modules[mod] = MagicMock()
|
||||||
|
|
||||||
|
import frigate.util.classification as classification # noqa: E402
|
||||||
|
from frigate.data_processing.real_time.custom_classification import ( # noqa: E402
|
||||||
|
CustomStateClassificationProcessor,
|
||||||
|
)
|
||||||
|
|
||||||
|
CAMERA = "cam"
|
||||||
|
MODEL_NAME = "parity"
|
||||||
|
FRAME_WIDTH = 1280
|
||||||
|
FRAME_HEIGHT = 720
|
||||||
|
# Below, near and above the 224 px model input, so crops are both upscaled
|
||||||
|
# and downscaled. 79 px is the crop size this was found on.
|
||||||
|
CROP_SIZES = (64, 79, 224, 600)
|
||||||
|
|
||||||
|
|
||||||
|
class _TrainingStopped(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _frame() -> np.ndarray:
|
||||||
|
"""A fixed random I420 frame."""
|
||||||
|
return np.random.default_rng(0).integers(
|
||||||
|
0, 256, (FRAME_HEIGHT * 3 // 2, FRAME_WIDTH), dtype=np.uint8
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _square_crop(size: int, x: int = 100, y: int = 60) -> list[float]:
|
||||||
|
"""Relative crop coordinates for a size x size px crop at (x, y).
|
||||||
|
|
||||||
|
The half pixel keeps the processor's int() truncation on the intended
|
||||||
|
pixel.
|
||||||
|
"""
|
||||||
|
return [
|
||||||
|
(x + 0.5) / FRAME_WIDTH,
|
||||||
|
(y + 0.5) / FRAME_HEIGHT,
|
||||||
|
(x + size + 0.5) / FRAME_WIDTH,
|
||||||
|
(y + size + 0.5) / FRAME_HEIGHT,
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _inference_input_and_saved_crop(
|
||||||
|
frame: np.ndarray, crop: list[float]
|
||||||
|
) -> tuple[np.ndarray, np.ndarray]:
|
||||||
|
"""Run the state processor on a frame.
|
||||||
|
|
||||||
|
Returns the image it classifies and the crop it saves for training.
|
||||||
|
"""
|
||||||
|
height = frame.shape[0] * 2 // 3
|
||||||
|
width = frame.shape[1]
|
||||||
|
|
||||||
|
config = MagicMock()
|
||||||
|
config.cameras[CAMERA].detect.width = width
|
||||||
|
config.cameras[CAMERA].detect.height = height
|
||||||
|
model_config = MagicMock()
|
||||||
|
model_config.name = MODEL_NAME
|
||||||
|
model_config.state_config.cameras = {CAMERA: MagicMock(crop=crop)}
|
||||||
|
model_config.state_config.interval = 1
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as model_cache:
|
||||||
|
with patch(
|
||||||
|
"frigate.data_processing.real_time.custom_classification.MODEL_CACHE_DIR",
|
||||||
|
model_cache,
|
||||||
|
):
|
||||||
|
processor = CustomStateClassificationProcessor(
|
||||||
|
config, model_config, MagicMock(), None
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
tasks = []
|
||||||
|
processor._enqueue_task = tasks.append
|
||||||
|
processor.last_run = -math.inf
|
||||||
|
processor.process_frame({"camera": CAMERA, "motion": []}, frame)
|
||||||
|
finally:
|
||||||
|
processor.shutdown()
|
||||||
|
|
||||||
|
assert len(tasks) == 1, "processor did not classify the frame"
|
||||||
|
_, _, _, resized_frame, crop_bgr = tasks[0]
|
||||||
|
return resized_frame, crop_bgr
|
||||||
|
|
||||||
|
|
||||||
|
def _training_batch(dataset_crop_bgr: np.ndarray) -> np.ndarray:
|
||||||
|
"""Run the training process up to the first batch it would train on."""
|
||||||
|
with tempfile.TemporaryDirectory() as root:
|
||||||
|
clips_dir = os.path.join(root, "clips")
|
||||||
|
model_cache = os.path.join(root, "model_cache")
|
||||||
|
|
||||||
|
# Two classes are required. Both get the same image, so every image
|
||||||
|
# in the batch must match the inference input regardless of shuffling.
|
||||||
|
for class_name in ("a", "b"):
|
||||||
|
class_dir = os.path.join(clips_dir, MODEL_NAME, "dataset", class_name)
|
||||||
|
os.makedirs(class_dir)
|
||||||
|
cv2.imwrite(os.path.join(class_dir, "crop.png"), dataset_crop_bgr)
|
||||||
|
|
||||||
|
import tensorflow as tf
|
||||||
|
|
||||||
|
captured = []
|
||||||
|
|
||||||
|
def fit(train_gen, *args, **kwargs):
|
||||||
|
# read the batch now, while the training images still exist
|
||||||
|
captured.append(train_gen[0][0])
|
||||||
|
raise _TrainingStopped()
|
||||||
|
|
||||||
|
model = MagicMock()
|
||||||
|
model.fit.side_effect = fit
|
||||||
|
|
||||||
|
process = classification.ClassificationTrainingProcess(MODEL_NAME)
|
||||||
|
process.BASE_WEIGHT_URL = ""
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(classification, "CLIPS_DIR", clips_dir),
|
||||||
|
patch.object(classification, "MODEL_CACHE_DIR", model_cache),
|
||||||
|
patch.object(tf.keras.applications, "MobileNetV2"),
|
||||||
|
patch.object(tf.keras.models, "Sequential", return_value=model),
|
||||||
|
):
|
||||||
|
process._ClassificationTrainingProcess__train_classification_model()
|
||||||
|
|
||||||
|
assert captured, "training did not reach model.fit"
|
||||||
|
return captured[0]
|
||||||
|
|
||||||
|
|
||||||
|
class TestTrainingMatchesInference(unittest.TestCase):
|
||||||
|
def test_training_input_matches_inference_input(self):
|
||||||
|
frame = _frame()
|
||||||
|
|
||||||
|
for size in CROP_SIZES:
|
||||||
|
with self.subTest(crop_size=size):
|
||||||
|
inference_input, saved_crop = _inference_input_and_saved_crop(
|
||||||
|
frame, _square_crop(size)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Saved training crops are webp, which cv2 writes losslessly
|
||||||
|
# by default, and categorising converts them to png. Writing
|
||||||
|
# png here gives the same dataset pixels.
|
||||||
|
batch = _training_batch(saved_crop)
|
||||||
|
|
||||||
|
# training rescales by 1/255; the quantised model takes uint8
|
||||||
|
# input
|
||||||
|
training_input = np.rint(batch * 255.0).astype(np.uint8)
|
||||||
|
|
||||||
|
self.assertEqual(training_input.shape[1:], inference_input.shape)
|
||||||
|
for image in training_input:
|
||||||
|
np.testing.assert_array_equal(image, inference_input)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPrepareTrainingDataset(unittest.TestCase):
|
||||||
|
def test_skips_files_that_are_not_images(self):
|
||||||
|
with (
|
||||||
|
tempfile.TemporaryDirectory() as dataset_dir,
|
||||||
|
tempfile.TemporaryDirectory() as output_dir,
|
||||||
|
):
|
||||||
|
os.makedirs(os.path.join(dataset_dir, "open"))
|
||||||
|
cv2.imwrite(
|
||||||
|
os.path.join(dataset_dir, "open", "a.png"),
|
||||||
|
np.zeros((79, 79, 3), dtype=np.uint8),
|
||||||
|
)
|
||||||
|
|
||||||
|
with open(os.path.join(dataset_dir, "open", "broken.png"), "w") as f:
|
||||||
|
f.write("not an image")
|
||||||
|
|
||||||
|
with open(os.path.join(dataset_dir, ".training_metadata.json"), "w") as f:
|
||||||
|
f.write("{}")
|
||||||
|
|
||||||
|
classification.prepare_training_dataset(dataset_dir, output_dir)
|
||||||
|
|
||||||
|
self.assertEqual(os.listdir(output_dir), ["open"])
|
||||||
|
self.assertEqual(os.listdir(os.path.join(output_dir, "open")), ["a.png"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -6,6 +6,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import shutil
|
import shutil
|
||||||
|
import tempfile
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
|
||||||
import cv2
|
import cv2
|
||||||
@@ -38,10 +39,48 @@ BATCH_SIZE = 16
|
|||||||
EPOCHS = 50
|
EPOCHS = 50
|
||||||
LEARNING_RATE = 0.001
|
LEARNING_RATE = 0.001
|
||||||
TRAINING_METADATA_FILE = ".training_metadata.json"
|
TRAINING_METADATA_FILE = ".training_metadata.json"
|
||||||
|
CLASSIFICATION_INPUT_SIZE = (224, 224)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def resize_classification_image(image: np.ndarray) -> np.ndarray:
|
||||||
|
"""Resize an image to the classification model input size.
|
||||||
|
|
||||||
|
Training and inference both need to use this, otherwise the model is
|
||||||
|
trained on different pixels than the ones it is later run on.
|
||||||
|
"""
|
||||||
|
return cv2.resize(image, CLASSIFICATION_INPUT_SIZE)
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_training_dataset(dataset_dir: str, output_dir: str) -> None:
|
||||||
|
"""Write a copy of the dataset that is resized the same way as inference.
|
||||||
|
|
||||||
|
The dataset holds crops at their original size. Keras would resize them
|
||||||
|
with nearest neighbor while inference uses cv2, so the images are resized
|
||||||
|
here and Keras is given images that are already the model input size.
|
||||||
|
"""
|
||||||
|
for class_name in os.listdir(dataset_dir):
|
||||||
|
class_dir = os.path.join(dataset_dir, class_name)
|
||||||
|
|
||||||
|
if not os.path.isdir(class_dir):
|
||||||
|
continue
|
||||||
|
|
||||||
|
output_class_dir = os.path.join(output_dir, class_name)
|
||||||
|
os.makedirs(output_class_dir, exist_ok=True)
|
||||||
|
|
||||||
|
for file in os.listdir(class_dir):
|
||||||
|
img = cv2.imread(os.path.join(class_dir, file))
|
||||||
|
|
||||||
|
if img is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
output_path = os.path.join(output_class_dir, file)
|
||||||
|
|
||||||
|
if not cv2.imwrite(output_path, resize_classification_image(img)):
|
||||||
|
raise OSError(f"Failed to write training image {output_path}")
|
||||||
|
|
||||||
|
|
||||||
def write_training_metadata(model_name: str, image_count: int) -> None:
|
def write_training_metadata(model_name: str, image_count: int) -> None:
|
||||||
"""
|
"""
|
||||||
Write training metadata to a hidden file in the model's clips directory.
|
Write training metadata to a hidden file in the model's clips directory.
|
||||||
@@ -160,7 +199,7 @@ class ClassificationTrainingProcess(FrigateProcess):
|
|||||||
for path in image_paths[:300]:
|
for path in image_paths[:300]:
|
||||||
img = cv2.imread(path)
|
img = cv2.imread(path)
|
||||||
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
||||||
img = cv2.resize(img, (224, 224))
|
img = resize_classification_image(img)
|
||||||
img_array = np.array(img, dtype=np.float32) / 255.0
|
img_array = np.array(img, dtype=np.float32) / 255.0
|
||||||
img_array = img_array[None, ...]
|
img_array = img_array[None, ...]
|
||||||
yield [img_array]
|
yield [img_array]
|
||||||
@@ -232,32 +271,35 @@ class ClassificationTrainingProcess(FrigateProcess):
|
|||||||
metrics=["accuracy"],
|
metrics=["accuracy"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# create training set
|
# create training set, resized the same way as inference
|
||||||
datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2)
|
with tempfile.TemporaryDirectory() as training_dir:
|
||||||
train_gen = datagen.flow_from_directory(
|
prepare_training_dataset(dataset_dir, training_dir)
|
||||||
dataset_dir,
|
datagen = ImageDataGenerator(rescale=1.0 / 255, validation_split=0.2)
|
||||||
target_size=(224, 224),
|
train_gen = datagen.flow_from_directory(
|
||||||
batch_size=BATCH_SIZE,
|
training_dir,
|
||||||
class_mode="categorical",
|
target_size=CLASSIFICATION_INPUT_SIZE,
|
||||||
subset="training",
|
batch_size=BATCH_SIZE,
|
||||||
)
|
class_mode="categorical",
|
||||||
|
subset="training",
|
||||||
|
)
|
||||||
|
|
||||||
total_images = train_gen.samples
|
total_images = train_gen.samples
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"Training {self.model_name}: {total_images} images across {num_classes} classes"
|
f"Training {self.model_name}: {total_images} images across {num_classes} classes"
|
||||||
)
|
)
|
||||||
|
|
||||||
# write labelmap
|
# write labelmap
|
||||||
class_indices = train_gen.class_indices
|
class_indices = train_gen.class_indices
|
||||||
index_to_class = {v: k for k, v in class_indices.items()}
|
index_to_class = {v: k for k, v in class_indices.items()}
|
||||||
sorted_classes = [index_to_class[i] for i in range(len(index_to_class))]
|
sorted_classes = [index_to_class[i] for i in range(len(index_to_class))]
|
||||||
with open(os.path.join(model_dir, "labelmap.txt"), "w") as f:
|
with open(os.path.join(model_dir, "labelmap.txt"), "w") as f:
|
||||||
for class_name in sorted_classes:
|
for class_name in sorted_classes:
|
||||||
f.write(f"{class_name}\n")
|
f.write(f"{class_name}\n")
|
||||||
|
|
||||||
|
# train the model
|
||||||
|
logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...")
|
||||||
|
model.fit(train_gen, epochs=EPOCHS, verbose=0)
|
||||||
|
|
||||||
# train the model
|
|
||||||
logger.debug(f"Training {self.model_name} for {EPOCHS} epochs...")
|
|
||||||
model.fit(train_gen, epochs=EPOCHS, verbose=0)
|
|
||||||
logger.debug(f"Converting {self.model_name} to TFLite...")
|
logger.debug(f"Converting {self.model_name} to TFLite...")
|
||||||
|
|
||||||
# convert model to tflite
|
# convert model to tflite
|
||||||
@@ -602,7 +644,7 @@ def _extract_keyframes(
|
|||||||
return idx, None
|
return idx, None
|
||||||
|
|
||||||
cropped = img[y1_clipped:y2_clipped, x1_clipped:x2_clipped]
|
cropped = img[y1_clipped:y2_clipped, x1_clipped:x2_clipped]
|
||||||
resized = cv2.resize(cropped, (224, 224))
|
resized = resize_classification_image(cropped)
|
||||||
|
|
||||||
output_path = os.path.join(output_dir, f"frame_{idx:04d}.jpg")
|
output_path = os.path.join(output_dir, f"frame_{idx:04d}.jpg")
|
||||||
cv2.imwrite(output_path, resized)
|
cv2.imwrite(output_path, resized)
|
||||||
@@ -880,7 +922,7 @@ def _extract_event_thumbnails(
|
|||||||
if img is None:
|
if img is None:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
resized = cv2.resize(img, (224, 224))
|
resized = resize_classification_image(img)
|
||||||
output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg")
|
output_path = os.path.join(output_dir, f"thumbnail_{idx:04d}.jpg")
|
||||||
cv2.imwrite(output_path, resized)
|
cv2.imwrite(output_path, resized)
|
||||||
image_paths.append(output_path)
|
image_paths.append(output_path)
|
||||||
|
|||||||
Reference in New Issue
Block a user