mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-20 04:23:01 +03:00
Resize classification training images the same way as inference (#24988)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
* Resize classification training images the same way as inference Training let Keras resize dataset images to 224x224, which uses nearest neighbor by default. Inference resizes the crop with cv2.resize (bilinear). For crops much smaller than 224 px the two results look very different, so a model could fit its training images perfectly and still report the same state for every live frame. Add resize_classification_image() and use it everywhere an image is resized for a classification model. Before training, the dataset is copied to a temporary directory through that same function, so Keras is given images that are already the model input size and does no resizing of its own. The new test runs a frame through the state processor and the crop it saves through the training process (stopped at model.fit), and checks the model receives the same pixels from both, for random frame and crop sizes. Adds hypothesis as a dev dependency. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Fail training when a prepared image cannot be written cv2.imwrite returns False instead of raising, so a full temporary directory would have dropped images from training without any error. Raise with the file path instead; the training process already logs the exception and fails. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Trim classification parity tests and drop hypothesis Per review: the parity test is now a plain loop over fixed crop sizes (64, 79, 224 and 600 px) on a fixed 1280x720 frame, which covers both upscaled and downscaled crops. The 224x224 check and the failed-write test are removed; the parity test already covers the prepared image size. hypothesis is no longer a dev dependency. The 64, 79 and 600 px cases still fail on dev and pass with the fix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Victor Bajanov <victorb@spunmonkey.org> Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Victor Bajanov
Claude Fable 5.2
parent
f22278bcfa
commit
f2a264c15e
@@ -0,0 +1,198 @@
|
||||
"""Training and inference must give a classification model the same input.
|
||||
|
||||
The realtime processor saves the crop at its own size for the user to
|
||||
categorise, and resizes it to the model input for inference. Training later
|
||||
loads the saved crop from the dataset. If the two resize it differently, the
|
||||
model learns pixels it never sees live.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
# Mock TFLite before importing the classification processor
|
||||
_MOCK_MODULES = [
|
||||
"tflite_runtime",
|
||||
"tflite_runtime.interpreter",
|
||||
"ai_edge_litert",
|
||||
"ai_edge_litert.interpreter",
|
||||
]
|
||||
for mod in _MOCK_MODULES:
|
||||
if mod not in sys.modules:
|
||||
sys.modules[mod] = MagicMock()
|
||||
|
||||
import frigate.util.classification as classification # noqa: E402
|
||||
from frigate.data_processing.real_time.custom_classification import ( # noqa: E402
|
||||
CustomStateClassificationProcessor,
|
||||
)
|
||||
|
||||
CAMERA = "cam"
|
||||
MODEL_NAME = "parity"
|
||||
FRAME_WIDTH = 1280
|
||||
FRAME_HEIGHT = 720
|
||||
# Below, near and above the 224 px model input, so crops are both upscaled
|
||||
# and downscaled. 79 px is the crop size this was found on.
|
||||
CROP_SIZES = (64, 79, 224, 600)
|
||||
|
||||
|
||||
class _TrainingStopped(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _frame() -> np.ndarray:
|
||||
"""A fixed random I420 frame."""
|
||||
return np.random.default_rng(0).integers(
|
||||
0, 256, (FRAME_HEIGHT * 3 // 2, FRAME_WIDTH), dtype=np.uint8
|
||||
)
|
||||
|
||||
|
||||
def _square_crop(size: int, x: int = 100, y: int = 60) -> list[float]:
|
||||
"""Relative crop coordinates for a size x size px crop at (x, y).
|
||||
|
||||
The half pixel keeps the processor's int() truncation on the intended
|
||||
pixel.
|
||||
"""
|
||||
return [
|
||||
(x + 0.5) / FRAME_WIDTH,
|
||||
(y + 0.5) / FRAME_HEIGHT,
|
||||
(x + size + 0.5) / FRAME_WIDTH,
|
||||
(y + size + 0.5) / FRAME_HEIGHT,
|
||||
]
|
||||
|
||||
|
||||
def _inference_input_and_saved_crop(
|
||||
frame: np.ndarray, crop: list[float]
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Run the state processor on a frame.
|
||||
|
||||
Returns the image it classifies and the crop it saves for training.
|
||||
"""
|
||||
height = frame.shape[0] * 2 // 3
|
||||
width = frame.shape[1]
|
||||
|
||||
config = MagicMock()
|
||||
config.cameras[CAMERA].detect.width = width
|
||||
config.cameras[CAMERA].detect.height = height
|
||||
model_config = MagicMock()
|
||||
model_config.name = MODEL_NAME
|
||||
model_config.state_config.cameras = {CAMERA: MagicMock(crop=crop)}
|
||||
model_config.state_config.interval = 1
|
||||
|
||||
with tempfile.TemporaryDirectory() as model_cache:
|
||||
with patch(
|
||||
"frigate.data_processing.real_time.custom_classification.MODEL_CACHE_DIR",
|
||||
model_cache,
|
||||
):
|
||||
processor = CustomStateClassificationProcessor(
|
||||
config, model_config, MagicMock(), None
|
||||
)
|
||||
|
||||
try:
|
||||
tasks = []
|
||||
processor._enqueue_task = tasks.append
|
||||
processor.last_run = -math.inf
|
||||
processor.process_frame({"camera": CAMERA, "motion": []}, frame)
|
||||
finally:
|
||||
processor.shutdown()
|
||||
|
||||
assert len(tasks) == 1, "processor did not classify the frame"
|
||||
_, _, _, resized_frame, crop_bgr = tasks[0]
|
||||
return resized_frame, crop_bgr
|
||||
|
||||
|
||||
def _training_batch(dataset_crop_bgr: np.ndarray) -> np.ndarray:
|
||||
"""Run the training process up to the first batch it would train on."""
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
clips_dir = os.path.join(root, "clips")
|
||||
model_cache = os.path.join(root, "model_cache")
|
||||
|
||||
# Two classes are required. Both get the same image, so every image
|
||||
# in the batch must match the inference input regardless of shuffling.
|
||||
for class_name in ("a", "b"):
|
||||
class_dir = os.path.join(clips_dir, MODEL_NAME, "dataset", class_name)
|
||||
os.makedirs(class_dir)
|
||||
cv2.imwrite(os.path.join(class_dir, "crop.png"), dataset_crop_bgr)
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
captured = []
|
||||
|
||||
def fit(train_gen, *args, **kwargs):
|
||||
# read the batch now, while the training images still exist
|
||||
captured.append(train_gen[0][0])
|
||||
raise _TrainingStopped()
|
||||
|
||||
model = MagicMock()
|
||||
model.fit.side_effect = fit
|
||||
|
||||
process = classification.ClassificationTrainingProcess(MODEL_NAME)
|
||||
process.BASE_WEIGHT_URL = ""
|
||||
|
||||
with (
|
||||
patch.object(classification, "CLIPS_DIR", clips_dir),
|
||||
patch.object(classification, "MODEL_CACHE_DIR", model_cache),
|
||||
patch.object(tf.keras.applications, "MobileNetV2"),
|
||||
patch.object(tf.keras.models, "Sequential", return_value=model),
|
||||
):
|
||||
process._ClassificationTrainingProcess__train_classification_model()
|
||||
|
||||
assert captured, "training did not reach model.fit"
|
||||
return captured[0]
|
||||
|
||||
|
||||
class TestTrainingMatchesInference(unittest.TestCase):
|
||||
def test_training_input_matches_inference_input(self):
|
||||
frame = _frame()
|
||||
|
||||
for size in CROP_SIZES:
|
||||
with self.subTest(crop_size=size):
|
||||
inference_input, saved_crop = _inference_input_and_saved_crop(
|
||||
frame, _square_crop(size)
|
||||
)
|
||||
|
||||
# Saved training crops are webp, which cv2 writes losslessly
|
||||
# by default, and categorising converts them to png. Writing
|
||||
# png here gives the same dataset pixels.
|
||||
batch = _training_batch(saved_crop)
|
||||
|
||||
# training rescales by 1/255; the quantised model takes uint8
|
||||
# input
|
||||
training_input = np.rint(batch * 255.0).astype(np.uint8)
|
||||
|
||||
self.assertEqual(training_input.shape[1:], inference_input.shape)
|
||||
for image in training_input:
|
||||
np.testing.assert_array_equal(image, inference_input)
|
||||
|
||||
|
||||
class TestPrepareTrainingDataset(unittest.TestCase):
|
||||
def test_skips_files_that_are_not_images(self):
|
||||
with (
|
||||
tempfile.TemporaryDirectory() as dataset_dir,
|
||||
tempfile.TemporaryDirectory() as output_dir,
|
||||
):
|
||||
os.makedirs(os.path.join(dataset_dir, "open"))
|
||||
cv2.imwrite(
|
||||
os.path.join(dataset_dir, "open", "a.png"),
|
||||
np.zeros((79, 79, 3), dtype=np.uint8),
|
||||
)
|
||||
|
||||
with open(os.path.join(dataset_dir, "open", "broken.png"), "w") as f:
|
||||
f.write("not an image")
|
||||
|
||||
with open(os.path.join(dataset_dir, ".training_metadata.json"), "w") as f:
|
||||
f.write("{}")
|
||||
|
||||
classification.prepare_training_dataset(dataset_dir, output_dir)
|
||||
|
||||
self.assertEqual(os.listdir(output_dir), ["open"])
|
||||
self.assertEqual(os.listdir(os.path.join(output_dir, "open")), ["a.png"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user