diff --git a/frigate/ptz/autotrack.py b/frigate/ptz/autotrack.py index fb76f6718d..55c8e6ecf8 100644 --- a/frigate/ptz/autotrack.py +++ b/frigate/ptz/autotrack.py @@ -48,6 +48,22 @@ def ptz_moving_at_frame_time(frame_time, ptz_start_time, ptz_stop_time): ) +def transform_is_finite(coord_transformations) -> bool: + """Return True if a norfair coordinate transform contains only finite values. + + A near-singular homography (common when the motion estimator can't find + enough stable features during zoom on a low-texture scene) can produce + inf/nan matrix entries. norfair accumulates the homography across frames, so + a single bad transform poisons every subsequent one and propagates nan into + the tracker's distance function, crashing the camera process. + """ + for attr in ("homography_matrix", "inverse_homography_matrix", "movement_vector"): + value = getattr(coord_transformations, attr, None) + if value is not None and not np.all(np.isfinite(value)): + return False + return True + + class PtzMotionEstimator: def __init__(self, config: CameraConfig, ptz_metrics: PTZMetrics) -> None: self.frame_manager = SharedMemoryFrameManager() @@ -135,6 +151,19 @@ class PtzMotionEstimator: ) self.coord_transformations = None + # A degenerate homography can yield non-finite transform values that + # norfair would accumulate and feed to the tracker as nan estimates. + # Drop the bad transform and request a reset so the estimator rebuilds + # a fresh reference frame instead of poisoning every following frame. + if self.coord_transformations is not None and not transform_is_finite( + self.coord_transformations + ): + logger.warning( + f"Autotracker: motion estimator produced a non-finite transform for {camera} at frame time {frame_time}, resetting" + ) + self.coord_transformations = None + self.ptz_metrics.reset.set() + try: logger.debug( f"{camera}: Motion estimator transformation: {self.coord_transformations.rel_to_abs([[0, 0]])}" diff --git a/frigate/test/test_norfair_distance.py b/frigate/test/test_norfair_distance.py new file mode 100644 index 0000000000..a79b91c3cf --- /dev/null +++ b/frigate/test/test_norfair_distance.py @@ -0,0 +1,91 @@ +import math +import unittest + +import numpy as np +from norfair.camera_motion import ( + HomographyTransformation, + TranslationTransformation, +) + +from frigate.ptz.autotrack import transform_is_finite +from frigate.track.norfair_tracker import distance + + +class TestNorfairDistance(unittest.TestCase): + """Regression tests for the tracker distance guard. + + norfair raises a hard ValueError on any nan distance, which kills the camera + process. During autotracking, an ill-conditioned homography can hand the + tracker a non-finite or degenerate estimate box, so distance() must never + return nan for any input. + """ + + def setUp(self) -> None: + # boxes are [[x1, y1], [x2, y2]] + self.detection = np.array([[805.0, 402.0], [864.0, 521.0]]) + self.estimate = np.array([[800.0, 400.0], [860.0, 520.0]]) + + def test_finite_boxes_give_finite_distance(self) -> None: + d = distance(self.detection, self.estimate) + self.assertTrue(math.isfinite(d)) + + def test_inf_estimate_corner_does_not_return_nan(self) -> None: + estimate = np.array([[np.inf, 400.0], [860.0, 520.0]]) + d = distance(self.detection, estimate) + self.assertFalse(math.isnan(d)) + self.assertEqual(d, float("inf")) + + def test_nan_estimate_corner_does_not_return_nan(self) -> None: + # the actual autotracking crash: a positive-only guard would miss this + # because nan <= 0 is False + estimate = np.array([[np.nan, 400.0], [860.0, 520.0]]) + d = distance(self.detection, estimate) + self.assertFalse(math.isnan(d)) + self.assertEqual(d, float("inf")) + + def test_zero_area_estimate_does_not_return_nan(self) -> None: + estimate = np.array([[900.0, 500.0], [900.0, 500.0]]) + d = distance(self.detection, estimate) + self.assertFalse(math.isnan(d)) + self.assertEqual(d, float("inf")) + + def test_zero_area_detection_does_not_return_nan(self) -> None: + detection = np.array([[805.0, 402.0], [805.0, 521.0]]) + d = distance(detection, self.estimate) + self.assertFalse(math.isnan(d)) + self.assertEqual(d, float("inf")) + + def test_inverted_estimate_corners_do_not_return_nan(self) -> None: + # Kalman estimates can occasionally cross corners (x2 < x1) + estimate = np.array([[860.0, 520.0], [800.0, 400.0]]) + d = distance(self.detection, estimate) + self.assertFalse(math.isnan(d)) + self.assertEqual(d, float("inf")) + + +class TestTransformIsFinite(unittest.TestCase): + def test_finite_homography_is_finite(self) -> None: + matrix = np.array([[1.0, 0.0, 5.0], [0.0, 1.0, 3.0], [0.0, 0.0, 1.0]]) + self.assertTrue(transform_is_finite(HomographyTransformation(matrix))) + + def test_finite_translation_is_finite(self) -> None: + self.assertTrue( + transform_is_finite(TranslationTransformation(np.array([12.0, -4.0]))) + ) + + def test_non_finite_homography_is_not_finite(self) -> None: + transform = HomographyTransformation(np.eye(3)) + # simulate accumulation overflowing to a non-finite matrix + transform.homography_matrix = np.array( + [[1.0, 0.0, np.inf], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]] + ) + self.assertFalse(transform_is_finite(transform)) + + def test_nan_translation_is_not_finite(self) -> None: + self.assertFalse( + transform_is_finite(TranslationTransformation(np.array([np.nan, 0.0]))) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py index 84a0f390a3..3f11823572 100644 --- a/frigate/track/norfair_tracker.py +++ b/frigate/track/norfair_tracker.py @@ -45,6 +45,17 @@ def distance(detection: np.ndarray, estimate: np.ndarray) -> float: estimate_dim = np.diff(estimate, axis=0).flatten() detection_dim = np.diff(detection, axis=0).flatten() + # Guard against degenerate or non-finite boxes + if ( + not np.all(np.isfinite(estimate_dim)) + or not np.all(np.isfinite(detection_dim)) + or estimate_dim[0] <= 0 + or estimate_dim[1] <= 0 + or detection_dim[0] <= 0 + or detection_dim[1] <= 0 + ): + return float("inf") + # get bottom center positions detection_position = np.array( [np.average(detection[:, 0]), np.max(detection[:, 1])]