diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index dc92ab0d54..7796482678 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -30,6 +30,7 @@ from frigate.const import ( UPSERT_REVIEW_SEGMENT, ) from frigate.models import ReviewSegment +from frigate.review.severity import get_review_severity from frigate.review.types import SeverityEnum from frigate.track.object_processing import ManualEventState from frigate.util.image import SharedMemoryFrameManager, calculate_16_9_crop @@ -217,57 +218,21 @@ class ActiveObjects: } for o in all_objects: - if ( - o["motionless_count"] >= camera_config.detect.stationary.threshold - and not o["pending_loitering"] - ): - # no stationary objects unless loitering - continue + severity = get_review_severity( + camera_config, + label=o["label"], + current_zones=o["current_zones"], + false_positive=o["false_positive"], + detected=o["frame_time"] == frame_time, + position_changes=o["position_changes"], + motionless_count=o["motionless_count"], + pending_loitering=o["pending_loitering"], + ) - if o["position_changes"] == 0: - # object must have moved at least once - continue - - if o["frame_time"] != frame_time: - # object must be detected in this frame - continue - - if o["false_positive"]: - # object must not be a false positive - continue - - if ( - o["label"] in camera_config.review.alerts.labels - and ( - not camera_config.review.alerts.required_zones - or ( - len(o["current_zones"]) > 0 - and set(o["current_zones"]) - & set(camera_config.review.alerts.required_zones) - ) - ) - and camera_config.review.alerts.enabled - ): + if severity == SeverityEnum.alert: self.categorized_objects["alerts"].append(o) - continue - - if ( - ( - camera_config.review.detections.labels is None - or o["label"] in camera_config.review.detections.labels - ) - and ( - not camera_config.review.detections.required_zones - or ( - len(o["current_zones"]) > 0 - and set(o["current_zones"]) - & set(camera_config.review.detections.required_zones) - ) - ) - and camera_config.review.detections.enabled - ): + elif severity == SeverityEnum.detection: self.categorized_objects["detections"].append(o) - continue def has_active_objects(self) -> bool: return ( diff --git a/frigate/review/severity.py b/frigate/review/severity.py new file mode 100644 index 0000000000..864e5c8b59 --- /dev/null +++ b/frigate/review/severity.py @@ -0,0 +1,80 @@ +"""Rules for which review severity a tracked object qualifies for.""" + +from typing import cast + +from frigate.config import CameraConfig +from frigate.review.types import SeverityEnum + + +def get_review_severity( + camera_config: CameraConfig, + *, + label: str, + current_zones: list[str], + false_positive: bool, + detected: bool, + position_changes: int, + motionless_count: int, + pending_loitering: bool, +) -> SeverityEnum | None: + """Get the review severity a tracked object qualifies for on the current frame. + + Review segments and a tracked object's max_severity both use this, so an + object is only reported as an alert or detection when it could actually + create or extend a review segment of that severity. + + Args: + camera_config: Config for the camera the object is on + label: The object's label + current_zones: Zones the object is in on the current frame + false_positive: Whether the object is still considered a false positive + detected: Whether the object was detected on the current frame + position_changes: Number of times the object has changed position + motionless_count: Number of frames the object has not moved + pending_loitering: Whether the object is in a loitering zone but has + not met the loitering time yet + + Returns: + The severity the object qualifies for, or None if it does not qualify + """ + # always populated from detect fps when the config is loaded + stationary_threshold = cast(int, camera_config.detect.stationary.threshold) + + # stationary objects don't count unless they are loitering + if motionless_count >= stationary_threshold and not pending_loitering: + return None + + # object must have moved at least once + if position_changes == 0: + return None + + if not detected or false_positive: + return None + + review_config = camera_config.review + zones = set(current_zones) + + if ( + review_config.alerts.enabled + and label in review_config.alerts.labels + and ( + not review_config.alerts.required_zones + or zones & set(review_config.alerts.required_zones) + ) + ): + return SeverityEnum.alert + + if ( + review_config.detections.enabled + and ( + review_config.detections.labels is None + or label in review_config.detections.labels + ) + and ( + not review_config.detections.required_zones + or zones & set(review_config.detections.required_zones) + ) + ): + return SeverityEnum.detection + + return None diff --git a/frigate/test/test_review_severity.py b/frigate/test/test_review_severity.py new file mode 100644 index 0000000000..c7eaf585f9 --- /dev/null +++ b/frigate/test/test_review_severity.py @@ -0,0 +1,230 @@ +"""Review severity rules shared by review segments and tracked objects.""" + +import unittest +from typing import Any + +from frigate.config import FrigateConfig +from frigate.review.maintainer import ActiveObjects +from frigate.review.severity import get_review_severity +from frigate.review.types import SeverityEnum +from frigate.track.tracked_object import TrackedObject + +CAMERA = "cam" +FPS = 5 +ZONE = "porch" +INSIDE_X = 480 +OUTSIDE_X = 160 + + +def _config(alerts_enabled: bool = True) -> FrigateConfig: + return FrigateConfig( + **{ + "mqtt": {"enabled": False}, + "cameras": { + CAMERA: { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": {"width": 640, "height": 480, "fps": FPS}, + "objects": {"track": ["person"]}, + "zones": {ZONE: {"coordinates": "0.5,0,1,0,1,1,0.5,1"}}, + "review": { + "alerts": { + "enabled": alerts_enabled, + "required_zones": [ZONE], + } + }, + } + }, + } + ) + + +class TestGetReviewSeverity(unittest.TestCase): + def setUp(self): + self.camera_config = _config().cameras[CAMERA] + self.stationary_threshold = self.camera_config.detect.stationary.threshold + + def _severity(self, camera_config=None, **overrides: Any) -> SeverityEnum | None: + kwargs = { + "label": "person", + "current_zones": [ZONE], + "false_positive": False, + "detected": True, + "position_changes": 1, + "motionless_count": 0, + "pending_loitering": False, + **overrides, + } + return get_review_severity(camera_config or self.camera_config, **kwargs) + + def test_alert_in_required_zone(self): + self.assertEqual(self._severity(), SeverityEnum.alert) + + def test_detection_outside_alert_required_zone(self): + self.assertEqual(self._severity(current_zones=[]), SeverityEnum.detection) + + def test_detection_when_alerts_disabled(self): + camera_config = _config(alerts_enabled=False).cameras[CAMERA] + self.assertEqual( + self._severity(camera_config=camera_config), SeverityEnum.detection + ) + + def test_stationary_object_does_not_qualify(self): + self.assertIsNone(self._severity(motionless_count=self.stationary_threshold)) + + def test_stationary_loitering_object_qualifies(self): + self.assertEqual( + self._severity( + motionless_count=self.stationary_threshold, pending_loitering=True + ), + SeverityEnum.alert, + ) + + def test_object_that_never_moved_does_not_qualify(self): + self.assertIsNone(self._severity(position_changes=0)) + + def test_undetected_object_does_not_qualify(self): + self.assertIsNone(self._severity(detected=False)) + + def test_false_positive_does_not_qualify(self): + self.assertIsNone(self._severity(false_positive=True)) + + +class TestTrackedObjectMaxSeverity(unittest.TestCase): + """max_severity must match what review segments would have been created.""" + + def setUp(self): + config = _config() + self.camera_config = config.cameras[CAMERA] + self.stationary_threshold = self.camera_config.detect.stationary.threshold + self.frame = 0 + self.position_changes = 0 + self.motionless_count = 0 + self.last_detection = self._obj_data(OUTSIDE_X, 0.6, 1000.0) + self.obj = TrackedObject( + config.model_for_camera(CAMERA), + self.camera_config, + config.ui, + {}, + {**self.last_detection, "score_history": [0.6]}, + ) + + def _obj_data(self, x: int, score: float, frame_time: float) -> dict[str, Any]: + return { + "id": "1000.0-abcdef", + "label": "person", + "frame_time": frame_time, + "start_time": 1000.0, + "score": score, + "box": (x - 40, 200, x + 40, 360), + "area": 12800, + "ratio": 0.5, + "region": (0, 0, 640, 480), + "centroid": (x, 280), + "position_changes": self.position_changes, + "motionless_count": self.motionless_count, + "attributes": [], + "sub_label": None, + } + + def _step( + self, x: int, score: float, moved: bool, detected: bool = True + ) -> SeverityEnum | None: + """Advance one frame and return the severity review segments would see.""" + self.frame += 1 + frame_time = 1000.0 + self.frame / FPS + + if detected: + # mirror the tracker, which counts the first move and any move + # after the object was stationary as a position change + if moved: + if ( + self.position_changes == 0 + or self.motionless_count >= self.stationary_threshold + ): + self.position_changes += 1 + self.motionless_count = 0 + else: + self.motionless_count += 1 + + self.last_detection = self._obj_data(x, score, frame_time) + + self.obj.update(frame_time, dict(self.last_detection), False) + activity = ActiveObjects(frame_time, self.camera_config, [self.obj.to_dict()]) + + if activity.has_activity_category(SeverityEnum.alert): + return SeverityEnum.alert + + if activity.has_activity_category(SeverityEnum.detection): + return SeverityEnum.detection + + return None + + def test_object_confirmed_while_stationary_in_zone_has_no_severity(self): + """An object that only becomes a true positive after it stopped moving + never creates a review, so it must not be reported as an alert even + though it entered a required zone.""" + review_severities = [] + + # walks into the zone with a low score, so it is a false positive + for i in range(10): + review_severities.append(self._step(OUTSIDE_X + i * 40, 0.6, moved=True)) + + # sits still until it is stationary + for _ in range(self.stationary_threshold): + review_severities.append(self._step(INSIDE_X, 0.6, moved=False)) + + # a re-detection while still stationary confirms it as a true positive + for _ in range(10): + review_severities.append(self._step(INSIDE_X, 0.9, moved=False)) + + # leaves and is no longer detected + for _ in range(10): + review_severities.append(self._step(INSIDE_X, 0.9, False, detected=False)) + + self.assertFalse(self.obj.false_positive) + self.assertEqual(self.obj.obj_data["position_changes"], 1) + self.assertEqual(self.obj.entered_zones, [ZONE]) + self.assertEqual(set(review_severities), {None}) + self.assertIsNone(self.obj.max_severity) + self.assertIsNone(self.obj.to_dict()["max_severity"]) + + def test_alert_is_kept_after_object_stops_qualifying(self): + for i in range(10): + self._step(OUTSIDE_X + i * 40, 0.9, moved=True) + + self.assertEqual(self.obj.current_zones, [ZONE]) + self.assertEqual(self.obj.max_severity, SeverityEnum.alert) + + for _ in range(self.stationary_threshold + 5): + self._step(INSIDE_X, 0.9, moved=False) + + self.assertTrue(self.obj.is_stationary()) + self.assertEqual(self.obj.max_severity, SeverityEnum.alert) + + def test_detection_upgrades_to_alert_and_is_not_downgraded(self): + review_severities = [] + + for _ in range(8): + review_severities.append(self._step(OUTSIDE_X, 0.9, moved=True)) + + self.assertEqual(self.obj.max_severity, SeverityEnum.detection) + + for _ in range(5): + review_severities.append(self._step(INSIDE_X, 0.9, moved=True)) + + self.assertEqual(self.obj.max_severity, SeverityEnum.alert) + + for _ in range(5): + review_severities.append(self._step(OUTSIDE_X, 0.9, moved=True)) + + self.assertEqual(review_severities[-1], SeverityEnum.detection) + self.assertEqual(self.obj.max_severity, SeverityEnum.alert) + self.assertIn(SeverityEnum.alert, review_severities) + + +if __name__ == "__main__": + unittest.main() diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index 9d0dd459fd..f87d605a73 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -17,6 +17,7 @@ from frigate.config import ( ) from frigate.const import CLIPS_DIR, REPLAY_CAMERA_PREFIX, THUMB_DIR from frigate.detectors.detector_config import ModelConfig +from frigate.review.severity import get_review_severity from frigate.review.types import SeverityEnum from frigate.util.builtin import sanitize_float from frigate.util.image import ( @@ -69,6 +70,8 @@ class TrackedObject: self.new_zone_entered: bool = False self.attributes: dict[str, float] = defaultdict(float) self.false_positive = True + # highest review severity the object qualified for on any frame + self.max_severity: SeverityEnum | None = None self.has_clip = False self.has_snapshot = False self.top_score = self.computed_score = 0.0 @@ -85,36 +88,6 @@ class TrackedObject: self.path_data: list[tuple[Any, float]] = [] self.previous = self.to_dict() - @property - def max_severity(self) -> str | None: - review_config = self.camera_config.review - - if ( - self.camera_config.review.alerts.enabled - and self.obj_data["label"] in review_config.alerts.labels - and ( - not review_config.alerts.required_zones - or set(self.entered_zones) & set(review_config.alerts.required_zones) - ) - ): - return SeverityEnum.alert - - if ( - self.camera_config.review.detections.enabled - and ( - not review_config.detections.labels - or self.obj_data["label"] in review_config.detections.labels - ) - and ( - not review_config.detections.required_zones - or set(self.entered_zones) - & set(review_config.detections.required_zones) - ) - ): - return SeverityEnum.detection - - return None - def _is_false_positive(self) -> bool: # once a true positive, always a true positive if not self.false_positive: @@ -386,6 +359,25 @@ class TrackedObject: self.obj_data.update(obj_data) self.current_zones = current_zones + + # use the same per-frame rules as review segments, so max_severity is + # never alert or detection unless the object could create that review + severity = get_review_severity( + self.camera_config, + label=self.obj_data["label"], + current_zones=current_zones, + false_positive=self.false_positive, + detected=detected, + position_changes=self.obj_data["position_changes"], + motionless_count=self.obj_data["motionless_count"], + pending_loitering=self.pending_loitering, + ) + + if severity == SeverityEnum.alert or ( + severity == SeverityEnum.detection and self.max_severity is None + ): + self.max_severity = severity + logger.debug( f"{self.camera_config.name}: Updating {obj_data['id']}: thumb update? {thumb_update}, significant change? {significant_change}, path update? {path_update}, autotracker update? {autotracker_update} " )