Compare commits

..
Author SHA1 Message Date
Josh Hawkins 6ab6f1be9f fix mypy and add docstring 2026-10-19 13:17:53 -05:00
Josh Hawkins 167aac8799 keep post capture segments after a review item ends
The recording maintainer only loaded review items whose end_time was after the oldest cached segment, so once an item closed in the database the segments in its post_capture window no longer matched it. With continuous recording off and no motion after the item, everything past roughly the review cutoff_time was deleted from the cache instead of being kept for the full post_capture. The query bound now subtracts the camera's largest post_capture, and the per-item check still decides the exact cutoff.
2026-10-19 13:06:18 -05:00
6 changed files with 109 additions and 307 deletions
@@ -176,12 +176,6 @@ http {
add_header Cache-Control "public"; add_header Cache-Control "public";
autoindex on; autoindex on;
root /media/frigate; root /media/frigate;
# review thumbnails are rewritten in place while a review is active,
# so a cached file size truncates the response
location /clips/review/ {
open_file_cache off;
}
} }
location /cache/ { location /cache/ {
+8
View File
@@ -224,6 +224,14 @@ class RecordConfig(FrigateBaseModel):
self.detections.pre_capture, self.detections.pre_capture,
) )
@property
def event_post_capture(self) -> int:
"""Largest post-capture window used to fetch candidate reviews."""
return max(
self.alerts.post_capture,
self.detections.post_capture,
)
def get_review_pre_capture(self, severity: SeverityEnum) -> int: def get_review_pre_capture(self, severity: SeverityEnum) -> int:
if severity == SeverityEnum.alert: if severity == SeverityEnum.alert:
return self.alerts.pre_capture return self.alerts.pre_capture
+23 -27
View File
@@ -198,56 +198,52 @@ class RecordingCleanup(threading.Thread):
deleted_recordings = set() deleted_recordings = set()
kept_recordings: list[tuple[float, float]] = [] kept_recordings: list[tuple[float, float]] = []
for recording in recordings: for recording in recordings:
modes: set[RetainModeEnum] = set() keep = False
# Now look for every reason to keep this recording segment mode = None
# Now look for a reason to keep this recording segment
for idx in range(review_start, len(reviews)): for idx in range(review_start, len(reviews)):
review = reviews[idx] review = reviews[idx]
severity = review.severity severity = review.severity
pre_capture = config.record.get_review_pre_capture(severity) pre_capture = config.record.get_review_pre_capture(severity)
post_capture = config.record.get_review_post_capture(severity) post_capture = config.record.get_review_post_capture(severity)
# stop once no later review's pre capture can reach this segment # if the review starts in the future, stop checking reviews
if ( # and let this recording segment expire
review.start_time - config.record.event_pre_capture if review.start_time - pre_capture > recording.end_time:
> recording.end_time keep = False
):
break break
if review.start_time - pre_capture > recording.end_time: # if the review is in progress or ends after the recording starts, keep it
continue # and stop looking at reviews
# collect every review that is in progress or ends after the
# recording starts, since their windows can overlap
if ( if (
review.end_time is None review.end_time is None
or review.end_time + post_capture >= recording.start_time or review.end_time + post_capture >= recording.start_time
): ):
modes.add( keep = True
mode = (
alerts_retain_mode alerts_retain_mode
if review.severity == "alert" if review.severity == "alert"
else detections_retain_mode else detections_retain_mode
) )
break
# if the review ends before this recording segment starts, skip # if the review ends before this recording segment starts, skip
# it on future segments too. reviews and recordings are sorted, # this review and check the next review for an overlap.
# but an earlier review with a longer post capture may still # since the review and recordings are sorted, we can skip review
# overlap, so only skip while nothing has overlapped yet # that end before the previous recording segment started on future segments
elif not modes: if review.end_time + post_capture < recording.start_time:
review_start = idx review_start = idx
# Delete recordings unless an overlapping review's retention mode keeps them # Delete recordings outside of the retention window or based on the retention mode
if not any( if (
mode == RetainModeEnum.all not keep
or ( or (
mode == RetainModeEnum.motion mode == RetainModeEnum.motion
and ( and recording.motion == 0
recording.motion != 0 and recording.objects == 0
or recording.objects != 0 and recording.dBFS == 0
or recording.dBFS != 0
)
) )
or (mode == RetainModeEnum.active_objects and recording.objects != 0) or (mode == RetainModeEnum.active_objects and recording.objects == 0)
for mode in modes
): ):
recording_path = Path(recording.path) recording_path = Path(recording.path)
recording_path.unlink(missing_ok=True) recording_path.unlink(missing_ok=True)
+28 -21
View File
@@ -471,9 +471,14 @@ class RecordingMaintainer(threading.Thread):
for key, recordings in grouped_recordings.items(): for key, recordings in grouped_recordings.items():
camera, stream_type = key camera, stream_type = key
# get all reviews with the end time after the start of the oldest # get all reviews whose post capture reaches the oldest cache file
# cache file or with end_time None; shared across stream types # or with end_time None; shared across stream types
if camera not in reviews_by_camera: if camera not in reviews_by_camera:
post_capture = (
self.config.cameras[camera].record.event_post_capture
if camera in self.config.cameras
else 0
)
reviews_by_camera[camera] = ( reviews_by_camera[camera] = (
ReviewSegment.select( ReviewSegment.select(
ReviewSegment.start_time, ReviewSegment.start_time,
@@ -484,7 +489,10 @@ class RecordingMaintainer(threading.Thread):
.where( .where(
ReviewSegment.camera == camera, ReviewSegment.camera == camera,
(ReviewSegment.end_time == None) (ReviewSegment.end_time == None)
| (ReviewSegment.end_time >= min_start_per_camera[camera]), | (
ReviewSegment.end_time
>= min_start_per_camera[camera] - post_capture
),
) )
.order_by(ReviewSegment.start_time) .order_by(ReviewSegment.start_time)
) )
@@ -759,40 +767,39 @@ class RecordingMaintainer(threading.Thread):
# we fell through the continuous / motion check, so we need to check the review items # we fell through the continuous / motion check, so we need to check the review items
# if the cached segment overlaps with the review items: # if the cached segment overlaps with the review items:
modes: set[RetainModeEnum] = set() overlaps = False
for review in reviews: for review in reviews:
severity = SeverityEnum[review.severity] severity = SeverityEnum[review.severity]
# stop once no later review item's pre capture can reach this segment # if the review item starts in the future, stop checking review items
if ( # and remove this segment
review.start_time - record_config.event_pre_capture
) > end_time.timestamp():
break
if ( if (
review.start_time - record_config.get_review_pre_capture(severity) review.start_time - record_config.get_review_pre_capture(severity)
) > end_time.timestamp(): ) > end_time.timestamp():
continue overlaps = False
break
# collect every review item that is in progress or ends after the # if the review item is in progress or ends after the recording starts, keep it
# recording starts, since their windows can overlap # and stop looking at review items
if ( if (
review.end_time is None review.end_time is None
or (review.end_time + record_config.get_review_post_capture(severity)) or (review.end_time + record_config.get_review_post_capture(severity))
>= start_time.timestamp() >= start_time.timestamp()
): ):
modes.add( overlaps = True
alerts_retain_mode break
if review.severity == "alert"
else detections_retain_mode if overlaps:
) record_mode = (
alerts_retain_mode
if review.severity == "alert"
else detections_retain_mode
)
if modes:
if segment_stats is None: if segment_stats is None:
segment_stats = self.segment_stats(camera, start_time, end_time) segment_stats = self.segment_stats(camera, start_time, end_time)
# keep the segment if any overlapping review item's mode keeps it if not segment_stats.should_discard_segment(record_mode):
if any(not segment_stats.should_discard_segment(m) for m in modes):
# move from cache to recordings immediately # move from cache to recordings immediately
return await self.move_segment( return await self.move_segment(
camera, camera,
+50 -111
View File
@@ -6,6 +6,8 @@ import unittest
from pathlib import Path from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch from unittest.mock import AsyncMock, MagicMock, patch
from peewee import SqliteDatabase
# Mock complex imports before importing maintainer, saving originals so we can # Mock complex imports before importing maintainer, saving originals so we can
# restore them after import and avoid polluting sys.modules for other tests. # restore them after import and avoid polluting sys.modules for other tests.
_MOCKED_MODULES = [ _MOCKED_MODULES = [
@@ -20,6 +22,7 @@ for name in _MOCKED_MODULES:
# Now import the class under test # Now import the class under test
from frigate.config import FrigateConfig, RetainModeEnum # noqa: E402 from frigate.config import FrigateConfig, RetainModeEnum # noqa: E402
from frigate.models import ReviewSegment # noqa: E402
from frigate.record.maintainer import RecordingMaintainer, SegmentInfo # noqa: E402 from frigate.record.maintainer import RecordingMaintainer, SegmentInfo # noqa: E402
# Restore original modules (or remove mock if there was no original) # Restore original modules (or remove mock if there was no original)
@@ -81,6 +84,53 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase):
f"Expected a single warning for unexpected files, got {len(matching)}", f"Expected a single warning for unexpected files, got {len(matching)}",
) )
async def test_ended_review_stays_eligible_through_post_capture(self):
config = MagicMock(spec=FrigateConfig)
camera_config = MagicMock()
camera_config.record.sub.enabled = False
camera_config.record.event_post_capture = 60
config.cameras = {"test_cam": camera_config}
maintainer = RecordingMaintainer(config, MagicMock())
maintainer.requestor = MagicMock()
maintainer.recordings_publisher = MagicMock()
maintainer.validate_and_move_segment = AsyncMock(return_value=None)
db = SqliteDatabase(":memory:")
with db.bind_ctx([ReviewSegment]):
db.create_tables([ReviewSegment])
# oldest cached segment starts 40s after the review ended
segment_start = datetime.datetime(
2026, 10, 19, 13, 1, 40, tzinfo=datetime.UTC
)
ReviewSegment.create(
id="review",
camera="test_cam",
start_time=segment_start.timestamp() - 50,
end_time=segment_start.timestamp() - 40,
severity="alert",
thumb_path="",
data={},
)
with tempfile.TemporaryDirectory() as cache_dir:
name = segment_start.strftime("%Y%m%d%H%M%S%z")
Path(cache_dir, f"test_cam@{name}.mp4").touch()
with (
patch("frigate.record.maintainer.CACHE_DIR", cache_dir),
patch(
"frigate.record.maintainer.psutil.process_iter", return_value=[]
),
):
await maintainer.move_files()
reviews = maintainer.validate_and_move_segment.call_args.args[1]
self.assertEqual(
[segment_start.timestamp() - 40], [r.end_time for r in reviews]
)
async def test_drops_quiet_segment_when_only_motion_retention(self): async def test_drops_quiet_segment_when_only_motion_retention(self):
# Regression: when motion retention is enabled but a segment has no # Regression: when motion retention is enabled but a segment has no
# motion and no review overlaps it, the segment must still be dropped. # motion and no review overlaps it, the segment must still be dropped.
@@ -127,116 +177,6 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase):
self.assertIsNone(result) self.assertIsNone(result)
maintainer.drop_segment.assert_called_once_with(cache_path) maintainer.drop_segment.assert_called_once_with(cache_path)
async def test_quiet_segment_kept_by_later_overlapping_alert(self):
config = MagicMock(spec=FrigateConfig)
camera_config = MagicMock()
camera_config.record.enabled = True
camera_config.record.continuous.days = 0
camera_config.record.motion.days = 0
camera_config.record.alerts.retain.mode = RetainModeEnum.all
camera_config.record.detections.retain.mode = RetainModeEnum.motion
camera_config.record.event_pre_capture = 60
camera_config.record.get_review_pre_capture.return_value = 60
camera_config.record.get_review_post_capture.return_value = 60
config.cameras = {"test_cam": camera_config}
maintainer = RecordingMaintainer(config, MagicMock())
now = datetime.datetime.now(datetime.UTC)
start_time = now - datetime.timedelta(seconds=20)
end_time = now - datetime.timedelta(seconds=10)
cache_path = "/tmp/cache/test_cam@20260417150000+0000.mp4"
maintainer.end_time_cache = {
cache_path: (end_time, 10.0, False, None, None, None, [])
}
maintainer.object_recordings_info["test_cam"] = [(now.timestamp(), [], [], [])]
maintainer.audio_recordings_info["test_cam"] = []
maintainer.drop_segment = MagicMock()
maintainer.move_segment = AsyncMock(return_value=None)
maintainer.recordings_publisher = MagicMock()
# the detection's post capture and the alert's window both cover the segment
detection = MagicMock()
detection.severity = "detection"
detection.start_time = start_time.timestamp() - 50
detection.end_time = start_time.timestamp() - 40
alert = MagicMock()
alert.severity = "alert"
alert.start_time = start_time.timestamp() - 10
alert.end_time = start_time.timestamp()
await maintainer.validate_and_move_segment(
"test_cam",
reviews=[detection, alert],
recording={
"start_time": start_time,
"cache_path": cache_path,
"stream_type": "main",
},
)
maintainer.move_segment.assert_awaited_once()
maintainer.drop_segment.assert_not_called()
async def test_quiet_segment_kept_by_later_alert_with_longer_pre_capture(self):
config = MagicMock(spec=FrigateConfig)
camera_config = MagicMock()
camera_config.record.enabled = True
camera_config.record.continuous.days = 0
camera_config.record.motion.days = 0
camera_config.record.alerts.retain.mode = RetainModeEnum.all
camera_config.record.detections.retain.mode = RetainModeEnum.all
camera_config.record.event_pre_capture = 30
camera_config.record.get_review_pre_capture.side_effect = lambda s: (
30 if s == "alert" else 5
)
camera_config.record.get_review_post_capture.return_value = 5
config.cameras = {"test_cam": camera_config}
maintainer = RecordingMaintainer(config, MagicMock())
now = datetime.datetime.now(datetime.UTC)
start_time = now - datetime.timedelta(seconds=20)
end_time = now - datetime.timedelta(seconds=10)
cache_path = "/tmp/cache/test_cam@20260417150000+0000.mp4"
maintainer.end_time_cache = {
cache_path: (end_time, 10.0, False, None, None, None, [])
}
maintainer.object_recordings_info["test_cam"] = [(now.timestamp(), [], [], [])]
maintainer.audio_recordings_info["test_cam"] = []
maintainer.drop_segment = MagicMock()
maintainer.move_segment = AsyncMock(return_value=None)
maintainer.recordings_publisher = MagicMock()
# the detection's pre capture starts after the segment, the alert's reaches it
detection = MagicMock()
detection.severity = "detection"
detection.start_time = end_time.timestamp() + 10
detection.end_time = end_time.timestamp() + 12
alert = MagicMock()
alert.severity = "alert"
alert.start_time = end_time.timestamp() + 20
alert.end_time = end_time.timestamp() + 25
await maintainer.validate_and_move_segment(
"test_cam",
reviews=[detection, alert],
recording={
"start_time": start_time,
"cache_path": cache_path,
"stream_type": "main",
},
)
maintainer.move_segment.assert_awaited_once()
maintainer.drop_segment.assert_not_called()
async def test_defers_review_overlap_segment_until_metadata_catches_up(self): async def test_defers_review_overlap_segment_until_metadata_catches_up(self):
# Regression: a segment overlapping an active_objects review must not # Regression: a segment overlapping an active_objects review must not
# be dropped while detection metadata lags behind the segment end, # be dropped while detection metadata lags behind the segment end,
@@ -249,7 +189,6 @@ class TestMaintainer(unittest.IsolatedAsyncioTestCase):
camera_config.record.continuous.days = 7 camera_config.record.continuous.days = 7
camera_config.record.motion.days = 0 camera_config.record.motion.days = 0
camera_config.record.alerts.retain.mode = RetainModeEnum.active_objects camera_config.record.alerts.retain.mode = RetainModeEnum.active_objects
camera_config.record.event_pre_capture = 5
camera_config.record.get_review_pre_capture.return_value = 5 camera_config.record.get_review_pre_capture.return_value = 5
camera_config.record.get_review_post_capture.return_value = 5 camera_config.record.get_review_post_capture.return_value = 5
config.cameras = {"test_cam": camera_config} config.cameras = {"test_cam": camera_config}
-142
View File
@@ -1,142 +0,0 @@
"""Tests for review-based retention in recording cleanup."""
import datetime
import unittest
from unittest.mock import MagicMock
from peewee import SqliteDatabase
from frigate.config import FrigateConfig
from frigate.models import Previews, Recordings, ReviewSegment, UserReviewStatus
from frigate.record.cleanup import RecordingCleanup
class TestRecordingCleanupReviewOverlap(unittest.TestCase):
def setUp(self):
self.db = SqliteDatabase(":memory:")
models = [Previews, Recordings, ReviewSegment, UserReviewStatus]
self.db.bind(models)
self.db.create_tables(models)
self.base = datetime.datetime.now().timestamp() - 3600
def tearDown(self):
self.db.close()
def _build_cleanup(self, alerts: dict, detections: dict) -> RecordingCleanup:
config = FrigateConfig(
**{
"mqtt": {"host": "mqtt"},
"cameras": {
"front_door": {
"ffmpeg": {
"inputs": [
{
"path": "rtsp://10.0.0.1:554/video",
"roles": ["detect", "record"],
}
]
},
"record": {
"enabled": True,
"continuous": {"days": 0},
"motion": {"days": 0},
"alerts": alerts,
"detections": detections,
},
}
},
}
)
return RecordingCleanup(config, MagicMock())
def _insert_review(self, id: str, severity: str, start: float, end: float):
ReviewSegment.create(
id=id,
camera="front_door",
start_time=self.base + start,
end_time=self.base + end,
severity=severity,
thumb_path=f"/media/frigate/clips/review/{id}.webp",
data={},
)
def _insert_recording(self, id: str, start: float):
Recordings.create(
id=id,
camera="front_door",
path=f"/media/frigate/recordings/{id}.mp4",
start_time=self.base + start,
end_time=self.base + start + 10,
duration=10,
motion=0,
objects=0,
dBFS=0,
segment_size=0,
stream_type="main",
)
def test_quiet_segment_kept_by_later_overlapping_alert(self):
cleanup = self._build_cleanup(
alerts={"pre_capture": 60, "post_capture": 60, "retain": {"mode": "all"}},
detections={"post_capture": 60, "retain": {"mode": "motion"}},
)
self._insert_review("detection", "detection", 90, 100)
self._insert_review("alert", "alert", 130, 140)
self._insert_recording("quiet", 140)
cleanup.expire_recordings()
assert Recordings.get_or_none(Recordings.id == "quiet") is not None
def test_quiet_segment_dropped_when_no_overlapping_mode_keeps_it(self):
cleanup = self._build_cleanup(
alerts={
"pre_capture": 60,
"post_capture": 60,
"retain": {"mode": "motion"},
},
detections={"post_capture": 60, "retain": {"mode": "motion"}},
)
self._insert_review("detection", "detection", 90, 100)
self._insert_review("alert", "alert", 130, 140)
self._insert_recording("quiet", 140)
cleanup.expire_recordings()
assert Recordings.get_or_none(Recordings.id == "quiet") is None
def test_longer_pre_capture_survives_an_earlier_shorter_review(self):
# the detection starts after the segment even with its pre capture,
# but the later alert's longer pre capture still reaches it
cleanup = self._build_cleanup(
alerts={"pre_capture": 30, "retain": {"mode": "all"}},
detections={"pre_capture": 5, "retain": {"mode": "all"}},
)
self._insert_review("detection", "detection", 10, 12)
self._insert_review("alert", "alert", 20, 25)
self._insert_recording("early", -6)
cleanup.expire_recordings()
assert Recordings.get_or_none(Recordings.id == "early") is not None
def test_longer_post_capture_survives_a_later_shorter_review(self):
# the alert's window ends before the detection's, so skipping past
# it must not skip the detection for later segments
cleanup = self._build_cleanup(
alerts={"post_capture": 5, "retain": {"mode": "motion"}},
detections={"post_capture": 60, "retain": {"mode": "all"}},
)
self._insert_review("detection", "detection", 90, 100)
self._insert_review("alert", "alert", 110, 115)
self._insert_recording("first", 125)
self._insert_recording("second", 135)
cleanup.expire_recordings()
assert Recordings.get_or_none(Recordings.id == "first") is not None
assert Recordings.get_or_none(Recordings.id == "second") is not None
if __name__ == "__main__":
unittest.main()