administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-02-14 15:15:22 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Refactor recording expiration to be based off of review items

Browse Source
This commit is contained in:
Nicolas Mowen 2024-08-26 07:24:00 -06:00
parent a19c1509e6
commit 308434ea49
1 changed files with 68 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
frigate/record/cleanup.py
View File

@ -12,7 +12,7 @@ from playhouse.sqlite_ext import SqliteExtDatabase
from frigate.config import CameraConfig, FrigateConfig, RetainModeEnum from frigate.config import CameraConfig, FrigateConfig, RetainModeEnum
from frigate.const import CACHE_DIR, CLIPS_DIR, MAX_WAL_SIZE, RECORD_DIR from frigate.const import CACHE_DIR, CLIPS_DIR, MAX_WAL_SIZE, RECORD_DIR
from frigate.models import Event, Previews, Recordings, ReviewSegment from frigate.models import Previews, Recordings, ReviewSegment
from frigate.record.util import remove_empty_directories, sync_recordings from frigate.record.util import remove_empty_directories, sync_recordings
from frigate.util.builtin import clear_and_unlink, get_tomorrow_at_time from frigate.util.builtin import clear_and_unlink, get_tomorrow_at_time
@ -61,8 +61,37 @@ class RecordingCleanup(threading.Thread):
db.execute_sql("PRAGMA wal_checkpoint(TRUNCATE);") db.execute_sql("PRAGMA wal_checkpoint(TRUNCATE);")
db.close() db.close()
def expire_review_segments(self, config: CameraConfig, now: datetime) -> None:
"""Delete review segments that are expired"""
alert_expire_date = (
now - datetime.timedelta(days=config.record.alerts.retain.days)
).timestamp()
detection_expire_date = (
now - datetime.timedelta(days=config.record.detections.retain.days)
).timestamp()
expired_reviews: ReviewSegment = ReviewSegment.select(ReviewSegment.id).where(
ReviewSegment.camera == config.name
and (
(
ReviewSegment.severity == "alert"
and ReviewSegment.end_time < alert_expire_date
)
or (
ReviewSegment.severity == "detection"
and ReviewSegment.end_time < detection_expire_date
)
)
)
max_deletes = 100000
deleted_reviews_list = list(expired_reviews)
for i in range(0, len(deleted_reviews_list), max_deletes):
ReviewSegment.delete().where(
ReviewSegment.id << deleted_reviews_list[i : i + max_deletes]
).execute()
def expire_existing_camera_recordings( def expire_existing_camera_recordings(
self, expire_date: float, config: CameraConfig, events: Event self, expire_date: float, config: CameraConfig, reviews: ReviewSegment
) -> None: ) -> None:
"""Delete recordings for existing camera based on retention config.""" """Delete recordings for existing camera based on retention config."""
# Get the timestamp for cutoff of retained days # Get the timestamp for cutoff of retained days
@ -86,47 +115,47 @@ class RecordingCleanup(threading.Thread):
.iterator() .iterator()
) )
# loop over recordings and see if they overlap with any non-expired events # loop over recordings and see if they overlap with any non-expired reviews
# TODO: expire segments based on segment stats according to config # TODO: expire segments based on segment stats according to config
event_start = 0 review_start = 0
deleted_recordings = set() deleted_recordings = set()
kept_recordings: list[tuple[float, float]] = [] kept_recordings: list[tuple[float, float]] = []
for recording in recordings: for recording in recordings:
keep = False keep = False
mode = None
# Now look for a reason to keep this recording segment # Now look for a reason to keep this recording segment
for idx in range(event_start, len(events)): for idx in range(review_start, len(reviews)):
event: Event = events[idx] review: ReviewSegment = reviews[idx]
# if the event starts in the future, stop checking events # if the review starts in the future, stop checking reviews
# and let this recording segment expire # and let this recording segment expire
if event.start_time > recording.end_time: if review.start_time > recording.end_time:
keep = False keep = False
break break
# if the event is in progress or ends after the recording starts, keep it # if the review is in progress or ends after the recording starts, keep it
# and stop looking at events # and stop looking at reviews
if event.end_time is None or event.end_time >= recording.start_time: if review.end_time is None or review.end_time >= recording.start_time:
keep = True keep = True
mode = (
config.record.alerts.retain.mode
if review.severity == "alert"
else config.record.detections.retain.mode
)
break break
# if the event ends before this recording segment starts, skip # if the review ends before this recording segment starts, skip
# this event and check the next event for an overlap. # this review and check the next review for an overlap.
# since the events and recordings are sorted, we can skip events # since the review and recordings are sorted, we can skip review
# that end before the previous recording segment started on future segments # that end before the previous recording segment started on future segments
if event.end_time < recording.start_time: if review.end_time < recording.start_time:
event_start = idx review_start = idx
# Delete recordings outside of the retention window or based on the retention mode # Delete recordings outside of the retention window or based on the retention mode
if ( if (
not keep not keep
or ( or (mode == RetainModeEnum.motion and recording.motion == 0)
config.record.events.retain.mode == RetainModeEnum.motion or (mode == RetainModeEnum.active_objects and recording.objects == 0)
and recording.motion == 0
)
or (
config.record.events.retain.mode == RetainModeEnum.active_objects
and recording.objects == 0
)
): ):
Path(recording.path).unlink(missing_ok=True) Path(recording.path).unlink(missing_ok=True)
deleted_recordings.add(recording.id) deleted_recordings.add(recording.id)
@ -202,65 +231,6 @@ class RecordingCleanup(threading.Thread):
Previews.id << deleted_previews_list[i : i + max_deletes] Previews.id << deleted_previews_list[i : i + max_deletes]
).execute() ).execute()
review_segments: list[ReviewSegment] = (
ReviewSegment.select(
ReviewSegment.id,
ReviewSegment.start_time,
ReviewSegment.end_time,
ReviewSegment.thumb_path,
)
.where(
ReviewSegment.camera == config.name,
ReviewSegment.end_time < expire_date,
)
.order_by(ReviewSegment.start_time)
.namedtuples()
.iterator()
)
# expire review segments
recording_start = 0
deleted_segments = set()
for segment in review_segments:
keep = False
# look for a reason to keep this segment
for idx in range(recording_start, len(kept_recordings)):
start_time, end_time = kept_recordings[idx]
# if the recording starts in the future, stop checking recordings
# and let this segment expire
if start_time > segment.end_time:
keep = False
break
# if the recording ends after the segment starts, keep it
# and stop looking at recordings
if end_time >= segment.start_time:
keep = True
break
# if the recording ends before this segment starts, skip
# this recording and check the next recording for an overlap.
# since the kept recordings and segments are sorted, we can skip recordings
# that end before the current segment started
if end_time < segment.start_time:
recording_start = idx
# Delete segments without any relevant recordings
if not keep:
Path(segment.thumb_path).unlink(missing_ok=True)
deleted_segments.add(segment.id)
# expire segments
logger.debug(f"Expiring {len(deleted_segments)} segments")
# delete up to 100,000 at a time
max_deletes = 100000
deleted_segments_list = list(deleted_segments)
for i in range(0, len(deleted_segments_list), max_deletes):
ReviewSegment.delete().where(
ReviewSegment.id << deleted_segments_list[i : i + max_deletes]
).execute()
def expire_recordings(self) -> None: def expire_recordings(self) -> None:
"""Delete recordings based on retention config.""" """Delete recordings based on retention config."""
logger.debug("Start expire recordings.") logger.debug("Start expire recordings.")
@ -302,30 +272,31 @@ class RecordingCleanup(threading.Thread):
logger.debug("Start all cameras.") logger.debug("Start all cameras.")
for camera, config in self.config.cameras.items(): for camera, config in self.config.cameras.items():
logger.debug(f"Start camera: {camera}.") logger.debug(f"Start camera: {camera}.")
now = datetime.datetime.now()
self.expire_review_segments(config, now)
expire_days = config.record.retain.days expire_days = config.record.retain.days
expire_date = ( expire_date = (now - datetime.timedelta(days=expire_days)).timestamp()
datetime.datetime.now() - datetime.timedelta(days=expire_days)
).timestamp()
# Get all the events to check against # Get all the reviews to check against
events: Event = ( reviews: ReviewSegment = (
Event.select( ReviewSegment.select(
Event.start_time, ReviewSegment.start_time,
Event.end_time, ReviewSegment.end_time,
ReviewSegment.severity,
) )
.where( .where(
Event.camera == camera, ReviewSegment.camera == camera,
# need to ensure segments for all events starting # need to ensure segments for all reviews starting
# before the expire date are included # before the expire date are included
Event.start_time < expire_date, ReviewSegment.start_time < expire_date,
Event.has_clip,
) )
.order_by(Event.start_time) .order_by(ReviewSegment.start_time)
.namedtuples() .namedtuples()
) )
self.expire_existing_camera_recordings(expire_date, config, events) self.expire_existing_camera_recordings(expire_date, config, reviews)
logger.debug(f"End camera: {camera}.") logger.debug(f"End camera: {camera}.")
logger.debug("End all cameras.") logger.debug("End all cameras.")