From 5bd45cf8581012a86ac9902e78dbb86b95476a0d Mon Sep 17 00:00:00 2001 From: Nick Mowen Date: Fri, 3 Nov 2023 06:37:33 -0600 Subject: [PATCH] Make automatic sync limited ot last 36 hours --- frigate/record/cleanup.py | 10 +++++----- frigate/record/util.py | 41 ++++++++++++++++++++++++++++++--------- frigate/util/builtin.py | 13 ++----------- frigate/video.py | 6 +++--- 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/frigate/record/cleanup.py b/frigate/record/cleanup.py index 619a1910e..e6bb41da3 100644 --- a/frigate/record/cleanup.py +++ b/frigate/record/cleanup.py @@ -11,7 +11,7 @@ from frigate.config import FrigateConfig, RetainModeEnum from frigate.const import CACHE_DIR, RECORD_DIR from frigate.models import Event, Recordings from frigate.record.util import remove_empty_directories, sync_recordings -from frigate.util.builtin import get_next_sunday_at_3 +from frigate.util.builtin import get_tomorrow_at_time logger = logging.getLogger(__name__) @@ -181,9 +181,9 @@ class RecordingCleanup(threading.Thread): def run(self) -> None: # on startup sync recordings with disk if enabled if self.config.record.sync_on_startup: - sync_recordings() + sync_recordings(limited=False) - next_sync = get_next_sunday_at_3() + next_sync = get_tomorrow_at_time(3) # Expire tmp clips every minute, recordings and clean directories every hour. for counter in itertools.cycle(range(self.config.record.expire_interval)): @@ -194,8 +194,8 @@ class RecordingCleanup(threading.Thread): self.clean_tmp_clips() if datetime.datetime.now().astimezone(datetime.timezone.utc) > next_sync: - sync_recordings() - next_sync = get_next_sunday_at_3() + sync_recordings(limited=True) + next_sync = get_tomorrow_at_time(3) if counter == 0: self.expire_recordings() diff --git a/frigate/record/util.py b/frigate/record/util.py index cfe5f5d62..4710b063e 100644 --- a/frigate/record/util.py +++ b/frigate/record/util.py @@ -1,5 +1,6 @@ """Recordings Utilities.""" +import datetime import logging import os @@ -27,13 +28,21 @@ def remove_empty_directories(directory: str) -> None: os.rmdir(path) -def sync_recordings() -> None: +def sync_recordings(limited: bool) -> None: """Check the db for stale recordings entries that don't exist in the filesystem.""" def delete_db_entries_without_file(files_on_disk: list[str]) -> bool: """Delete db entries where file was deleted outside of frigate.""" - # get all recordings in the db - recordings = Recordings.select(Recordings.id, Recordings.path) + + if limited: + recordings = Recordings.select(Recordings.id, Recordings.path).where( + Recordings.start_time + >= (datetime.datetime.now() - datetime.timedelta(hours=36)).timestamp() + ) + else: + # get all recordings in the db + recordings = Recordings.select(Recordings.id, Recordings.path) + # Use pagination to process records in chunks page_size = 1000 num_pages = (recordings.count() + page_size - 1) // page_size @@ -97,12 +106,26 @@ def sync_recordings() -> None: logger.debug("Start sync recordings.") - # get all recordings files on disk and put them in a set - files_on_disk = { - os.path.join(root, file) - for root, _, files in os.walk(RECORD_DIR) - for file in files - } + if limited: + # get recording files from last 36 hours + hour_check = ( + datetime.datetime.now().astimezone(datetime.timezone.utc) + - datetime.timedelta(hours=36) + ).strftime("%Y-%m-%d/%H") + files_on_disk = { + os.path.join(root, file) + for root, _, files in os.walk(RECORD_DIR) + for file in files + if file > hour_check + } + else: + # get all recordings files on disk and put them in a set + files_on_disk = { + os.path.join(root, file) + for root, _, files in os.walk(RECORD_DIR) + for file in files + } + db_success = delete_db_entries_without_file(files_on_disk) # only try to cleanup files if db cleanup was successful diff --git a/frigate/util/builtin.py b/frigate/util/builtin.py index c353211c0..a0f0d7725 100644 --- a/frigate/util/builtin.py +++ b/frigate/util/builtin.py @@ -263,18 +263,9 @@ def find_by_key(dictionary, target_key): return None -def get_tomorrow_at_2() -> datetime.datetime: +def get_tomorrow_at_time(hour: int) -> datetime.datetime: """Returns the datetime of the following day at 2am.""" tomorrow = datetime.datetime.now(get_localzone()) + datetime.timedelta(days=1) - return tomorrow.replace(hour=2, minute=0, second=0).astimezone( + return tomorrow.replace(hour=hour, minute=0, second=0).astimezone( datetime.timezone.utc ) - - -def get_next_sunday_at_3() -> datetime.datetime: - """Returns the datetime of the next Sunday at 3am.""" - # adapted from https://stackoverflow.com/a/16770463 - now = datetime.datetime.now(get_localzone()) - diff = datetime.timedelta((13 - now.weekday()) % 7) - sunday = now + diff - return sunday.replace(hour=3, minute=0, second=0).astimezone(datetime.timezone.utc) diff --git a/frigate/video.py b/frigate/video.py index 23b3481cf..7ab41a5a6 100755 --- a/frigate/video.py +++ b/frigate/video.py @@ -26,7 +26,7 @@ from frigate.ptz.autotrack import ptz_moving_at_frame_time from frigate.track import ObjectTracker from frigate.track.norfair_tracker import NorfairTracker from frigate.types import PTZMetricsTypes -from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_2 +from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_time from frigate.util.image import ( FrameManager, SharedMemoryFrameManager, @@ -528,7 +528,7 @@ def process_frames( fps = process_info["process_fps"] detection_fps = process_info["detection_fps"] current_frame_time = process_info["detection_frame"] - next_region_update = get_tomorrow_at_2() + next_region_update = get_tomorrow_at_time(2) fps_tracker = EventsPerSecond() fps_tracker.start() @@ -550,7 +550,7 @@ def process_frames( except queue.Empty: logger.error(f"Unable to get updated region grid for {camera_name}") - next_region_update = get_tomorrow_at_2() + next_region_update = get_tomorrow_at_time(2) try: if exit_on_empty: