Make automatic sync limited ot last 36 hours

frigate/record/cleanup.py

@@ -11,7 +11,7 @@ from frigate.config import FrigateConfig, RetainModeEnum
 from frigate.const import CACHE_DIR, RECORD_DIR
 from frigate.models import Event, Recordings
 from frigate.record.util import remove_empty_directories, sync_recordings
-from frigate.util.builtin import get_next_sunday_at_3
+from frigate.util.builtin import get_tomorrow_at_time
 
 logger = logging.getLogger(__name__)
 
@@ -181,9 +181,9 @@ class RecordingCleanup(threading.Thread):
     def run(self) -> None:
         # on startup sync recordings with disk if enabled
         if self.config.record.sync_on_startup:
-            sync_recordings()
+            sync_recordings(limited=False)
 
-        next_sync = get_next_sunday_at_3()
+        next_sync = get_tomorrow_at_time(3)
 
         # Expire tmp clips every minute, recordings and clean directories every hour.
         for counter in itertools.cycle(range(self.config.record.expire_interval)):
@@ -194,8 +194,8 @@ class RecordingCleanup(threading.Thread):
             self.clean_tmp_clips()
 
             if datetime.datetime.now().astimezone(datetime.timezone.utc) > next_sync:
-                sync_recordings()
-                next_sync = get_next_sunday_at_3()
+                sync_recordings(limited=True)
+                next_sync = get_tomorrow_at_time(3)
 
             if counter == 0:
                 self.expire_recordings()

frigate/record/util.py

@@ -1,5 +1,6 @@
 """Recordings Utilities."""
 
+import datetime
 import logging
 import os
 
@@ -27,13 +28,21 @@ def remove_empty_directories(directory: str) -> None:
             os.rmdir(path)
 
 
-def sync_recordings() -> None:
+def sync_recordings(limited: bool) -> None:
     """Check the db for stale recordings entries that don't exist in the filesystem."""
 
     def delete_db_entries_without_file(files_on_disk: list[str]) -> bool:
         """Delete db entries where file was deleted outside of frigate."""
-        # get all recordings in the db
-        recordings = Recordings.select(Recordings.id, Recordings.path)
+
+        if limited:
+            recordings = Recordings.select(Recordings.id, Recordings.path).where(
+                Recordings.start_time
+                >= (datetime.datetime.now() - datetime.timedelta(hours=36)).timestamp()
+            )
+        else:
+            # get all recordings in the db
+            recordings = Recordings.select(Recordings.id, Recordings.path)
+
         # Use pagination to process records in chunks
         page_size = 1000
         num_pages = (recordings.count() + page_size - 1) // page_size
@@ -97,12 +106,26 @@ def sync_recordings() -> None:
 
     logger.debug("Start sync recordings.")
 
-    # get all recordings files on disk and put them in a set
-    files_on_disk = {
-        os.path.join(root, file)
-        for root, _, files in os.walk(RECORD_DIR)
-        for file in files
-    }
+    if limited:
+        # get recording files from last 36 hours
+        hour_check = (
+            datetime.datetime.now().astimezone(datetime.timezone.utc)
+            - datetime.timedelta(hours=36)
+        ).strftime("%Y-%m-%d/%H")
+        files_on_disk = {
+            os.path.join(root, file)
+            for root, _, files in os.walk(RECORD_DIR)
+            for file in files
+            if file > hour_check
+        }
+    else:
+        # get all recordings files on disk and put them in a set
+        files_on_disk = {
+            os.path.join(root, file)
+            for root, _, files in os.walk(RECORD_DIR)
+            for file in files
+        }
+
     db_success = delete_db_entries_without_file(files_on_disk)
 
     # only try to cleanup files if db cleanup was successful

frigate/util/builtin.py

@@ -263,18 +263,9 @@ def find_by_key(dictionary, target_key):
     return None
 
 
-def get_tomorrow_at_2() -> datetime.datetime:
+def get_tomorrow_at_time(hour: int) -> datetime.datetime:
     """Returns the datetime of the following day at 2am."""
     tomorrow = datetime.datetime.now(get_localzone()) + datetime.timedelta(days=1)
-    return tomorrow.replace(hour=2, minute=0, second=0).astimezone(
+    return tomorrow.replace(hour=hour, minute=0, second=0).astimezone(
         datetime.timezone.utc
     )
-
-
-def get_next_sunday_at_3() -> datetime.datetime:
-    """Returns the datetime of the next Sunday at 3am."""
-    # adapted from https://stackoverflow.com/a/16770463
-    now = datetime.datetime.now(get_localzone())
-    diff = datetime.timedelta((13 - now.weekday()) % 7)
-    sunday = now + diff
-    return sunday.replace(hour=3, minute=0, second=0).astimezone(datetime.timezone.utc)

frigate/video.py

@@ -26,7 +26,7 @@ from frigate.ptz.autotrack import ptz_moving_at_frame_time
 from frigate.track import ObjectTracker
 from frigate.track.norfair_tracker import NorfairTracker
 from frigate.types import PTZMetricsTypes
-from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_2
+from frigate.util.builtin import EventsPerSecond, get_tomorrow_at_time
 from frigate.util.image import (
     FrameManager,
     SharedMemoryFrameManager,
@@ -528,7 +528,7 @@ def process_frames(
     fps = process_info["process_fps"]
     detection_fps = process_info["detection_fps"]
     current_frame_time = process_info["detection_frame"]
-    next_region_update = get_tomorrow_at_2()
+    next_region_update = get_tomorrow_at_time(2)
 
     fps_tracker = EventsPerSecond()
     fps_tracker.start()
@@ -550,7 +550,7 @@ def process_frames(
             except queue.Empty:
                 logger.error(f"Unable to get updated region grid for {camera_name}")
 
-            next_region_update = get_tomorrow_at_2()
+            next_region_update = get_tomorrow_at_time(2)
 
         try:
             if exit_on_empty: