Revert "Early 0.18 work (#22138)" (#22142)

This reverts commit d24b96d3bb.
2026-08-03 01:22:17 +03:00 · 2026-02-26 21:27:31 -07:00
parent d24b96d3bb
commit 451d6f5c22
107 changed files with 1049 additions and 6765 deletions
@@ -13,8 +13,9 @@ from playhouse.sqlite_ext import SqliteExtDatabase
 from frigate.config import CameraConfig, FrigateConfig, RetainModeEnum
 from frigate.const import CACHE_DIR, CLIPS_DIR, MAX_WAL_SIZE, RECORD_DIR
 from frigate.models import Previews, Recordings, ReviewSegment, UserReviewStatus
+from frigate.record.util import remove_empty_directories, sync_recordings
 from frigate.util.builtin import clear_and_unlink
-from frigate.util.media import remove_empty_directories
+from frigate.util.time import get_tomorrow_at_time

 logger = logging.getLogger(__name__)

@@ -60,7 +61,7 @@ class RecordingCleanup(threading.Thread):
            db.execute_sql("PRAGMA wal_checkpoint(TRUNCATE);")
            db.close()

-    def expire_review_segments(self, config: CameraConfig, now: datetime) -> set[Path]:
+    def expire_review_segments(self, config: CameraConfig, now: datetime) -> None:
        """Delete review segments that are expired"""
        alert_expire_date = (
            now - datetime.timedelta(days=config.record.alerts.retain.days)
@@ -84,12 +85,9 @@ class RecordingCleanup(threading.Thread):
            .namedtuples()
        )

-        maybe_empty_dirs = set()
        thumbs_to_delete = list(map(lambda x: x[1], expired_reviews))
        for thumb_path in thumbs_to_delete:
-            thumb_path = Path(thumb_path)
-            thumb_path.unlink(missing_ok=True)
-            maybe_empty_dirs.add(thumb_path.parent)
+            Path(thumb_path).unlink(missing_ok=True)

        max_deletes = 100000
        deleted_reviews_list = list(map(lambda x: x[0], expired_reviews))
@@ -102,15 +100,13 @@ class RecordingCleanup(threading.Thread):
                << deleted_reviews_list[i : i + max_deletes]
            ).execute()

-        return maybe_empty_dirs
-
    def expire_existing_camera_recordings(
        self,
        continuous_expire_date: float,
        motion_expire_date: float,
        config: CameraConfig,
        reviews: ReviewSegment,
-    ) -> set[Path]:
+    ) -> None:
        """Delete recordings for existing camera based on retention config."""
        # Get the timestamp for cutoff of retained days

@@ -141,8 +137,6 @@ class RecordingCleanup(threading.Thread):
            .iterator()
        )

-        maybe_empty_dirs = set()
-
        # loop over recordings and see if they overlap with any non-expired reviews
        # TODO: expire segments based on segment stats according to config
        review_start = 0
@@ -197,10 +191,8 @@ class RecordingCleanup(threading.Thread):
                )
                or (mode == RetainModeEnum.active_objects and recording.objects == 0)
            ):
-                recording_path = Path(recording.path)
-                recording_path.unlink(missing_ok=True)
+                Path(recording.path).unlink(missing_ok=True)
                deleted_recordings.add(recording.id)
-                maybe_empty_dirs.add(recording_path.parent)
            else:
                kept_recordings.append((recording.start_time, recording.end_time))

@@ -261,10 +253,8 @@ class RecordingCleanup(threading.Thread):

            # Delete previews without any relevant recordings
            if not keep:
-                preview_path = Path(preview.path)
-                preview_path.unlink(missing_ok=True)
+                Path(preview.path).unlink(missing_ok=True)
                deleted_previews.add(preview.id)
-                maybe_empty_dirs.add(preview_path.parent)

        # expire previews
        logger.debug(f"Expiring {len(deleted_previews)} previews")
@@ -276,9 +266,7 @@ class RecordingCleanup(threading.Thread):
                Previews.id << deleted_previews_list[i : i + max_deletes]
            ).execute()

-        return maybe_empty_dirs
-
-    def expire_recordings(self) -> set[Path]:
+    def expire_recordings(self) -> None:
        """Delete recordings based on retention config."""
        logger.debug("Start expire recordings.")
        logger.debug("Start deleted cameras.")
@@ -303,14 +291,10 @@ class RecordingCleanup(threading.Thread):
            .iterator()
        )

-        maybe_empty_dirs = set()
-
        deleted_recordings = set()
        for recording in no_camera_recordings:
-            recording_path = Path(recording.path)
-            recording_path.unlink(missing_ok=True)
+            Path(recording.path).unlink(missing_ok=True)
            deleted_recordings.add(recording.id)
-            maybe_empty_dirs.add(recording_path.parent)

        logger.debug(f"Expiring {len(deleted_recordings)} recordings")
        # delete up to 100,000 at a time
@@ -327,7 +311,7 @@ class RecordingCleanup(threading.Thread):
            logger.debug(f"Start camera: {camera}.")
            now = datetime.datetime.now()

-            maybe_empty_dirs |= self.expire_review_segments(config, now)
+            self.expire_review_segments(config, now)
            continuous_expire_date = (
                now - datetime.timedelta(days=config.record.continuous.days)
            ).timestamp()
@@ -357,7 +341,7 @@ class RecordingCleanup(threading.Thread):
                .namedtuples()
            )

-            maybe_empty_dirs |= self.expire_existing_camera_recordings(
+            self.expire_existing_camera_recordings(
                continuous_expire_date, motion_expire_date, config, reviews
            )
            logger.debug(f"End camera: {camera}.")
@@ -365,9 +349,12 @@ class RecordingCleanup(threading.Thread):
        logger.debug("End all cameras.")
        logger.debug("End expire recordings.")

-        return maybe_empty_dirs
-
    def run(self) -> None:
+        # on startup sync recordings with disk if enabled
+        if self.config.record.sync_recordings:
+            sync_recordings(limited=False)
+            next_sync = get_tomorrow_at_time(3)
+
        # Expire tmp clips every minute, recordings and clean directories every hour.
        for counter in itertools.cycle(range(self.config.record.expire_interval)):
            if self.stop_event.wait(60):
@@ -376,8 +363,16 @@ class RecordingCleanup(threading.Thread):

            self.clean_tmp_previews()

+            if (
+                self.config.record.sync_recordings
+                and datetime.datetime.now().astimezone(datetime.timezone.utc)
+                > next_sync
+            ):
+                sync_recordings(limited=True)
+                next_sync = get_tomorrow_at_time(3)
+
            if counter == 0:
                self.clean_tmp_clips()
-                maybe_empty_dirs = self.expire_recordings()
-                remove_empty_directories(Path(RECORD_DIR), maybe_empty_dirs)
+                self.expire_recordings()
+                remove_empty_directories(RECORD_DIR)
                self.truncate_wal()
@@ -33,7 +33,6 @@ from frigate.util.time import is_current_hour
 logger = logging.getLogger(__name__)


-DEFAULT_TIME_LAPSE_FFMPEG_ARGS = "-vf setpts=0.04*PTS -r 30"
 TIMELAPSE_DATA_INPUT_ARGS = "-an -skip_frame nokey"


@@ -41,6 +40,11 @@ def lower_priority():
    os.nice(PROCESS_PRIORITY_LOW)


+class PlaybackFactorEnum(str, Enum):
+    realtime = "realtime"
+    timelapse_25x = "timelapse_25x"
+
+
 class PlaybackSourceEnum(str, Enum):
    recordings = "recordings"
    preview = "preview"
@@ -58,11 +62,8 @@ class RecordingExporter(threading.Thread):
        image: Optional[str],
        start_time: int,
        end_time: int,
+        playback_factor: PlaybackFactorEnum,
        playback_source: PlaybackSourceEnum,
-        export_case_id: Optional[str] = None,
-        ffmpeg_input_args: Optional[str] = None,
-        ffmpeg_output_args: Optional[str] = None,
-        cpu_fallback: bool = False,
    ) -> None:
        super().__init__()
        self.config = config
@@ -72,11 +73,8 @@ class RecordingExporter(threading.Thread):
        self.user_provided_image = image
        self.start_time = start_time
        self.end_time = end_time
+        self.playback_factor = playback_factor
        self.playback_source = playback_source
-        self.export_case_id = export_case_id
-        self.ffmpeg_input_args = ffmpeg_input_args
-        self.ffmpeg_output_args = ffmpeg_output_args
-        self.cpu_fallback = cpu_fallback

        # ensure export thumb dir
        Path(os.path.join(CLIPS_DIR, "export")).mkdir(exist_ok=True)
@@ -181,16 +179,9 @@ class RecordingExporter(threading.Thread):

        return thumb_path

-    def get_record_export_command(
-        self, video_path: str, use_hwaccel: bool = True
-    ) -> list[str]:
-        # handle case where internal port is a string with ip:port
-        internal_port = self.config.networking.listen.internal
-        if type(internal_port) is str:
-            internal_port = int(internal_port.split(":")[-1])
-
+    def get_record_export_command(self, video_path: str) -> list[str]:
        if (self.end_time - self.start_time) <= MAX_PLAYLIST_SECONDS:
-            playlist_lines = f"http://127.0.0.1:{internal_port}/vod/{self.camera}/start/{self.start_time}/end/{self.end_time}/index.m3u8"
+            playlist_lines = f"http://127.0.0.1:5000/vod/{self.camera}/start/{self.start_time}/end/{self.end_time}/index.m3u8"
            ffmpeg_input = (
                f"-y -protocol_whitelist pipe,file,http,tcp -i {playlist_lines}"
            )
@@ -222,30 +213,25 @@ class RecordingExporter(threading.Thread):
            for page in range(1, num_pages + 1):
                playlist = export_recordings.paginate(page, page_size)
                playlist_lines.append(
-                    f"file 'http://127.0.0.1:{internal_port}/vod/{self.camera}/start/{float(playlist[0].start_time)}/end/{float(playlist[-1].end_time)}/index.m3u8'"
+                    f"file 'http://127.0.0.1:5000/vod/{self.camera}/start/{float(playlist[0].start_time)}/end/{float(playlist[-1].end_time)}/index.m3u8'"
                )

            ffmpeg_input = "-y -protocol_whitelist pipe,file,http,tcp -f concat -safe 0 -i /dev/stdin"

-        if self.ffmpeg_input_args is not None and self.ffmpeg_output_args is not None:
-            hwaccel_args = (
-                self.config.cameras[self.camera].record.export.hwaccel_args
-                if use_hwaccel
-                else None
-            )
+        if self.playback_factor == PlaybackFactorEnum.realtime:
+            ffmpeg_cmd = (
+                f"{self.config.ffmpeg.ffmpeg_path} -hide_banner {ffmpeg_input} -c copy -movflags +faststart"
+            ).split(" ")
+        elif self.playback_factor == PlaybackFactorEnum.timelapse_25x:
            ffmpeg_cmd = (
                parse_preset_hardware_acceleration_encode(
                    self.config.ffmpeg.ffmpeg_path,
-                    hwaccel_args,
-                    f"{self.ffmpeg_input_args} -an {ffmpeg_input}".strip(),
-                    f"{self.ffmpeg_output_args} -movflags +faststart".strip(),
+                    self.config.ffmpeg.hwaccel_args,
+                    f"-an {ffmpeg_input}",
+                    f"{self.config.cameras[self.camera].record.export.timelapse_args} -movflags +faststart",
                    EncodeTypeEnum.timelapse,
                )
            ).split(" ")
-        else:
-            ffmpeg_cmd = (
-                f"{self.config.ffmpeg.ffmpeg_path} -hide_banner {ffmpeg_input} -c copy -movflags +faststart"
-            ).split(" ")

        # add metadata
        title = f"Frigate Recording for {self.camera}, {self.get_datetime_from_timestamp(self.start_time)} - {self.get_datetime_from_timestamp(self.end_time)}"
@@ -255,9 +241,7 @@ class RecordingExporter(threading.Thread):

        return ffmpeg_cmd, playlist_lines

-    def get_preview_export_command(
-        self, video_path: str, use_hwaccel: bool = True
-    ) -> list[str]:
+    def get_preview_export_command(self, video_path: str) -> list[str]:
        playlist_lines = []
        codec = "-c copy"

@@ -325,25 +309,20 @@ class RecordingExporter(threading.Thread):
            "-y -protocol_whitelist pipe,file,tcp -f concat -safe 0 -i /dev/stdin"
        )

-        if self.ffmpeg_input_args is not None and self.ffmpeg_output_args is not None:
-            hwaccel_args = (
-                self.config.cameras[self.camera].record.export.hwaccel_args
-                if use_hwaccel
-                else None
-            )
+        if self.playback_factor == PlaybackFactorEnum.realtime:
+            ffmpeg_cmd = (
+                f"{self.config.ffmpeg.ffmpeg_path} -hide_banner {ffmpeg_input} {codec} -movflags +faststart {video_path}"
+            ).split(" ")
+        elif self.playback_factor == PlaybackFactorEnum.timelapse_25x:
            ffmpeg_cmd = (
                parse_preset_hardware_acceleration_encode(
                    self.config.ffmpeg.ffmpeg_path,
-                    hwaccel_args,
-                    f"{self.ffmpeg_input_args} {TIMELAPSE_DATA_INPUT_ARGS} {ffmpeg_input}".strip(),
-                    f"{self.ffmpeg_output_args} -movflags +faststart {video_path}".strip(),
+                    self.config.ffmpeg.hwaccel_args,
+                    f"{TIMELAPSE_DATA_INPUT_ARGS} {ffmpeg_input}",
+                    f"{self.config.cameras[self.camera].record.export.timelapse_args} -movflags +faststart {video_path}",
                    EncodeTypeEnum.timelapse,
                )
            ).split(" ")
-        else:
-            ffmpeg_cmd = (
-                f"{self.config.ffmpeg.ffmpeg_path} -hide_banner {ffmpeg_input} {codec} -movflags +faststart {video_path}"
-            ).split(" ")

        # add metadata
        title = f"Frigate Preview for {self.camera}, {self.get_datetime_from_timestamp(self.start_time)} - {self.get_datetime_from_timestamp(self.end_time)}"
@@ -369,20 +348,17 @@ class RecordingExporter(threading.Thread):
        video_path = f"{EXPORT_DIR}/{self.camera}_{filename_start_datetime}-{filename_end_datetime}_{cleaned_export_id}.mp4"
        thumb_path = self.save_thumbnail(self.export_id)

-        export_values = {
-            Export.id: self.export_id,
-            Export.camera: self.camera,
-            Export.name: export_name,
-            Export.date: self.start_time,
-            Export.video_path: video_path,
-            Export.thumb_path: thumb_path,
-            Export.in_progress: True,
-        }
-
-        if self.export_case_id is not None:
-            export_values[Export.export_case] = self.export_case_id
-
-        Export.insert(export_values).execute()
+        Export.insert(
+            {
+                Export.id: self.export_id,
+                Export.camera: self.camera,
+                Export.name: export_name,
+                Export.date: self.start_time,
+                Export.video_path: video_path,
+                Export.thumb_path: thumb_path,
+                Export.in_progress: True,
+            }
+        ).execute()

        try:
            if self.playback_source == PlaybackSourceEnum.recordings:
@@ -400,34 +376,6 @@ class RecordingExporter(threading.Thread):
            capture_output=True,
        )

-        # If export failed and cpu_fallback is enabled, retry without hwaccel
-        if (
-            p.returncode != 0
-            and self.cpu_fallback
-            and self.ffmpeg_input_args is not None
-            and self.ffmpeg_output_args is not None
-        ):
-            logger.warning(
-                f"Export with hardware acceleration failed, retrying without hwaccel for {self.export_id}"
-            )
-
-            if self.playback_source == PlaybackSourceEnum.recordings:
-                ffmpeg_cmd, playlist_lines = self.get_record_export_command(
-                    video_path, use_hwaccel=False
-                )
-            else:
-                ffmpeg_cmd, playlist_lines = self.get_preview_export_command(
-                    video_path, use_hwaccel=False
-                )
-
-            p = sp.run(
-                ffmpeg_cmd,
-                input="\n".join(playlist_lines),
-                encoding="ascii",
-                preexec_fn=lower_priority,
-                capture_output=True,
-            )
-
        if p.returncode != 0:
            logger.error(
                f"Failed to export {self.playback_source.value} for command {' '.join(ffmpeg_cmd)}"
@@ -0,0 +1,147 @@
+"""Recordings Utilities."""
+
+import datetime
+import logging
+import os
+
+from peewee import DatabaseError, chunked
+
+from frigate.const import RECORD_DIR
+from frigate.models import Recordings, RecordingsToDelete
+
+logger = logging.getLogger(__name__)
+
+
+def remove_empty_directories(directory: str) -> None:
+    # list all directories recursively and sort them by path,
+    # longest first
+    paths = sorted(
+        [x[0] for x in os.walk(directory)],
+        key=lambda p: len(str(p)),
+        reverse=True,
+    )
+    for path in paths:
+        # don't delete the parent
+        if path == directory:
+            continue
+        if len(os.listdir(path)) == 0:
+            os.rmdir(path)
+
+
+def sync_recordings(limited: bool) -> None:
+    """Check the db for stale recordings entries that don't exist in the filesystem."""
+
+    def delete_db_entries_without_file(check_timestamp: float) -> bool:
+        """Delete db entries where file was deleted outside of frigate."""
+
+        if limited:
+            recordings = Recordings.select(Recordings.id, Recordings.path).where(
+                Recordings.start_time >= check_timestamp
+            )
+        else:
+            # get all recordings in the db
+            recordings = Recordings.select(Recordings.id, Recordings.path)
+
+        # Use pagination to process records in chunks
+        page_size = 1000
+        num_pages = (recordings.count() + page_size - 1) // page_size
+        recordings_to_delete = set()
+
+        for page in range(num_pages):
+            for recording in recordings.paginate(page, page_size):
+                if not os.path.exists(recording.path):
+                    recordings_to_delete.add(recording.id)
+
+        if len(recordings_to_delete) == 0:
+            return True
+
+        logger.info(
+            f"Deleting {len(recordings_to_delete)} recording DB entries with missing files"
+        )
+
+        # convert back to list of dictionaries for insertion
+        recordings_to_delete = [
+            {"id": recording_id} for recording_id in recordings_to_delete
+        ]
+
+        if float(len(recordings_to_delete)) / max(1, recordings.count()) > 0.5:
+            logger.warning(
+                f"Deleting {(len(recordings_to_delete) / max(1, recordings.count()) * 100):.2f}% of recordings DB entries, could be due to configuration error. Aborting..."
+            )
+            return False
+
+        # create a temporary table for deletion
+        RecordingsToDelete.create_table(temporary=True)
+
+        # insert ids to the temporary table
+        max_inserts = 1000
+        for batch in chunked(recordings_to_delete, max_inserts):
+            RecordingsToDelete.insert_many(batch).execute()
+
+        try:
+            # delete records in the main table that exist in the temporary table
+            query = Recordings.delete().where(
+                Recordings.id.in_(RecordingsToDelete.select(RecordingsToDelete.id))
+            )
+            query.execute()
+        except DatabaseError as e:
+            logger.error(f"Database error during recordings db cleanup: {e}")
+
+        return True
+
+    def delete_files_without_db_entry(files_on_disk: list[str]):
+        """Delete files where file is not inside frigate db."""
+        files_to_delete = []
+
+        for file in files_on_disk:
+            if not Recordings.select().where(Recordings.path == file).exists():
+                files_to_delete.append(file)
+
+        if len(files_to_delete) == 0:
+            return True
+
+        logger.info(
+            f"Deleting {len(files_to_delete)} recordings files with missing DB entries"
+        )
+
+        if float(len(files_to_delete)) / max(1, len(files_on_disk)) > 0.5:
+            logger.debug(
+                f"Deleting {(len(files_to_delete) / max(1, len(files_on_disk)) * 100):.2f}% of recordings DB entries, could be due to configuration error. Aborting..."
+            )
+            return False
+
+        for file in files_to_delete:
+            os.unlink(file)
+
+        return True
+
+    logger.debug("Start sync recordings.")
+
+    # start checking on the hour 36 hours ago
+    check_point = datetime.datetime.now().replace(
+        minute=0, second=0, microsecond=0
+    ).astimezone(datetime.timezone.utc) - datetime.timedelta(hours=36)
+    db_success = delete_db_entries_without_file(check_point.timestamp())
+
+    # only try to cleanup files if db cleanup was successful
+    if db_success:
+        if limited:
+            # get recording files from last 36 hours
+            hour_check = f"{RECORD_DIR}/{check_point.strftime('%Y-%m-%d/%H')}"
+            files_on_disk = {
+                os.path.join(root, file)
+                for root, _, files in os.walk(RECORD_DIR)
+                for file in files
+                if root > hour_check
+            }
+        else:
+            # get all recordings files on disk and put them in a set
+            files_on_disk = {
+                os.path.join(root, file)
+                for root, _, files in os.walk(RECORD_DIR)
+                for file in files
+            }
+
+        delete_files_without_db_entry(files_on_disk)
+
+    logger.debug("End sync recordings.")