Store camera labels in dict and other optimizations

frigate/events/cleanup.py

@@ -4,6 +4,7 @@ import datetime
 import logging
 import os
 import threading
+from enum import Enum
 from multiprocessing.synchronize import Event as MpEvent
 from pathlib import Path
 
@@ -14,6 +15,11 @@ from frigate.models import Event
 logger = logging.getLogger(__name__)
 
 
+class EventCleanupType(str, Enum):
+    clips = "clips"
+    snapshots = "snapshots"
+
+
 class EventCleanup(threading.Thread):
     def __init__(self, config: FrigateConfig, stop_event: MpEvent):
         threading.Thread.__init__(self)
@@ -21,25 +27,45 @@ class EventCleanup(threading.Thread):
         self.config = config
         self.stop_event = stop_event
         self.camera_keys = list(self.config.cameras.keys())
+        self.removed_camera_labels: list[str] = None
+        self.camera_labels: dict[str, list] = {}
 
-    def expire(self, media_type: str) -> None:
+    def get_removed_camera_labels(self) -> list[Event]:
-        # TODO: Refactor media_type to enum
+        if self.removed_camera_labels is None:
+            self.removed_camera_labels = list(
+                Event.select(Event.label)
+                .where(Event.camera.not_in(self.camera_keys))
+                .distinct()
+                .execute()
+            )
+
+        return self.removed_camera_labels
+
+    def get_camera_labels(self, camera: str) -> list[Event]:
+        if self.camera_labels.get(camera) is None:
+            self.camera_labels[camera] = list(
+                Event.select(Event.label)
+                .where(Event.camera == camera)
+                .distinct()
+                .execute()
+            )
+
+        return self.camera_labels[camera]
+
+    def expire(self, media_type: EventCleanupType) -> None:
         ## Expire events from unlisted cameras based on the global config
-        if media_type == "clips":
+        if media_type == EventCleanupType.clips:
             retain_config = self.config.record.events.retain
-            file_extension = "mp4"
+            file_extension = None  # mp4 clips are no longer stored in /clips
             update_params = {"has_clip": False}
         else:
             retain_config = self.config.snapshots.retain
             file_extension = "jpg"
             update_params = {"has_snapshot": False}
 
-        distinct_labels = (
+        distinct_labels = self.get_removed_camera_labels()
-            Event.select(Event.label)
-            .where(Event.camera.not_in(self.camera_keys))
-            .distinct()
-        )
 
+        ## Expire events from cameras no longer in the config
         # loop over object types in db
         for event in distinct_labels:
             # get expiration time for this label
@@ -78,14 +104,13 @@ class EventCleanup(threading.Thread):
 
         ## Expire events from cameras based on the camera config
         for name, camera in self.config.cameras.items():
-            if media_type == "clips":
+            if media_type == EventCleanupType.clips:
                 retain_config = camera.record.events.retain
             else:
                 retain_config = camera.snapshots.retain
+
             # get distinct objects in database for this camera
-            distinct_labels = (
+            distinct_labels = self.get_camera_labels(name)
-                Event.select(Event.label).where(Event.camera == name).distinct()
-            )
 
             # loop over object types in db
             for event in distinct_labels:
@@ -103,18 +128,22 @@ class EventCleanup(threading.Thread):
                     Event.label == event.label,
                     Event.retain_indefinitely == False,
                 )
+
                 # delete the grabbed clips from disk
+                # only snapshots are stored in /clips
+                # so no need to delete mp4 files
                 for event in expired_events:
-                    media_name = f"{event.camera}-{event.id}"
+                    if media_type == EventCleanupType.snapshots:
-                    media_path = Path(
+                        media_name = f"{event.camera}-{event.id}"
-                        f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}"
+                        media_path = Path(
-                    )
+                            f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}"
-                    media_path.unlink(missing_ok=True)
+                        )
-                    if file_extension == "jpg":
+                        media_path.unlink(missing_ok=True)
                         media_path = Path(
                             f"{os.path.join(CLIPS_DIR, media_name)}-clean.png"
                         )
                         media_path.unlink(missing_ok=True)
+
                 # update the clips attribute for the db entry
                 update_query = Event.update(update_params).where(
                     Event.camera == name,
@@ -149,8 +178,6 @@ class EventCleanup(threading.Thread):
             media_path.unlink(missing_ok=True)
             media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}-clean.png")
             media_path.unlink(missing_ok=True)
-            media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.mp4")
-            media_path.unlink(missing_ok=True)
 
         (
             Event.delete()
@@ -161,8 +188,8 @@ class EventCleanup(threading.Thread):
     def run(self) -> None:
         # only expire events every 5 minutes
         while not self.stop_event.wait(300):
-            self.expire("clips")
+            self.expire(EventCleanupType.clips)
-            self.expire("snapshots")
+            self.expire(EventCleanupType.snapshots)
             self.purge_duplicates()
 
             # drop events from db where has_clip and has_snapshot are false

frigate/storage.py

@@ -42,21 +42,21 @@ class StorageMaintainer(threading.Thread):
                     )
                 }
 
-            # calculate MB/hr
+                # calculate MB/hr
-            try:
+                try:
-                bandwidth = round(
+                    bandwidth = round(
-                    Recordings.select(fn.AVG(bandwidth_equation))
+                        Recordings.select(fn.AVG(bandwidth_equation))
-                    .where(Recordings.camera == camera, Recordings.segment_size > 0)
+                        .where(Recordings.camera == camera, Recordings.segment_size > 0)
-                    .limit(100)
+                        .limit(100)
-                    .scalar()
+                        .scalar()
-                    * 3600,
+                        * 3600,
-                    2,
+                        2,
-                )
+                    )
-            except TypeError:
+                except TypeError:
-                bandwidth = 0
+                    bandwidth = 0
 
-            self.camera_storage_stats[camera]["bandwidth"] = bandwidth
+                self.camera_storage_stats[camera]["bandwidth"] = bandwidth
-            logger.debug(f"{camera} has a bandwidth of {bandwidth} MiB/hr.")
+                logger.debug(f"{camera} has a bandwidth of {bandwidth} MiB/hr.")
 
     def calculate_camera_usages(self) -> dict[str, dict]:
         """Calculate the storage usage of each camera."""