administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-02-19 01:17:06 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Add configurable disk space cleanup trigger/target by time or space

Browse Source
This commit is contained in:
Jake 2024-12-13 11:08:19 -05:00
parent 6b12a45a95
commit cf27811323
6 changed files with 118 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/docs/configuration/record.md
View File

@ -68,7 +68,20 @@ record:
## Will Frigate delete old recordings if my storage runs out?
As of Frigate 0.12 if there is less than an hour left of storage, the oldest 2 hours of recordings will be deleted.
As of Frigate 0.12 if there is less than an hour left of storage, the oldest 2 hours of recordings will be deleted. As of Frigate 0.15 this can be configured to trigger when free disk space drops below a specified minutes of recording time or raw disk space available. You can also specify the amount of disk space to free, either in recording time or raw disk space. When both `_minutes` and `_space` configuration values are specified, the larger calculated number is used.
```yaml
record:
cleanup:
trigger_minutes: 60 # <- Trigger cleanup when 60 minutes of recording time is remaining
target_minutes: 120 # <- Free up 120 minutes of total recording time
trigger_space: 5000 # <- Trigger cleanup when 5GB of disk space is remaining
target_space: 100000 # <- Free up 100GB of total disk space
```
In the above example, a cleanup will trigger when either 60 minutes of recording time remains on disk, or when free space drops below 5GB. It will then free up space to ensure that a minimum of 120 minutes of recording time is available, or 100GB, whichever is larger.
**WARNING**: Do not set both `_minutes` and `_space` configuration values to zero as you may run out of disk space unless other recording retention policies are sufficient to maintain free space.
## Configuring Recording Retention

9
docs/docs/configuration/reference.md
View File

@ -415,6 +415,15 @@ record:
# Optional: Number of minutes to wait between cleanup runs (default: shown below)
# This can be used to reduce the frequency of deleting recording segments from disk if you want to minimize i/o
expire_interval: 60
cleanup:
# Optional: Minutes of remaining recording time on disk to start recording cleanup (default: shown below)
trigger_minutes: 60
# Optional: Minutes of remaining recording time on disk to free (default: shown below)
target_minutes: 120
# Optional: The free disk space in MiB to start recording cleanup (default: shown below)
trigger_space: 0
# Optional: The amount of free disk space in MiB to free (default: shown below)
target_space: 0
# Optional: Sync recordings with disk on startup and once a day (default: shown below).
sync_recordings: False
# Optional: Retention settings for recording

27
frigate/config/camera/record.py
View File

@ -15,6 +15,7 @@ __all__ = [
"EventsConfig",
"ReviewRetainConfig",
"RecordRetainConfig",
"RecordCleanupConfig",
"RetainModeEnum",
]
@ -31,6 +32,28 @@ class RecordRetainConfig(FrigateBaseModel):
days: float = Field(default=0, title="Default retention period.")
mode: RetainModeEnum = Field(default=RetainModeEnum.all, title="Retain mode.")
class RecordCleanupConfig(FrigateBaseModel):
trigger_minutes: int = Field(
default=60,
title="Minutes of remaining recording time remaining on disk to perform recording cleanup",
ge=0,
)
trigger_space: int = Field(
default=0,
title="Space remaining on disk to perform recording cleanup.",
ge=0,
)
target_minutes: int = Field(
default=120,
title="Minutes of remaining recording time to remove once below threshold.",
ge=0,
)
target_space: int = Field(
default=0,
title="Amount of space to free once below threshold.",
ge=0,
)
class ReviewRetainConfig(FrigateBaseModel):
days: float = Field(default=10, title="Default retention period.")
@ -75,6 +98,7 @@ class RecordConfig(FrigateBaseModel):
expire_interval: int = Field(
default=60,
title="Number of minutes to wait between cleanup runs.",
ge=0,
)
retain: RecordRetainConfig = Field(
default_factory=RecordRetainConfig, title="Record retention settings."
@ -88,6 +112,9 @@ class RecordConfig(FrigateBaseModel):
export: RecordExportConfig = Field(
default_factory=RecordExportConfig, title="Recording Export Config"
)
cleanup: RecordCleanupConfig = Field(
default_factory=RecordCleanupConfig, title="Recording Cleanup Config"
)
preview: RecordPreviewConfig = Field(
default_factory=RecordPreviewConfig, title="Recording Preview Config"
)

60
frigate/storage.py
View File

@ -17,6 +17,8 @@ bandwidth_equation = Recordings.segment_size / (
Recordings.end_time - Recordings.start_time
)
MAX_CALCULATED_BANDWIDTH = 10000 # 10Gb/hr
class StorageMaintainer(threading.Thread):
"""Maintain frigates recording storage."""
@ -52,6 +54,12 @@ class StorageMaintainer(threading.Thread):
* 3600,
2,
)
if bandwidth > MAX_CALCULATED_BANDWIDTH:
logger.warning(
f"{camera} has a bandwidth of {bandwidth} MB/hr which exceeds the expected maximum. This typically indicates an issue with the cameras recordings."
)
bandwidth = MAX_CALCULATED_BANDWIDTH
except TypeError:
bandwidth = 0
@ -78,26 +86,48 @@ class StorageMaintainer(threading.Thread):
return usages
def calculate_storage_recovery_target(self) -> int:
hourly_bandwidth = sum(
[b["bandwidth"] for b in self.camera_storage_stats.values()]
)
target_space_recording_time = (
hourly_bandwidth / 60
) * self.config.record.cleanup.target_minutes
target_space_minimum = self.config.record.cleanup.target_space
cleanup_target = round(
max(target_space_recording_time, target_space_minimum), 1
)
remaining_storage = round(shutil.disk_usage(RECORD_DIR).free / pow(2, 20), 1)
# The target is the total free space, so need to consider what is already free on disk
space_to_clean = cleanup_target - remaining_storage
logger.debug(
f"Will attempt to remove {space_to_clean} MB of recordings (target of {cleanup_target} MB - currently free {remaining_storage} MB). Config: {target_space_minimum} MB by space, {target_space_recording_time} MB by recording time ({self.config.record.cleanup.target_minutes} minutes)"
)
return space_to_clean
def check_storage_needs_cleanup(self) -> bool:
"""Return if storage needs cleanup."""
# currently runs cleanup if less than 1 hour of space is left
# disk_usage should not spin up disks
hourly_bandwidth = sum(
[b["bandwidth"] for b in self.camera_storage_stats.values()]
)
trigger_recording_space = (
hourly_bandwidth / 60
) * self.config.record.cleanup.trigger_minutes
trigger_space_min = self.config.record.cleanup.trigger_space
free_storage_target = round(max(trigger_recording_space, trigger_space_min), 1)
remaining_storage = round(shutil.disk_usage(RECORD_DIR).free / pow(2, 20), 1)
needs_cleanup = remaining_storage < free_storage_target
logger.debug(
f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage}."
f"Storage cleanup needed: {needs_cleanup}. {free_storage_target} MB to trigger cleanup (recording time: {hourly_bandwidth} MB * {self.config.record.cleanup.trigger_minutes} minutes = {trigger_recording_space}, min space: {trigger_space_min}) with remaining storage: {remaining_storage} MB."
)
return remaining_storage < hourly_bandwidth
return needs_cleanup
def reduce_storage_consumption(self) -> None:
"""Remove oldest hour of recordings."""
"""Remove oldest recordings to meet cleanup target."""
logger.debug("Starting storage cleanup.")
deleted_segments_size = 0
hourly_bandwidth = sum(
[b["bandwidth"] for b in self.camera_storage_stats.values()]
)
storage_clear_target = self.calculate_storage_recovery_target()
recordings: Recordings = (
Recordings.select(
@ -128,8 +158,8 @@ class StorageMaintainer(threading.Thread):
event_start = 0
deleted_recordings = set()
for recording in recordings:
# check if 1 hour of storage has been reclaimed
if deleted_segments_size > hourly_bandwidth:
# check if sufficient storage has been reclaimed
if deleted_segments_size > storage_clear_target:
break
keep = False
@ -168,9 +198,9 @@ class StorageMaintainer(threading.Thread):
pass
# check if need to delete retained segments
if deleted_segments_size < hourly_bandwidth:
if deleted_segments_size < storage_clear_target:
logger.error(
f"Could not clear {hourly_bandwidth} MB, currently {deleted_segments_size} MB have been cleared. Retained recordings must be deleted."
f"Could not clear {storage_clear_target} MB, currently {deleted_segments_size} MB have been cleared. Retained recordings must be deleted."
)
recordings = (
Recordings.select(
@ -184,7 +214,7 @@ class StorageMaintainer(threading.Thread):
)
for recording in recordings:
if deleted_segments_size > hourly_bandwidth:
if deleted_segments_size > storage_clear_target:
break
try:
@ -195,7 +225,9 @@ class StorageMaintainer(threading.Thread):
# this file was not found so we must assume no space was cleaned up
pass
else:
logger.info(f"Cleaned up {deleted_segments_size} MB of recordings")
logger.info(
f"Cleaned up {round(deleted_segments_size, 1)} MB of recordings (target was {storage_clear_target} MB)"
)
logger.debug(f"Expiring {len(deleted_recordings)} recordings")
# delete up to 100,000 at a time
@ -218,7 +250,7 @@ class StorageMaintainer(threading.Thread):
if self.check_storage_needs_cleanup():
logger.info(
"Less than 1 hour of recording space left, running storage maintenance..."
"Less than desired recording space left, running storage maintenance..."
)
self.reduce_storage_consumption()

10
frigate/test/test_storage.py
View File

@ -3,7 +3,9 @@ import logging
import os
import tempfile
import unittest
import shutil
from unittest.mock import MagicMock
from unittest.mock import patch
from peewee import DoesNotExist
from peewee_migrate import Router
@ -139,10 +141,12 @@ class TestHttp(unittest.TestCase):
"front_door": {"bandwidth": 0, "needs_refresh": True},
}
def test_storage_cleanup(self):
@patch('frigate.storage.shutil.disk_usage')
def test_storage_cleanup(self, mock_disk_usage):
"""Ensure that all recordings are cleaned up when necessary."""
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock())
mock_disk_usage.return_value = shutil._ntuple_diskusage(total=10000, used=5000, free=5000)
id = "123456.keep"
time_keep = datetime.datetime.now().timestamp()
@ -209,10 +213,12 @@ class TestHttp(unittest.TestCase):
Recordings.get(Recordings.id == rec_d2_id)
Recordings.get(Recordings.id == rec_d3_id)
def test_storage_cleanup_keeps_retained(self):
@patch('frigate.storage.shutil.disk_usage')
def test_storage_cleanup_keeps_retained(self, mock_disk_usage):
"""Ensure that all recordings are cleaned up when necessary."""
config = FrigateConfig(**self.minimal_config)
storage = StorageMaintainer(config, MagicMock())
mock_disk_usage.return_value = shutil._ntuple_diskusage(total=10000, used=5000, free=5000)
id = "123456.keep"
time_keep = datetime.datetime.now().timestamp()

12
web/src/types/frigateConfig.ts
View File

@ -158,6 +158,12 @@ export interface CameraConfig {
};
};
expire_interval: number;
cleanup: {
trigger_minutes: number;
target_minutes: number;
trigger_space: number;
target_space: number;
}
export: {
timelapse_args: string;
};
@ -400,6 +406,12 @@ export interface FrigateConfig {
};
};
expire_interval: number;
cleanup: {
trigger_minutes: number;
target_minutes: number;
trigger_space: number;
target_space: number;
}
export: {
timelapse_args: string;
};