mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-03-18 06:08:22 +03:00
Fix multi-path storage: clean all paths per cycle, prevent death loop
- storage.py: refactor check_storage_needs_cleanup(root) to check a specific path instead of returning the first needy one; run() now iterates all configured recording roots per 5-minute cycle so a stuck path can no longer starve the others - storage.py: skip stale camera entries in _get_path_bandwidths to avoid KeyError when a camera is removed from config at runtime - maintainer.py: delete partial output file when ffmpeg fails (ENOSPC), preventing orphaned files that consume disk space without a DB entry and block future conversion attempts https://claude.ai/code/session_016bxjbVpx8DqpjysnGYmXdx
This commit is contained in:
parent
eb2a684de1
commit
c8ac840b85
@ -620,6 +620,7 @@ class RecordingMaintainer(threading.Thread):
|
|||||||
if p.returncode != 0:
|
if p.returncode != 0:
|
||||||
logger.error(f"Unable to convert {cache_path} to {file_path}")
|
logger.error(f"Unable to convert {cache_path} to {file_path}")
|
||||||
logger.error((await p.stderr.read()).decode("ascii"))
|
logger.error((await p.stderr.read()).decode("ascii"))
|
||||||
|
Path(file_path).unlink(missing_ok=True)
|
||||||
return None
|
return None
|
||||||
else:
|
else:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
|||||||
@ -205,6 +205,8 @@ class StorageMaintainer(threading.Thread):
|
|||||||
bandwidth_per_path: dict[str, float] = {}
|
bandwidth_per_path: dict[str, float] = {}
|
||||||
|
|
||||||
for camera, stats in self.camera_storage_stats.items():
|
for camera, stats in self.camera_storage_stats.items():
|
||||||
|
if camera not in self.config.cameras:
|
||||||
|
continue
|
||||||
path = self.config.get_camera_recordings_path(camera)
|
path = self.config.get_camera_recordings_path(camera)
|
||||||
bandwidth_per_path[path] = bandwidth_per_path.get(path, 0) + stats.get(
|
bandwidth_per_path[path] = bandwidth_per_path.get(path, 0) + stats.get(
|
||||||
"bandwidth", 0
|
"bandwidth", 0
|
||||||
@ -212,24 +214,25 @@ class StorageMaintainer(threading.Thread):
|
|||||||
|
|
||||||
return bandwidth_per_path
|
return bandwidth_per_path
|
||||||
|
|
||||||
def check_storage_needs_cleanup(self) -> str | None:
|
def check_storage_needs_cleanup(self, recordings_root: str) -> bool:
|
||||||
"""Return recordings root path that needs cleanup, if any."""
|
"""Return True if the given recordings root path needs cleanup."""
|
||||||
# currently runs cleanup if less than 1 hour of space is left
|
# currently runs cleanup if less than 1 hour of space is left
|
||||||
# disk_usage should not spin up disks
|
# disk_usage should not spin up disks
|
||||||
for path, hourly_bandwidth in self._get_path_bandwidths().items():
|
hourly_bandwidth = self._get_path_bandwidths().get(recordings_root, 0)
|
||||||
try:
|
if not hourly_bandwidth:
|
||||||
remaining_storage = round(shutil.disk_usage(path).free / pow(2, 20), 1)
|
return False
|
||||||
except (FileNotFoundError, OSError):
|
try:
|
||||||
continue
|
remaining_storage = round(
|
||||||
|
shutil.disk_usage(recordings_root).free / pow(2, 20), 1
|
||||||
logger.debug(
|
|
||||||
f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage} for path {path}."
|
|
||||||
)
|
)
|
||||||
|
except (FileNotFoundError, OSError):
|
||||||
|
return False
|
||||||
|
|
||||||
if remaining_storage < hourly_bandwidth:
|
logger.debug(
|
||||||
return path
|
f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage} for path {recordings_root}."
|
||||||
|
)
|
||||||
|
|
||||||
return None
|
return remaining_storage < hourly_bandwidth
|
||||||
|
|
||||||
def reduce_storage_consumption(self, recordings_root: str) -> None:
|
def reduce_storage_consumption(self, recordings_root: str) -> None:
|
||||||
"""Remove oldest hour of recordings."""
|
"""Remove oldest hour of recordings."""
|
||||||
@ -403,11 +406,11 @@ class StorageMaintainer(threading.Thread):
|
|||||||
self.calculate_camera_bandwidth()
|
self.calculate_camera_bandwidth()
|
||||||
logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.")
|
logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.")
|
||||||
|
|
||||||
cleanup_root = self.check_storage_needs_cleanup()
|
for recordings_root in self.config.get_recordings_paths():
|
||||||
if cleanup_root:
|
if self.check_storage_needs_cleanup(recordings_root):
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Less than 1 hour of recording space left for {cleanup_root}, running storage maintenance..."
|
f"Less than 1 hour of recording space left for {recordings_root}, running storage maintenance..."
|
||||||
)
|
)
|
||||||
self.reduce_storage_consumption(cleanup_root)
|
self.reduce_storage_consumption(recordings_root)
|
||||||
|
|
||||||
logger.info("Exiting storage maintainer...")
|
logger.info("Exiting storage maintainer...")
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user