Fix multi-path storage: clean all paths per cycle, prevent death loop

- storage.py: refactor check_storage_needs_cleanup(root) to check a
  specific path instead of returning the first needy one; run() now
  iterates all configured recording roots per 5-minute cycle so a
  stuck path can no longer starve the others
- storage.py: skip stale camera entries in _get_path_bandwidths to
  avoid KeyError when a camera is removed from config at runtime
- maintainer.py: delete partial output file when ffmpeg fails
  (ENOSPC), preventing orphaned files that consume disk space without
  a DB entry and block future conversion attempts

https://claude.ai/code/session_016bxjbVpx8DqpjysnGYmXdx
This commit is contained in:
Claude 2026-03-13 03:33:36 +00:00
parent eb2a684de1
commit c8ac840b85
No known key found for this signature in database
2 changed files with 23 additions and 19 deletions

View File

@ -620,6 +620,7 @@ class RecordingMaintainer(threading.Thread):
if p.returncode != 0:
logger.error(f"Unable to convert {cache_path} to {file_path}")
logger.error((await p.stderr.read()).decode("ascii"))
Path(file_path).unlink(missing_ok=True)
return None
else:
logger.debug(

View File

@ -205,6 +205,8 @@ class StorageMaintainer(threading.Thread):
bandwidth_per_path: dict[str, float] = {}
for camera, stats in self.camera_storage_stats.items():
if camera not in self.config.cameras:
continue
path = self.config.get_camera_recordings_path(camera)
bandwidth_per_path[path] = bandwidth_per_path.get(path, 0) + stats.get(
"bandwidth", 0
@ -212,24 +214,25 @@ class StorageMaintainer(threading.Thread):
return bandwidth_per_path
def check_storage_needs_cleanup(self) -> str | None:
"""Return recordings root path that needs cleanup, if any."""
def check_storage_needs_cleanup(self, recordings_root: str) -> bool:
"""Return True if the given recordings root path needs cleanup."""
# currently runs cleanup if less than 1 hour of space is left
# disk_usage should not spin up disks
for path, hourly_bandwidth in self._get_path_bandwidths().items():
try:
remaining_storage = round(shutil.disk_usage(path).free / pow(2, 20), 1)
except (FileNotFoundError, OSError):
continue
logger.debug(
f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage} for path {path}."
hourly_bandwidth = self._get_path_bandwidths().get(recordings_root, 0)
if not hourly_bandwidth:
return False
try:
remaining_storage = round(
shutil.disk_usage(recordings_root).free / pow(2, 20), 1
)
except (FileNotFoundError, OSError):
return False
if remaining_storage < hourly_bandwidth:
return path
logger.debug(
f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage} for path {recordings_root}."
)
return None
return remaining_storage < hourly_bandwidth
def reduce_storage_consumption(self, recordings_root: str) -> None:
"""Remove oldest hour of recordings."""
@ -403,11 +406,11 @@ class StorageMaintainer(threading.Thread):
self.calculate_camera_bandwidth()
logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.")
cleanup_root = self.check_storage_needs_cleanup()
if cleanup_root:
logger.info(
f"Less than 1 hour of recording space left for {cleanup_root}, running storage maintenance..."
)
self.reduce_storage_consumption(cleanup_root)
for recordings_root in self.config.get_recordings_paths():
if self.check_storage_needs_cleanup(recordings_root):
logger.info(
f"Less than 1 hour of recording space left for {recordings_root}, running storage maintenance..."
)
self.reduce_storage_consumption(recordings_root)
logger.info("Exiting storage maintainer...")