Fix multi-path storage: clean all paths per cycle, prevent death loop

- storage.py: refactor check_storage_needs_cleanup(root) to check a
  specific path instead of returning the first needy one; run() now
  iterates all configured recording roots per 5-minute cycle so a
  stuck path can no longer starve the others
- storage.py: skip stale camera entries in _get_path_bandwidths to
  avoid KeyError when a camera is removed from config at runtime
- maintainer.py: delete partial output file when ffmpeg fails
  (ENOSPC), preventing orphaned files that consume disk space without
  a DB entry and block future conversion attempts

https://claude.ai/code/session_016bxjbVpx8DqpjysnGYmXdx
This commit is contained in:
Claude 2026-03-13 03:33:36 +00:00
parent eb2a684de1
commit c8ac840b85
No known key found for this signature in database
2 changed files with 23 additions and 19 deletions

View File

@ -620,6 +620,7 @@ class RecordingMaintainer(threading.Thread):
if p.returncode != 0: if p.returncode != 0:
logger.error(f"Unable to convert {cache_path} to {file_path}") logger.error(f"Unable to convert {cache_path} to {file_path}")
logger.error((await p.stderr.read()).decode("ascii")) logger.error((await p.stderr.read()).decode("ascii"))
Path(file_path).unlink(missing_ok=True)
return None return None
else: else:
logger.debug( logger.debug(

View File

@ -205,6 +205,8 @@ class StorageMaintainer(threading.Thread):
bandwidth_per_path: dict[str, float] = {} bandwidth_per_path: dict[str, float] = {}
for camera, stats in self.camera_storage_stats.items(): for camera, stats in self.camera_storage_stats.items():
if camera not in self.config.cameras:
continue
path = self.config.get_camera_recordings_path(camera) path = self.config.get_camera_recordings_path(camera)
bandwidth_per_path[path] = bandwidth_per_path.get(path, 0) + stats.get( bandwidth_per_path[path] = bandwidth_per_path.get(path, 0) + stats.get(
"bandwidth", 0 "bandwidth", 0
@ -212,24 +214,25 @@ class StorageMaintainer(threading.Thread):
return bandwidth_per_path return bandwidth_per_path
def check_storage_needs_cleanup(self) -> str | None: def check_storage_needs_cleanup(self, recordings_root: str) -> bool:
"""Return recordings root path that needs cleanup, if any.""" """Return True if the given recordings root path needs cleanup."""
# currently runs cleanup if less than 1 hour of space is left # currently runs cleanup if less than 1 hour of space is left
# disk_usage should not spin up disks # disk_usage should not spin up disks
for path, hourly_bandwidth in self._get_path_bandwidths().items(): hourly_bandwidth = self._get_path_bandwidths().get(recordings_root, 0)
try: if not hourly_bandwidth:
remaining_storage = round(shutil.disk_usage(path).free / pow(2, 20), 1) return False
except (FileNotFoundError, OSError): try:
continue remaining_storage = round(
shutil.disk_usage(recordings_root).free / pow(2, 20), 1
logger.debug(
f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage} for path {path}."
) )
except (FileNotFoundError, OSError):
return False
if remaining_storage < hourly_bandwidth: logger.debug(
return path f"Storage cleanup check: {hourly_bandwidth} hourly with remaining storage: {remaining_storage} for path {recordings_root}."
)
return None return remaining_storage < hourly_bandwidth
def reduce_storage_consumption(self, recordings_root: str) -> None: def reduce_storage_consumption(self, recordings_root: str) -> None:
"""Remove oldest hour of recordings.""" """Remove oldest hour of recordings."""
@ -403,11 +406,11 @@ class StorageMaintainer(threading.Thread):
self.calculate_camera_bandwidth() self.calculate_camera_bandwidth()
logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.") logger.debug(f"Default camera bandwidths: {self.camera_storage_stats}.")
cleanup_root = self.check_storage_needs_cleanup() for recordings_root in self.config.get_recordings_paths():
if cleanup_root: if self.check_storage_needs_cleanup(recordings_root):
logger.info( logger.info(
f"Less than 1 hour of recording space left for {cleanup_root}, running storage maintenance..." f"Less than 1 hour of recording space left for {recordings_root}, running storage maintenance..."
) )
self.reduce_storage_consumption(cleanup_root) self.reduce_storage_consumption(recordings_root)
logger.info("Exiting storage maintainer...") logger.info("Exiting storage maintainer...")