mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-03-20 07:08:23 +03:00
fix: address code review findings for rollover storage
- C1: Rewrite breakdown endpoint with JOIN queries (eliminates N+1) - C2: Use getUnitSize() for consistent MiB/GiB display - C3: Add retain_policy to global FrigateConfig TypeScript type - I1: Only fetch breakdown data when rollover mode is active - I2: Handle NULL end_time for in-progress events/reviews - I5: Add model_validator warning when rollover + days are both set - S4: Hide breakdown when total is 0
This commit is contained in:
parent
154246331c
commit
dad34a71cc
@ -63,91 +63,90 @@ def get_recordings_storage_usage(request: Request):
|
|||||||
dependencies=[Depends(allow_any_authenticated())],
|
dependencies=[Depends(allow_any_authenticated())],
|
||||||
)
|
)
|
||||||
def get_recordings_storage_breakdown(request: Request):
|
def get_recordings_storage_breakdown(request: Request):
|
||||||
"""Return storage usage broken down by category: overwritable, event retention, and protected."""
|
"""Return storage usage in MB broken down by category: overwritable, event retention, and protected."""
|
||||||
|
|
||||||
now = datetime.now().timestamp()
|
now = datetime.now().timestamp()
|
||||||
|
|
||||||
# 1. Protected (indefinite): recordings overlapping retain_indefinitely events
|
# 1. Protected (indefinite): recordings overlapping retain_indefinitely events
|
||||||
retained_events = (
|
# Uses a JOIN to avoid N+1 query problem
|
||||||
Event.select(Event.camera, Event.start_time, Event.end_time)
|
protected_query = (
|
||||||
.where(Event.retain_indefinitely == True, Event.has_clip == True)
|
Recordings.select(
|
||||||
|
Recordings.id,
|
||||||
|
Recordings.segment_size,
|
||||||
|
)
|
||||||
|
.join(
|
||||||
|
Event,
|
||||||
|
on=(
|
||||||
|
(Event.camera == Recordings.camera)
|
||||||
|
& (Event.start_time < Recordings.end_time)
|
||||||
|
& (
|
||||||
|
(Event.end_time.is_null())
|
||||||
|
| (Event.end_time > Recordings.start_time)
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
Event.retain_indefinitely == True,
|
||||||
|
Event.has_clip == True,
|
||||||
|
Recordings.segment_size > 0,
|
||||||
|
)
|
||||||
|
.distinct()
|
||||||
.namedtuples()
|
.namedtuples()
|
||||||
)
|
)
|
||||||
|
|
||||||
protected_size = 0.0
|
protected_size = 0.0
|
||||||
protected_recording_ids = set()
|
protected_recording_ids = set()
|
||||||
for event in retained_events:
|
for rec in protected_query:
|
||||||
end_time_clause = (
|
protected_recording_ids.add(rec.id)
|
||||||
Recordings.start_time < event.end_time
|
protected_size += rec.segment_size
|
||||||
if event.end_time is not None
|
|
||||||
else True
|
|
||||||
)
|
|
||||||
overlapping = (
|
|
||||||
Recordings.select(Recordings.id, Recordings.segment_size)
|
|
||||||
.where(
|
|
||||||
Recordings.camera == event.camera,
|
|
||||||
end_time_clause,
|
|
||||||
Recordings.end_time > event.start_time,
|
|
||||||
Recordings.segment_size > 0,
|
|
||||||
)
|
|
||||||
.namedtuples()
|
|
||||||
)
|
|
||||||
for rec in overlapping:
|
|
||||||
if rec.id not in protected_recording_ids:
|
|
||||||
protected_recording_ids.add(rec.id)
|
|
||||||
protected_size += rec.segment_size
|
|
||||||
|
|
||||||
# 2. Event retention (aging out): recordings overlapping non-expired review segments
|
# 2. Event retention (aging out): recordings overlapping non-expired review segments
|
||||||
|
# Use global config for expiry thresholds
|
||||||
config = request.app.frigate_config
|
config = request.app.frigate_config
|
||||||
alert_expire = now - timedelta(days=config.record.alerts.retain.days).total_seconds()
|
alert_expire = now - timedelta(days=config.record.alerts.retain.days).total_seconds()
|
||||||
detection_expire = (
|
detection_expire = (
|
||||||
now - timedelta(days=config.record.detections.retain.days).total_seconds()
|
now - timedelta(days=config.record.detections.retain.days).total_seconds()
|
||||||
)
|
)
|
||||||
|
|
||||||
active_reviews = (
|
# Include in-progress reviews (end_time IS NULL) as they are not overwritable
|
||||||
ReviewSegment.select(
|
event_query = (
|
||||||
ReviewSegment.camera,
|
Recordings.select(
|
||||||
ReviewSegment.start_time,
|
Recordings.id,
|
||||||
ReviewSegment.end_time,
|
Recordings.segment_size,
|
||||||
|
)
|
||||||
|
.join(
|
||||||
|
ReviewSegment,
|
||||||
|
on=(
|
||||||
|
(ReviewSegment.camera == Recordings.camera)
|
||||||
|
& (ReviewSegment.start_time < Recordings.end_time)
|
||||||
|
& (
|
||||||
|
(ReviewSegment.end_time.is_null())
|
||||||
|
| (ReviewSegment.end_time > Recordings.start_time)
|
||||||
|
)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.where(
|
.where(
|
||||||
|
Recordings.segment_size > 0,
|
||||||
(
|
(
|
||||||
(ReviewSegment.severity == "alert")
|
(ReviewSegment.end_time.is_null())
|
||||||
& (ReviewSegment.end_time >= alert_expire)
|
| (
|
||||||
)
|
(ReviewSegment.severity == "alert")
|
||||||
| (
|
& (ReviewSegment.end_time >= alert_expire)
|
||||||
(ReviewSegment.severity == "detection")
|
)
|
||||||
& (ReviewSegment.end_time >= detection_expire)
|
| (
|
||||||
)
|
(ReviewSegment.severity == "detection")
|
||||||
|
& (ReviewSegment.end_time >= detection_expire)
|
||||||
|
)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
.distinct()
|
||||||
.namedtuples()
|
.namedtuples()
|
||||||
)
|
)
|
||||||
|
|
||||||
event_size = 0.0
|
event_size = 0.0
|
||||||
event_recording_ids = set()
|
for rec in event_query:
|
||||||
for review in active_reviews:
|
if rec.id not in protected_recording_ids:
|
||||||
end_time_clause = (
|
event_size += rec.segment_size
|
||||||
Recordings.start_time < review.end_time
|
|
||||||
if review.end_time is not None
|
|
||||||
else True
|
|
||||||
)
|
|
||||||
overlapping = (
|
|
||||||
Recordings.select(Recordings.id, Recordings.segment_size)
|
|
||||||
.where(
|
|
||||||
Recordings.camera == review.camera,
|
|
||||||
end_time_clause,
|
|
||||||
Recordings.end_time > review.start_time,
|
|
||||||
Recordings.segment_size > 0,
|
|
||||||
)
|
|
||||||
.namedtuples()
|
|
||||||
)
|
|
||||||
for rec in overlapping:
|
|
||||||
if (
|
|
||||||
rec.id not in protected_recording_ids
|
|
||||||
and rec.id not in event_recording_ids
|
|
||||||
):
|
|
||||||
event_recording_ids.add(rec.id)
|
|
||||||
event_size += rec.segment_size
|
|
||||||
|
|
||||||
# 3. Total recordings size
|
# 3. Total recordings size
|
||||||
total_size = (
|
total_size = (
|
||||||
@ -159,12 +158,14 @@ def get_recordings_storage_breakdown(request: Request):
|
|||||||
# Overwritable = total - protected - event
|
# Overwritable = total - protected - event
|
||||||
overwritable_size = max(0.0, total_size - protected_size - event_size)
|
overwritable_size = max(0.0, total_size - protected_size - event_size)
|
||||||
|
|
||||||
|
# All values are in MB (matching segment_size column units)
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
content={
|
content={
|
||||||
"total": round(total_size, 2),
|
"total": round(total_size, 2),
|
||||||
"overwritable": round(overwritable_size, 2),
|
"overwritable": round(overwritable_size, 2),
|
||||||
"event_retention": round(event_size, 2),
|
"event_retention": round(event_size, 2),
|
||||||
"protected": round(protected_size, 2),
|
"protected": round(protected_size, 2),
|
||||||
|
"units": "MB",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@ -1,13 +1,16 @@
|
|||||||
|
import logging
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional, Union
|
from typing import Optional, Union
|
||||||
|
|
||||||
from pydantic import Field
|
from pydantic import Field, model_validator
|
||||||
|
|
||||||
from frigate.const import MAX_PRE_CAPTURE
|
from frigate.const import MAX_PRE_CAPTURE
|
||||||
from frigate.review.types import SeverityEnum
|
from frigate.review.types import SeverityEnum
|
||||||
|
|
||||||
from ..base import FrigateBaseModel
|
from ..base import FrigateBaseModel
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"RecordConfig",
|
"RecordConfig",
|
||||||
"RecordExportConfig",
|
"RecordExportConfig",
|
||||||
@ -152,6 +155,17 @@ class RecordConfig(FrigateBaseModel):
|
|||||||
description="Indicates whether recording was enabled in the original static configuration.",
|
description="Indicates whether recording was enabled in the original static configuration.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def warn_rollover_with_days(self) -> "RecordConfig":
|
||||||
|
if self.retain_policy == RetainPolicyEnum.continuous_rollover:
|
||||||
|
if self.continuous.days > 0 or self.motion.days > 0:
|
||||||
|
logger.warning(
|
||||||
|
"retain_policy is 'continuous_rollover' - continuous.days and "
|
||||||
|
"motion.days are ignored. Recordings will fill available disk "
|
||||||
|
"space and oldest footage will be overwritten when needed."
|
||||||
|
)
|
||||||
|
return self
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def event_pre_capture(self) -> int:
|
def event_pre_capture(self) -> int:
|
||||||
return max(
|
return max(
|
||||||
|
|||||||
@ -543,6 +543,7 @@ export interface FrigateConfig {
|
|||||||
record: {
|
record: {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
enabled_in_config: boolean | null;
|
enabled_in_config: boolean | null;
|
||||||
|
retain_policy: "time" | "continuous_rollover";
|
||||||
events: {
|
events: {
|
||||||
objects: string[] | null;
|
objects: string[] | null;
|
||||||
post_capture: number;
|
post_capture: number;
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
import { CombinedStorageGraph } from "@/components/graph/CombinedStorageGraph";
|
import { CombinedStorageGraph } from "@/components/graph/CombinedStorageGraph";
|
||||||
import { StorageGraph } from "@/components/graph/StorageGraph";
|
import { StorageGraph } from "@/components/graph/StorageGraph";
|
||||||
|
import { getUnitSize } from "@/utils/storageUtil";
|
||||||
import { FrigateStats } from "@/types/stats";
|
import { FrigateStats } from "@/types/stats";
|
||||||
import { useMemo } from "react";
|
import { useMemo } from "react";
|
||||||
import {
|
import {
|
||||||
@ -35,12 +36,14 @@ export default function StorageMetrics({
|
|||||||
}: StorageMetricsProps) {
|
}: StorageMetricsProps) {
|
||||||
const { data: cameraStorage } = useSWR<CameraStorage>("recordings/storage");
|
const { data: cameraStorage } = useSWR<CameraStorage>("recordings/storage");
|
||||||
const { data: stats } = useSWR<FrigateStats>("stats");
|
const { data: stats } = useSWR<FrigateStats>("stats");
|
||||||
const { data: storageBreakdown } = useSWR<StorageBreakdown>(
|
|
||||||
"recordings/storage/breakdown",
|
|
||||||
);
|
|
||||||
const { data: config } = useSWR<FrigateConfig>("config", {
|
const { data: config } = useSWR<FrigateConfig>("config", {
|
||||||
revalidateOnFocus: false,
|
revalidateOnFocus: false,
|
||||||
});
|
});
|
||||||
|
const isRollover =
|
||||||
|
config?.record?.retain_policy === "continuous_rollover";
|
||||||
|
const { data: storageBreakdown } = useSWR<StorageBreakdown>(
|
||||||
|
isRollover ? "recordings/storage/breakdown" : null,
|
||||||
|
);
|
||||||
const { t } = useTranslation(["views/system"]);
|
const { t } = useTranslation(["views/system"]);
|
||||||
const timezone = useTimezone(config);
|
const timezone = useTimezone(config);
|
||||||
const { getLocaleDocUrl } = useDocDomain();
|
const { getLocaleDocUrl } = useDocDomain();
|
||||||
@ -131,7 +134,7 @@ export default function StorageMetrics({
|
|||||||
used={totalStorage.camera}
|
used={totalStorage.camera}
|
||||||
total={totalStorage.total}
|
total={totalStorage.total}
|
||||||
/>
|
/>
|
||||||
{storageBreakdown && (
|
{storageBreakdown && storageBreakdown.total > 0 && (
|
||||||
<div className="mt-3 space-y-1 text-xs">
|
<div className="mt-3 space-y-1 text-xs">
|
||||||
<div className="flex justify-between text-primary-variant">
|
<div className="flex justify-between text-primary-variant">
|
||||||
<span>
|
<span>
|
||||||
@ -140,9 +143,7 @@ export default function StorageMetrics({
|
|||||||
"Continuous (overwritable)",
|
"Continuous (overwritable)",
|
||||||
)}
|
)}
|
||||||
</span>
|
</span>
|
||||||
<span>
|
<span>{getUnitSize(storageBreakdown.overwritable)}</span>
|
||||||
{(storageBreakdown.overwritable / 1024).toFixed(1)} GB
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div className="flex justify-between text-primary-variant">
|
<div className="flex justify-between text-primary-variant">
|
||||||
<span>
|
<span>
|
||||||
@ -151,9 +152,7 @@ export default function StorageMetrics({
|
|||||||
"Events (aging out)",
|
"Events (aging out)",
|
||||||
)}
|
)}
|
||||||
</span>
|
</span>
|
||||||
<span>
|
<span>{getUnitSize(storageBreakdown.event_retention)}</span>
|
||||||
{(storageBreakdown.event_retention / 1024).toFixed(1)} GB
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div className="flex justify-between text-primary-variant">
|
<div className="flex justify-between text-primary-variant">
|
||||||
<span>
|
<span>
|
||||||
@ -162,9 +161,7 @@ export default function StorageMetrics({
|
|||||||
"Protected (indefinite)",
|
"Protected (indefinite)",
|
||||||
)}
|
)}
|
||||||
</span>
|
</span>
|
||||||
<span>
|
<span>{getUnitSize(storageBreakdown.protected)}</span>
|
||||||
{(storageBreakdown.protected / 1024).toFixed(1)} GB
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user