administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-08-02 17:12:16 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Media sync API refactor and UI (#21542)

Browse Source 
* generic job infrastructure

* types and dispatcher changes for jobs

* save data in memory only for completed jobs

* implement media sync job and endpoints

* change logs to debug

* websocket hook and types

* frontend

* i18n

* docs tweaks

* endpoint descriptions

* tweak docs
This commit is contained in:
Josh Hawkins
2026-02-26 21:27:56 -07:00
committed by Nicolas Mowen
parent 63e7bf8b28
commit b66e69efc9
15 changed files with 951 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frigate/api/app.py
+80 -45
View File
@@ -38,8 +38,14 @@ from frigate.config.camera.updater import (
CameraConfigUpdateTopic,
)
from frigate.ffmpeg_presets import FFMPEG_HWACCEL_VAAPI, _gpu_selector
from frigate.jobs.media_sync import (
get_current_media_sync_job,
get_media_sync_job_by_id,
start_media_sync_job,
)
from frigate.models import Event, Timeline
from frigate.stats.prometheus import get_metrics, update_metrics
from frigate.types import JobStatusTypesEnum
from frigate.util.builtin import (
clean_camera_user_pass,
flatten_config_data,
@@ -47,7 +53,6 @@ from frigate.util.builtin import (
update_yaml_file_bulk,
)
from frigate.util.config import find_config_file
from frigate.util.media import sync_all_media
from frigate.util.services import (
get_nvidia_driver_info,
process_logs,
@@ -608,12 +613,19 @@ def restart():
)
@router.post("/media/sync", dependencies=[Depends(require_role(["admin"]))])
@router.post(
"/media/sync",
dependencies=[Depends(require_role(["admin"]))],
summary="Start media sync job",
description="""Start an asynchronous media sync job to find and (optionally) remove orphaned media files.
Returns 202 with job details when queued, or 409 if a job is already running.""",
)
def sync_media(body: MediaSyncBody = Body(...)):
"""Sync media files with database - remove orphaned files.
"""Start async media sync job - remove orphaned files.
Syncs specified media types: event snapshots, event thumbnails, review thumbnails,
previews, exports, and/or recordings.
previews, exports, and/or recordings. Job runs in background; use /media/sync/current
or /media/sync/status/{job_id} to check status.
Args:
body: MediaSyncBody with dry_run flag and media_types list.
@@ -621,54 +633,77 @@ def sync_media(body: MediaSyncBody = Body(...)):
'review_thumbnails', 'previews', 'exports', 'recordings'
Returns:
JSON response with sync results for each requested media type.
202 Accepted with job_id, or 409 Conflict if job already running.
"""
try:
results = sync_all_media(
dry_run=body.dry_run, media_types=body.media_types, force=body.force
)
job_id = start_media_sync_job(
dry_run=body.dry_run, media_types=body.media_types, force=body.force
)
# Check if any operations were aborted or had errors
has_errors = False
for result_name in [
"event_snapshots",
"event_thumbnails",
"review_thumbnails",
"previews",
"exports",
"recordings",
]:
result = getattr(results, result_name, None)
if result and (result.aborted or result.error):
has_errors = True
break
content = {
"success": not has_errors,
"dry_run": body.dry_run,
"media_types": body.media_types,
"results": results.to_dict(),
}
if has_errors:
content["message"] = (
"Some sync operations were aborted or had errors; check logs for details."
)
return JSONResponse(
content=content,
status_code=200,
)
except Exception as e:
logger.error(f"Error syncing media files: {e}")
if job_id is None:
# A job is already running
current = get_current_media_sync_job()
return JSONResponse(
content={
"success": False,
"message": f"Error syncing media files: {str(e)}",
"error": "A media sync job is already running",
"current_job_id": current.id if current else None,
},
status_code=500,
status_code=409,
)
return JSONResponse(
content={
"job": {
"job_type": "media_sync",
"status": JobStatusTypesEnum.queued,
"id": job_id,
}
},
status_code=202,
)
@router.get(
"/media/sync/current",
dependencies=[Depends(require_role(["admin"]))],
summary="Get current media sync job",
description="""Retrieve the current running media sync job, if any. Returns the job details
or null when no job is active.""",
)
def get_media_sync_current():
"""Get the current running media sync job, if any."""
job = get_current_media_sync_job()
if job is None:
return JSONResponse(content={"job": None}, status_code=200)
return JSONResponse(
content={"job": job.to_dict()},
status_code=200,
)
@router.get(
"/media/sync/status/{job_id}",
dependencies=[Depends(require_role(["admin"]))],
summary="Get media sync job status",
description="""Get status and results for the specified media sync job id. Returns 200 with
job details including results, or 404 if the job is not found.""",
)
def get_media_sync_status(job_id: str):
"""Get the status of a specific media sync job."""
job = get_media_sync_job_by_id(job_id)
if job is None:
return JSONResponse(
content={"error": "Job not found"},
status_code=404,
)
return JSONResponse(
content={"job": job.to_dict()},
status_code=200,
)
@router.get("/labels", dependencies=[Depends(allow_any_authenticated())])
def get_labels(camera: str = ""):
frigate/comms/dispatcher.py
+17
View File
@@ -28,6 +28,7 @@ from frigate.const import (
UPDATE_CAMERA_ACTIVITY,
UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
UPDATE_EVENT_DESCRIPTION,
UPDATE_JOB_STATE,
UPDATE_MODEL_STATE,
UPDATE_REVIEW_DESCRIPTION,
UPSERT_REVIEW_SEGMENT,
@@ -60,6 +61,7 @@ class Dispatcher:
self.camera_activity = CameraActivityManager(config, self.publish)
self.audio_activity = AudioActivityManager(config, self.publish)
self.model_state: dict[str, ModelStatusTypesEnum] = {}
self.job_state: dict[str, dict[str, Any]] = {} # {job_type: job_data}
self.embeddings_reindex: dict[str, Any] = {}
self.birdseye_layout: dict[str, Any] = {}
self.audio_transcription_state: str = "idle"
@@ -180,6 +182,19 @@ class Dispatcher:
def handle_model_state() -> None:
self.publish("model_state", json.dumps(self.model_state.copy()))
def handle_update_job_state() -> None:
if payload and isinstance(payload, dict):
job_type = payload.get("job_type")
if job_type:
self.job_state[job_type] = payload
self.publish(
"job_state",
json.dumps(self.job_state),
)
def handle_job_state() -> None:
self.publish("job_state", json.dumps(self.job_state.copy()))
def handle_update_audio_transcription_state() -> None:
if payload:
self.audio_transcription_state = payload
@@ -277,6 +292,7 @@ class Dispatcher:
UPDATE_EVENT_DESCRIPTION: handle_update_event_description,
UPDATE_REVIEW_DESCRIPTION: handle_update_review_description,
UPDATE_MODEL_STATE: handle_update_model_state,
UPDATE_JOB_STATE: handle_update_job_state,
UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress,
UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout,
UPDATE_AUDIO_TRANSCRIPTION_STATE: handle_update_audio_transcription_state,
@@ -284,6 +300,7 @@ class Dispatcher:
"restart": handle_restart,
"embeddingsReindexProgress": handle_embeddings_reindex_progress,
"modelState": handle_model_state,
"jobState": handle_job_state,
"audioTranscriptionState": handle_audio_transcription_state,
"birdseyeLayout": handle_birdseye_layout,
"onConnect": handle_on_connect,
frigate/const.py
+1
View File
@@ -122,6 +122,7 @@ UPDATE_REVIEW_DESCRIPTION = "update_review_description"
UPDATE_MODEL_STATE = "update_model_state"
UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress"
UPDATE_BIRDSEYE_LAYOUT = "update_birdseye_layout"
UPDATE_JOB_STATE = "update_job_state"
NOTIFICATION_TEST = "notification_test"
# IO Nice Values
frigate/jobs/__init__.py
View File
frigate/jobs/job.py
+21
View File
@@ -0,0 +1,21 @@
"""Generic base class for long-running background jobs."""
from dataclasses import asdict, dataclass, field
from typing import Any, Optional
@dataclass
class Job:
"""Base class for long-running background jobs."""
id: str = field(default_factory=lambda: __import__("uuid").uuid4().__str__()[:12])
job_type: str = "" # Must be set by subclasses
status: str = "queued" # queued, running, success, failed, cancelled
results: Optional[dict[str, Any]] = None
start_time: Optional[float] = None
end_time: Optional[float] = None
error_message: Optional[str] = None
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary for WebSocket transmission."""
return asdict(self)
frigate/jobs/manager.py
+70
View File
@@ -0,0 +1,70 @@
"""Generic job management for long-running background tasks."""
import threading
from typing import Optional
from frigate.jobs.job import Job
from frigate.types import JobStatusTypesEnum
# Global state and locks for enforcing single concurrent job per job type
_job_locks: dict[str, threading.Lock] = {}
_current_jobs: dict[str, Optional[Job]] = {}
# Keep completed jobs for retrieval, keyed by (job_type, job_id)
_completed_jobs: dict[tuple[str, str], Job] = {}
def _get_lock(job_type: str) -> threading.Lock:
"""Get or create a lock for the specified job type."""
if job_type not in _job_locks:
_job_locks[job_type] = threading.Lock()
return _job_locks[job_type]
def set_current_job(job: Job) -> None:
"""Set the current job for a given job type."""
lock = _get_lock(job.job_type)
with lock:
# Store the previous job if it was completed
old_job = _current_jobs.get(job.job_type)
if old_job and old_job.status in (
JobStatusTypesEnum.success,
JobStatusTypesEnum.failed,
JobStatusTypesEnum.cancelled,
):
_completed_jobs[(job.job_type, old_job.id)] = old_job
_current_jobs[job.job_type] = job
def clear_current_job(job_type: str, job_id: Optional[str] = None) -> None:
"""Clear the current job for a given job type, optionally checking the ID."""
lock = _get_lock(job_type)
with lock:
if job_type in _current_jobs:
current = _current_jobs[job_type]
if current is None or (job_id is None or current.id == job_id):
_current_jobs[job_type] = None
def get_current_job(job_type: str) -> Optional[Job]:
"""Get the current running/queued job for a given job type, if any."""
lock = _get_lock(job_type)
with lock:
return _current_jobs.get(job_type)
def get_job_by_id(job_type: str, job_id: str) -> Optional[Job]:
"""Get job by ID. Checks current job first, then completed jobs."""
lock = _get_lock(job_type)
with lock:
# Check if it's the current job
current = _current_jobs.get(job_type)
if current and current.id == job_id:
return current
# Check if it's a completed job
return _completed_jobs.get((job_type, job_id))
def job_is_running(job_type: str) -> bool:
"""Check if a job of the given type is currently running or queued."""
job = get_current_job(job_type)
return job is not None and job.status in ("queued", "running")
frigate/jobs/media_sync.py
+135
View File
@@ -0,0 +1,135 @@
"""Media sync job management with background execution."""
import logging
import threading
from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional
from frigate.comms.inter_process import InterProcessRequestor
from frigate.const import UPDATE_JOB_STATE
from frigate.jobs.job import Job
from frigate.jobs.manager import (
get_current_job,
get_job_by_id,
job_is_running,
set_current_job,
)
from frigate.types import JobStatusTypesEnum
from frigate.util.media import sync_all_media
logger = logging.getLogger(__name__)
@dataclass
class MediaSyncJob(Job):
"""In-memory job state for media sync operations."""
job_type: str = "media_sync"
dry_run: bool = False
media_types: list[str] = field(default_factory=lambda: ["all"])
force: bool = False
class MediaSyncRunner(threading.Thread):
"""Thread-based runner for media sync jobs."""
def __init__(self, job: MediaSyncJob) -> None:
super().__init__(daemon=True, name="media_sync")
self.job = job
self.requestor = InterProcessRequestor()
def run(self) -> None:
"""Execute the media sync job and broadcast status updates."""
try:
# Update job status to running
self.job.status = JobStatusTypesEnum.running
self.job.start_time = datetime.now().timestamp()
self._broadcast_status()
# Execute sync with provided parameters
logger.debug(
f"Starting media sync job {self.job.id}: "
f"media_types={self.job.media_types}, "
f"dry_run={self.job.dry_run}, "
f"force={self.job.force}"
)
results = sync_all_media(
dry_run=self.job.dry_run,
media_types=self.job.media_types,
force=self.job.force,
)
# Store results and mark as complete
self.job.results = results.to_dict()
self.job.status = JobStatusTypesEnum.success
self.job.end_time = datetime.now().timestamp()
logger.debug(f"Media sync job {self.job.id} completed successfully")
self._broadcast_status()
except Exception as e:
logger.error(f"Media sync job {self.job.id} failed: {e}", exc_info=True)
self.job.status = JobStatusTypesEnum.failed
self.job.error_message = str(e)
self.job.end_time = datetime.now().timestamp()
self._broadcast_status()
finally:
if self.requestor:
self.requestor.stop()
def _broadcast_status(self) -> None:
"""Broadcast job status update via IPC to all WebSocket subscribers."""
try:
self.requestor.send_data(
UPDATE_JOB_STATE,
self.job.to_dict(),
)
except Exception as e:
logger.warning(f"Failed to broadcast media sync status: {e}")
def start_media_sync_job(
dry_run: bool = False,
media_types: Optional[list[str]] = None,
force: bool = False,
) -> Optional[str]:
"""Start a new media sync job if none is currently running.
Returns job ID on success, None if job already running.
"""
# Check if a job is already running
if job_is_running("media_sync"):
current = get_current_job("media_sync")
logger.warning(
f"Media sync job {current.id} is already running. Rejecting new request."
)
return None
# Create and start new job
job = MediaSyncJob(
dry_run=dry_run,
media_types=media_types or ["all"],
force=force,
)
logger.debug(f"Creating new media sync job: {job.id}")
set_current_job(job)
# Start the background runner
runner = MediaSyncRunner(job)
runner.start()
return job.id
def get_current_media_sync_job() -> Optional[MediaSyncJob]:
"""Get the current running/queued media sync job, if any."""
return get_current_job("media_sync")
def get_media_sync_job_by_id(job_id: str) -> Optional[MediaSyncJob]:
"""Get media sync job by ID. Currently only tracks the current job."""
return get_job_by_id("media_sync", job_id)
frigate/types.py
+9
View File
@@ -26,6 +26,15 @@ class ModelStatusTypesEnum(str, Enum):
failed = "failed"
class JobStatusTypesEnum(str, Enum):
pending = "pending"
queued = "queued"
running = "running"
success = "success"
failed = "failed"
cancelled = "cancelled"
class TrackedObjectUpdateTypesEnum(str, Enum):
description = "description"
face = "face"