From 8226429baf343868853f6e45a6e3cc0942c35b80 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:30:02 -0500 Subject: [PATCH] thumbnail file management --- frigate/api/event.py | 106 ++++++++++++++---- frigate/app.py | 4 + frigate/comms/embeddings_updater.py | 2 + frigate/const.py | 1 + frigate/embeddings/__init__.py | 21 ++++ frigate/embeddings/embeddings.py | 162 ++++++++++++++++++++++++---- frigate/embeddings/maintainer.py | 11 ++ 7 files changed, 265 insertions(+), 42 deletions(-) diff --git a/frigate/api/event.py b/frigate/api/event.py index e1a6cfbfb..2766f0814 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -47,7 +47,7 @@ from frigate.api.defs.response.event_response import ( from frigate.api.defs.response.generic_response import GenericResponse from frigate.api.defs.tags import Tags from frigate.comms.event_metadata_updater import EventMetadataTypeEnum -from frigate.const import CLIPS_DIR +from frigate.const import CLIPS_DIR, TRIGGER_DIR from frigate.embeddings import EmbeddingsContext from frigate.models import Event, ReviewSegment, Timeline, Trigger from frigate.track.object_processing import TrackedObject @@ -1512,11 +1512,18 @@ def create_trigger_embedding( query_embedding = row[0] embedding = np.frombuffer(query_embedding, dtype=np.float32) else: - # TODO: fixme # Extract valid thumbnail thumbnail = get_event_thumbnail_bytes(event) - # TODO: save image to the triggers directory + if thumbnail is None: + return JSONResponse( + content={ + "success": False, + "message": f"Failed to get thumbnail for {body.data} for {body.type} trigger", + }, + status_code=400, + ) + embedding = context.generate_image_embedding( body.data, (base64.b64encode(thumbnail).decode("ASCII")) ) @@ -1530,6 +1537,12 @@ def create_trigger_embedding( status_code=400, ) + if body.type == "thumbnail": + # Save image to the triggers directory + context.save_trigger_thumbnail( + camera, body.data, (base64.b64encode(thumbnail).decode("ASCII")) + ) + Trigger.create( camera=camera, name=name, @@ -1587,26 +1600,40 @@ def update_trigger_embedding( if body.type == "description": embedding = context.generate_description_embedding(body.data) elif body.type == "thumbnail": + webp_file = body.data + ".webp" + webp_path = os.path.join(TRIGGER_DIR, camera, webp_file) + try: event: Event = Event.get(Event.id == body.data) + # Skip the event if not an object + if event.data.get("type") != "object": + return JSONResponse( + content={ + "success": False, + "message": f"Event {body.data} is not a tracked object for {body.type} trigger", + }, + status_code=400, + ) + # Extract valid thumbnail + thumbnail = get_event_thumbnail_bytes(event) + + with open(webp_path, "wb") as f: + f.write(thumbnail) except DoesNotExist: - # TODO: check triggers directory for image - return JSONResponse( - content={ - "success": False, - "message": f"Failed to fetch event for {body.type} trigger", - }, - status_code=400, - ) + # check triggers directory for image + if not os.path.exists(webp_path): + return JSONResponse( + content={ + "success": False, + "message": f"Failed to fetch event for {body.type} trigger", + }, + status_code=400, + ) + else: + # Load the image from the triggers directory + with open(webp_path, "rb") as f: + thumbnail = f.read() - # Skip the event if not an object - if event.data.get("type") != "object": - return - - # Extract valid thumbnail - thumbnail = get_event_thumbnail_bytes(event) - - # TODO: save image to the triggers directory embedding = context.generate_image_embedding( body.data, (base64.b64encode(thumbnail).decode("ASCII")) ) @@ -1620,6 +1647,23 @@ def update_trigger_embedding( status_code=400, ) + old = list( + Trigger.select(Trigger.camera, Trigger.name, Trigger.data) + .where(Trigger.camera == camera, Trigger.name == name) + .dicts() + .iterator() + ) + if not old: + return JSONResponse( + content={ + "success": False, + "message": f"Trigger {camera}:{name} not found", + }, + status_code=404, + ) + + context.delete_trigger_thumbnail(camera, old[0]["data"]) + updated = ( Trigger.update( data=body.data, @@ -1642,6 +1686,12 @@ def update_trigger_embedding( status_code=404, ) + if body.type == "thumbnail": + # Save image to the triggers directory + context.save_trigger_thumbnail( + camera, body.data, (base64.b64encode(thumbnail).decode("ASCII")) + ) + return JSONResponse( content={ "success": True, @@ -1666,10 +1716,21 @@ def update_trigger_embedding( dependencies=[Depends(require_role(["admin"]))], ) def delete_trigger_embedding( + request: Request, camera: str, name: str, ): try: + trigger = Trigger.get_or_none(Trigger.camera == camera, Trigger.name == name) + if trigger is None: + return JSONResponse( + content={ + "success": False, + "message": f"Trigger {camera}:{name} not found", + }, + status_code=500, + ) + deleted = ( Trigger.delete() .where(Trigger.camera == camera, Trigger.name == name) @@ -1679,11 +1740,14 @@ def delete_trigger_embedding( return JSONResponse( content={ "success": False, - "message": f"Trigger {camera}:{name} not found", + "message": f"Error deleting trigger {camera}:{name}", }, - status_code=404, + status_code=401, ) + context: EmbeddingsContext = request.app.embeddings + context.delete_trigger_thumbnail(camera, trigger.data) + return JSONResponse( content={ "success": True, diff --git a/frigate/app.py b/frigate/app.py index 2a4a823ee..ee2bf924d 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -38,6 +38,7 @@ from frigate.const import ( MODEL_CACHE_DIR, RECORD_DIR, THUMB_DIR, + TRIGGER_DIR, ) from frigate.data_processing.types import DataProcessorMetrics from frigate.db.sqlitevecq import SqliteVecQueueDatabase @@ -122,6 +123,9 @@ class FrigateApp: if self.config.face_recognition.enabled: dirs.append(FACE_DIR) + if self.config.semantic_search.enabled: + dirs.append(TRIGGER_DIR) + for d in dirs: if not os.path.exists(d) and not os.path.islink(d): logger.info(f"Creating directory: {d}") diff --git a/frigate/comms/embeddings_updater.py b/frigate/comms/embeddings_updater.py index f97319051..5e8d0b96b 100644 --- a/frigate/comms/embeddings_updater.py +++ b/frigate/comms/embeddings_updater.py @@ -23,6 +23,8 @@ class EmbeddingsRequestEnum(Enum): embed_thumbnail = "embed_thumbnail" generate_search = "generate_search" reindex = "reindex" + write_trigger_thumbnail = "write_trigger_thumbnail" + remove_trigger_thumbnail = "remove_trigger_thumbnail" # LPR reprocess_plate = "reprocess_plate" diff --git a/frigate/const.py b/frigate/const.py index 69335902e..f4bfee3d1 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -11,6 +11,7 @@ EXPORT_DIR = f"{BASE_DIR}/exports" FACE_DIR = f"{CLIPS_DIR}/faces" THUMB_DIR = f"{CLIPS_DIR}/thumbs" RECORD_DIR = f"{BASE_DIR}/recordings" +TRIGGER_DIR = f"{CLIPS_DIR}/triggers" BIRDSEYE_PIPE = "/tmp/cache/birdseye" CACHE_DIR = "/tmp/cache" FRIGATE_LOCALHOST = "http://127.0.0.1:5000" diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index f3acbbcf7..94c971939 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -299,3 +299,24 @@ class EmbeddingsContext: EmbeddingsRequestEnum.embed_thumbnail.value, {"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False}, ) + + def save_trigger_thumbnail( + self, camera: str, event_id: str, thumbnail: bytes + ) -> None: + self.requestor.send_data( + EmbeddingsRequestEnum.write_trigger_thumbnail.value, + { + "camera": str(camera), + "event_id": str(event_id), + "thumbnail": str(thumbnail), + }, + ) + + def delete_trigger_thumbnail(self, camera: str, event_id: str) -> None: + self.requestor.send_data( + EmbeddingsRequestEnum.remove_trigger_thumbnail.value, + { + "camera": str(camera), + "event_id": str(event_id), + }, + ) diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py index 0a44f1407..663b11ae7 100644 --- a/frigate/embeddings/embeddings.py +++ b/frigate/embeddings/embeddings.py @@ -7,7 +7,7 @@ import threading import time import numpy as np -from peewee import IntegrityError +from peewee import DoesNotExist, IntegrityError from playhouse.shortcuts import model_to_dict from frigate.comms.embeddings_updater import ( @@ -18,6 +18,7 @@ from frigate.config import FrigateConfig from frigate.config.classification import SemanticSearchModelEnum from frigate.const import ( CONFIG_DIR, + TRIGGER_DIR, UPDATE_EMBEDDINGS_REINDEX_PROGRESS, UPDATE_MODEL_STATE, ) @@ -407,8 +408,6 @@ class Embeddings: self.reindex_thread = None def sync_triggers(self) -> None: - # TODO: fixme - return for camera in self.config.cameras.values(): # Get all existing triggers for this camera existing_triggers = { @@ -417,18 +416,52 @@ class Embeddings: } # Get all configured trigger names - configured_trigger_names = { - trigger.name for trigger in camera.semantic_search.triggers - } + configured_trigger_names = set(camera.semantic_search.triggers or {}) # Create or update triggers from config - # TODO: copy event thumbnail to triggers image directory - for trigger in camera.semantic_search.triggers: - if trigger.name in existing_triggers: - # Update existing trigger if data has changed - existing_trigger = existing_triggers[trigger.name] + for trigger_name, trigger in ( + camera.semantic_search.triggers or {} + ).items(): + if trigger_name in existing_triggers: + existing_trigger = existing_triggers[trigger_name] needs_embedding_update = False + thumbnail_missing = False + # Check if data has changed or thumbnail is missing for thumbnail type + if trigger.type == "thumbnail": + thumbnail_path = os.path.join( + TRIGGER_DIR, camera.name, f"{trigger.data}.webp" + ) + try: + event = Event.get(Event.id == trigger.data) + if event.data.get("type") != "object": + logger.warning( + f"Event {trigger.data} is not a tracked object for {trigger.type} trigger" + ) + continue # Skip if not an object + + # Check if thumbnail needs to be updated (data changed or missing) + if ( + existing_trigger.data != trigger.data + or not os.path.exists(thumbnail_path) + ): + thumbnail = get_event_thumbnail_bytes(event) + if not thumbnail: + logger.warning( + f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}." + ) + continue + self.write_trigger_thumbnail( + camera.name, trigger.data, thumbnail + ) + thumbnail_missing = True + except DoesNotExist: + logger.warning( + f"Event ID {trigger.data} for trigger {trigger_name} does not exist." + ) + continue + + # Update existing trigger if data has changed if ( existing_trigger.type != trigger.type or existing_trigger.data != trigger.data @@ -440,7 +473,11 @@ class Embeddings: needs_embedding_update = True # Check if embedding is missing or needs update - if not existing_trigger.embedding or needs_embedding_update: + if ( + not existing_trigger.embedding + or needs_embedding_update + or thumbnail_missing + ): existing_trigger.embedding = self._calculate_trigger_embedding( trigger ) @@ -451,12 +488,39 @@ class Embeddings: else: # Create new trigger try: + try: + event: Event = Event.get(Event.id == trigger.data) + except DoesNotExist: + logger.warning( + f"Event ID {trigger.data} for trigger {trigger_name} does not exist." + ) + continue + + # Skip the event if not an object + if event.data.get("type") != "object": + logger.warning( + f"Event ID {trigger.data} for trigger {trigger_name} is not a tracked object." + ) + continue + + thumbnail = get_event_thumbnail_bytes(event) + + if not thumbnail: + logger.warning( + f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}." + ) + continue + + self.write_trigger_thumbnail( + camera.name, trigger.data, thumbnail + ) + # Calculate embedding for new trigger embedding = self._calculate_trigger_embedding(trigger) Trigger.create( camera=camera.name, - name=trigger.name, + name=trigger_name, type=trigger.type, data=trigger.data, threshold=trigger.threshold, @@ -465,6 +529,7 @@ class Embeddings: triggering_event_id="", last_triggered=None, ) + except IntegrityError: pass # Handle duplicate creation attempts @@ -476,22 +541,50 @@ class Embeddings: Trigger.delete().where( Trigger.camera == camera.name, Trigger.name.in_(triggers_to_remove) ).execute() + for trigger_name in triggers_to_remove: + self.remove_trigger_thumbnail(camera.name, trigger_name) + + def write_trigger_thumbnail( + self, camera: str, event_id: str, thumbnail: bytes + ) -> None: + """Write the thumbnail to the trigger directory.""" + try: + os.makedirs(os.path.join(TRIGGER_DIR, camera), exist_ok=True) + with open(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp"), "wb") as f: + f.write(thumbnail) + logger.debug( + f"Writing thumbnail for trigger with data {event_id} in {camera}." + ) + except Exception as e: + logger.error( + f"Failed to write thumbnail for trigger with data {event_id} in {camera}: {e}" + ) + + def remove_trigger_thumbnail(self, camera: str, event_id: str) -> None: + """Write the thumbnail to the trigger directory.""" + try: + os.remove(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp")) + logger.debug( + f"Deleted thumbnail for trigger with data {event_id} in {camera}." + ) + except Exception as e: + logger.error( + f"Failed to delete thumbnail for trigger with data {event_id} in {camera}: {e}" + ) def _calculate_trigger_embedding(self, trigger) -> bytes: """Calculate embedding for a trigger based on its type and data.""" if trigger.type == "description": + logger.debug(f"Generating embedding for trigger description {trigger.name}") embedding = self.requestor.send_data( EmbeddingsRequestEnum.embed_description.value, {"id": None, "description": trigger.data, "upsert": False}, ) return embedding.astype(np.float32).tobytes() + elif trigger.type == "thumbnail": - # return self.requestor.send_data( - # EmbeddingsRequestEnum.embed_thumbnail.value, - # {"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False}, - # ) # For image triggers, trigger.data should be an image ID - # Get embedding from vec_thumbnails table + # Try to get embedding from vec_thumbnails table first cursor = self.db.execute_sql( "SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?", [trigger.data], @@ -500,10 +593,37 @@ class Embeddings: if row: return row[0] # Already in bytes format else: - logger.warning( - f"No thumbnail embedding found for image ID: {trigger.data}" + logger.debug( + f"No thumbnail embedding found for image ID: {trigger.data}, generating from saved trigger thumbnail" ) - return b"" + + try: + with open( + os.path.join( + TRIGGER_DIR, trigger.camera, f"{trigger.data}.webp" + ), + "rb", + ) as f: + thumbnail = f.read() + except Exception as e: + logger.error( + f"Failed to read thumbnail for trigger {trigger.name} with ID {trigger.data}: {e}" + ) + return b"" + + logger.debug( + f"Generating embedding for trigger thumbnail {trigger.name} with ID {trigger.data}" + ) + embedding = self.requestor.send_data( + EmbeddingsRequestEnum.embed_thumbnail.value, + { + "id": str(trigger.data), + "thumbnail": str(thumbnail), + "upsert": False, + }, + ) + return embedding.astype(np.float32).tobytes() + else: logger.warning(f"Unknown trigger type: {trigger.type}") return b"" diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 90a60e314..8fd3e853e 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -288,6 +288,17 @@ class EmbeddingMaintainer(threading.Thread): elif topic == EmbeddingsRequestEnum.reindex.value: response = self.embeddings.start_reindex() return "started" if response else "in_progress" + elif topic == EmbeddingsRequestEnum.write_trigger_thumbnail.value: + thumbnail = base64.b64decode(data["thumbnail"]) + self.embeddings.write_trigger_thumbnail( + data["camera"], data["event_id"], thumbnail + ) + return + elif topic == EmbeddingsRequestEnum.remove_trigger_thumbnail.value: + self.embeddings.remove_trigger_thumbnail( + data["camera"], data["event_id"] + ) + return processors = [self.realtime_processors, self.post_processors] for processor_list in processors: