administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-04-27 17:17:40 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

thumbnail file management

Browse Source
This commit is contained in:
Josh Hawkins 2025-07-01 16:30:02 -05:00
parent cf2640452c
commit 8226429baf
7 changed files with 265 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
frigate/api/event.py
View File

@ -47,7 +47,7 @@ from frigate.api.defs.response.event_response import (
from frigate.api.defs.response.generic_response import GenericResponse
from frigate.api.defs.tags import Tags
from frigate.comms.event_metadata_updater import EventMetadataTypeEnum
from frigate.const import CLIPS_DIR
from frigate.const import CLIPS_DIR, TRIGGER_DIR
from frigate.embeddings import EmbeddingsContext
from frigate.models import Event, ReviewSegment, Timeline, Trigger
from frigate.track.object_processing import TrackedObject
@ -1512,11 +1512,18 @@ def create_trigger_embedding(
query_embedding = row[0]
embedding = np.frombuffer(query_embedding, dtype=np.float32)
else:
# TODO: fixme
# Extract valid thumbnail
thumbnail = get_event_thumbnail_bytes(event)
# TODO: save image to the triggers directory
if thumbnail is None:
return JSONResponse(
content={
"success": False,
"message": f"Failed to get thumbnail for {body.data} for {body.type} trigger",
},
status_code=400,
)
embedding = context.generate_image_embedding(
body.data, (base64.b64encode(thumbnail).decode("ASCII"))
)
@ -1530,6 +1537,12 @@ def create_trigger_embedding(
status_code=400,
)
if body.type == "thumbnail":
# Save image to the triggers directory
context.save_trigger_thumbnail(
camera, body.data, (base64.b64encode(thumbnail).decode("ASCII"))
)
Trigger.create(
camera=camera,
name=name,
@ -1587,26 +1600,40 @@ def update_trigger_embedding(
if body.type == "description":
embedding = context.generate_description_embedding(body.data)
elif body.type == "thumbnail":
webp_file = body.data + ".webp"
webp_path = os.path.join(TRIGGER_DIR, camera, webp_file)
try:
event: Event = Event.get(Event.id == body.data)
# Skip the event if not an object
if event.data.get("type") != "object":
return JSONResponse(
content={
"success": False,
"message": f"Event {body.data} is not a tracked object for {body.type} trigger",
},
status_code=400,
)
# Extract valid thumbnail
thumbnail = get_event_thumbnail_bytes(event)
with open(webp_path, "wb") as f:
f.write(thumbnail)
except DoesNotExist:
# TODO: check triggers directory for image
return JSONResponse(
content={
"success": False,
"message": f"Failed to fetch event for {body.type} trigger",
},
status_code=400,
)
# check triggers directory for image
if not os.path.exists(webp_path):
return JSONResponse(
content={
"success": False,
"message": f"Failed to fetch event for {body.type} trigger",
},
status_code=400,
)
else:
# Load the image from the triggers directory
with open(webp_path, "rb") as f:
thumbnail = f.read()
# Skip the event if not an object
if event.data.get("type") != "object":
return
# Extract valid thumbnail
thumbnail = get_event_thumbnail_bytes(event)
# TODO: save image to the triggers directory
embedding = context.generate_image_embedding(
body.data, (base64.b64encode(thumbnail).decode("ASCII"))
)
@ -1620,6 +1647,23 @@ def update_trigger_embedding(
status_code=400,
)
old = list(
Trigger.select(Trigger.camera, Trigger.name, Trigger.data)
.where(Trigger.camera == camera, Trigger.name == name)
.dicts()
.iterator()
)
if not old:
return JSONResponse(
content={
"success": False,
"message": f"Trigger {camera}:{name} not found",
},
status_code=404,
)
context.delete_trigger_thumbnail(camera, old[0]["data"])
updated = (
Trigger.update(
data=body.data,
@ -1642,6 +1686,12 @@ def update_trigger_embedding(
status_code=404,
)
if body.type == "thumbnail":
# Save image to the triggers directory
context.save_trigger_thumbnail(
camera, body.data, (base64.b64encode(thumbnail).decode("ASCII"))
)
return JSONResponse(
content={
"success": True,
@ -1666,10 +1716,21 @@ def update_trigger_embedding(
dependencies=[Depends(require_role(["admin"]))],
)
def delete_trigger_embedding(
request: Request,
camera: str,
name: str,
):
try:
trigger = Trigger.get_or_none(Trigger.camera == camera, Trigger.name == name)
if trigger is None:
return JSONResponse(
content={
"success": False,
"message": f"Trigger {camera}:{name} not found",
},
status_code=500,
)
deleted = (
Trigger.delete()
.where(Trigger.camera == camera, Trigger.name == name)
@ -1679,11 +1740,14 @@ def delete_trigger_embedding(
return JSONResponse(
content={
"success": False,
"message": f"Trigger {camera}:{name} not found",
"message": f"Error deleting trigger {camera}:{name}",
},
status_code=404,
status_code=401,
)
context: EmbeddingsContext = request.app.embeddings
context.delete_trigger_thumbnail(camera, trigger.data)
return JSONResponse(
content={
"success": True,

4
frigate/app.py
View File

@ -38,6 +38,7 @@ from frigate.const import (
MODEL_CACHE_DIR,
RECORD_DIR,
THUMB_DIR,
TRIGGER_DIR,
)
from frigate.data_processing.types import DataProcessorMetrics
from frigate.db.sqlitevecq import SqliteVecQueueDatabase
@ -122,6 +123,9 @@ class FrigateApp:
if self.config.face_recognition.enabled:
dirs.append(FACE_DIR)
if self.config.semantic_search.enabled:
dirs.append(TRIGGER_DIR)
for d in dirs:
if not os.path.exists(d) and not os.path.islink(d):
logger.info(f"Creating directory: {d}")

2
frigate/comms/embeddings_updater.py
View File

@ -23,6 +23,8 @@ class EmbeddingsRequestEnum(Enum):
embed_thumbnail = "embed_thumbnail"
generate_search = "generate_search"
reindex = "reindex"
write_trigger_thumbnail = "write_trigger_thumbnail"
remove_trigger_thumbnail = "remove_trigger_thumbnail"
# LPR
reprocess_plate = "reprocess_plate"

1
frigate/const.py
View File

@ -11,6 +11,7 @@ EXPORT_DIR = f"{BASE_DIR}/exports"
FACE_DIR = f"{CLIPS_DIR}/faces"
THUMB_DIR = f"{CLIPS_DIR}/thumbs"
RECORD_DIR = f"{BASE_DIR}/recordings"
TRIGGER_DIR = f"{CLIPS_DIR}/triggers"
BIRDSEYE_PIPE = "/tmp/cache/birdseye"
CACHE_DIR = "/tmp/cache"
FRIGATE_LOCALHOST = "http://127.0.0.1:5000"

21
frigate/embeddings/__init__.py
View File

@ -299,3 +299,24 @@ class EmbeddingsContext:
EmbeddingsRequestEnum.embed_thumbnail.value,
{"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False},
)
def save_trigger_thumbnail(
self, camera: str, event_id: str, thumbnail: bytes
) -> None:
self.requestor.send_data(
EmbeddingsRequestEnum.write_trigger_thumbnail.value,
{
"camera": str(camera),
"event_id": str(event_id),
"thumbnail": str(thumbnail),
},
)
def delete_trigger_thumbnail(self, camera: str, event_id: str) -> None:
self.requestor.send_data(
EmbeddingsRequestEnum.remove_trigger_thumbnail.value,
{
"camera": str(camera),
"event_id": str(event_id),
},
)

162
frigate/embeddings/embeddings.py
View File

@ -7,7 +7,7 @@ import threading
import time
import numpy as np
from peewee import IntegrityError
from peewee import DoesNotExist, IntegrityError
from playhouse.shortcuts import model_to_dict
from frigate.comms.embeddings_updater import (
@ -18,6 +18,7 @@ from frigate.config import FrigateConfig
from frigate.config.classification import SemanticSearchModelEnum
from frigate.const import (
CONFIG_DIR,
TRIGGER_DIR,
UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
UPDATE_MODEL_STATE,
)
@ -407,8 +408,6 @@ class Embeddings:
self.reindex_thread = None
def sync_triggers(self) -> None:
# TODO: fixme
return
for camera in self.config.cameras.values():
# Get all existing triggers for this camera
existing_triggers = {
@ -417,18 +416,52 @@ class Embeddings:
}
# Get all configured trigger names
configured_trigger_names = {
trigger.name for trigger in camera.semantic_search.triggers
}
configured_trigger_names = set(camera.semantic_search.triggers or {})
# Create or update triggers from config
# TODO: copy event thumbnail to triggers image directory
for trigger in camera.semantic_search.triggers:
if trigger.name in existing_triggers:
# Update existing trigger if data has changed
existing_trigger = existing_triggers[trigger.name]
for trigger_name, trigger in (
camera.semantic_search.triggers or {}
).items():
if trigger_name in existing_triggers:
existing_trigger = existing_triggers[trigger_name]
needs_embedding_update = False
thumbnail_missing = False
# Check if data has changed or thumbnail is missing for thumbnail type
if trigger.type == "thumbnail":
thumbnail_path = os.path.join(
TRIGGER_DIR, camera.name, f"{trigger.data}.webp"
)
try:
event = Event.get(Event.id == trigger.data)
if event.data.get("type") != "object":
logger.warning(
f"Event {trigger.data} is not a tracked object for {trigger.type} trigger"
)
continue # Skip if not an object
# Check if thumbnail needs to be updated (data changed or missing)
if (
existing_trigger.data != trigger.data
or not os.path.exists(thumbnail_path)
):
thumbnail = get_event_thumbnail_bytes(event)
if not thumbnail:
logger.warning(
f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}."
)
continue
self.write_trigger_thumbnail(
camera.name, trigger.data, thumbnail
)
thumbnail_missing = True
except DoesNotExist:
logger.warning(
f"Event ID {trigger.data} for trigger {trigger_name} does not exist."
)
continue
# Update existing trigger if data has changed
if (
existing_trigger.type != trigger.type
or existing_trigger.data != trigger.data
@ -440,7 +473,11 @@ class Embeddings:
needs_embedding_update = True
# Check if embedding is missing or needs update
if not existing_trigger.embedding or needs_embedding_update:
if (
not existing_trigger.embedding
or needs_embedding_update
or thumbnail_missing
):
existing_trigger.embedding = self._calculate_trigger_embedding(
trigger
)
@ -451,12 +488,39 @@ class Embeddings:
else:
# Create new trigger
try:
try:
event: Event = Event.get(Event.id == trigger.data)
except DoesNotExist:
logger.warning(
f"Event ID {trigger.data} for trigger {trigger_name} does not exist."
)
continue
# Skip the event if not an object
if event.data.get("type") != "object":
logger.warning(
f"Event ID {trigger.data} for trigger {trigger_name} is not a tracked object."
)
continue
thumbnail = get_event_thumbnail_bytes(event)
if not thumbnail:
logger.warning(
f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}."
)
continue
self.write_trigger_thumbnail(
camera.name, trigger.data, thumbnail
)
# Calculate embedding for new trigger
embedding = self._calculate_trigger_embedding(trigger)
Trigger.create(
camera=camera.name,
name=trigger.name,
name=trigger_name,
type=trigger.type,
data=trigger.data,
threshold=trigger.threshold,
@ -465,6 +529,7 @@ class Embeddings:
triggering_event_id="",
last_triggered=None,
)
except IntegrityError:
pass # Handle duplicate creation attempts
@ -476,22 +541,50 @@ class Embeddings:
Trigger.delete().where(
Trigger.camera == camera.name, Trigger.name.in_(triggers_to_remove)
).execute()
for trigger_name in triggers_to_remove:
self.remove_trigger_thumbnail(camera.name, trigger_name)
def write_trigger_thumbnail(
self, camera: str, event_id: str, thumbnail: bytes
) -> None:
"""Write the thumbnail to the trigger directory."""
try:
os.makedirs(os.path.join(TRIGGER_DIR, camera), exist_ok=True)
with open(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp"), "wb") as f:
f.write(thumbnail)
logger.debug(
f"Writing thumbnail for trigger with data {event_id} in {camera}."
)
except Exception as e:
logger.error(
f"Failed to write thumbnail for trigger with data {event_id} in {camera}: {e}"
)
def remove_trigger_thumbnail(self, camera: str, event_id: str) -> None:
"""Write the thumbnail to the trigger directory."""
try:
os.remove(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp"))
logger.debug(
f"Deleted thumbnail for trigger with data {event_id} in {camera}."
)
except Exception as e:
logger.error(
f"Failed to delete thumbnail for trigger with data {event_id} in {camera}: {e}"
)
def _calculate_trigger_embedding(self, trigger) -> bytes:
"""Calculate embedding for a trigger based on its type and data."""
if trigger.type == "description":
logger.debug(f"Generating embedding for trigger description {trigger.name}")
embedding = self.requestor.send_data(
EmbeddingsRequestEnum.embed_description.value,
{"id": None, "description": trigger.data, "upsert": False},
)
return embedding.astype(np.float32).tobytes()
elif trigger.type == "thumbnail":
# return self.requestor.send_data(
# EmbeddingsRequestEnum.embed_thumbnail.value,
# {"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False},
# )
# For image triggers, trigger.data should be an image ID
# Get embedding from vec_thumbnails table
# Try to get embedding from vec_thumbnails table first
cursor = self.db.execute_sql(
"SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?",
[trigger.data],
@ -500,10 +593,37 @@ class Embeddings:
if row:
return row[0] # Already in bytes format
else:
logger.warning(
f"No thumbnail embedding found for image ID: {trigger.data}"
logger.debug(
f"No thumbnail embedding found for image ID: {trigger.data}, generating from saved trigger thumbnail"
)
return b""
try:
with open(
os.path.join(
TRIGGER_DIR, trigger.camera, f"{trigger.data}.webp"
),
"rb",
) as f:
thumbnail = f.read()
except Exception as e:
logger.error(
f"Failed to read thumbnail for trigger {trigger.name} with ID {trigger.data}: {e}"
)
return b""
logger.debug(
f"Generating embedding for trigger thumbnail {trigger.name} with ID {trigger.data}"
)
embedding = self.requestor.send_data(
EmbeddingsRequestEnum.embed_thumbnail.value,
{
"id": str(trigger.data),
"thumbnail": str(thumbnail),
"upsert": False,
},
)
return embedding.astype(np.float32).tobytes()
else:
logger.warning(f"Unknown trigger type: {trigger.type}")
return b""

11
frigate/embeddings/maintainer.py
View File

@ -288,6 +288,17 @@ class EmbeddingMaintainer(threading.Thread):
elif topic == EmbeddingsRequestEnum.reindex.value:
response = self.embeddings.start_reindex()
return "started" if response else "in_progress"
elif topic == EmbeddingsRequestEnum.write_trigger_thumbnail.value:
thumbnail = base64.b64decode(data["thumbnail"])
self.embeddings.write_trigger_thumbnail(
data["camera"], data["event_id"], thumbnail
)
return
elif topic == EmbeddingsRequestEnum.remove_trigger_thumbnail.value:
self.embeddings.remove_trigger_thumbnail(
data["camera"], data["event_id"]
)
return
processors = [self.realtime_processors, self.post_processors]
for processor_list in processors: