mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-10-10 16:52:47 +03:00
Semantic Search Triggers (#18969)
* semantic trigger test * database and model * config * embeddings maintainer and trigger post-processor * api to create, edit, delete triggers * frontend and i18n keys * use thumbnail and description for trigger types * image picker tweaks * initial sync * thumbnail file management * clean up logs and use saved thumbnail on frontend * publish mqtt messages * webpush changes to enable trigger notifications * add enabled switch * add triggers from explore * renaming and deletion fixes * fix typing * UI updates and add last triggering event time and link * log exception instead of return in endpoint * highlight entry in UI when triggered * save and delete thumbnails directly * remove alert action for now and add descriptions * tweaks * clean up * fix types * docs * docs tweaks * docs * reuse enum
This commit is contained in:
Josh Hawkins
committed by
Blake Blackshear
Blake Blackshear
parent
28f816b49a
commit
3609b41217
@@ -7,21 +7,26 @@ import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
from numpy import ndarray
|
||||
import numpy as np
|
||||
from peewee import DoesNotExist, IntegrityError
|
||||
from PIL import Image
|
||||
from playhouse.shortcuts import model_to_dict
|
||||
|
||||
from frigate.comms.embeddings_updater import (
|
||||
EmbeddingsRequestEnum,
|
||||
)
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.classification import SemanticSearchModelEnum
|
||||
from frigate.const import (
|
||||
CONFIG_DIR,
|
||||
TRIGGER_DIR,
|
||||
UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
|
||||
UPDATE_MODEL_STATE,
|
||||
)
|
||||
from frigate.data_processing.types import DataProcessorMetrics
|
||||
from frigate.db.sqlitevecq import SqliteVecQueueDatabase
|
||||
from frigate.models import Event
|
||||
from frigate.models import Event, Trigger
|
||||
from frigate.types import ModelStatusTypesEnum
|
||||
from frigate.util.builtin import EventsPerSecond, InferenceSpeed, serialize
|
||||
from frigate.util.path import get_event_thumbnail_bytes
|
||||
@@ -167,7 +172,7 @@ class Embeddings:
|
||||
|
||||
def embed_thumbnail(
|
||||
self, event_id: str, thumbnail: bytes, upsert: bool = True
|
||||
) -> ndarray:
|
||||
) -> np.ndarray:
|
||||
"""Embed thumbnail and optionally insert into DB.
|
||||
|
||||
@param: event_id in Events DB
|
||||
@@ -194,7 +199,7 @@ class Embeddings:
|
||||
|
||||
def batch_embed_thumbnail(
|
||||
self, event_thumbs: dict[str, bytes], upsert: bool = True
|
||||
) -> list[ndarray]:
|
||||
) -> list[np.ndarray]:
|
||||
"""Embed thumbnails and optionally insert into DB.
|
||||
|
||||
@param: event_thumbs Map of Event IDs in DB to thumbnail bytes in jpg format
|
||||
@@ -244,7 +249,7 @@ class Embeddings:
|
||||
|
||||
def embed_description(
|
||||
self, event_id: str, description: str, upsert: bool = True
|
||||
) -> ndarray:
|
||||
) -> np.ndarray:
|
||||
start = datetime.datetime.now().timestamp()
|
||||
embedding = self.text_embedding([description])[0]
|
||||
|
||||
@@ -264,7 +269,7 @@ class Embeddings:
|
||||
|
||||
def batch_embed_description(
|
||||
self, event_descriptions: dict[str, str], upsert: bool = True
|
||||
) -> ndarray:
|
||||
) -> np.ndarray:
|
||||
start = datetime.datetime.now().timestamp()
|
||||
# upsert embeddings one by one to avoid token limit
|
||||
embeddings = []
|
||||
@@ -417,3 +422,224 @@ class Embeddings:
|
||||
with self.reindex_lock:
|
||||
self.reindex_running = False
|
||||
self.reindex_thread = None
|
||||
|
||||
def sync_triggers(self) -> None:
|
||||
for camera in self.config.cameras.values():
|
||||
# Get all existing triggers for this camera
|
||||
existing_triggers = {
|
||||
trigger.name: trigger
|
||||
for trigger in Trigger.select().where(Trigger.camera == camera.name)
|
||||
}
|
||||
|
||||
# Get all configured trigger names
|
||||
configured_trigger_names = set(camera.semantic_search.triggers or {})
|
||||
|
||||
# Create or update triggers from config
|
||||
for trigger_name, trigger in (
|
||||
camera.semantic_search.triggers or {}
|
||||
).items():
|
||||
if trigger_name in existing_triggers:
|
||||
existing_trigger = existing_triggers[trigger_name]
|
||||
needs_embedding_update = False
|
||||
thumbnail_missing = False
|
||||
|
||||
# Check if data has changed or thumbnail is missing for thumbnail type
|
||||
if trigger.type == "thumbnail":
|
||||
thumbnail_path = os.path.join(
|
||||
TRIGGER_DIR, camera.name, f"{trigger.data}.webp"
|
||||
)
|
||||
try:
|
||||
event = Event.get(Event.id == trigger.data)
|
||||
if event.data.get("type") != "object":
|
||||
logger.warning(
|
||||
f"Event {trigger.data} is not a tracked object for {trigger.type} trigger"
|
||||
)
|
||||
continue # Skip if not an object
|
||||
|
||||
# Check if thumbnail needs to be updated (data changed or missing)
|
||||
if (
|
||||
existing_trigger.data != trigger.data
|
||||
or not os.path.exists(thumbnail_path)
|
||||
):
|
||||
thumbnail = get_event_thumbnail_bytes(event)
|
||||
if not thumbnail:
|
||||
logger.warning(
|
||||
f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}."
|
||||
)
|
||||
continue
|
||||
self.write_trigger_thumbnail(
|
||||
camera.name, trigger.data, thumbnail
|
||||
)
|
||||
thumbnail_missing = True
|
||||
except DoesNotExist:
|
||||
logger.warning(
|
||||
f"Event ID {trigger.data} for trigger {trigger_name} does not exist."
|
||||
)
|
||||
continue
|
||||
|
||||
# Update existing trigger if data has changed
|
||||
if (
|
||||
existing_trigger.type != trigger.type
|
||||
or existing_trigger.data != trigger.data
|
||||
or existing_trigger.threshold != trigger.threshold
|
||||
):
|
||||
existing_trigger.type = trigger.type
|
||||
existing_trigger.data = trigger.data
|
||||
existing_trigger.threshold = trigger.threshold
|
||||
needs_embedding_update = True
|
||||
|
||||
# Check if embedding is missing or needs update
|
||||
if (
|
||||
not existing_trigger.embedding
|
||||
or needs_embedding_update
|
||||
or thumbnail_missing
|
||||
):
|
||||
existing_trigger.embedding = self._calculate_trigger_embedding(
|
||||
trigger
|
||||
)
|
||||
needs_embedding_update = True
|
||||
|
||||
if needs_embedding_update:
|
||||
existing_trigger.save()
|
||||
else:
|
||||
# Create new trigger
|
||||
try:
|
||||
try:
|
||||
event: Event = Event.get(Event.id == trigger.data)
|
||||
except DoesNotExist:
|
||||
logger.warning(
|
||||
f"Event ID {trigger.data} for trigger {trigger_name} does not exist."
|
||||
)
|
||||
continue
|
||||
|
||||
# Skip the event if not an object
|
||||
if event.data.get("type") != "object":
|
||||
logger.warning(
|
||||
f"Event ID {trigger.data} for trigger {trigger_name} is not a tracked object."
|
||||
)
|
||||
continue
|
||||
|
||||
thumbnail = get_event_thumbnail_bytes(event)
|
||||
|
||||
if not thumbnail:
|
||||
logger.warning(
|
||||
f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}."
|
||||
)
|
||||
continue
|
||||
|
||||
self.write_trigger_thumbnail(
|
||||
camera.name, trigger.data, thumbnail
|
||||
)
|
||||
|
||||
# Calculate embedding for new trigger
|
||||
embedding = self._calculate_trigger_embedding(trigger)
|
||||
|
||||
Trigger.create(
|
||||
camera=camera.name,
|
||||
name=trigger_name,
|
||||
type=trigger.type,
|
||||
data=trigger.data,
|
||||
threshold=trigger.threshold,
|
||||
model=self.config.semantic_search.model,
|
||||
embedding=embedding,
|
||||
triggering_event_id="",
|
||||
last_triggered=None,
|
||||
)
|
||||
|
||||
except IntegrityError:
|
||||
pass # Handle duplicate creation attempts
|
||||
|
||||
# Remove triggers that are no longer in config
|
||||
triggers_to_remove = (
|
||||
set(existing_triggers.keys()) - configured_trigger_names
|
||||
)
|
||||
if triggers_to_remove:
|
||||
Trigger.delete().where(
|
||||
Trigger.camera == camera.name, Trigger.name.in_(triggers_to_remove)
|
||||
).execute()
|
||||
for trigger_name in triggers_to_remove:
|
||||
self.remove_trigger_thumbnail(camera.name, trigger_name)
|
||||
|
||||
def write_trigger_thumbnail(
|
||||
self, camera: str, event_id: str, thumbnail: bytes
|
||||
) -> None:
|
||||
"""Write the thumbnail to the trigger directory."""
|
||||
try:
|
||||
os.makedirs(os.path.join(TRIGGER_DIR, camera), exist_ok=True)
|
||||
with open(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp"), "wb") as f:
|
||||
f.write(thumbnail)
|
||||
logger.debug(
|
||||
f"Writing thumbnail for trigger with data {event_id} in {camera}."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to write thumbnail for trigger with data {event_id} in {camera}: {e}"
|
||||
)
|
||||
|
||||
def remove_trigger_thumbnail(self, camera: str, event_id: str) -> None:
|
||||
"""Write the thumbnail to the trigger directory."""
|
||||
try:
|
||||
os.remove(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp"))
|
||||
logger.debug(
|
||||
f"Deleted thumbnail for trigger with data {event_id} in {camera}."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to delete thumbnail for trigger with data {event_id} in {camera}: {e}"
|
||||
)
|
||||
|
||||
def _calculate_trigger_embedding(self, trigger) -> bytes:
|
||||
"""Calculate embedding for a trigger based on its type and data."""
|
||||
if trigger.type == "description":
|
||||
logger.debug(f"Generating embedding for trigger description {trigger.name}")
|
||||
embedding = self.requestor.send_data(
|
||||
EmbeddingsRequestEnum.embed_description.value,
|
||||
{"id": None, "description": trigger.data, "upsert": False},
|
||||
)
|
||||
return embedding.astype(np.float32).tobytes()
|
||||
|
||||
elif trigger.type == "thumbnail":
|
||||
# For image triggers, trigger.data should be an image ID
|
||||
# Try to get embedding from vec_thumbnails table first
|
||||
cursor = self.db.execute_sql(
|
||||
"SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?",
|
||||
[trigger.data],
|
||||
)
|
||||
row = cursor.fetchone() if cursor else None
|
||||
if row:
|
||||
return row[0] # Already in bytes format
|
||||
else:
|
||||
logger.debug(
|
||||
f"No thumbnail embedding found for image ID: {trigger.data}, generating from saved trigger thumbnail"
|
||||
)
|
||||
|
||||
try:
|
||||
with open(
|
||||
os.path.join(
|
||||
TRIGGER_DIR, trigger.camera, f"{trigger.data}.webp"
|
||||
),
|
||||
"rb",
|
||||
) as f:
|
||||
thumbnail = f.read()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to read thumbnail for trigger {trigger.name} with ID {trigger.data}: {e}"
|
||||
)
|
||||
return b""
|
||||
|
||||
logger.debug(
|
||||
f"Generating embedding for trigger thumbnail {trigger.name} with ID {trigger.data}"
|
||||
)
|
||||
embedding = self.requestor.send_data(
|
||||
EmbeddingsRequestEnum.embed_thumbnail.value,
|
||||
{
|
||||
"id": str(trigger.data),
|
||||
"thumbnail": str(thumbnail),
|
||||
"upsert": False,
|
||||
},
|
||||
)
|
||||
return embedding.astype(np.float32).tobytes()
|
||||
|
||||
else:
|
||||
logger.warning(f"Unknown trigger type: {trigger.type}")
|
||||
return b""
|
||||
|
||||
Reference in New Issue
Block a user