administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-10-02 04:46:50 +03:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Semantic Search Triggers (#18969)

Browse Source 
* semantic trigger test

* database and model

* config

* embeddings maintainer and trigger post-processor

* api to create, edit, delete triggers

* frontend and i18n keys

* use thumbnail and description for trigger types

* image picker tweaks

* initial sync

* thumbnail file management

* clean up logs and use saved thumbnail on frontend

* publish mqtt messages

* webpush changes to enable trigger notifications

* add enabled switch

* add triggers from explore

* renaming and deletion fixes

* fix typing

* UI updates and add last triggering event time and link

* log exception instead of return in endpoint

* highlight entry in UI when triggered

* save and delete thumbnails directly

* remove alert action for now and add descriptions

* tweaks

* clean up

* fix types

* docs

* docs tweaks

* docs

* reuse enum
This commit is contained in:
Josh Hawkins
2025-08-16 10:20:33 -05:00
committed by Blake Blackshear
parent 28f816b49a
commit 3609b41217
37 changed files with 2736 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frigate/embeddings/__init__.py
+12
View File
@@ -296,3 +296,15 @@ class EmbeddingsContext:
return self.requestor.send_data(
EmbeddingsRequestEnum.transcribe_audio.value, {"event": event}
)
def generate_description_embedding(self, text: str) -> None:
return self.requestor.send_data(
EmbeddingsRequestEnum.embed_description.value,
{"id": None, "description": text, "upsert": False},
)
def generate_image_embedding(self, event_id: str, thumbnail: bytes) -> None:
return self.requestor.send_data(
EmbeddingsRequestEnum.embed_thumbnail.value,
{"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False},
)
frigate/embeddings/embeddings.py
+232 -6
View File
@@ -7,21 +7,26 @@ import os
import threading
import time
from numpy import ndarray
import numpy as np
from peewee import DoesNotExist, IntegrityError
from PIL import Image
from playhouse.shortcuts import model_to_dict
from frigate.comms.embeddings_updater import (
EmbeddingsRequestEnum,
)
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import FrigateConfig
from frigate.config.classification import SemanticSearchModelEnum
from frigate.const import (
CONFIG_DIR,
TRIGGER_DIR,
UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
UPDATE_MODEL_STATE,
)
from frigate.data_processing.types import DataProcessorMetrics
from frigate.db.sqlitevecq import SqliteVecQueueDatabase
from frigate.models import Event
from frigate.models import Event, Trigger
from frigate.types import ModelStatusTypesEnum
from frigate.util.builtin import EventsPerSecond, InferenceSpeed, serialize
from frigate.util.path import get_event_thumbnail_bytes
@@ -167,7 +172,7 @@ class Embeddings:
def embed_thumbnail(
self, event_id: str, thumbnail: bytes, upsert: bool = True
) -> ndarray:
) -> np.ndarray:
"""Embed thumbnail and optionally insert into DB.
@param: event_id in Events DB
@@ -194,7 +199,7 @@ class Embeddings:
def batch_embed_thumbnail(
self, event_thumbs: dict[str, bytes], upsert: bool = True
) -> list[ndarray]:
) -> list[np.ndarray]:
"""Embed thumbnails and optionally insert into DB.
@param: event_thumbs Map of Event IDs in DB to thumbnail bytes in jpg format
@@ -244,7 +249,7 @@ class Embeddings:
def embed_description(
self, event_id: str, description: str, upsert: bool = True
) -> ndarray:
) -> np.ndarray:
start = datetime.datetime.now().timestamp()
embedding = self.text_embedding([description])[0]
@@ -264,7 +269,7 @@ class Embeddings:
def batch_embed_description(
self, event_descriptions: dict[str, str], upsert: bool = True
) -> ndarray:
) -> np.ndarray:
start = datetime.datetime.now().timestamp()
# upsert embeddings one by one to avoid token limit
embeddings = []
@@ -417,3 +422,224 @@ class Embeddings:
with self.reindex_lock:
self.reindex_running = False
self.reindex_thread = None
def sync_triggers(self) -> None:
for camera in self.config.cameras.values():
# Get all existing triggers for this camera
existing_triggers = {
trigger.name: trigger
for trigger in Trigger.select().where(Trigger.camera == camera.name)
}
# Get all configured trigger names
configured_trigger_names = set(camera.semantic_search.triggers or {})
# Create or update triggers from config
for trigger_name, trigger in (
camera.semantic_search.triggers or {}
).items():
if trigger_name in existing_triggers:
existing_trigger = existing_triggers[trigger_name]
needs_embedding_update = False
thumbnail_missing = False
# Check if data has changed or thumbnail is missing for thumbnail type
if trigger.type == "thumbnail":
thumbnail_path = os.path.join(
TRIGGER_DIR, camera.name, f"{trigger.data}.webp"
)
try:
event = Event.get(Event.id == trigger.data)
if event.data.get("type") != "object":
logger.warning(
f"Event {trigger.data} is not a tracked object for {trigger.type} trigger"
)
continue # Skip if not an object
# Check if thumbnail needs to be updated (data changed or missing)
if (
existing_trigger.data != trigger.data
or not os.path.exists(thumbnail_path)
):
thumbnail = get_event_thumbnail_bytes(event)
if not thumbnail:
logger.warning(
f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}."
)
continue
self.write_trigger_thumbnail(
camera.name, trigger.data, thumbnail
)
thumbnail_missing = True
except DoesNotExist:
logger.warning(
f"Event ID {trigger.data} for trigger {trigger_name} does not exist."
)
continue
# Update existing trigger if data has changed
if (
existing_trigger.type != trigger.type
or existing_trigger.data != trigger.data
or existing_trigger.threshold != trigger.threshold
):
existing_trigger.type = trigger.type
existing_trigger.data = trigger.data
existing_trigger.threshold = trigger.threshold
needs_embedding_update = True
# Check if embedding is missing or needs update
if (
not existing_trigger.embedding
or needs_embedding_update
or thumbnail_missing
):
existing_trigger.embedding = self._calculate_trigger_embedding(
trigger
)
needs_embedding_update = True
if needs_embedding_update:
existing_trigger.save()
else:
# Create new trigger
try:
try:
event: Event = Event.get(Event.id == trigger.data)
except DoesNotExist:
logger.warning(
f"Event ID {trigger.data} for trigger {trigger_name} does not exist."
)
continue
# Skip the event if not an object
if event.data.get("type") != "object":
logger.warning(
f"Event ID {trigger.data} for trigger {trigger_name} is not a tracked object."
)
continue
thumbnail = get_event_thumbnail_bytes(event)
if not thumbnail:
logger.warning(
f"Unable to retrieve thumbnail for event ID {trigger.data} for {trigger_name}."
)
continue
self.write_trigger_thumbnail(
camera.name, trigger.data, thumbnail
)
# Calculate embedding for new trigger
embedding = self._calculate_trigger_embedding(trigger)
Trigger.create(
camera=camera.name,
name=trigger_name,
type=trigger.type,
data=trigger.data,
threshold=trigger.threshold,
model=self.config.semantic_search.model,
embedding=embedding,
triggering_event_id="",
last_triggered=None,
)
except IntegrityError:
pass # Handle duplicate creation attempts
# Remove triggers that are no longer in config
triggers_to_remove = (
set(existing_triggers.keys()) - configured_trigger_names
)
if triggers_to_remove:
Trigger.delete().where(
Trigger.camera == camera.name, Trigger.name.in_(triggers_to_remove)
).execute()
for trigger_name in triggers_to_remove:
self.remove_trigger_thumbnail(camera.name, trigger_name)
def write_trigger_thumbnail(
self, camera: str, event_id: str, thumbnail: bytes
) -> None:
"""Write the thumbnail to the trigger directory."""
try:
os.makedirs(os.path.join(TRIGGER_DIR, camera), exist_ok=True)
with open(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp"), "wb") as f:
f.write(thumbnail)
logger.debug(
f"Writing thumbnail for trigger with data {event_id} in {camera}."
)
except Exception as e:
logger.error(
f"Failed to write thumbnail for trigger with data {event_id} in {camera}: {e}"
)
def remove_trigger_thumbnail(self, camera: str, event_id: str) -> None:
"""Write the thumbnail to the trigger directory."""
try:
os.remove(os.path.join(TRIGGER_DIR, camera, f"{event_id}.webp"))
logger.debug(
f"Deleted thumbnail for trigger with data {event_id} in {camera}."
)
except Exception as e:
logger.error(
f"Failed to delete thumbnail for trigger with data {event_id} in {camera}: {e}"
)
def _calculate_trigger_embedding(self, trigger) -> bytes:
"""Calculate embedding for a trigger based on its type and data."""
if trigger.type == "description":
logger.debug(f"Generating embedding for trigger description {trigger.name}")
embedding = self.requestor.send_data(
EmbeddingsRequestEnum.embed_description.value,
{"id": None, "description": trigger.data, "upsert": False},
)
return embedding.astype(np.float32).tobytes()
elif trigger.type == "thumbnail":
# For image triggers, trigger.data should be an image ID
# Try to get embedding from vec_thumbnails table first
cursor = self.db.execute_sql(
"SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?",
[trigger.data],
)
row = cursor.fetchone() if cursor else None
if row:
return row[0] # Already in bytes format
else:
logger.debug(
f"No thumbnail embedding found for image ID: {trigger.data}, generating from saved trigger thumbnail"
)
try:
with open(
os.path.join(
TRIGGER_DIR, trigger.camera, f"{trigger.data}.webp"
),
"rb",
) as f:
thumbnail = f.read()
except Exception as e:
logger.error(
f"Failed to read thumbnail for trigger {trigger.name} with ID {trigger.data}: {e}"
)
return b""
logger.debug(
f"Generating embedding for trigger thumbnail {trigger.name} with ID {trigger.data}"
)
embedding = self.requestor.send_data(
EmbeddingsRequestEnum.embed_thumbnail.value,
{
"id": str(trigger.data),
"thumbnail": str(thumbnail),
"upsert": False,
},
)
return embedding.astype(np.float32).tobytes()
else:
logger.warning(f"Unknown trigger type: {trigger.type}")
return b""
frigate/embeddings/maintainer.py
+71 -31
View File
@@ -14,7 +14,10 @@ import numpy as np
from peewee import DoesNotExist
from frigate.comms.detections_updater import DetectionSubscriber, DetectionTypeEnum
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum, EmbeddingsResponder
from frigate.comms.embeddings_updater import (
EmbeddingsRequestEnum,
EmbeddingsResponder,
)
from frigate.comms.event_metadata_updater import (
EventMetadataPublisher,
EventMetadataSubscriber,
@@ -46,6 +49,7 @@ from frigate.data_processing.post.audio_transcription import (
from frigate.data_processing.post.license_plate import (
LicensePlatePostProcessor,
)
from frigate.data_processing.post.semantic_trigger import SemanticTriggerProcessor
from frigate.data_processing.real_time.api import RealTimeProcessorApi
from frigate.data_processing.real_time.bird import BirdRealTimeProcessor
from frigate.data_processing.real_time.custom_classification import (
@@ -60,7 +64,7 @@ from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataE
from frigate.db.sqlitevecq import SqliteVecQueueDatabase
from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum
from frigate.genai import get_genai_client
from frigate.models import Event, Recordings
from frigate.models import Event, Recordings, Trigger
from frigate.types import TrackedObjectUpdateTypesEnum
from frigate.util.builtin import serialize
from frigate.util.image import (
@@ -93,7 +97,11 @@ class EmbeddingMaintainer(threading.Thread):
self.config_updater = CameraConfigUpdateSubscriber(
self.config,
self.config.cameras,
[CameraConfigUpdateEnum.add, CameraConfigUpdateEnum.remove],
[
CameraConfigUpdateEnum.add,
CameraConfigUpdateEnum.remove,
CameraConfigUpdateEnum.semantic_search,
],
)
# Configure Frigate DB
@@ -109,7 +117,7 @@ class EmbeddingMaintainer(threading.Thread):
),
load_vec_extension=True,
)
models = [Event, Recordings]
models = [Event, Recordings, Trigger]
db.bind(models)
if config.semantic_search.enabled:
@@ -119,6 +127,9 @@ class EmbeddingMaintainer(threading.Thread):
if config.semantic_search.reindex:
self.embeddings.reindex()
# Sync semantic search triggers in db with config
self.embeddings.sync_triggers()
# create communication for updating event descriptions
self.requestor = InterProcessRequestor()
@@ -211,6 +222,17 @@ class EmbeddingMaintainer(threading.Thread):
AudioTranscriptionPostProcessor(self.config, self.requestor, metrics)
)
if self.config.semantic_search.enabled:
self.post_processors.append(
SemanticTriggerProcessor(
db,
self.config,
self.requestor,
metrics,
self.embeddings,
)
)
self.stop_event = stop_event
self.tracked_events: dict[str, list[Any]] = {}
self.early_request_sent: dict[str, bool] = {}
@@ -387,33 +409,6 @@ class EmbeddingMaintainer(threading.Thread):
event_id, camera, updated_db = ended
camera_config = self.config.cameras[camera]
# call any defined post processors
for processor in self.post_processors:
if isinstance(processor, LicensePlatePostProcessor):
recordings_available = self.recordings_available_through.get(camera)
if (
recordings_available is not None
and event_id in self.detected_license_plates
and self.config.cameras[camera].type != "lpr"
):
processor.process_data(
{
"event_id": event_id,
"camera": camera,
"recordings_available": self.recordings_available_through[
camera
],
"obj_data": self.detected_license_plates[event_id][
"obj_data"
],
},
PostProcessDataEnum.recording,
)
elif isinstance(processor, AudioTranscriptionPostProcessor):
continue
else:
processor.process_data(event_id, PostProcessDataEnum.event_id)
# expire in realtime processors
for processor in self.realtime_processors:
processor.expire_object(event_id, camera)
@@ -450,6 +445,41 @@ class EmbeddingMaintainer(threading.Thread):
):
self._process_genai_description(event, camera_config, thumbnail)
# call any defined post processors
for processor in self.post_processors:
if isinstance(processor, LicensePlatePostProcessor):
recordings_available = self.recordings_available_through.get(camera)
if (
recordings_available is not None
and event_id in self.detected_license_plates
and self.config.cameras[camera].type != "lpr"
):
processor.process_data(
{
"event_id": event_id,
"camera": camera,
"recordings_available": self.recordings_available_through[
camera
],
"obj_data": self.detected_license_plates[event_id][
"obj_data"
],
},
PostProcessDataEnum.recording,
)
elif isinstance(processor, AudioTranscriptionPostProcessor):
continue
elif isinstance(processor, SemanticTriggerProcessor):
processor.process_data(
{"event_id": event_id, "camera": camera, "type": "image"},
PostProcessDataEnum.tracked_object,
)
else:
processor.process_data(
{"event_id": event_id, "camera": camera},
PostProcessDataEnum.tracked_object,
)
# Delete tracked events based on the event_id
if event_id in self.tracked_events:
del self.tracked_events[event_id]
@@ -658,6 +688,16 @@ class EmbeddingMaintainer(threading.Thread):
if self.config.semantic_search.enabled:
self.embeddings.embed_description(event.id, description)
# Check semantic trigger for this description
for processor in self.post_processors:
if isinstance(processor, SemanticTriggerProcessor):
processor.process_data(
{"event_id": event.id, "camera": event.camera, "type": "text"},
PostProcessDataEnum.tracked_object,
)
else:
continue
logger.debug(
"Generated description for %s (%d images): %s",
event.id,