administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-04-29 01:57:41 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

publish mqtt messages

Browse Source
This commit is contained in:
Josh Hawkins 2025-07-02 08:43:48 -05:00
parent 3006d9fceb
commit c97be95173
1 changed files with 61 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
frigate/data_processing/post/semantic_trigger.py
View File

@ -63,40 +63,6 @@ class SemanticTriggerProcessor(PostProcessorApi):
process_type = data["type"] process_type = data["type"]
if self.config.cameras[camera].semantic_search.triggers is None: if self.config.cameras[camera].semantic_search.triggers is None:
logger.debug(
f"No semantic triggers configured for camera: {camera}, skipping processing."
)
return
# Get embeddings based on type
thumbnail_embedding = None
description_embedding = None
if process_type == "image":
cursor = self.db.execute_sql(
"""
SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?
""",
[event_id],
)
row = cursor.fetchone() if cursor else None
if row:
thumbnail_embedding = np.frombuffer(row[0], dtype=np.float32)
if process_type == "text":
cursor = self.db.execute_sql(
"""
SELECT description_embedding FROM vec_descriptions WHERE id = ?
""",
[event_id],
)
row = cursor.fetchone() if cursor else None
if row:
description_embedding = np.frombuffer(row[0], dtype=np.float32)
# Skip processing if we don't have any embeddings
if thumbnail_embedding is None and description_embedding is None:
logger.debug(f"No embeddings found for event id: {event_id}")
return return
triggers = ( triggers = (
@ -115,21 +81,52 @@ class SemanticTriggerProcessor(PostProcessorApi):
for trigger in triggers: for trigger in triggers:
logger.debug( logger.debug(
f"Processing trigger for {trigger['camera']}: {trigger['name']}" f"Processing {trigger['type']} trigger for {event_id} on {trigger['camera']}: {trigger['name']}"
) )
trigger_embedding = np.frombuffer(trigger["embedding"], dtype=np.float32) trigger_embedding = np.frombuffer(trigger["embedding"], dtype=np.float32)
# Get embeddings based on type
thumbnail_embedding = None
description_embedding = None
if process_type == "image":
cursor = self.db.execute_sql(
"""
SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?
""",
[event_id],
)
row = cursor.fetchone() if cursor else None
if row:
thumbnail_embedding = np.frombuffer(row[0], dtype=np.float32)
if process_type == "text":
cursor = self.db.execute_sql(
"""
SELECT description_embedding FROM vec_descriptions WHERE id = ?
""",
[event_id],
)
row = cursor.fetchone() if cursor else None
if row:
description_embedding = np.frombuffer(row[0], dtype=np.float32)
# Skip processing if we don't have any embeddings
if thumbnail_embedding is None and description_embedding is None:
logger.debug(f"No embeddings found for {event_id}")
return
# Determine which embedding to compare based on trigger type # Determine which embedding to compare based on trigger type
if trigger["type"] == "text" and thumbnail_embedding is not None: if (
trigger["type"] in ["text", "thumbnail"]
and thumbnail_embedding is not None
):
data_embedding = thumbnail_embedding data_embedding = thumbnail_embedding
normalized_distance = self.thumb_stats.normalize( normalized_distance = self.thumb_stats.normalize(
[cosine_distance(data_embedding, trigger_embedding)], [cosine_distance(data_embedding, trigger_embedding)],
save_stats=False, save_stats=False,
)[0] )[0]
elif trigger["type"] == "thumbnail" and thumbnail_embedding is not None:
data_embedding = thumbnail_embedding
normalized_distance = cosine_distance(data_embedding, trigger_embedding)
elif trigger["type"] == "description" and description_embedding is not None: elif trigger["type"] == "description" and description_embedding is not None:
data_embedding = description_embedding data_embedding = description_embedding
normalized_distance = self.desc_stats.normalize( normalized_distance = self.desc_stats.normalize(
@ -138,23 +135,43 @@ class SemanticTriggerProcessor(PostProcessorApi):
)[0] )[0]
else: else:
# Skip trigger if embedding type doesn't match available data
continue continue
similarity = 1 - normalized_distance similarity = 1 - normalized_distance
logger.debug( logger.debug(
f"Trigger: {trigger['data'] if trigger['type'] == 'text' or trigger['type'] == 'description' else 'image'} " f"Trigger {trigger['name']} ({trigger['data'] if trigger['type'] == 'text' or trigger['type'] == 'description' else 'image'}): "
f"(camera: {trigger['camera']}): normalized distance: {normalized_distance:.4f}, " f"normalized distance: {normalized_distance:.4f}, "
f"similarity: {similarity:.4f}, threshold: {trigger['threshold']}" f"similarity: {similarity:.4f}, threshold: {trigger['threshold']}"
) )
# Check if similarity meets threshold # Check if similarity meets threshold
if similarity >= trigger["threshold"]: if similarity >= trigger["threshold"]:
logger.info( logger.info(
f"Trigger '{trigger['name']}' activated with similarity {similarity:.4f}" f"Trigger {trigger['name']} activated with similarity {similarity:.4f}"
) )
# TODO: handle actions for the trigger
# Always publish MQTT message
self.requestor.send_data(
"triggers",
json.dumps(
{
"name": trigger["name"],
"camera": camera,
"event_id": event_id,
"type": trigger["type"],
"score": similarity,
}
),
)
if (
self.config.cameras[camera]
.semantic_search.triggers[trigger["name"]]
.actions
):
# TODO: handle actions for the trigger
pass
if WRITE_DEBUG_IMAGES: if WRITE_DEBUG_IMAGES:
try: try: