Semantic Search Triggers (#18969)

* semantic trigger test

* database and model

* config

* embeddings maintainer and trigger post-processor

* api to create, edit, delete triggers

* frontend and i18n keys

* use thumbnail and description for trigger types

* image picker tweaks

* initial sync

* thumbnail file management

* clean up logs and use saved thumbnail on frontend

* publish mqtt messages

* webpush changes to enable trigger notifications

* add enabled switch

* add triggers from explore

* renaming and deletion fixes

* fix typing

* UI updates and add last triggering event time and link

* log exception instead of return in endpoint

* highlight entry in UI when triggered

* save and delete thumbnails directly

* remove alert action for now and add descriptions

* tweaks

* clean up

* fix types

* docs

* docs tweaks

* docs

* reuse enum

frigate/embeddings/maintainer.py

@@ -14,7 +14,10 @@ import numpy as np
 from peewee import DoesNotExist
 
 from frigate.comms.detections_updater import DetectionSubscriber, DetectionTypeEnum
-from frigate.comms.embeddings_updater import EmbeddingsRequestEnum, EmbeddingsResponder
+from frigate.comms.embeddings_updater import (
+    EmbeddingsRequestEnum,
+    EmbeddingsResponder,
+)
 from frigate.comms.event_metadata_updater import (
     EventMetadataPublisher,
     EventMetadataSubscriber,
@@ -46,6 +49,7 @@ from frigate.data_processing.post.audio_transcription import (
 from frigate.data_processing.post.license_plate import (
     LicensePlatePostProcessor,
 )
+from frigate.data_processing.post.semantic_trigger import SemanticTriggerProcessor
 from frigate.data_processing.real_time.api import RealTimeProcessorApi
 from frigate.data_processing.real_time.bird import BirdRealTimeProcessor
 from frigate.data_processing.real_time.custom_classification import (
@@ -60,7 +64,7 @@ from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataE
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum
 from frigate.genai import get_genai_client
-from frigate.models import Event, Recordings
+from frigate.models import Event, Recordings, Trigger
 from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
 from frigate.util.image import (
@@ -93,7 +97,11 @@ class EmbeddingMaintainer(threading.Thread):
         self.config_updater = CameraConfigUpdateSubscriber(
             self.config,
             self.config.cameras,
-            [CameraConfigUpdateEnum.add, CameraConfigUpdateEnum.remove],
+            [
+                CameraConfigUpdateEnum.add,
+                CameraConfigUpdateEnum.remove,
+                CameraConfigUpdateEnum.semantic_search,
+            ],
         )
 
         # Configure Frigate DB
@@ -109,7 +117,7 @@ class EmbeddingMaintainer(threading.Thread):
             ),
             load_vec_extension=True,
         )
-        models = [Event, Recordings]
+        models = [Event, Recordings, Trigger]
         db.bind(models)
 
         if config.semantic_search.enabled:
@@ -119,6 +127,9 @@ class EmbeddingMaintainer(threading.Thread):
             if config.semantic_search.reindex:
                 self.embeddings.reindex()
 
+            # Sync semantic search triggers in db with config
+            self.embeddings.sync_triggers()
+
         # create communication for updating event descriptions
         self.requestor = InterProcessRequestor()
 
@@ -211,6 +222,17 @@ class EmbeddingMaintainer(threading.Thread):
                 AudioTranscriptionPostProcessor(self.config, self.requestor, metrics)
             )
 
+        if self.config.semantic_search.enabled:
+            self.post_processors.append(
+                SemanticTriggerProcessor(
+                    db,
+                    self.config,
+                    self.requestor,
+                    metrics,
+                    self.embeddings,
+                )
+            )
+
         self.stop_event = stop_event
         self.tracked_events: dict[str, list[Any]] = {}
         self.early_request_sent: dict[str, bool] = {}
@@ -387,33 +409,6 @@ class EmbeddingMaintainer(threading.Thread):
             event_id, camera, updated_db = ended
             camera_config = self.config.cameras[camera]
 
-            # call any defined post processors
-            for processor in self.post_processors:
-                if isinstance(processor, LicensePlatePostProcessor):
-                    recordings_available = self.recordings_available_through.get(camera)
-                    if (
-                        recordings_available is not None
-                        and event_id in self.detected_license_plates
-                        and self.config.cameras[camera].type != "lpr"
-                    ):
-                        processor.process_data(
-                            {
-                                "event_id": event_id,
-                                "camera": camera,
-                                "recordings_available": self.recordings_available_through[
-                                    camera
-                                ],
-                                "obj_data": self.detected_license_plates[event_id][
-                                    "obj_data"
-                                ],
-                            },
-                            PostProcessDataEnum.recording,
-                        )
-                elif isinstance(processor, AudioTranscriptionPostProcessor):
-                    continue
-                else:
-                    processor.process_data(event_id, PostProcessDataEnum.event_id)
-
             # expire in realtime processors
             for processor in self.realtime_processors:
                 processor.expire_object(event_id, camera)
@@ -450,6 +445,41 @@ class EmbeddingMaintainer(threading.Thread):
                 ):
                     self._process_genai_description(event, camera_config, thumbnail)
 
+            # call any defined post processors
+            for processor in self.post_processors:
+                if isinstance(processor, LicensePlatePostProcessor):
+                    recordings_available = self.recordings_available_through.get(camera)
+                    if (
+                        recordings_available is not None
+                        and event_id in self.detected_license_plates
+                        and self.config.cameras[camera].type != "lpr"
+                    ):
+                        processor.process_data(
+                            {
+                                "event_id": event_id,
+                                "camera": camera,
+                                "recordings_available": self.recordings_available_through[
+                                    camera
+                                ],
+                                "obj_data": self.detected_license_plates[event_id][
+                                    "obj_data"
+                                ],
+                            },
+                            PostProcessDataEnum.recording,
+                        )
+                elif isinstance(processor, AudioTranscriptionPostProcessor):
+                    continue
+                elif isinstance(processor, SemanticTriggerProcessor):
+                    processor.process_data(
+                        {"event_id": event_id, "camera": camera, "type": "image"},
+                        PostProcessDataEnum.tracked_object,
+                    )
+                else:
+                    processor.process_data(
+                        {"event_id": event_id, "camera": camera},
+                        PostProcessDataEnum.tracked_object,
+                    )
+
             # Delete tracked events based on the event_id
             if event_id in self.tracked_events:
                 del self.tracked_events[event_id]
@@ -658,6 +688,16 @@ class EmbeddingMaintainer(threading.Thread):
         if self.config.semantic_search.enabled:
             self.embeddings.embed_description(event.id, description)
 
+        # Check semantic trigger for this description
+        for processor in self.post_processors:
+            if isinstance(processor, SemanticTriggerProcessor):
+                processor.process_data(
+                    {"event_id": event.id, "camera": event.camera, "type": "text"},
+                    PostProcessDataEnum.tracked_object,
+                )
+            else:
+                continue
+
         logger.debug(
             "Generated description for %s (%d images): %s",
             event.id,