Semantic Search Triggers (#18969)

* semantic trigger test

* database and model

* config

* embeddings maintainer and trigger post-processor

* api to create, edit, delete triggers

* frontend and i18n keys

* use thumbnail and description for trigger types

* image picker tweaks

* initial sync

* thumbnail file management

* clean up logs and use saved thumbnail on frontend

* publish mqtt messages

* webpush changes to enable trigger notifications

* add enabled switch

* add triggers from explore

* renaming and deletion fixes

* fix typing

* UI updates and add last triggering event time and link

* log exception instead of return in endpoint

* highlight entry in UI when triggered

* save and delete thumbnails directly

* remove alert action for now and add descriptions

* tweaks

* clean up

* fix types

* docs

* docs tweaks

* docs

* reuse enum

frigate/config/camera/camera.py

@@ -22,6 +22,7 @@ from ..classification import (
     AudioTranscriptionConfig,
     CameraFaceRecognitionConfig,
     CameraLicensePlateRecognitionConfig,
+    CameraSemanticSearchConfig,
 )
 from .audio import AudioConfig
 from .birdseye import BirdseyeCameraConfig
@@ -91,6 +92,10 @@ class CameraConfig(FrigateBaseModel):
     review: ReviewConfig = Field(
         default_factory=ReviewConfig, title="Review configuration."
     )
+    semantic_search: CameraSemanticSearchConfig = Field(
+        default_factory=CameraSemanticSearchConfig,
+        title="Semantic search configuration.",
+    )
     snapshots: SnapshotsConfig = Field(
         default_factory=SnapshotsConfig, title="Snapshot configuration."
     )

frigate/config/camera/updater.py

@@ -23,6 +23,7 @@ class CameraConfigUpdateEnum(str, Enum):
     record = "record"
     remove = "remove"  # for removing a camera
     review = "review"
+    semantic_search = "semantic_search"  # for semantic search triggers
     snapshots = "snapshots"
     zones = "zones"
 
@@ -106,6 +107,8 @@ class CameraConfigUpdateSubscriber:
             config.record = updated_config
         elif update_type == CameraConfigUpdateEnum.review:
             config.review = updated_config
+        elif update_type == CameraConfigUpdateEnum.semantic_search:
+            config.semantic_search = updated_config
         elif update_type == CameraConfigUpdateEnum.snapshots:
             config.snapshots = updated_config
         elif update_type == CameraConfigUpdateEnum.zones:

frigate/config/classification.py

@@ -10,6 +10,7 @@ __all__ = [
     "CameraLicensePlateRecognitionConfig",
     "FaceRecognitionConfig",
     "SemanticSearchConfig",
+    "CameraSemanticSearchConfig",
     "LicensePlateRecognitionConfig",
 ]
 
@@ -24,6 +25,15 @@ class EnrichmentsDeviceEnum(str, Enum):
     CPU = "CPU"
 
 
+class TriggerType(str, Enum):
+    THUMBNAIL = "thumbnail"
+    DESCRIPTION = "description"
+
+
+class TriggerAction(str, Enum):
+    NOTIFICATION = "notification"
+
+
 class AudioTranscriptionConfig(FrigateBaseModel):
     enabled: bool = Field(default=False, title="Enable audio transcription.")
     language: str = Field(
@@ -113,6 +123,32 @@ class SemanticSearchConfig(FrigateBaseModel):
     )
 
 
+class TriggerConfig(FrigateBaseModel):
+    enabled: bool = Field(default=True, title="Enable this trigger")
+    type: TriggerType = Field(default=TriggerType.DESCRIPTION, title="Type of trigger")
+    data: str = Field(title="Trigger content (text phrase or image ID)")
+    threshold: float = Field(
+        title="Confidence score required to run the trigger",
+        default=0.8,
+        gt=0.0,
+        le=1.0,
+    )
+    actions: Optional[List[TriggerAction]] = Field(
+        default=[], title="Actions to perform when trigger is matched"
+    )
+
+    model_config = ConfigDict(extra="forbid", protected_namespaces=())
+
+
+class CameraSemanticSearchConfig(FrigateBaseModel):
+    triggers: Optional[Dict[str, TriggerConfig]] = Field(
+        default=None,
+        title="Trigger actions on tracked objects that match existing thumbnails or descriptions",
+    )
+
+    model_config = ConfigDict(extra="forbid", protected_namespaces=())
+
+
 class FaceRecognitionConfig(FrigateBaseModel):
     enabled: bool = Field(default=False, title="Enable face recognition.")
     model_size: str = Field(