Semantic Search for Detections (#11899)

* Initial re-implementation of semantic search

* put docker-compose back and make reindex match docs

* remove debug code and fix import

* fix docs

* manually build pysqlite3 as binaries are only available for x86-64

* update comment in build_pysqlite3.sh

* only embed objects

* better error handling when genai fails

* ask ollama to pull requested model at startup

* update ollama docs

* address some PR review comments

* fix lint

* use IPC to write description, update docs for reindex

* remove gemini-pro-vision from docs as it will be unavailable soon

* fix OpenAI doc available models

* fix api error in gemini and metadata for embeddings

frigate/events/audio.py

@@ -223,7 +223,7 @@ class AudioEventMaintainer(threading.Thread):
                     audio_detections.append(label)
 
             # send audio detection data
-            self.detection_publisher.send_data(
+            self.detection_publisher.publish(
                 (
                     self.config.name,
                     datetime.datetime.now().timestamp(),

frigate/events/cleanup.py

@@ -10,6 +10,7 @@ from pathlib import Path
 
 from frigate.config import FrigateConfig
 from frigate.const import CLIPS_DIR
+from frigate.embeddings.embeddings import Embeddings
 from frigate.models import Event, Timeline
 
 logger = logging.getLogger(__name__)
@@ -26,6 +27,7 @@ class EventCleanup(threading.Thread):
         self.name = "event_cleanup"
         self.config = config
         self.stop_event = stop_event
+        self.embeddings = Embeddings()
         self.camera_keys = list(self.config.cameras.keys())
         self.removed_camera_labels: list[str] = None
         self.camera_labels: dict[str, dict[str, any]] = {}
@@ -197,9 +199,20 @@ class EventCleanup(threading.Thread):
             self.expire(EventCleanupType.snapshots)
 
             # drop events from db where has_clip and has_snapshot are false
-            delete_query = Event.delete().where(
-                Event.has_clip == False, Event.has_snapshot == False
+            events = (
+                Event.select()
+                .where(Event.has_clip == False, Event.has_snapshot == False)
+                .iterator()
             )
-            delete_query.execute()
+            events_to_delete = [e.id for e in events]
+            if len(events_to_delete) > 0:
+                chunk_size = 50
+                for i in range(0, len(events_to_delete), chunk_size):
+                    chunk = events_to_delete[i : i + chunk_size]
+                    Event.delete().where(Event.id << chunk).execute()
+
+                    if self.config.semantic_search.enabled:
+                        self.embeddings.thumbnail.delete(ids=chunk)
+                        self.embeddings.description.delete(ids=chunk)
 
         logger.info("Exiting event cleanup...")

frigate/events/external.py

@@ -86,7 +86,7 @@ class ExternalEventProcessor:
 
         if source_type == "api":
             self.event_camera[event_id] = camera
-            self.detection_updater.send_data(
+            self.detection_updater.publish(
                 (
                     camera,
                     now,
@@ -115,7 +115,7 @@ class ExternalEventProcessor:
         )
 
         if event_id in self.event_camera:
-            self.detection_updater.send_data(
+            self.detection_updater.publish(
                 (
                     self.event_camera[event_id],
                     end_time,

frigate/events/maintainer.py

@@ -237,7 +237,7 @@ class EventProcessor(threading.Thread):
 
         if event_type == EventStateEnum.end:
             del self.events_in_process[event_data["id"]]
-            self.event_end_publisher.publish((event_data["id"], camera))
+            self.event_end_publisher.publish((event_data["id"], camera, updated_db))
 
     def handle_external_detection(
         self, event_type: EventStateEnum, event_data: Event