Fix semantic search reindex (#24407)

* fix semantic search reindex

sqlite-vec added the `_info` shadow table in 0.1.6 and drops it unconditionally when a vec0 table is destroyed, so `DROP TABLE` on a table written by 0.17 failed with "SQL logic error" once 0.18 moved to 0.1.9. `SqliteQueueDatabase` queues non-SELECT statements and stores the exception on the cursor it returns, and nothing read those cursors, so the failed drop and every write after it went unreported while reindex still logged "Embedded N thumbnails". `drop_embeddings_tables()` now recreates the missing `_info` stub before dropping, and writes go through `execute_write()`, which waits on the cursor so failures raise. `INSERT OR REPLACE` is gone too, since vec0 implements neither REPLACE nor UPSERT and it always failed on an id already in the table, including under the 0.1.3 build 0.17 shipped.

* use lock

* show reindex failure in status bar

frigate/db/sqlitevecq.py

@@ -1,8 +1,10 @@
 import logging
 import sqlite3
+import threading
 from typing import Any
 
 import regex
+from peewee import DatabaseError
 from playhouse.sqliteq import SqliteQueueDatabase
 
 logger = logging.getLogger(__name__)
@@ -17,6 +19,7 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
         self.load_vec_extension: bool = load_vec_extension
         # no extension necessary, sqlite will load correctly for each platform
         self.sqlite_vec_path = "/usr/local/lib/vec0"
+        self.upsert_lock = threading.Lock()
         super().__init__(*args, **kwargs)
 
     def _connect(self, *args: Any, **kwargs: Any) -> sqlite3.Connection:
@@ -53,6 +56,22 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
 
         conn.create_function("REGEXP", 2, regexp)
 
+    def execute_write(self, sql: str, params: Any = None) -> None:
+        """Run a write and wait for it, so that failures are raised here.
+
+        SqliteQueueDatabase hands non-SELECT statements to a writer thread and
+        stores any exception on the cursor it returns, so callers that ignore
+        that cursor never learn the write failed.
+        """
+        self.execute_sql(sql, params).fetchall()
+
+    def _table_exists(self, table: str) -> bool:
+        cursor = self.execute_sql(
+            "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
+            (table,),
+        )
+        return cursor.fetchone() is not None
+
     def _delete_embeddings(self, table: str, event_ids: list[str]) -> None:
         """Delete embeddings for the given events, if the table exists.
 
@@ -63,17 +82,17 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
             return
 
         # the embeddings tables are only created once semantic search has run
-        cursor = self.execute_sql(
-            "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
-            (table,),
-        )
-
-        if cursor.fetchone() is None:
+        if not self._table_exists(table):
             logger.debug("Skipping %s cleanup, table does not exist", table)
             return
 
         ids = ",".join(["?" for _ in event_ids])
-        self.execute_sql(f"DELETE FROM {table} WHERE id IN ({ids})", event_ids)
+
+        # callers treat cleanup as best effort, so log rather than propagate
+        try:
+            self.execute_write(f"DELETE FROM {table} WHERE id IN ({ids})", event_ids)
+        except DatabaseError:
+            logger.exception("Failed to delete embeddings from %s", table)
 
     def delete_embeddings_thumbnail(self, event_ids: list[str]) -> None:
         self._delete_embeddings("vec_thumbnails", event_ids)
@@ -81,25 +100,67 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
     def delete_embeddings_description(self, event_ids: list[str]) -> None:
         self._delete_embeddings("vec_descriptions", event_ids)
 
+    def _restore_vec_info_table(self, table: str) -> None:
+        """Recreate the _info shadow table a legacy vec0 table is missing.
+
+        sqlite-vec added _info in 0.1.6 and drops it unconditionally when a
+        table is destroyed, so tables written by Frigate 0.17 and earlier fail
+        to drop. An empty stub is enough, and leaving it unseeded keeps the
+        table reading as pre-0.1.10 if the drop does not follow.
+        """
+        if not self._table_exists(table) or self._table_exists(f"{table}_info"):
+            return
+
+        logger.debug("Restoring the %s_info shadow table before dropping", table)
+        self.execute_write(
+            f'CREATE TABLE "{table}_info" (key TEXT PRIMARY KEY, value ANY)'
+        )
+
     def drop_embeddings_tables(self) -> None:
-        self.execute_sql("""
-            DROP TABLE vec_descriptions;
-        """)
-        self.execute_sql("""
-            DROP TABLE vec_thumbnails;
-        """)
+        for table in ("vec_descriptions", "vec_thumbnails"):
+            self._restore_vec_info_table(table)
+            self.execute_write(f"DROP TABLE IF EXISTS {table}")
 
     def create_embeddings_tables(self) -> None:
         """Create vec0 virtual table for embeddings"""
-        self.execute_sql("""
+        self.execute_write("""
             CREATE VIRTUAL TABLE IF NOT EXISTS vec_thumbnails USING vec0(
                 id TEXT PRIMARY KEY,
                 thumbnail_embedding FLOAT[768] distance_metric=cosine
             );
         """)
-        self.execute_sql("""
+        self.execute_write("""
             CREATE VIRTUAL TABLE IF NOT EXISTS vec_descriptions USING vec0(
                 id TEXT PRIMARY KEY,
                 description_embedding FLOAT[768] distance_metric=cosine
             );
         """)
+
+    def upsert_embeddings(
+        self, table: str, column: str, embeddings: dict[str, bytes]
+    ) -> None:
+        """Write embeddings for the given event ids, replacing any that exist.
+
+        vec0 implements neither REPLACE nor UPSERT, so rows that are already
+        there have to be deleted first.
+        """
+        if not embeddings:
+            return
+
+        event_ids = list(embeddings.keys())
+        ids = ",".join(["?" for _ in event_ids])
+        params: list[Any] = []
+
+        for event_id in event_ids:
+            params.extend((event_id, embeddings[event_id]))
+
+        values = ", ".join(["(?, ?)"] * len(event_ids))
+
+        # reindexing and live embedding run on separate threads, and each write
+        # is queued separately, so the delete and the insert have to be held
+        # together or an interleaved pair fails on the vec0 primary key
+        with self.upsert_lock:
+            self.execute_write(f"DELETE FROM {table} WHERE id IN ({ids})", event_ids)
+            self.execute_write(
+                f"INSERT INTO {table}(id, {column}) VALUES {values}", params
+            )