Fix semantic search reindex (#24407)

* fix semantic search reindex

sqlite-vec added the `_info` shadow table in 0.1.6 and drops it unconditionally when a vec0 table is destroyed, so `DROP TABLE` on a table written by 0.17 failed with "SQL logic error" once 0.18 moved to 0.1.9. `SqliteQueueDatabase` queues non-SELECT statements and stores the exception on the cursor it returns, and nothing read those cursors, so the failed drop and every write after it went unreported while reindex still logged "Embedded N thumbnails". `drop_embeddings_tables()` now recreates the missing `_info` stub before dropping, and writes go through `execute_write()`, which waits on the cursor so failures raise. `INSERT OR REPLACE` is gone too, since vec0 implements neither REPLACE nor UPSERT and it always failed on an id already in the table, including under the 0.1.3 build 0.17 shipped.

* use lock

* show reindex failure in status bar

frigate/db/sqlitevecq.py

@@ -1,8 +1,10 @@
 import logging
 import sqlite3
+import threading
 from typing import Any
 
 import regex
+from peewee import DatabaseError
 from playhouse.sqliteq import SqliteQueueDatabase
 
 logger = logging.getLogger(__name__)
@@ -17,6 +19,7 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
         self.load_vec_extension: bool = load_vec_extension
         # no extension necessary, sqlite will load correctly for each platform
         self.sqlite_vec_path = "/usr/local/lib/vec0"
+        self.upsert_lock = threading.Lock()
         super().__init__(*args, **kwargs)
 
     def _connect(self, *args: Any, **kwargs: Any) -> sqlite3.Connection:
@@ -53,6 +56,22 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
 
         conn.create_function("REGEXP", 2, regexp)
 
+    def execute_write(self, sql: str, params: Any = None) -> None:
+        """Run a write and wait for it, so that failures are raised here.
+
+        SqliteQueueDatabase hands non-SELECT statements to a writer thread and
+        stores any exception on the cursor it returns, so callers that ignore
+        that cursor never learn the write failed.
+        """
+        self.execute_sql(sql, params).fetchall()
+
+    def _table_exists(self, table: str) -> bool:
+        cursor = self.execute_sql(
+            "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
+            (table,),
+        )
+        return cursor.fetchone() is not None
+
     def _delete_embeddings(self, table: str, event_ids: list[str]) -> None:
         """Delete embeddings for the given events, if the table exists.
 
@@ -63,17 +82,17 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
             return
 
         # the embeddings tables are only created once semantic search has run
-        cursor = self.execute_sql(
-            "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
-            (table,),
-        )
-
-        if cursor.fetchone() is None:
+        if not self._table_exists(table):
             logger.debug("Skipping %s cleanup, table does not exist", table)
             return
 
         ids = ",".join(["?" for _ in event_ids])
-        self.execute_sql(f"DELETE FROM {table} WHERE id IN ({ids})", event_ids)
+
+        # callers treat cleanup as best effort, so log rather than propagate
+        try:
+            self.execute_write(f"DELETE FROM {table} WHERE id IN ({ids})", event_ids)
+        except DatabaseError:
+            logger.exception("Failed to delete embeddings from %s", table)
 
     def delete_embeddings_thumbnail(self, event_ids: list[str]) -> None:
         self._delete_embeddings("vec_thumbnails", event_ids)
@@ -81,25 +100,67 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
     def delete_embeddings_description(self, event_ids: list[str]) -> None:
         self._delete_embeddings("vec_descriptions", event_ids)
 
+    def _restore_vec_info_table(self, table: str) -> None:
+        """Recreate the _info shadow table a legacy vec0 table is missing.
+
+        sqlite-vec added _info in 0.1.6 and drops it unconditionally when a
+        table is destroyed, so tables written by Frigate 0.17 and earlier fail
+        to drop. An empty stub is enough, and leaving it unseeded keeps the
+        table reading as pre-0.1.10 if the drop does not follow.
+        """
+        if not self._table_exists(table) or self._table_exists(f"{table}_info"):
+            return
+
+        logger.debug("Restoring the %s_info shadow table before dropping", table)
+        self.execute_write(
+            f'CREATE TABLE "{table}_info" (key TEXT PRIMARY KEY, value ANY)'
+        )
+
     def drop_embeddings_tables(self) -> None:
-        self.execute_sql("""
-            DROP TABLE vec_descriptions;
-        """)
-        self.execute_sql("""
-            DROP TABLE vec_thumbnails;
-        """)
+        for table in ("vec_descriptions", "vec_thumbnails"):
+            self._restore_vec_info_table(table)
+            self.execute_write(f"DROP TABLE IF EXISTS {table}")
 
     def create_embeddings_tables(self) -> None:
         """Create vec0 virtual table for embeddings"""
-        self.execute_sql("""
+        self.execute_write("""
             CREATE VIRTUAL TABLE IF NOT EXISTS vec_thumbnails USING vec0(
                 id TEXT PRIMARY KEY,
                 thumbnail_embedding FLOAT[768] distance_metric=cosine
             );
         """)
-        self.execute_sql("""
+        self.execute_write("""
             CREATE VIRTUAL TABLE IF NOT EXISTS vec_descriptions USING vec0(
                 id TEXT PRIMARY KEY,
                 description_embedding FLOAT[768] distance_metric=cosine
             );
         """)
+
+    def upsert_embeddings(
+        self, table: str, column: str, embeddings: dict[str, bytes]
+    ) -> None:
+        """Write embeddings for the given event ids, replacing any that exist.
+
+        vec0 implements neither REPLACE nor UPSERT, so rows that are already
+        there have to be deleted first.
+        """
+        if not embeddings:
+            return
+
+        event_ids = list(embeddings.keys())
+        ids = ",".join(["?" for _ in event_ids])
+        params: list[Any] = []
+
+        for event_id in event_ids:
+            params.extend((event_id, embeddings[event_id]))
+
+        values = ", ".join(["(?, ?)"] * len(event_ids))
+
+        # reindexing and live embedding run on separate threads, and each write
+        # is queued separately, so the delete and the insert have to be held
+        # together or an interleaved pair fails on the vec0 primary key
+        with self.upsert_lock:
+            self.execute_write(f"DELETE FROM {table} WHERE id IN ({ids})", event_ids)
+            self.execute_write(
+                f"INSERT INTO {table}(id, {column}) VALUES {values}", params
+            )

frigate/embeddings/embeddings.py

@@ -6,9 +6,10 @@ import logging
 import os
 import threading
 import time
+from typing import Any
 
 import numpy as np
-from peewee import DoesNotExist, IntegrityError
+from peewee import DatabaseError, DoesNotExist, IntegrityError
 from PIL import Image
 from playhouse.shortcuts import model_to_dict
 
@@ -207,12 +208,10 @@ class Embeddings:
         embedding = self.vision_embedding([thumbnail])[0]
 
         if upsert:
-            self.db.execute_sql(
-                """
-                INSERT OR REPLACE INTO vec_thumbnails(id, thumbnail_embedding)
-                VALUES(?, ?)
-                """,
-                (event_id, serialize(embedding)),
+            self.db.upsert_embeddings(
+                "vec_thumbnails",
+                "thumbnail_embedding",
+                {event_id: serialize(embedding)},
             )
 
         self.image_inference_speed.update(datetime.datetime.now().timestamp() - start)
@@ -251,19 +250,12 @@ class Embeddings:
         embeddings = self.vision_embedding(valid_thumbs)
 
         if upsert:
-            items = []
+            items = {}
             for i in range(len(valid_ids)):
-                items.append(valid_ids[i])
-                items.append(serialize(embeddings[i]))
+                items[valid_ids[i]] = serialize(embeddings[i])
                 self.image_eps.update()
 
-            self.db.execute_sql(
-                """
-                INSERT OR REPLACE INTO vec_thumbnails(id, thumbnail_embedding)
-                VALUES {}
-                """.format(", ".join(["(?, ?)"] * len(valid_ids))),
-                items,
-            )
+            self.db.upsert_embeddings("vec_thumbnails", "thumbnail_embedding", items)
 
         duration = datetime.datetime.now().timestamp() - start
         self.image_inference_speed.update(duration / len(valid_ids))
@@ -277,12 +269,10 @@ class Embeddings:
         embedding = self.text_embedding([description])[0]
 
         if upsert:
-            self.db.execute_sql(
-                """
-                INSERT OR REPLACE INTO vec_descriptions(id, description_embedding)
-                VALUES(?, ?)
-                """,
-                (event_id, serialize(embedding)),
+            self.db.upsert_embeddings(
+                "vec_descriptions",
+                "description_embedding",
+                {event_id: serialize(embedding)},
             )
 
         self.text_inference_speed.update(datetime.datetime.now().timestamp() - start)
@@ -302,19 +292,14 @@ class Embeddings:
 
         if upsert:
             ids = list(event_descriptions.keys())
-            items = []
+            items = {}
 
             for i in range(len(ids)):
-                items.append(ids[i])
-                items.append(serialize(embeddings[i]))
+                items[ids[i]] = serialize(embeddings[i])
                 self.text_eps.update()
 
-            self.db.execute_sql(
-                """
-                INSERT OR REPLACE INTO vec_descriptions(id, description_embedding)
-                VALUES {}
-                """.format(", ".join(["(?, ?)"] * len(ids))),
-                items,
+            self.db.upsert_embeddings(
+                "vec_descriptions", "description_embedding", items
             )
 
         self.text_inference_speed.update(datetime.datetime.now().timestamp() - start)
@@ -322,6 +307,17 @@ class Embeddings:
         return embeddings
 
     def reindex(self) -> None:
+        """Rebuild every tracked object embedding from scratch."""
+        totals: dict[str, Any] = {"status": "indexing"}
+
+        try:
+            self._reindex(totals)
+        except DatabaseError:
+            logger.exception("Unable to reindex tracked object embeddings")
+            totals["status"] = "failed"
+            self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
+
+    def _reindex(self, totals: dict[str, Any]) -> None:
         logger.info("Indexing tracked object embeddings...")
 
         self.db.drop_embeddings_tables()
@@ -346,14 +342,18 @@ class Embeddings:
             batch_size = 32
         current_page = 1
 
-        totals = {
-            "thumbnails": 0,
-            "descriptions": 0,
-            "processed_objects": total_events - 1 if total_events < batch_size else 0,
-            "total_objects": total_events,
-            "time_remaining": 0 if total_events < batch_size else -1,
-            "status": "indexing",
-        }
+        totals.update(
+            {
+                "thumbnails": 0,
+                "descriptions": 0,
+                "processed_objects": total_events - 1
+                if total_events < batch_size
+                else 0,
+                "total_objects": total_events,
+                "time_remaining": 0 if total_events < batch_size else -1,
+                "status": "indexing",
+            }
+        )
 
         self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
 

frigate/test/test_sqlitevecq_embeddings.py

@@ -1,16 +1,24 @@
-"""Tests for embedding cleanup on the main Frigate database.
+"""Tests for embedding storage and cleanup on the main Frigate database.
 
 Embeddings are deleted whether or not semantic search is currently enabled, so
 the delete path has to tolerate databases where the vec0 tables were never
 created and installs where the sqlite-vec extension is unavailable.
+
+The write paths need the real extension, since the behavior under test belongs
+to vec0 itself, so those tests are skipped when it is not installed.
 """
 
 import os
+import struct
 import tempfile
 import unittest
 
+from peewee import OperationalError
+
 from frigate.db.sqlitevecq import SqliteVecQueueDatabase
 
+VEC_EXTENSION_PATH = "/usr/local/lib/vec0.so"
+
 
 class TestDeleteEmbeddings(unittest.TestCase):
     def setUp(self) -> None:
@@ -52,6 +60,21 @@ class TestDeleteEmbeddings(unittest.TestCase):
 
         self.assertEqual(self._thumbnail_ids(), ["b"])
 
+    def test_delete_failure_is_logged_not_raised(self) -> None:
+        self._create_thumbnails_table()
+        self.db.execute_sql(
+            """
+            CREATE TRIGGER vec_thumbnails_no_delete BEFORE DELETE ON vec_thumbnails
+            BEGIN SELECT RAISE(ABORT, 'delete blocked'); END
+            """
+        ).fetchall()
+
+        with self.assertLogs("frigate.db.sqlitevecq", level="ERROR") as logs:
+            self.db.delete_embeddings_thumbnail(event_ids=["a"])
+
+        self.assertIn("Failed to delete embeddings", logs.output[0])
+        self.assertEqual(self._thumbnail_ids(), ["a", "b"])
+
     def test_delete_skipped_without_extension(self) -> None:
         self._create_thumbnails_table()
         self.db.load_vec_extension = False
@@ -61,3 +84,104 @@ class TestDeleteEmbeddings(unittest.TestCase):
 
         # the vec0 tables cannot be written without the extension
         self.assertEqual(self._thumbnail_ids(), ["a", "b"])
+
+
+def _vector(value: float) -> bytes:
+    return struct.pack("768f", *([value] * 768))
+
+
+@unittest.skipUnless(
+    os.path.exists(VEC_EXTENSION_PATH), "sqlite-vec extension is not installed"
+)
+class TestEmbeddingsTableWrites(unittest.TestCase):
+    """Covers the vec0 writes behind semantic search reindexing."""
+
+    def setUp(self) -> None:
+        self.tmp_dir = tempfile.TemporaryDirectory()
+        self.db = SqliteVecQueueDatabase(
+            os.path.join(self.tmp_dir.name, "test.db"), load_vec_extension=True
+        )
+        self.db.start()
+        self.db.create_embeddings_tables()
+
+    def tearDown(self) -> None:
+        self.db.stop()
+        self.db.close()
+        self.tmp_dir.cleanup()
+
+    def _vec_tables(self) -> list[str]:
+        return [
+            row[0]
+            for row in self.db.execute_sql(
+                "SELECT name FROM sqlite_master WHERE name LIKE 'vec_%' ORDER BY name"
+            )
+        ]
+
+    def _make_legacy(self, table: str) -> None:
+        # sqlite-vec added the _info shadow table in 0.1.6, so tables written by
+        # Frigate 0.17 and earlier do not have one
+        self.db.execute_sql(f"DROP TABLE {table}_info").fetchall()
+
+    def _stored(self, table: str, column: str, event_id: str) -> str | None:
+        row = self.db.execute_sql(
+            f"SELECT vec_to_json({column}) FROM {table} WHERE id = ?", (event_id,)
+        ).fetchone()
+        return row[0] if row else None
+
+    def test_write_error_is_raised(self) -> None:
+        # queued writes hide their exception in the returned cursor
+        with self.assertRaises(OperationalError):
+            self.db.execute_write("INSERT INTO vec_missing(id) VALUES ('a')")
+
+    def test_drop_tables_removes_legacy_tables(self) -> None:
+        self._make_legacy("vec_thumbnails")
+        self._make_legacy("vec_descriptions")
+
+        self.db.drop_embeddings_tables()
+
+        self.assertEqual(self._vec_tables(), [])
+
+    def test_drop_tables_without_any_tables_does_not_raise(self) -> None:
+        self.db.drop_embeddings_tables()
+
+        self.db.drop_embeddings_tables()
+
+    def test_upsert_replaces_existing_embedding(self) -> None:
+        self.db.upsert_embeddings(
+            "vec_thumbnails", "thumbnail_embedding", {"evt1": _vector(0.01)}
+        )
+
+        self.db.upsert_embeddings(
+            "vec_thumbnails", "thumbnail_embedding", {"evt1": _vector(0.99)}
+        )
+
+        stored = self._stored("vec_thumbnails", "thumbnail_embedding", "evt1")
+        self.assertTrue(stored.startswith("[0.990000"), stored)
+
+    def test_upsert_keeps_one_row_per_event(self) -> None:
+        for _ in range(3):
+            self.db.upsert_embeddings(
+                "vec_descriptions", "description_embedding", {"evt1": _vector(0.5)}
+            )
+
+        count = self.db.execute_sql(
+            "SELECT count(*) FROM vec_descriptions WHERE id = 'evt1'"
+        ).fetchone()[0]
+        self.assertEqual(count, 1)
+
+    def test_reindex_cycle_rewrites_legacy_tables(self) -> None:
+        """The 0.18 upgrade path: old vectors in, new vectors out."""
+        self.db.upsert_embeddings(
+            "vec_thumbnails", "thumbnail_embedding", {"evt1": _vector(0.01)}
+        )
+        self._make_legacy("vec_thumbnails")
+        self._make_legacy("vec_descriptions")
+
+        self.db.drop_embeddings_tables()
+        self.db.create_embeddings_tables()
+        self.db.upsert_embeddings(
+            "vec_thumbnails", "thumbnail_embedding", {"evt1": _vector(0.99)}
+        )
+
+        stored = self._stored("vec_thumbnails", "thumbnail_embedding", "evt1")
+        self.assertTrue(stored.startswith("[0.990000"), stored)

web/public/locales/en/views/system.json

@@ -233,6 +233,7 @@
     "detectHighCpuUsage": "{{camera}} has high detect CPU usage ({{detectAvg}}%)",
     "healthy": "System is healthy",
     "reindexingEmbeddings": "Reindexing embeddings ({{processed}}% complete)",
+    "reindexEmbeddingsFailed": "Reindexing embeddings failed, check the logs",
     "cameraIsOffline": "{{camera}} is offline",
     "detectIsSlow": "{{detect}} is slow ({{speed}} ms)",
     "detectIsVerySlow": "{{detect}} is very slow ({{speed}} ms)",

web/src/components/Statusbar.tsx

@@ -71,8 +71,9 @@ export default function Statusbar() {
 
   useEffect(() => {
     if (reindexState) {
-      if (reindexState.status == "indexing") {
-        clearMessages("embeddings-reindex");
+      clearMessages("embeddings-reindex");
+
+      if (reindexState.status === "indexing") {
         addMessage(
           "embeddings-reindex",
           t("stats.reindexingEmbeddings", {
@@ -82,9 +83,8 @@ export default function Statusbar() {
             ),
           }),
         );
-      }
-      if (reindexState.status === "completed") {
-        clearMessages("embeddings-reindex");
+      } else if (reindexState.status === "failed") {
+        addMessage("embeddings-reindex", t("stats.reindexEmbeddingsFailed"));
       }
     }
   }, [reindexState, addMessage, clearMessages, t]);

web/src/components/navigation/Bottombar.tsx

@@ -133,8 +133,9 @@ function StatusAlertNav({ className, large }: StatusAlertNavProps) {
 
   useEffect(() => {
     if (reindexState) {
-      if (reindexState.status == "indexing") {
-        clearMessages("embeddings-reindex");
+      clearMessages("embeddings-reindex");
+
+      if (reindexState.status === "indexing") {
         addMessage(
           "embeddings-reindex",
           t("stats.reindexingEmbeddings", {
@@ -144,9 +145,8 @@ function StatusAlertNav({ className, large }: StatusAlertNavProps) {
             ),
           }),
         );
-      }
-      if (reindexState.status === "completed") {
-        clearMessages("embeddings-reindex");
+      } else if (reindexState.status === "failed") {
+        addMessage("embeddings-reindex", t("stats.reindexEmbeddingsFailed"));
       }
     }
   }, [reindexState, addMessage, clearMessages, t]);