Dynamic embeddings reindexing (#17418)

* reindex with api endpoint and zmq

* threading

* frontend

* require admin role
This commit is contained in:
Josh Hawkins
2025-03-27 11:29:34 -06:00
committed by GitHub
parent 67dd50a7f7
commit 23c3323871
7 changed files with 193 additions and 73 deletions
+3
View File
@@ -250,3 +250,6 @@ class EmbeddingsContext:
return self.requestor.send_data(
EmbeddingsRequestEnum.reprocess_plate.value, {"event": event}
)
def reindex_embeddings(self) -> dict[str, any]:
return self.requestor.send_data(EmbeddingsRequestEnum.reindex.value, {})
+29
View File
@@ -3,6 +3,7 @@
import datetime
import logging
import os
import threading
import time
from numpy import ndarray
@@ -74,6 +75,10 @@ class Embeddings:
self.metrics = metrics
self.requestor = InterProcessRequestor()
self.reindex_lock = threading.Lock()
self.reindex_thread = None
self.reindex_running = False
# Create tables if they don't exist
self.db.create_embeddings_tables()
@@ -368,3 +373,27 @@ class Embeddings:
totals["status"] = "completed"
self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
def start_reindex(self) -> bool:
"""Start reindexing in a separate thread if not already running."""
with self.reindex_lock:
if self.reindex_running:
logger.warning("Reindex embeddings is already running.")
return False
# Mark as running and start the thread
self.reindex_running = True
self.reindex_thread = threading.Thread(
target=self._reindex_wrapper, daemon=True
)
self.reindex_thread.start()
return True
def _reindex_wrapper(self) -> None:
"""Wrapper to run reindex and reset running flag when done."""
try:
self.reindex()
finally:
with self.reindex_lock:
self.reindex_running = False
self.reindex_thread = None
+3
View File
@@ -206,6 +206,9 @@ class EmbeddingMaintainer(threading.Thread):
self.embeddings.embed_description("", data, upsert=False),
pack=False,
)
elif topic == EmbeddingsRequestEnum.reindex.value:
response = self.embeddings.start_reindex()
return "started" if response else "in_progress"
processors = [self.realtime_processors, self.post_processors]
for processor_list in processors: