diff --git a/frigate/const.py b/frigate/const.py index e8e841f4f..ad1aacd0f 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -85,6 +85,7 @@ CLEAR_ONGOING_REVIEW_SEGMENTS = "clear_ongoing_review_segments" UPDATE_CAMERA_ACTIVITY = "update_camera_activity" UPDATE_EVENT_DESCRIPTION = "update_event_description" UPDATE_MODEL_STATE = "update_model_state" +UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress" # Stats Values diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py index 9bcf2e6c0..19cdcf392 100644 --- a/frigate/embeddings/embeddings.py +++ b/frigate/embeddings/embeddings.py @@ -10,7 +10,7 @@ from playhouse.shortcuts import model_to_dict from frigate.comms.inter_process import InterProcessRequestor from frigate.config.semantic_search import SemanticSearchConfig -from frigate.const import UPDATE_MODEL_STATE +from frigate.const import UPDATE_EMBEDDINGS_REINDEX_PROGRESS, UPDATE_MODEL_STATE from frigate.db.sqlitevecq import SqliteVecQueueDatabase from frigate.models import Event from frigate.types import ModelStatusTypesEnum @@ -165,19 +165,36 @@ class Embeddings: return embedding def reindex(self) -> None: - logger.info("Indexing event embeddings...") + logger.info("Indexing tracked object embeddings...") self._drop_tables() self._create_tables() st = time.time() totals = { - "thumb": 0, - "desc": 0, + "thumbnails": 0, + "descriptions": 0, + "processed_objects": 0, + "total_objects": 0, } + self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals) + + # Get total count of events to process + total_events = ( + Event.select() + .where( + (Event.has_clip == True | Event.has_snapshot == True) + & Event.thumbnail.is_null(False) + ) + .count() + ) + totals["total_objects"] = total_events + batch_size = 100 current_page = 1 + processed_events = 0 + events = ( Event.select() .where( @@ -193,11 +210,27 @@ class Embeddings: for event in events: thumbnail = base64.b64decode(event.thumbnail) self.upsert_thumbnail(event.id, thumbnail) - totals["thumb"] += 1 + totals["thumbnails"] += 1 + if description := event.data.get("description", "").strip(): - totals["desc"] += 1 + totals["descriptions"] += 1 self.upsert_description(event.id, description) + totals["processed_objects"] += 1 + + # display progress debug message every batch_size events + progress = (processed_events / total_events) * 100 + logger.debug( + "Processed %d/%d events (%.2f%% complete) | Thumbnails: %d, Descriptions: %d", + processed_events, + total_events, + progress, + totals["thumbnails"], + totals["descriptions"], + ) + self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals) + + # Move to the next page current_page += 1 events = ( Event.select() @@ -211,7 +244,8 @@ class Embeddings: logger.info( "Embedded %d thumbnails and %d descriptions in %s seconds", - totals["thumb"], - totals["desc"], + totals["thumbnails"], + totals["descriptions"], time.time() - st, ) + self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals) diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index a78722b66..2e083cf83 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -2,6 +2,7 @@ import { baseUrl } from "./baseUrl"; import { useCallback, useEffect, useState } from "react"; import useWebSocket, { ReadyState } from "react-use-websocket"; import { + EmbeddingsReindexProgressType, FrigateCameraState, FrigateEvent, FrigateReview, @@ -302,6 +303,42 @@ export function useModelState( return { payload: data ? data[model] : undefined }; } +export function useEmbeddingsReindexProgress( + revalidateOnFocus: boolean = true, +): { + payload: EmbeddingsReindexProgressType; +} { + const { + value: { payload }, + send: sendCommand, + } = useWs("embeddings_reindex_progress", "embeddingsReindexProgress"); + + const data = useDeepMemo(JSON.parse(payload as string)); + + useEffect(() => { + let listener = undefined; + if (revalidateOnFocus) { + sendCommand("embeddingsReindexProgress"); + listener = () => { + if (document.visibilityState == "visible") { + sendCommand("embeddingsReindexProgress"); + } + }; + addEventListener("visibilitychange", listener); + } + + return () => { + if (listener) { + removeEventListener("visibilitychange", listener); + } + }; + // we know that these deps are correct + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [revalidateOnFocus]); + + return { payload: data }; +} + export function useMotionActivity(camera: string): { payload: string } { const { value: { payload }, diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index 5a1ed6145..8607c8760 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -1,5 +1,10 @@ -import { useEventUpdate, useModelState } from "@/api/ws"; +import { + useEmbeddingsReindexProgress, + useEventUpdate, + useModelState, +} from "@/api/ws"; import ActivityIndicator from "@/components/indicators/activity-indicator"; +import AnimatedCircularProgressBar from "@/components/ui/circular-progress-bar"; import { useApiFilterArgs } from "@/hooks/use-api-filter"; import { useTimezone } from "@/hooks/use-date-utils"; import { FrigateConfig } from "@/types/frigateConfig"; @@ -182,6 +187,18 @@ export default function Explore() { // eslint-disable-next-line react-hooks/exhaustive-deps }, [eventUpdate]); + // embeddings reindex progress + + const { payload: reindexProgress } = useEmbeddingsReindexProgress(); + + const embeddingsReindexing = useMemo( + () => + reindexProgress + ? reindexProgress.total_objects - reindexProgress.processed_objects > 0 + : undefined, + [reindexProgress], + ); + // model states const { payload: textModelState } = useModelState( @@ -238,59 +255,101 @@ export default function Explore() { return ( <> - {config?.semantic_search.enabled && !allModelsLoaded ? ( + {config?.semantic_search.enabled && + (!allModelsLoaded || embeddingsReindexing) ? (
-
+
Search Unavailable
-
- Frigate is downloading the necessary embeddings models to support - semantic searching. This may take several minutes depending on the - speed of your network connection. -
-
-
- {renderModelStateIcon(visionModelState)} - Vision model -
-
- {renderModelStateIcon(visionFeatureExtractorState)} - Vision model feature extractor -
-
- {renderModelStateIcon(textModelState)} - Text model -
-
- {renderModelStateIcon(textTokenizerState)} - Text tokenizer -
-
- {(textModelState === "error" || - textTokenizerState === "error" || - visionModelState === "error" || - visionFeatureExtractorState === "error") && ( -
- An error has occurred. Check Frigate logs. -
+ {embeddingsReindexing && ( + <> +
+ Search can be used after tracked object embeddings have + finished reindexing. +
+
+ +
+
+
+ + Thumbnails embedded: + + {reindexProgress.thumbnails} +
+
+ + Descriptions embedded: + + {reindexProgress.descriptions} +
+
+ + Tracked objects processed: + + {reindexProgress.processed_objects} +
+
+ + )} + {!allModelsLoaded && ( + <> +
+ Frigate is downloading the necessary embeddings models to + support semantic searching. This may take several minutes + depending on the speed of your network connection. +
+
+
+ {renderModelStateIcon(visionModelState)} + Vision model +
+
+ {renderModelStateIcon(visionFeatureExtractorState)} + Vision model feature extractor +
+
+ {renderModelStateIcon(textModelState)} + Text model +
+
+ {renderModelStateIcon(textTokenizerState)} + Text tokenizer +
+
+ {(textModelState === "error" || + textTokenizerState === "error" || + visionModelState === "error" || + visionFeatureExtractorState === "error") && ( +
+ An error has occurred. Check Frigate logs. +
+ )} +
+ You may want to reindex the embeddings of your tracked objects + once the models are downloaded. +
+
+ + Read the documentation{" "} + + +
+ )} -
- You may want to reindex the embeddings of your tracked objects - once the models are downloaded. -
-
- - Read the documentation{" "} - - -
) : ( diff --git a/web/src/types/ws.ts b/web/src/types/ws.ts index a8211d269..cab759074 100644 --- a/web/src/types/ws.ts +++ b/web/src/types/ws.ts @@ -62,4 +62,11 @@ export type ModelState = | "downloaded" | "error"; +export type EmbeddingsReindexProgressType = { + thumbnails: number; + descriptions: number; + processed_objects: number; + total_objects: number; +}; + export type ToggleableSetting = "ON" | "OFF";