From 2f0ef1dc6922e3f5cfc6ebb253a0094b42786ba0 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 13 Oct 2024 18:47:15 -0500 Subject: [PATCH 1/6] Publish model state and embeddings reindex in dispatcher onConnect --- frigate/comms/dispatcher.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 4a3862eaf..1f480fa9c 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -179,6 +179,11 @@ class Dispatcher: } self.publish("camera_activity", json.dumps(camera_status)) + self.publish("model_state", json.dumps(self.model_state.copy())) + self.publish( + "embeddings_reindex_progress", + json.dumps(self.embeddings_reindex.copy()), + ) # Dictionary mapping topic to handlers topic_handlers = { From 4ede289a4e9cfd5a51a310a031a7ee0511aacd9c Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 13 Oct 2024 18:47:43 -0500 Subject: [PATCH 2/6] remove unneeded from explore --- web/src/pages/Explore.tsx | 53 ++++++++++++--------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index e4bb49521..03a60a8d0 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -2,7 +2,6 @@ import { useEmbeddingsReindexProgress, useEventUpdate, useModelState, - useWs, } from "@/api/ws"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import AnimatedCircularProgressBar from "@/components/ui/circular-progress-bar"; @@ -193,22 +192,11 @@ export default function Explore() { // embeddings reindex progress - const { send: sendReindexCommand } = useWs( - "embeddings_reindex_progress", - "embeddingsReindexProgress", - ); - - useEffect(() => { - sendReindexCommand("embeddingsReindexProgress"); - // only run on mount - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - const { payload: reindexProgress } = useEmbeddingsReindexProgress(); + const { payload: reindexState } = useEmbeddingsReindexProgress(); const embeddingsReindexing = useMemo(() => { - if (reindexProgress) { - switch (reindexProgress.status) { + if (reindexState) { + switch (reindexState.status) { case "indexing": return true; case "completed": @@ -217,18 +205,10 @@ export default function Explore() { return undefined; } } - }, [reindexProgress]); + }, [reindexState]); // model states - const { send: sendModelCommand } = useWs("model_state", "modelState"); - - useEffect(() => { - sendModelCommand("modelState"); - // only run on mount - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - const { payload: textModelState } = useModelState( "jinaai/jina-clip-v1-text_model_fp16.onnx", ); @@ -274,7 +254,8 @@ export default function Explore() { if ( config?.semantic_search.enabled && - (!textModelState || + (!reindexState || + !textModelState || !textTokenizerState || !visionModelState || !visionFeatureExtractorState) @@ -303,24 +284,22 @@ export default function Explore() {
- {reindexProgress.time_remaining !== null && ( + {reindexState.time_remaining !== null && (
- {reindexProgress.time_remaining === -1 + {reindexState.time_remaining === -1 ? "Starting up..." : "Estimated time remaining:"}
- {reindexProgress.time_remaining >= 0 && - (formatSecondsToDuration( - reindexProgress.time_remaining, - ) || + {reindexState.time_remaining >= 0 && + (formatSecondsToDuration(reindexState.time_remaining) || "Finishing shortly")}
)} @@ -328,20 +307,20 @@ export default function Explore() { Thumbnails embedded: - {reindexProgress.thumbnails} + {reindexState.thumbnails}
Descriptions embedded: - {reindexProgress.descriptions} + {reindexState.descriptions}
Tracked objects processed: - {reindexProgress.processed_objects} /{" "} - {reindexProgress.total_objects} + {reindexState.processed_objects} /{" "} + {reindexState.total_objects}
From 6fe2c619d1bbbf5ac70e64c2084d19110e229ce0 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 13 Oct 2024 18:48:42 -0500 Subject: [PATCH 3/6] add embeddings reindex progress to statusbar --- web/src/components/Statusbar.tsx | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/web/src/components/Statusbar.tsx b/web/src/components/Statusbar.tsx index 41bd9372f..fbdce28c4 100644 --- a/web/src/components/Statusbar.tsx +++ b/web/src/components/Statusbar.tsx @@ -1,3 +1,4 @@ +import { useEmbeddingsReindexProgress } from "@/api/ws"; import { StatusBarMessagesContext, StatusMessage, @@ -10,7 +11,7 @@ import { MdCircle } from "react-icons/md"; import { Link } from "react-router-dom"; export default function Statusbar() { - const { messages, addMessage, clearMessages } = useContext( + const { messages, addMessage, removeMessage, clearMessages } = useContext( StatusBarMessagesContext, )!; @@ -41,6 +42,24 @@ export default function Statusbar() { }); }, [potentialProblems, addMessage, clearMessages]); + const { payload: reindexState } = useEmbeddingsReindexProgress(); + + useEffect(() => { + if (reindexState) { + if (reindexState.status == "indexing") { + addMessage( + "embeddings-reindex", + `Reindexing embeddings (${Math.floor((reindexState.processed_objects / reindexState.total_objects) * 100)}% complete)`, + undefined, + "status", + ); + } + if (reindexState.status === "completed") { + removeMessage("embeddings-reindex", "status"); + } + } + }, [reindexState, addMessage, removeMessage]); + return (
From 5e7f946a586514ccf32c0924de5cef7ef8391074 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 13 Oct 2024 18:58:25 -0500 Subject: [PATCH 4/6] don't allow right click or show similar button if semantic search is disabled --- .../overlay/detail/SearchDetailDialog.tsx | 22 ++++++++++--------- web/src/views/search/SearchView.tsx | 6 ++++- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 94d28c7c8..1cee70aaa 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -396,17 +396,19 @@ function ObjectDetailsTab({ draggable={false} src={`${apiHost}api/events/${search.id}/thumbnail.jpg`} /> - + if (setSimilarity) { + setSimilarity(); + } + }} + > + Find Similar + + )}
diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx index 4c33f7dc8..e64affa36 100644 --- a/web/src/views/search/SearchView.tsx +++ b/web/src/views/search/SearchView.tsx @@ -393,7 +393,11 @@ export default function SearchView({ > setSimilaritySearch(value)} + findSimilar={() => { + if (config?.semantic_search.enabled) { + setSimilaritySearch(value); + } + }} onClick={() => onSelectSearch(value, index)} /> {(searchTerm || From fd40a1f2e2cdf16713f451b6607f18f9de893d7b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 13 Oct 2024 19:04:52 -0500 Subject: [PATCH 5/6] fix status bar --- web/src/components/Statusbar.tsx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/web/src/components/Statusbar.tsx b/web/src/components/Statusbar.tsx index fbdce28c4..1b20b26f6 100644 --- a/web/src/components/Statusbar.tsx +++ b/web/src/components/Statusbar.tsx @@ -11,7 +11,7 @@ import { MdCircle } from "react-icons/md"; import { Link } from "react-router-dom"; export default function Statusbar() { - const { messages, addMessage, removeMessage, clearMessages } = useContext( + const { messages, addMessage, clearMessages } = useContext( StatusBarMessagesContext, )!; @@ -47,18 +47,17 @@ export default function Statusbar() { useEffect(() => { if (reindexState) { if (reindexState.status == "indexing") { + clearMessages("embeddings-reindex"); addMessage( "embeddings-reindex", `Reindexing embeddings (${Math.floor((reindexState.processed_objects / reindexState.total_objects) * 100)}% complete)`, - undefined, - "status", ); } if (reindexState.status === "completed") { - removeMessage("embeddings-reindex", "status"); + clearMessages("embeddings-reindex"); } } - }, [reindexState, addMessage, removeMessage]); + }, [reindexState, addMessage, clearMessages]); return (
From 4a1ae5dbde5fa81e4c17b0c2d46c26dca347781c Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sun, 13 Oct 2024 21:59:59 -0500 Subject: [PATCH 6/6] Convert peewee model to dict before formatting for genai description --- frigate/genai/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index dccb74c1d..e2d509383 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -4,6 +4,8 @@ import importlib import os from typing import Optional +from playhouse.shortcuts import model_to_dict + from frigate.config import CameraConfig, GenAIConfig, GenAIProviderEnum from frigate.models import Event @@ -36,8 +38,9 @@ class GenAIClient: ) -> Optional[str]: """Generate a description for the frame.""" prompt = camera_config.genai.object_prompts.get( - event.label, camera_config.genai.prompt - ).format(**event) + event.label, + camera_config.genai.prompt, + ).format(**model_to_dict(event)) return self._send(prompt, thumbnails) def _init_provider(self):