From e5e074ab84c8227ee80deee3ccb152921fd5b1cf Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 11 Sep 2024 06:42:49 -0500 Subject: [PATCH] explore view with new api endpoint --- frigate/api/event.py | 55 ++++ .../components/filter/SearchFilterGroup.tsx | 20 +- web/src/pages/Explore.tsx | 75 +++--- web/src/pages/Search.tsx | 242 ------------------ web/src/views/explore/ExploreView.tsx | 85 ++++-- web/src/views/search/SearchView.tsx | 114 +++++---- 6 files changed, 218 insertions(+), 373 deletions(-) delete mode 100644 web/src/pages/Search.tsx diff --git a/frigate/api/event.py b/frigate/api/event.py index ca821c07a..6e75602e9 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -251,6 +251,61 @@ def events(): return jsonify(list(events)) +@EventBp.route("/events/explore") +def events_explore(): + limit = request.args.get("limit", 10, type=int) + + subquery = Event.select( + Event.id, + Event.camera, + Event.label, + Event.zones, + Event.start_time, + Event.end_time, + Event.has_clip, + Event.has_snapshot, + Event.plus_id, + Event.retain_indefinitely, + Event.sub_label, + Event.top_score, + Event.false_positive, + Event.box, + Event.data, + fn.rank() + .over(partition_by=[Event.label], order_by=[Event.start_time.desc()]) + .alias("rank"), + fn.COUNT(Event.id).over(partition_by=[Event.label]).alias("event_count"), + ).alias("subquery") + + query = ( + Event.select( + subquery.c.id, + subquery.c.camera, + subquery.c.label, + subquery.c.zones, + subquery.c.start_time, + subquery.c.end_time, + subquery.c.has_clip, + subquery.c.has_snapshot, + subquery.c.plus_id, + subquery.c.retain_indefinitely, + subquery.c.sub_label, + subquery.c.top_score, + subquery.c.false_positive, + subquery.c.box, + subquery.c.data, + subquery.c.event_count, + ) + .from_(subquery) + .where(subquery.c.rank <= limit) + .order_by(subquery.c.event_count.desc(), subquery.c.start_time.desc()) + .dicts() + ) + + events = query.iterator() + return jsonify(list(events)) + + @EventBp.route("/event_ids") def event_ids(): idString = request.args.get("ids") diff --git a/web/src/components/filter/SearchFilterGroup.tsx b/web/src/components/filter/SearchFilterGroup.tsx index c66ab9438..c684907df 100644 --- a/web/src/components/filter/SearchFilterGroup.tsx +++ b/web/src/components/filter/SearchFilterGroup.tsx @@ -179,15 +179,6 @@ export default function SearchFilterGroup({ updateSelectedRange={onUpdateSelectedRange} /> )} - {filters.includes("general") && ( - { - onUpdateFilter({ ...filter, labels: newLabels }); - }} - /> - )} {filters.includes("zone") && allZones.length > 0 && ( )} + {filters.includes("general") && ( + { + onUpdateFilter({ ...filter, labels: newLabels }); + }} + /> + )} {filters.includes("sub") && ( - Filter + All Labels ); diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index e26cd067e..3ccda67e1 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -11,7 +11,6 @@ import { import { TimeRange } from "@/types/timeline"; import { RecordingView } from "@/views/recording/RecordingView"; import SearchView from "@/views/search/SearchView"; -import ExploreView from "@/views/explore/ExploreView"; import { useCallback, useEffect, useMemo, useState } from "react"; import useSWR from "swr"; @@ -119,21 +118,25 @@ export default function Explore() { ]; } - return [ - "events", - { - cameras: searchSearchParams["cameras"], - labels: searchSearchParams["labels"], - sub_labels: searchSearchParams["subLabels"], - zones: searchSearchParams["zones"], - before: searchSearchParams["before"], - after: searchSearchParams["after"], - search_type: searchSearchParams["search_type"], - limit: Object.keys(searchSearchParams).length == 0 ? 20 : null, - in_progress: 0, - include_thumbnails: 0, - }, - ]; + if (searchSearchParams && Object.keys(searchSearchParams).length !== 0) { + return [ + "events", + { + cameras: searchSearchParams["cameras"], + labels: searchSearchParams["labels"], + sub_labels: searchSearchParams["subLabels"], + zones: searchSearchParams["zones"], + before: searchSearchParams["before"], + after: searchSearchParams["after"], + search_type: searchSearchParams["search_type"], + limit: Object.keys(searchSearchParams).length == 0 ? 20 : null, + in_progress: 0, + include_thumbnails: 0, + }, + ]; + } + + return null; }, [searchTerm, searchSearchParams, similaritySearch]); const { data: searchResults, isLoading } = @@ -225,31 +228,19 @@ export default function Explore() { ); } } else { - if ( - search || - similaritySearch || - (searchFilter && Object.keys(searchFilter).length != 0) - ) { - return ( - - ); - } else { - return ( -
- -
- ); - } + return ( + + ); } } diff --git a/web/src/pages/Search.tsx b/web/src/pages/Search.tsx deleted file mode 100644 index 636f34e06..000000000 --- a/web/src/pages/Search.tsx +++ /dev/null @@ -1,242 +0,0 @@ -import { useApiFilterArgs } from "@/hooks/use-api-filter"; -import { useCameraPreviews } from "@/hooks/use-camera-previews"; -import { useOverlayState } from "@/hooks/use-overlay-state"; -import { FrigateConfig } from "@/types/frigateConfig"; -import { RecordingStartingPoint } from "@/types/record"; -import { - PartialSearchResult, - SearchFilter, - SearchResult, -} from "@/types/search"; -import { TimeRange } from "@/types/timeline"; -import { RecordingView } from "@/views/recording/RecordingView"; -import SearchView from "@/views/search/SearchView"; -import { useCallback, useEffect, useMemo, useState } from "react"; -import useSWR from "swr"; - -export default function Search() { - const { data: config } = useSWR("config", { - revalidateOnFocus: false, - }); - - // search field handler - - const [searchTimeout, setSearchTimeout] = useState(); - const [search, setSearch] = useState(""); - const [searchTerm, setSearchTerm] = useState(""); - - const [recording, setRecording] = - useOverlayState("recording"); - - // search filter - - const [searchFilter, setSearchFilter, searchSearchParams] = - useApiFilterArgs(); - - const onUpdateFilter = useCallback( - (newFilter: SearchFilter) => { - setSearchFilter(newFilter); - }, - [setSearchFilter], - ); - - // search api - - const [similaritySearch, setSimilaritySearch] = - useState(); - - useEffect(() => { - if ( - config?.semantic_search.enabled && - searchSearchParams["search_type"] == "similarity" && - searchSearchParams["event_id"]?.length != 0 && - searchFilter - ) { - setSimilaritySearch({ - id: searchSearchParams["event_id"], - }); - - // remove event id from url params - const { event_id: _event_id, ...newFilter } = searchFilter; - setSearchFilter(newFilter); - } - // only run similarity search with event_id in the url when coming from review - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - useEffect(() => { - if (similaritySearch) { - setSimilaritySearch(undefined); - } - - if (searchTimeout) { - clearTimeout(searchTimeout); - } - - setSearchTimeout( - setTimeout(() => { - setSearchTimeout(undefined); - setSearchTerm(search); - }, 750), - ); - // we only want to update the searchTerm when search changes - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [search]); - - const searchQuery = useMemo(() => { - if (similaritySearch) { - return [ - "events/search", - { - query: similaritySearch.id, - cameras: searchSearchParams["cameras"], - labels: searchSearchParams["labels"], - sub_labels: searchSearchParams["subLabels"], - zones: searchSearchParams["zones"], - before: searchSearchParams["before"], - after: searchSearchParams["after"], - include_thumbnails: 0, - search_type: "similarity", - }, - ]; - } - - if (searchTerm) { - return [ - "events/search", - { - query: searchTerm, - cameras: searchSearchParams["cameras"], - labels: searchSearchParams["labels"], - sub_labels: searchSearchParams["subLabels"], - zones: searchSearchParams["zones"], - before: searchSearchParams["before"], - after: searchSearchParams["after"], - search_type: searchSearchParams["search_type"], - include_thumbnails: 0, - }, - ]; - } - - return [ - "events", - { - cameras: searchSearchParams["cameras"], - labels: searchSearchParams["labels"], - sub_labels: searchSearchParams["subLabels"], - zones: searchSearchParams["zones"], - before: searchSearchParams["before"], - after: searchSearchParams["after"], - search_type: searchSearchParams["search_type"], - limit: Object.keys(searchSearchParams).length == 0 ? 20 : null, - in_progress: 0, - include_thumbnails: 0, - }, - ]; - }, [searchTerm, searchSearchParams, similaritySearch]); - - const { data: searchResults, isLoading } = - useSWR(searchQuery); - - const previewTimeRange = useMemo(() => { - if (!searchResults) { - return { after: 0, before: 0 }; - } - - return { - after: Math.min(...searchResults.map((res) => res.start_time)), - before: Math.max( - ...searchResults.map((res) => res.end_time ?? Date.now() / 1000), - ), - }; - }, [searchResults]); - - const allPreviews = useCameraPreviews(previewTimeRange, { - autoRefresh: false, - fetchPreviews: searchResults != undefined, - }); - - // selection - - const onOpenSearch = useCallback( - (item: SearchResult) => { - setRecording({ - camera: item.camera, - startTime: item.start_time, - severity: "alert", - }); - }, - [setRecording], - ); - - const selectedReviewData = useMemo(() => { - if (!recording) { - return undefined; - } - - if (!config) { - return undefined; - } - - if (!searchResults) { - return undefined; - } - - const allCameras = searchFilter?.cameras ?? Object.keys(config.cameras); - - return { - camera: recording.camera, - start_time: recording.startTime, - allCameras: allCameras, - }; - - // previews will not update after item is selected - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [recording, searchResults]); - - const selectedTimeRange = useMemo(() => { - if (!recording) { - return undefined; - } - - const time = new Date(recording.startTime * 1000); - time.setUTCMinutes(0, 0, 0); - const start = time.getTime() / 1000; - time.setHours(time.getHours() + 2); - const end = time.getTime() / 1000; - return { - after: start, - before: end, - }; - }, [recording]); - - if (recording) { - if (selectedReviewData && selectedTimeRange) { - return ( - - ); - } - } else { - return ( - - ); - } -} diff --git a/web/src/views/explore/ExploreView.tsx b/web/src/views/explore/ExploreView.tsx index eeeb639c4..1ee415549 100644 --- a/web/src/views/explore/ExploreView.tsx +++ b/web/src/views/explore/ExploreView.tsx @@ -1,6 +1,5 @@ -import { Event } from "@/types/event"; import { useEffect, useMemo, useState } from "react"; -import { isIOS } from "react-device-detect"; +import { isIOS, isMobileOnly } from "react-device-detect"; import useSWR from "swr"; import { useApiHost } from "@/api"; import { cn } from "@/lib/utils"; @@ -12,8 +11,13 @@ import { TooltipTrigger, } from "@/components/ui/tooltip"; import { TooltipPortal } from "@radix-ui/react-tooltip"; +import { SearchResult } from "@/types/search"; -export default function ImageAccordion() { +type ExploreViewProps = { + onSelectSearch: (searchResult: SearchResult, detail: boolean) => void; +}; + +export default function ExploreView({ onSelectSearch }: ExploreViewProps) { // title useEffect(() => { @@ -22,11 +26,11 @@ export default function ImageAccordion() { // data - const { data: events } = useSWR( + const { data: events } = useSWR( [ - "events", + "events/explore", { - limit: 100, + limit: isMobileOnly ? 5 : 10, }, ], { @@ -36,7 +40,7 @@ export default function ImageAccordion() { const eventsByLabel = useMemo(() => { if (!events) return {}; - return events.reduce>((acc, event) => { + return events.reduce>((acc, event) => { const label = event.label || "Unknown"; if (!acc[label]) { acc[label] = []; @@ -49,19 +53,28 @@ export default function ImageAccordion() { return (
{Object.entries(eventsByLabel).map(([label, filteredEvents]) => ( - + ))}
); } +type ThumbnailRowType = { + objectType: string; + searchResults?: SearchResult[]; + onSelectSearch: (searchResult: SearchResult, detail: boolean) => void; +}; + function ThumbnailRow({ objectType, - events, -}: { - objectType: string; - events?: Event[]; -}) { + searchResults, + onSelectSearch, +}: ThumbnailRowType) { const apiHost = useApiHost(); const navigate = useNavigate(); const [hoveredIndex, setHoveredIndex] = useState(null); @@ -74,20 +87,34 @@ function ThumbnailRow({ }; return ( -
-

{objectType}

-
- {events?.map((event, index) => ( +
+
+ {objectType.replaceAll("_", " ")} + {searchResults && ( + + ( + { + // @ts-expect-error we know this is correct + searchResults[0].event_count + }{" "} + detected objects){" "} + + )} +
+
+ {searchResults?.map((event, index) => (
setHoveredIndex(index)} onMouseLeave={() => setHoveredIndex(null)} > {`${objectType} onSelectSearch(event, true)} />
))} @@ -114,13 +138,13 @@ function ThumbnailRow({ - Explore More {objectType}s + @@ -129,3 +153,12 @@ function ThumbnailRow({
); } + +function ExploreMoreLink({ objectType }: { objectType: string }) { + const formattedType = objectType.replaceAll("_", " "); + const label = formattedType.endsWith("s") + ? `${formattedType}es` + : `${formattedType}s`; + + return
Explore More {label}
; +} diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx index 5e2783d05..471498d78 100644 --- a/web/src/views/search/SearchView.tsx +++ b/web/src/views/search/SearchView.tsx @@ -21,6 +21,7 @@ import { useCallback, useMemo, useState } from "react"; import { isMobileOnly } from "react-device-detect"; import { LuImage, LuSearchX, LuText, LuXCircle } from "react-icons/lu"; import useSWR from "swr"; +import ExploreView from "../explore/ExploreView"; type SearchViewProps = { search: string; @@ -109,7 +110,7 @@ export default function SearchView({
)} -
- {uniqueResults && - uniqueResults.map((value) => { - const selected = false; + {uniqueResults && ( +
+ {uniqueResults && + uniqueResults.map((value) => { + const selected = false; - return ( -
+ return (
- setSimilaritySearch(value)} - onClick={onSelectSearch} - /> - {(searchTerm || similaritySearch) && ( -
- - - - {value.search_source == "thumbnail" ? ( - - ) : ( - - )} +
+ setSimilaritySearch(value)} + onClick={onSelectSearch} + /> + {(searchTerm || similaritySearch) && ( +
+ + + + {value.search_source == "thumbnail" ? ( + + ) : ( + + )} + {zScoreToConfidence( + value.search_distance, + value.search_source, + )} + % + + + + Matched {value.search_source} at{" "} {zScoreToConfidence( value.search_distance, value.search_source, )} % - - - - Matched {value.search_source} at{" "} - {zScoreToConfidence( - value.search_distance, - value.search_source, - )} - % - - -
- )} + + +
+ )} +
+
-
-
- ); - })} -
+ ); + })} +
+ )} + {!uniqueResults && !isLoading && ( +
+ +
+ )}
);