From d578fceac824abaa11b0853beefeb6520e33594e Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sat, 4 Jan 2025 15:44:15 -0700 Subject: [PATCH] Enable UI for feature metrics --- frigate/embeddings/maintainer.py | 38 ++++---- web/src/pages/System.tsx | 32 ++++++- web/src/types/stats.ts | 8 ++ web/src/views/system/FeatureMetrics.tsx | 122 ++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 21 deletions(-) create mode 100644 web/src/views/system/FeatureMetrics.tsx diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index ca1f0a0d3..cfa6adef1 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -235,11 +235,13 @@ class EmbeddingMaintainer(threading.Thread): if self.lpr_config.enabled: start = datetime.datetime.now().timestamp() - self._process_license_plate(data, yuv_frame) - duration = datetime.datetime.now().timestamp() - start - self.metrics.alpr_pps.value = ( - self.metrics.alpr_pps.value * 9 + duration - ) / 10 + processed = self._process_license_plate(data, yuv_frame) + + if processed: + duration = datetime.datetime.now().timestamp() - start + self.metrics.alpr_pps.value = ( + self.metrics.alpr_pps.value * 9 + duration + ) / 10 # no need to save our own thumbnails if genai is not enabled # or if the object has become stationary @@ -558,19 +560,19 @@ class EmbeddingMaintainer(threading.Thread): def _process_license_plate( self, obj_data: dict[str, any], frame: np.ndarray - ) -> None: + ) -> bool: """Look for license plates in image.""" id = obj_data["id"] # don't run for non car objects if obj_data.get("label") != "car": logger.debug("Not a processing license plate for non car object.") - return + return False # don't run for stationary car objects if obj_data.get("stationary") == True: logger.debug("Not a processing license plate for a stationary car object.") - return + return False # don't overwrite sub label for objects that have a sub label # that is not a license plate @@ -578,7 +580,7 @@ class EmbeddingMaintainer(threading.Thread): logger.debug( f"Not processing license plate due to existing sub label: {obj_data.get('sub_label')}." ) - return + return False license_plate: Optional[dict[str, any]] = None @@ -587,7 +589,7 @@ class EmbeddingMaintainer(threading.Thread): car_box = obj_data.get("box") if not car_box: - return None + return False rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420) left, top, right, bottom = car_box @@ -596,7 +598,7 @@ class EmbeddingMaintainer(threading.Thread): if not license_plate: logger.debug("Detected no license plates for car object.") - return + return False license_plate_frame = car[ license_plate[1] : license_plate[3], license_plate[0] : license_plate[2] @@ -606,7 +608,7 @@ class EmbeddingMaintainer(threading.Thread): # don't run for object without attributes if not obj_data.get("current_attributes"): logger.debug("No attributes to parse.") - return + return False attributes: list[dict[str, any]] = obj_data.get("current_attributes", []) for attr in attributes: @@ -620,7 +622,7 @@ class EmbeddingMaintainer(threading.Thread): # no license plates detected in this frame if not license_plate: - return + return False license_plate_box = license_plate.get("box") @@ -630,7 +632,7 @@ class EmbeddingMaintainer(threading.Thread): or area(license_plate_box) < self.config.lpr.min_area ): logger.debug(f"Invalid license plate box {license_plate}") - return + return False license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420) license_plate_frame = license_plate_frame[ @@ -659,7 +661,7 @@ class EmbeddingMaintainer(threading.Thread): else: # no plates found logger.debug("No text detected") - return + return True top_plate, top_char_confidences, top_area = ( license_plates[0], @@ -705,14 +707,14 @@ class EmbeddingMaintainer(threading.Thread): f"length={len(top_plate)}, avg_conf={avg_confidence:.2f}, area={top_area} " f"vs Previous: length={len(prev_plate)}, avg_conf={prev_avg_confidence:.2f}, area={prev_area}" ) - return + return True # Check against minimum confidence threshold if avg_confidence < self.lpr_config.threshold: logger.debug( f"Average confidence {avg_confidence} is less than threshold ({self.lpr_config.threshold})" ) - return + return True # Determine subLabel based on known plates, use regex matching # Default to the detected plate, use label name if there's a match @@ -742,6 +744,8 @@ class EmbeddingMaintainer(threading.Thread): "area": top_area, } + return True + def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]: """Return jpg thumbnail of a region of the frame.""" frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2BGR_I420) diff --git a/web/src/pages/System.tsx b/web/src/pages/System.tsx index 23d1b7e6a..491149be2 100644 --- a/web/src/pages/System.tsx +++ b/web/src/pages/System.tsx @@ -1,12 +1,12 @@ import useSWR from "swr"; import { FrigateStats } from "@/types/stats"; -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import TimeAgo from "@/components/dynamic/TimeAgo"; import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; import { isDesktop, isMobile } from "react-device-detect"; import GeneralMetrics from "@/views/system/GeneralMetrics"; import StorageMetrics from "@/views/system/StorageMetrics"; -import { LuActivity, LuHardDrive } from "react-icons/lu"; +import { LuActivity, LuHardDrive, LuSearchCode } from "react-icons/lu"; import { FaVideo } from "react-icons/fa"; import Logo from "@/components/Logo"; import useOptimisticState from "@/hooks/use-optimistic-state"; @@ -14,11 +14,28 @@ import CameraMetrics from "@/views/system/CameraMetrics"; import { useHashState } from "@/hooks/use-overlay-state"; import { capitalizeFirstLetter } from "@/utils/stringUtil"; import { Toaster } from "@/components/ui/sonner"; +import { FrigateConfig } from "@/types/frigateConfig"; +import FeatureMetrics from "@/views/system/FeatureMetrics"; -const metrics = ["general", "storage", "cameras"] as const; -type SystemMetric = (typeof metrics)[number]; +const allMetrics = ["general", "features", "storage", "cameras"] as const; +type SystemMetric = (typeof allMetrics)[number]; function System() { + const { data: config } = useSWR("config", { + revalidateOnFocus: false, + }); + + const metrics = useMemo(() => { + const metrics = [...allMetrics]; + + if (!config?.semantic_search.enabled) { + const index = metrics.indexOf("features"); + metrics.splice(index, 1); + } + + return metrics; + }, [config]); + // stats page const [page, setPage] = useHashState(); @@ -67,6 +84,7 @@ function System() { aria-label={`Select ${item}`} > {item == "general" && } + {item == "features" && } {item == "storage" && } {item == "cameras" && } {isDesktop &&
{item}
} @@ -96,6 +114,12 @@ function System() { setLastUpdated={setLastUpdated} /> )} + {page == "features" && ( + + )} {page == "storage" && } {page == "cameras" && ( void; +}; +export default function FeatureMetrics({ + lastUpdated, + setLastUpdated, +}: FeatureMetricsProps) { + // stats + + const { data: initialStats } = useSWR( + ["stats/history", { keys: "embeddings,service" }], + { + revalidateOnFocus: false, + }, + ); + + const [statsHistory, setStatsHistory] = useState([]); + const updatedStats = useFrigateStats(); + + useEffect(() => { + if (initialStats == undefined || initialStats.length == 0) { + return; + } + + if (statsHistory.length == 0) { + setStatsHistory(initialStats); + return; + } + + if (!updatedStats) { + return; + } + + if (updatedStats.service.last_updated > lastUpdated) { + setStatsHistory([...statsHistory.slice(1), updatedStats]); + setLastUpdated(Date.now() / 1000); + } + }, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]); + + // timestamps + + const updateTimes = useMemo( + () => statsHistory.map((stats) => stats.service.last_updated), + [statsHistory], + ); + + // features stats + + const embeddingInferenceTimeSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: number }[] }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats?.embeddings) { + return; + } + + Object.entries(stats.embeddings).forEach(([rawKey, stat]) => { + const key = rawKey.replaceAll("_", " "); + + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + series[key].data.push({ x: statsIdx + 1, y: stat }); + }); + }); + return Object.values(series); + }, [statsHistory]); + + return ( + <> +
+
+ Features +
+
+ {statsHistory.length != 0 ? ( + <> + {embeddingInferenceTimeSeries.map((series) => ( +
+
{series.name}
+ +
+ ))} + + ) : ( + + )} +
+
+ + ); +}