From 6116604ef8f8f302c9b201388d7b3efe3f38b3b5 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:03:13 -0500 Subject: [PATCH] revamp object debug view --- frigate/api/app.py | 1 + frigate/object_processing.py | 9 +- .../components/settings/ObjectSettings.tsx | 225 +++++++++++++++++- web/src/hooks/use-camera-activity.ts | 42 ++-- web/src/types/frigateConfig.ts | 1 + web/src/types/ws.ts | 3 + 6 files changed, 259 insertions(+), 22 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 5d0bce78b..67ad072a7 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -164,6 +164,7 @@ def config(): camera_dict["zones"][zone_name]["color"] = zone.color config["plus"] = {"enabled": current_app.plus_api.is_active()} + config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): detector_config["model"]["labelmap"] = ( diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 6dd70781e..e03cf005e 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -747,7 +747,14 @@ class CameraState: label = f"{object_type}-verified" camera_activity["objects"].append( - {"id": obj.obj_data["id"], "label": label, "stationary": not active} + { + "id": obj.obj_data["id"], + "label": label, + "stationary": not active, + "area": obj.obj_data["area"], + "ratio": obj.obj_data["ratio"], + "score": obj.obj_data["score"], + } ) # if the object's thumbnail is not from the current frame diff --git a/web/src/components/settings/ObjectSettings.tsx b/web/src/components/settings/ObjectSettings.tsx index 92dac2d7d..6f53f7e1e 100644 --- a/web/src/components/settings/ObjectSettings.tsx +++ b/web/src/components/settings/ObjectSettings.tsx @@ -1,24 +1,79 @@ -import { useEffect, useMemo } from "react"; -import DebugCameraImage from "../camera/DebugCameraImage"; -import { FrigateConfig } from "@/types/frigateConfig"; +import { useCallback, useEffect, useMemo } from "react"; +import ActivityIndicator from "@/components/indicators/activity-indicator"; +import AutoUpdatingCameraImage from "@/components/camera/AutoUpdatingCameraImage"; +import { CameraConfig, FrigateConfig } from "@/types/frigateConfig"; +import { Toaster } from "@/components/ui/sonner"; +import { Label } from "@/components/ui/label"; import useSWR from "swr"; -import ActivityIndicator from "../indicators/activity-indicator"; +import Heading from "../ui/heading"; +import { Switch } from "../ui/switch"; +import { usePersistence } from "@/hooks/use-persistence"; +import { Skeleton } from "../ui/skeleton"; +import { useCameraActivity } from "@/hooks/use-camera-activity"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs"; +import { ObjectType } from "@/types/ws"; +import useDeepMemo from "@/hooks/use-deep-memo"; +import { Card } from "../ui/card"; +import { getIconForLabel } from "@/utils/iconUtil"; +import { capitalizeFirstLetter } from "@/utils/stringUtil"; type ObjectSettingsProps = { selectedCamera?: string; }; +type Options = { [key: string]: boolean }; + +const emptyObject = Object.freeze({}); + export default function ObjectSettings({ selectedCamera, }: ObjectSettingsProps) { const { data: config } = useSWR("config"); + const DEBUG_OPTIONS = [ + { param: "bbox", title: "Bounding boxes" }, + { param: "timestamp", title: "Timestamp" }, + { param: "zones", title: "Zones" }, + { param: "mask", title: "Masks" }, + { param: "motion", title: "Motion boxes" }, + { param: "regions", title: "Regions" }, + ]; + + const [options, setOptions] = usePersistence( + `${selectedCamera}-feed`, + emptyObject, + ); + + const handleSetOption = useCallback( + (id: string, value: boolean) => { + const newOptions = { ...options, [id]: value }; + setOptions(newOptions); + }, + [options, setOptions], + ); + const cameraConfig = useMemo(() => { if (config && selectedCamera) { return config.cameras[selectedCamera]; } }, [config, selectedCamera]); + const { objects } = useCameraActivity(cameraConfig ?? ({} as CameraConfig)); + + const memoizedObjects = useDeepMemo(objects); + + const searchParams = useMemo( + () => + new URLSearchParams( + Object.keys(options || {}).reduce((memo, key) => { + //@ts-expect-error we know this is correct + memo.push([key, options[key] === true ? "1" : "0"]); + return memo; + }, []), + ), + [options], + ); + useEffect(() => { document.title = "Object Settings - Frigate"; }, []); @@ -28,8 +83,166 @@ export default function ObjectSettings({ } return ( -
- +
+ +
+ + Object Detection + +
+

+ Frigate uses your detectors{" "} + {config + ? "(" + + Object.keys(config?.detectors) + .map((detector) => capitalizeFirstLetter(detector)) + .join(",") + + ")" + : ""}{" "} + to detect objects in your camera's video stream. +

+

+ Debugging view shows a real-time view of detected objects and their + statistics. The object list shows a time-delayed summary of detected + objects. +

+
+ + + + Debugging + Object List + + +
+
+
+ {DEBUG_OPTIONS.map(({ param, title }) => ( +
+ + { + handleSetOption(param, isChecked); + }} + /> +
+ ))} +
+
+
+ + + {ObjectList(memoizedObjects)} + + +
+ + {cameraConfig ? ( +
+
+ +
+
+ ) : ( + + )} +
+ ); +} + +function ObjectList(objects?: ObjectType[]) { + const { data: config } = useSWR("config"); + + const colormap = useMemo(() => { + if (!config) { + return; + } + + return config.model?.colormap; + }, [config]); + + const getColorForObjectName = useCallback( + (objectName: string) => { + return colormap && colormap[objectName] + ? `rgb(${colormap[objectName][2]}, ${colormap[objectName][1]}, ${colormap[objectName][0]})` + : "rgb(128, 128, 128)"; + }, + [colormap], + ); + + return ( +
+ {objects && objects.length > 0 ? ( + objects.map((obj) => { + return ( + +
+
+
+ {getIconForLabel(obj.label, "size-5 text-white")} +
+
+ {capitalizeFirstLetter(obj.label)} +
+
+
+
+
+

+ Score +

+ {(obj.score * 100).toFixed(1).toString()}% +
+
+
+
+

+ Ratio +

+ {obj.ratio.toFixed(2).toString()} +
+
+
+
+

+ Area +

+ {obj.area.toString()} +
+
+
+
+ + ); + }) + ) : ( +
No objects
+ )}
); } diff --git a/web/src/hooks/use-camera-activity.ts b/web/src/hooks/use-camera-activity.ts index 4e3b8b4be..4ca6e1fac 100644 --- a/web/src/hooks/use-camera-activity.ts +++ b/web/src/hooks/use-camera-activity.ts @@ -5,10 +5,11 @@ import { } from "@/api/ws"; import { ATTRIBUTE_LABELS, CameraConfig } from "@/types/frigateConfig"; import { MotionData, ReviewSegment } from "@/types/review"; -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useTimelineUtils } from "./use-timeline-utils"; import { ObjectType } from "@/types/ws"; import useDeepMemo from "./use-deep-memo"; +import { isEqual } from "lodash"; type useCameraActivityReturn = { activeTracking: boolean; @@ -29,7 +30,6 @@ export function useCameraActivity( useEffect(() => { if (updatedCameraState) { - console.log(`the initial objects are ${JSON.stringify(updatedCameraState.objects)}`) setObjects(updatedCameraState.objects); } }, [updatedCameraState]); @@ -45,12 +45,21 @@ export function useCameraActivity( const { payload: event } = useFrigateEvents(); const updatedEvent = useDeepMemo(event); + const handleSetObjects = useCallback( + (newObjects: ObjectType[]) => { + if (!isEqual(objects, newObjects)) { + setObjects(newObjects); + } + }, + [objects], + ); + useEffect(() => { if (!updatedEvent) { return; } - if (updatedEvent.after.camera != camera.name) { + if (updatedEvent.after.camera !== camera.name) { return; } @@ -58,23 +67,25 @@ export function useCameraActivity( (obj) => obj.id === updatedEvent.after.id, ); - if (updatedEvent.type == "end") { - if (updatedEventIndex != -1) { - const newActiveObjects = [...objects]; - newActiveObjects.splice(updatedEventIndex, 1); - setObjects(newActiveObjects); + let newObjects: ObjectType[] = [...objects]; + + if (updatedEvent.type === "end") { + if (updatedEventIndex !== -1) { + newObjects.splice(updatedEventIndex, 1); } } else { - if (updatedEventIndex == -1) { + if (updatedEventIndex === -1) { // add unknown updatedEvent to list if not stationary if (!updatedEvent.after.stationary) { const newActiveObject: ObjectType = { id: updatedEvent.after.id, label: updatedEvent.after.label, stationary: updatedEvent.after.stationary, + area: updatedEvent.after.area, + ratio: updatedEvent.after.ratio, + score: updatedEvent.after.score, }; - const newActiveObjects = [...objects, newActiveObject]; - setObjects(newActiveObjects); + newObjects = [...objects, newActiveObject]; } } else { const newObjects = [...objects]; @@ -94,16 +105,17 @@ export function useCameraActivity( newObjects[updatedEventIndex].label = label; newObjects[updatedEventIndex].stationary = updatedEvent.after.stationary; - setObjects(newObjects); } } - }, [camera, updatedEvent, objects]); + + handleSetObjects(newObjects); + }, [camera, updatedEvent, objects, handleSetObjects]); return { activeTracking: hasActiveObjects, activeMotion: detectingMotion - ? detectingMotion == "ON" - : initialCameraState?.motion == true, + ? detectingMotion === "ON" + : initialCameraState?.motion === true, objects, }; } diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts index b5c89f5db..517a00761 100644 --- a/web/src/types/frigateConfig.ts +++ b/web/src/types/frigateConfig.ts @@ -324,6 +324,7 @@ export interface FrigateConfig { model_type: string; path: string | null; width: number; + colormap: { [key: string]: [number, number, number] }; }; motion: Record | null; diff --git a/web/src/types/ws.ts b/web/src/types/ws.ts index fb7a963c4..82b8750a7 100644 --- a/web/src/types/ws.ts +++ b/web/src/types/ws.ts @@ -45,6 +45,9 @@ export type ObjectType = { id: string; label: string; stationary: boolean; + area: number; + ratio: number; + score: number; }; export interface FrigateCameraState {