Store and show boxes for attributes in timeline (#20513)

* Store and show boxes for attributes in timeline

* Simplify

frigate/timeline.py

@@ -142,6 +142,11 @@ class TimelineProcessor(threading.Thread):
                 timeline_entry[Timeline.data]["attribute"] = list(
                     event_data["attributes"].keys()
                 )[0]
+                timeline_entry[Timeline.data]["attribute_box"] = to_relative_box(
+                    camera_config.detect.width,
+                    camera_config.detect.height,
+                    event_data["current_attributes"][0]["box"],
+                )
                 save = True
         elif event_type == EventStateEnum.end:
             timeline_entry[Timeline.class_type] = "gone"

web/src/components/overlay/TimelineDataOverlay.tsx

@@ -1,101 +0,0 @@
-import { ObjectLifecycleSequence } from "@/types/timeline";
-import { useState } from "react";
-
-type TimelineEventOverlayProps = {
-  timeline: ObjectLifecycleSequence;
-  cameraConfig: {
-    detect: {
-      width: number;
-      height: number;
-    };
-  };
-};
-
-export default function TimelineEventOverlay({
-  timeline,
-  cameraConfig,
-}: TimelineEventOverlayProps) {
-  const [isHovering, setIsHovering] = useState<boolean>(false);
-  const getHoverStyle = () => {
-    if (!timeline.data.box) {
-      return {};
-    }
-
-    if (boxLeftEdge < 15) {
-      // show object stats on right side
-      return {
-        left: `${boxLeftEdge + timeline.data.box[2] * 100 + 1}%`,
-        top: `${boxTopEdge}%`,
-      };
-    }
-
-    return {
-      right: `${boxRightEdge + timeline.data.box[2] * 100 + 1}%`,
-      top: `${boxTopEdge}%`,
-    };
-  };
-
-  const getObjectArea = () => {
-    if (!timeline.data.box) {
-      return 0;
-    }
-
-    const width = timeline.data.box[2] * cameraConfig.detect.width;
-    const height = timeline.data.box[3] * cameraConfig.detect.height;
-    return Math.round(width * height);
-  };
-
-  const getObjectRatio = () => {
-    if (!timeline.data.box) {
-      return 0.0;
-    }
-
-    const width = timeline.data.box[2] * cameraConfig.detect.width;
-    const height = timeline.data.box[3] * cameraConfig.detect.height;
-    return Math.round(100 * (width / height)) / 100;
-  };
-
-  if (!timeline.data.box) {
-    return null;
-  }
-
-  const boxLeftEdge = Math.round(timeline.data.box[0] * 100);
-  const boxTopEdge = Math.round(timeline.data.box[1] * 100);
-  const boxRightEdge = Math.round(
-    (1 - timeline.data.box[2] - timeline.data.box[0]) * 100,
-  );
-  const boxBottomEdge = Math.round(
-    (1 - timeline.data.box[3] - timeline.data.box[1]) * 100,
-  );
-
-  return (
-    <>
-      <div
-        className="absolute border-4 border-red-600"
-        onMouseEnter={() => setIsHovering(true)}
-        onMouseLeave={() => setIsHovering(false)}
-        onTouchStart={() => setIsHovering(true)}
-        onTouchEnd={() => setIsHovering(false)}
-        style={{
-          left: `${boxLeftEdge}%`,
-          top: `${boxTopEdge}%`,
-          right: `${boxRightEdge}%`,
-          bottom: `${boxBottomEdge}%`,
-        }}
-      >
-        {timeline.class_type == "entered_zone" ? (
-          <div className="absolute bottom-0 left-[50%] h-2 w-2 -translate-x-1/2 translate-y-3/4 bg-yellow-500" />
-        ) : null}
-      </div>
-      {isHovering && (
-        <div
-          className="absolute block bg-white p-4 text-lg text-black dark:bg-slate-800 dark:text-white"
-          style={getHoverStyle()}
-        >
-          <div>{`Area: ${getObjectArea()} px`}</div>
-          <div>{`Ratio: ${getObjectRatio()}`}</div>
-        </div>
-      )}
-    </>
-  );
-}

web/src/components/overlay/detail/ObjectLifecycle.tsx

@@ -151,6 +151,8 @@ export default function ObjectLifecycle({
   );
 
   const [boxStyle, setBoxStyle] = useState<React.CSSProperties | null>(null);
+  const [attributeBoxStyle, setAttributeBoxStyle] =
+    useState<React.CSSProperties | null>(null);
 
   const configAnnotationOffset = useMemo(() => {
     if (!config) {
@@ -218,7 +220,7 @@ export default function ObjectLifecycle({
   const [timeIndex, setTimeIndex] = useState(0);
 
   const handleSetBox = useCallback(
-    (box: number[]) => {
+    (box: number[], attrBox: number[] | undefined) => {
       if (imgRef.current && Array.isArray(box) && box.length === 4) {
         const imgElement = imgRef.current;
         const imgRect = imgElement.getBoundingClientRect();
@@ -231,6 +233,19 @@ export default function ObjectLifecycle({
           borderColor: `rgb(${getObjectColor(event.label)?.join(",")})`,
         };
 
+        if (attrBox) {
+          const attrStyle = {
+            left: `${attrBox[0] * imgRect.width}px`,
+            top: `${attrBox[1] * imgRect.height}px`,
+            width: `${attrBox[2] * imgRect.width}px`,
+            height: `${attrBox[3] * imgRect.height}px`,
+            borderColor: `rgb(${getObjectColor(event.label)?.join(",")})`,
+          };
+          setAttributeBoxStyle(attrStyle);
+        } else {
+          setAttributeBoxStyle(null);
+        }
+
         setBoxStyle(style);
       }
     },
@@ -292,7 +307,10 @@ export default function ObjectLifecycle({
       } else {
         // lifecycle point
         setTimeIndex(eventSequence?.[current].timestamp);
-        handleSetBox(eventSequence?.[current].data.box ?? []);
+        handleSetBox(
+          eventSequence?.[current].data.box ?? [],
+          eventSequence?.[current].data?.attribute_box,
+        );
         setLifecycleZones(eventSequence?.[current].data.zones);
       }
       setSelectedZone("");
@@ -448,6 +466,9 @@ export default function ObjectLifecycle({
                   <div className="absolute bottom-[-3px] left-1/2 h-[5px] w-[5px] -translate-x-1/2 transform bg-yellow-500" />
                 </div>
               )}
+              {attributeBoxStyle && (
+                <div className="absolute border-2" style={attributeBoxStyle} />
+              )}
               {imgRef.current?.width &&
                 imgRef.current?.height &&
                 pathPoints &&

web/src/types/timeline.ts

@@ -20,6 +20,7 @@ export type ObjectLifecycleSequence = {
     box?: [number, number, number, number];
     region: [number, number, number, number];
     attribute: string;
+    attribute_box?: [number, number, number, number];
     zones: string[];
   };
   class_type: LifecycleClassType;