Add timeline overlay selector

web-new/src/components/card/HistoryCard.tsx

@@ -3,21 +3,13 @@ import PreviewThumbnailPlayer from "../player/PreviewThumbnailPlayer";
 import { Card } from "../ui/card";
 import { FrigateConfig } from "@/types/frigateConfig";
 import ActivityIndicator from "../ui/activity-indicator";
-import {
-  LuCircle,
-  LuClock,
-  LuPlay,
-  LuPlayCircle,
-  LuTruck,
-} from "react-icons/lu";
-import { IoMdExit } from "react-icons/io";
-import {
-  MdFaceUnlock,
-  MdOutlineLocationOn,
-  MdOutlinePictureInPictureAlt,
-} from "react-icons/md";
+import { LuClock } from "react-icons/lu";
 import { HiOutlineVideoCamera } from "react-icons/hi";
 import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
+import {
+  getTimelineIcon,
+  getTimelineItemDescription,
+} from "@/utils/timelineUtil";
 
 type HistoryCardProps = {
   timeline: Card;
@@ -78,76 +70,3 @@ export default function HistoryCard({
     </Card>
   );
 }
-
-function getTimelineIcon(timelineItem: Timeline) {
-  switch (timelineItem.class_type) {
-    case "visible":
-      return <LuPlay className="w-4 mr-1" />;
-    case "gone":
-      return <IoMdExit className="w-4 mr-1" />;
-    case "active":
-      return <LuPlayCircle className="w-4 mr-1" />;
-    case "stationary":
-      return <LuCircle className="w-4 mr-1" />;
-    case "entered_zone":
-      return <MdOutlineLocationOn className="w-4 mr-1" />;
-    case "attribute":
-      switch (timelineItem.data.attribute) {
-        case "face":
-          return <MdFaceUnlock className="w-4 mr-1" />;
-        case "license_plate":
-          return <MdOutlinePictureInPictureAlt className="w-4 mr-1" />;
-        default:
-          return <LuTruck className="w-4 mr-1" />;
-      }
-    case "sub_label":
-      switch (timelineItem.data.label) {
-        case "person":
-          return <MdFaceUnlock className="w-4 mr-1" />;
-        case "car":
-          return <MdOutlinePictureInPictureAlt className="w-4 mr-1" />;
-      }
-  }
-}
-
-function getTimelineItemDescription(timelineItem: Timeline) {
-  const label = (
-    (Array.isArray(timelineItem.data.sub_label)
-      ? timelineItem.data.sub_label[0]
-      : timelineItem.data.sub_label) || timelineItem.data.label
-  ).replaceAll("_", " ");
-
-  switch (timelineItem.class_type) {
-    case "visible":
-      return `${label} detected`;
-    case "entered_zone":
-      return `${label} entered ${timelineItem.data.zones
-        .join(" and ")
-        .replaceAll("_", " ")}`;
-    case "active":
-      return `${label} became active`;
-    case "stationary":
-      return `${label} became stationary`;
-    case "attribute": {
-      let title = "";
-      if (
-        timelineItem.data.attribute == "face" ||
-        timelineItem.data.attribute == "license_plate"
-      ) {
-        title = `${timelineItem.data.attribute.replaceAll(
-          "_",
-          " "
-        )} detected for ${label}`;
-      } else {
-        title = `${
-          timelineItem.data.sub_label
-        } recognized as ${timelineItem.data.attribute.replaceAll("_", " ")}`;
-      }
-      return title;
-    }
-    case "sub_label":
-      return `${timelineItem.data.label} recognized as ${timelineItem.data.sub_label}`;
-    case "gone":
-      return `${label} left`;
-  }
-}

web-new/src/components/card/TimelineCardPlayer.tsx

@@ -3,8 +3,23 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "../ui/dialog";
 import useSWR from "swr";
 import { FrigateConfig } from "@/types/frigateConfig";
 import VideoPlayer from "../player/VideoPlayer";
-import { useMemo } from "react";
+import { useMemo, useRef, useState } from "react";
 import { useApiHost } from "@/api";
+import TimelineEventOverlay from "../overlay/TimelineDataOverlay";
+import ActivityIndicator from "../ui/activity-indicator";
+import { Button } from "../ui/button";
+import {
+  getTimelineIcon,
+  getTimelineItemDescription,
+} from "@/utils/timelineUtil";
+import { LuAlertCircle } from "react-icons/lu";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "../ui/tooltip";
+import Player from "video.js/dist/types/player";
 
 type TimelinePlayerCardProps = {
   timeline?: Card;
@@ -17,6 +32,18 @@ export default function TimelinePlayerCard({
 }: TimelinePlayerCardProps) {
   const { data: config } = useSWR<FrigateConfig>("config");
   const apiHost = useApiHost();
+  const playerRef = useRef<Player | undefined>();
+
+  const annotationOffset = useMemo(() => {
+    if (!config || !timeline) {
+      return 0;
+    }
+
+    return (
+      (config.cameras[timeline.camera]?.detect?.annotation_offset || 0) / 1000
+    );
+  }, [config, timeline]);
+  const [selectedItem, setSelectedItem] = useState<Timeline | undefined>();
 
   const recordingParams = useMemo(() => {
     if (!timeline) {
@@ -28,15 +55,37 @@ export default function TimelinePlayerCard({
       after: timeline.entries.at(0)!!.timestamp,
     };
   }, [timeline]);
+
   const { data: recordings } = useSWR<Recording[]>(
     timeline ? [`${timeline.camera}/recordings`, recordingParams] : null,
     { revalidateOnFocus: false }
   );
 
+  const playbackUri = useMemo(() => {
+    if (!timeline) {
+      return "";
+    }
+
+    const end = timeline.entries.at(-1)!!.timestamp + 30;
+    const start = timeline.entries.at(0)!!.timestamp;
+    return `${apiHost}vod/${timeline?.camera}/start/${
+      Number.isInteger(start) ? start.toFixed(1) : start
+    }/end/${Number.isInteger(end) ? end.toFixed(1) : end}/master.m3u8`;
+  }, [timeline]);
+
   return (
     <>
-      <Dialog open={timeline != null} onOpenChange={onDismiss}>
-        <DialogContent className="md:max-w-xl lg:max-w-2xl xl:max-w-3xl 2xl:max-w-4xl">
+      <Dialog
+        open={timeline != null}
+        onOpenChange={(_) => {
+          setSelectedItem(undefined);
+          onDismiss();
+        }}
+      >
+        <DialogContent
+          className="md:max-w-2xl lg:max-w-3xl xl:max-w-4xl 2xl:max-w-5xl"
+          onOpenAutoFocus={(e) => e.preventDefault()}
+        >
           <DialogHeader>
             <DialogTitle className="capitalize">
               {`${timeline?.camera?.replaceAll(
@@ -48,27 +97,166 @@ export default function TimelinePlayerCard({
               })}`}
             </DialogTitle>
           </DialogHeader>
-          {recordings && recordings.length > 0 && (
-            <VideoPlayer
-              options={{
-                preload: "auto",
-                autoplay: true,
-                sources: [
-                  {
-                    src: `${apiHost}vod/${timeline?.camera}/start/${recordings
-                      .at(0)
-                      ?.start_time.toFixed(2)}/end/${recordings
-                      .at(-1)
-                      ?.end_time.toFixed(2)}/master.m3u8`,
-                    type: "application/vnd.apple.mpegurl",
-                  },
-                ],
-              }}
-              seekOptions={{ forward: 10, backward: 5 }}
-            />
+          {config && timeline && recordings && recordings.length > 0 && (
+            <>
+              <TimelineSummary
+                timeline={timeline}
+                annotationOffset={annotationOffset}
+                recordings={recordings}
+                onFrameSelected={(selected, seekTime) => {
+                  setSelectedItem(selected);
+                  playerRef.current?.pause();
+                  playerRef.current?.currentTime(seekTime);
+                }}
+              />
+              <div className="relative">
+                <VideoPlayer
+                  options={{
+                    preload: "auto",
+                    autoplay: true,
+                    sources: [
+                      {
+                        src: playbackUri,
+                        type: "application/vnd.apple.mpegurl",
+                      },
+                    ],
+                  }}
+                  seekOptions={{ forward: 10, backward: 5 }}
+                  onReady={(player) => {
+                    playerRef.current = player;
+                    player.on("playing", () => {
+                      setSelectedItem(undefined);
+                    });
+                  }}
+                  onDispose={() => {
+                    playerRef.current = undefined;
+                  }}
+                >
+                  {selectedItem ? (
+                    <TimelineEventOverlay
+                      timeline={selectedItem}
+                      cameraConfig={config.cameras[timeline.camera]}
+                    />
+                  ) : undefined}
+                </VideoPlayer>
+              </div>
+            </>
           )}
         </DialogContent>
       </Dialog>
     </>
   );
 }
+
+type TimelineSummaryProps = {
+  timeline: Card;
+  annotationOffset: number;
+  recordings: Recording[];
+  onFrameSelected: (timeline: Timeline, frameTime: number) => void;
+};
+
+function TimelineSummary({
+  timeline,
+  annotationOffset,
+  recordings,
+  onFrameSelected,
+}: TimelineSummaryProps) {
+  const [timeIndex, setTimeIndex] = useState<number>(-1);
+
+  // calculates the seek seconds by adding up all the seconds in the segments prior to the playback time
+  const getSeekSeconds = (seekUnix: number) => {
+    if (!recordings) {
+      return 0;
+    }
+
+    let seekSeconds = 0;
+    recordings.every((segment) => {
+      // if the next segment is past the desired time, stop calculating
+      if (segment.start_time > seekUnix) {
+        return false;
+      }
+
+      if (segment.end_time < seekUnix) {
+        seekSeconds += segment.end_time - segment.start_time;
+        return true;
+      }
+
+      seekSeconds +=
+        segment.end_time - segment.start_time - (segment.end_time - seekUnix);
+      return true;
+    });
+
+    return seekSeconds;
+  };
+
+  const onSelectMoment = async (index: number) => {
+    setTimeIndex(index);
+    onFrameSelected(
+      timeline.entries[index],
+      getSeekSeconds(timeline.entries[index].timestamp + annotationOffset)
+    );
+  };
+
+  if (!timeline || !recordings) {
+    return <ActivityIndicator />;
+  }
+
+  return (
+    <div className="flex flex-col">
+      <div className="h-12 flex justify-center">
+        <div className="flex flex-row flex-nowrap justify-between overflow-auto">
+          {timeline.entries.map((item, index) => (
+            <TooltipProvider key={item.timestamp}>
+              <Tooltip>
+                <TooltipTrigger asChild>
+                  <Button
+                    className={`m-1 blue ${
+                      index == timeIndex ? "text-blue-500" : "text-gray-500"
+                    }`}
+                    variant="secondary"
+                    autoFocus={false}
+                    onClick={() => onSelectMoment(index)}
+                  >
+                    {getTimelineIcon(item)}
+                  </Button>
+                </TooltipTrigger>
+                <TooltipContent>
+                  <p>{getTimelineItemDescription(item)}</p>
+                </TooltipContent>
+              </Tooltip>
+            </TooltipProvider>
+          ))}
+        </div>
+      </div>
+      {timeIndex >= 0 ? (
+        <div className="max-w-md self-center">
+          <div className="flex justify-start">
+            <div className="text-sm flex justify-between py-1 items-center">
+              Bounding boxes may not align
+            </div>
+            <TooltipProvider>
+              <Tooltip>
+                <TooltipTrigger asChild>
+                  <Button size="icon" variant="ghost">
+                    <LuAlertCircle />
+                  </Button>
+                </TooltipTrigger>
+                <TooltipContent>
+                  <p>
+                    Disclaimer: This data comes from the detect feed but is
+                    shown on the recordings.
+                  </p>
+                  <p>
+                    It is unlikely that the streams are perfectly in sync so the
+                    bounding box and the footage will not line up perfectly.
+                  </p>
+                  <p>The annotation_offset field can be used to adjust this.</p>
+                </TooltipContent>
+              </Tooltip>
+            </TooltipProvider>
+          </div>
+        </div>
+      ) : null}
+    </div>
+  );
+}

web-new/src/components/overlay/TimelineDataOverlay.tsx

@@ -0,0 +1,84 @@
+import { useState } from "react";
+
+type TimelineEventOverlayProps = {
+  timeline: Timeline;
+  cameraConfig: {
+    detect: {
+      width: number;
+      height: number;
+    };
+  };
+};
+
+export default function TimelineEventOverlay({
+  timeline,
+  cameraConfig,
+}: TimelineEventOverlayProps) {
+  const boxLeftEdge = Math.round(timeline.data.box[0] * 100);
+  const boxTopEdge = Math.round(timeline.data.box[1] * 100);
+  const boxRightEdge = Math.round(
+    (1 - timeline.data.box[2] - timeline.data.box[0]) * 100
+  );
+  const boxBottomEdge = Math.round(
+    (1 - timeline.data.box[3] - timeline.data.box[1]) * 100
+  );
+
+  const [isHovering, setIsHovering] = useState<boolean>(false);
+  const getHoverStyle = () => {
+    if (boxLeftEdge < 15) {
+      // show object stats on right side
+      return {
+        left: `${boxLeftEdge + timeline.data.box[2] * 100 + 1}%`,
+        top: `${boxTopEdge}%`,
+      };
+    }
+
+    return {
+      right: `${boxRightEdge + timeline.data.box[2] * 100 + 1}%`,
+      top: `${boxTopEdge}%`,
+    };
+  };
+
+  const getObjectArea = () => {
+    const width = timeline.data.box[2] * cameraConfig.detect.width;
+    const height = timeline.data.box[3] * cameraConfig.detect.height;
+    return Math.round(width * height);
+  };
+
+  const getObjectRatio = () => {
+    const width = timeline.data.box[2] * cameraConfig.detect.width;
+    const height = timeline.data.box[3] * cameraConfig.detect.height;
+    return Math.round(100 * (width / height)) / 100;
+  };
+
+  return (
+    <>
+      <div
+        className="absolute border-4 border-red-600"
+        onMouseEnter={() => setIsHovering(true)}
+        onMouseLeave={() => setIsHovering(false)}
+        onTouchStart={() => setIsHovering(true)}
+        onTouchEnd={() => setIsHovering(false)}
+        style={{
+          left: `${boxLeftEdge}%`,
+          top: `${boxTopEdge}%`,
+          right: `${boxRightEdge}%`,
+          bottom: `${boxBottomEdge}%`,
+        }}
+      >
+        {timeline.class_type == "entered_zone" ? (
+          <div className="absolute w-2 h-2 bg-yellow-500 left-[50%] -translate-x-1/2 translate-y-3/4 bottom-0" />
+        ) : null}
+      </div>
+      {isHovering && (
+        <div
+          className="absolute bg-white dark:bg-slate-800 p-4 block text-black dark:text-white text-lg"
+          style={getHoverStyle()}
+        >
+          <div>{`Area: ${getObjectArea()} px`}</div>
+          <div>{`Ratio: ${getObjectRatio()}`}</div>
+        </div>
+      )}
+    </>
+  );
+}

web-new/src/components/player/VideoPlayer.tsx

@@ -5,7 +5,7 @@ import 'video.js/dist/video-js.css';
 import Player from "video.js/dist/types/player";
 
 type VideoPlayerProps = {
-    children?: ReactElement | ReactElement[],
+  children?: ReactElement | ReactElement[],
     options?: {
         [key: string]: any
     },

web-new/src/pages/History.tsx

@@ -9,7 +9,7 @@ import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
 import axios from "axios";
 import TimelinePlayerCard from "@/components/card/TimelineCardPlayer";
 
-const API_LIMIT = 100;
+const API_LIMIT = 120;
 
 function History() {
   const { data: config } = useSWR<FrigateConfig>("config");

web-new/src/pages/Storage.tsx

@@ -71,7 +71,7 @@ function Storage() {
       <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
         <Card>
           <CardHeader>
-            <div className="flex bg-center snap-center text-center items-center">
+            <div className="flex items-center">
               <CardTitle>Data</CardTitle>
               <TooltipProvider>
                 <Tooltip>
@@ -137,7 +137,7 @@ function Storage() {
 
         <Card>
           <CardHeader>
-            <div className="flex bg-center snap-center text-center items-center">
+            <div className="flex items-center">
               <CardTitle>Memory</CardTitle>
               <TooltipProvider>
                 <Tooltip>
@@ -187,7 +187,7 @@ function Storage() {
         </Card>
       </div>
 
-      <div className="flex bg-center snap-center text-center items-center my-4">
+      <div className="flex items-center my-4">
         <Heading as="h4">Cameras</Heading>
         <TooltipProvider>
           <Tooltip>

web-new/src/utils/timelineUtil.tsx

@@ -0,0 +1,80 @@
+import { LuCircle, LuPlay, LuPlayCircle, LuTruck } from "react-icons/lu";
+import { IoMdExit } from "react-icons/io";
+import {
+  MdFaceUnlock,
+  MdOutlineLocationOn,
+  MdOutlinePictureInPictureAlt,
+} from "react-icons/md";
+
+export function getTimelineIcon(timelineItem: Timeline) {
+  switch (timelineItem.class_type) {
+    case "visible":
+      return <LuPlay className="w-4 mr-1" />;
+    case "gone":
+      return <IoMdExit className="w-4 mr-1" />;
+    case "active":
+      return <LuPlayCircle className="w-4 mr-1" />;
+    case "stationary":
+      return <LuCircle className="w-4 mr-1" />;
+    case "entered_zone":
+      return <MdOutlineLocationOn className="w-4 mr-1" />;
+    case "attribute":
+      switch (timelineItem.data.attribute) {
+        case "face":
+          return <MdFaceUnlock className="w-4 mr-1" />;
+        case "license_plate":
+          return <MdOutlinePictureInPictureAlt className="w-4 mr-1" />;
+        default:
+          return <LuTruck className="w-4 mr-1" />;
+      }
+    case "sub_label":
+      switch (timelineItem.data.label) {
+        case "person":
+          return <MdFaceUnlock className="w-4 mr-1" />;
+        case "car":
+          return <MdOutlinePictureInPictureAlt className="w-4 mr-1" />;
+      }
+  }
+}
+
+export function getTimelineItemDescription(timelineItem: Timeline) {
+  const label = (
+    (Array.isArray(timelineItem.data.sub_label)
+      ? timelineItem.data.sub_label[0]
+      : timelineItem.data.sub_label) || timelineItem.data.label
+  ).replaceAll("_", " ");
+
+  switch (timelineItem.class_type) {
+    case "visible":
+      return `${label} detected`;
+    case "entered_zone":
+      return `${label} entered ${timelineItem.data.zones
+        .join(" and ")
+        .replaceAll("_", " ")}`;
+    case "active":
+      return `${label} became active`;
+    case "stationary":
+      return `${label} became stationary`;
+    case "attribute": {
+      let title = "";
+      if (
+        timelineItem.data.attribute == "face" ||
+        timelineItem.data.attribute == "license_plate"
+      ) {
+        title = `${timelineItem.data.attribute.replaceAll(
+          "_",
+          " "
+        )} detected for ${label}`;
+      } else {
+        title = `${
+          timelineItem.data.sub_label
+        } recognized as ${timelineItem.data.attribute.replaceAll("_", " ")}`;
+      }
+      return title;
+    }
+    case "sub_label":
+      return `${timelineItem.data.label} recognized as ${timelineItem.data.sub_label}`;
+    case "gone":
+      return `${label} left`;
+  }
+}