From a2396db2aa21eeacc9106691d57afd75ec40ba65 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 18 Oct 2025 13:19:21 -0500 Subject: [PATCH] Detail Stream tweaks (#20553) * show audio events in detail stream * refactor object lifecycle to look similar to detail stream * pass detail stream as prop to avoid context error * fix highlighting timing * add view in explore to menu --- .../overlay/detail/ObjectLifecycle.tsx | 527 +++++++++--------- web/src/components/player/HlsVideoPlayer.tsx | 18 +- .../player/dynamic/DynamicVideoPlayer.tsx | 5 + web/src/components/timeline/DetailStream.tsx | 42 +- web/src/components/timeline/EventMenu.tsx | 7 + web/src/context/detail-stream-context.tsx | 2 +- 6 files changed, 310 insertions(+), 291 deletions(-) diff --git a/web/src/components/overlay/detail/ObjectLifecycle.tsx b/web/src/components/overlay/detail/ObjectLifecycle.tsx index c06afd1e9..d335e35c9 100644 --- a/web/src/components/overlay/detail/ObjectLifecycle.tsx +++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx @@ -2,14 +2,6 @@ import useSWR from "swr"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Event } from "@/types/event"; import ActivityIndicator from "@/components/indicators/activity-indicator"; -import { - Carousel, - CarouselApi, - CarouselContent, - CarouselItem, - CarouselNext, - CarouselPrevious, -} from "@/components/ui/carousel"; import { Button } from "@/components/ui/button"; import { ObjectLifecycleSequence } from "@/types/timeline"; import Heading from "@/components/ui/heading"; @@ -33,7 +25,6 @@ import { MdOutlinePictureInPictureAlt, } from "react-icons/md"; import { cn } from "@/lib/utils"; -import { Card, CardContent } from "@/components/ui/card"; import { useApiHost } from "@/api"; import { isDesktop, isIOS, isSafari } from "react-device-detect"; import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator"; @@ -55,6 +46,7 @@ import { ObjectPath } from "./ObjectPath"; import { getLifecycleItemDescription } from "@/utils/lifecycleUtil"; import { IoPlayCircleOutline } from "react-icons/io5"; import { useTranslation } from "react-i18next"; +import { getTranslatedLabel } from "@/utils/i18n"; type ObjectLifecycleProps = { className?: string; @@ -166,16 +158,6 @@ export default function ObjectLifecycle({ configAnnotationOffset, ); - const detectArea = useMemo(() => { - if (!config) { - return 0; - } - return ( - config.cameras[event.camera]?.detect?.width * - config.cameras[event.camera]?.detect?.height - ); - }, [config, event.camera]); - const savedPathPoints = useMemo(() => { return ( event.data.path_data?.map(([coords, timestamp]: [number[], number]) => ({ @@ -272,91 +254,108 @@ export default function ObjectLifecycle({ // carousels - const [mainApi, setMainApi] = useState(); - const [thumbnailApi, setThumbnailApi] = useState(); - const [current, setCurrent] = useState(0); - - const handleThumbnailClick = (index: number) => { - if (!mainApi || !thumbnailApi) { - return; - } - mainApi.scrollTo(index); - setCurrent(index); - }; - - const handleThumbnailNavigation = useCallback( - (direction: "next" | "previous") => { - if (!mainApi || !thumbnailApi || !eventSequence) return; - const newIndex = - direction === "next" - ? Math.min(current + 1, eventSequence.length - 1) - : Math.max(current - 1, 0); - mainApi.scrollTo(newIndex); - thumbnailApi.scrollTo(newIndex); - setCurrent(newIndex); - }, - [mainApi, thumbnailApi, current, eventSequence], - ); - - useEffect(() => { - if (eventSequence && eventSequence.length > 0) { - if (current == -1) { - // normal path point - setBoxStyle(null); - setLifecycleZones([]); - } else { - // lifecycle point - setTimeIndex(eventSequence?.[current].timestamp); - handleSetBox( - eventSequence?.[current].data.box ?? [], - eventSequence?.[current].data?.attribute_box, - ); - setLifecycleZones(eventSequence?.[current].data.zones); - } - setSelectedZone(""); - } - }, [current, imgLoaded, handleSetBox, eventSequence]); - - useEffect(() => { - if (!mainApi || !thumbnailApi || !eventSequence || !event) { - return; - } - - const handleTopSelect = () => { - const selected = mainApi.selectedScrollSnap(); - setCurrent(selected); - thumbnailApi.scrollTo(selected); - }; - - mainApi.on("select", handleTopSelect).on("reInit", handleTopSelect); - - return () => { - mainApi.off("select", handleTopSelect); - }; - // we know that these deps are correct - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [mainApi, thumbnailApi]); + // Selected lifecycle item index; -1 when viewing a path-only point const handlePathPointClick = useCallback( (index: number) => { - if (!mainApi || !thumbnailApi || !eventSequence) return; + if (!eventSequence) return; const sequenceIndex = eventSequence.findIndex( (item) => item.timestamp === pathPoints[index].timestamp, ); if (sequenceIndex !== -1) { - mainApi.scrollTo(sequenceIndex); - thumbnailApi.scrollTo(sequenceIndex); - setCurrent(sequenceIndex); + setTimeIndex(eventSequence[sequenceIndex].timestamp); + handleSetBox( + eventSequence[sequenceIndex]?.data.box ?? [], + eventSequence[sequenceIndex]?.data?.attribute_box, + ); + setLifecycleZones(eventSequence[sequenceIndex]?.data.zones); } else { // click on a normal path point, not a lifecycle point - setCurrent(-1); setTimeIndex(pathPoints[index].timestamp); + setBoxStyle(null); + setLifecycleZones([]); } }, - [mainApi, thumbnailApi, eventSequence, pathPoints], + [eventSequence, pathPoints, handleSetBox], ); - if (!event.id || !eventSequence || !config || !timeIndex) { + const formattedStart = config + ? formatUnixTimestampToDateTime(event.start_time ?? 0, { + timezone: config.ui.timezone, + date_format: + config.ui.time_format == "24hour" + ? t("time.formattedTimestampHourMinuteSecond.24hour", { + ns: "common", + }) + : t("time.formattedTimestampHourMinuteSecond.12hour", { + ns: "common", + }), + time_style: "medium", + date_style: "medium", + }) + : ""; + + const formattedEnd = config + ? formatUnixTimestampToDateTime(event.end_time ?? 0, { + timezone: config.ui.timezone, + date_format: + config.ui.time_format == "24hour" + ? t("time.formattedTimestampHourMinuteSecond.24hour", { + ns: "common", + }) + : t("time.formattedTimestampHourMinuteSecond.12hour", { + ns: "common", + }), + time_style: "medium", + date_style: "medium", + }) + : ""; + + useEffect(() => { + if (!eventSequence || eventSequence.length === 0) return; + // If timeIndex hasn't been set to a non-zero value, prefer the first lifecycle timestamp + if (!timeIndex) { + setTimeIndex(eventSequence[0].timestamp); + handleSetBox( + eventSequence[0]?.data.box ?? [], + eventSequence[0]?.data?.attribute_box, + ); + setLifecycleZones(eventSequence[0]?.data.zones); + } + }, [eventSequence, timeIndex, handleSetBox]); + + // When timeIndex changes or image finishes loading, sync the box/zones to matching lifecycle, else clear + useEffect(() => { + if (!eventSequence || timeIndex == null) return; + const idx = eventSequence.findIndex((i) => i.timestamp === timeIndex); + if (idx !== -1) { + if (imgLoaded) { + handleSetBox( + eventSequence[idx]?.data.box ?? [], + eventSequence[idx]?.data?.attribute_box, + ); + } + setLifecycleZones(eventSequence[idx]?.data.zones); + } else { + // Non-lifecycle point (e.g., saved path point) + setBoxStyle(null); + setLifecycleZones([]); + } + }, [timeIndex, imgLoaded, eventSequence, handleSetBox]); + + const selectedLifecycle = useMemo(() => { + if (!eventSequence || eventSequence.length === 0) return undefined; + const idx = eventSequence.findIndex((i) => i.timestamp === timeIndex); + return idx !== -1 ? eventSequence[idx] : eventSequence[0]; + }, [eventSequence, timeIndex]); + + const selectedIndex = useMemo(() => { + if (!eventSequence || eventSequence.length === 0) return 0; + const idx = eventSequence.findIndex((i) => i.timestamp === timeIndex); + return idx === -1 ? 0 : idx; + }, [eventSequence, timeIndex]); + + if (!config) { return ; } @@ -502,7 +501,7 @@ export default function ObjectLifecycle({ className="flex w-full cursor-pointer items-center justify-start gap-2 p-2" onClick={() => navigate( - `/settings?page=masksAndZones&camera=${event.camera}&object_mask=${eventSequence?.[current].data.box}`, + `/settings?page=masksAndZones&camera=${event.camera}&object_mask=${selectedLifecycle?.data.box}`, ) } > @@ -547,8 +546,8 @@ export default function ObjectLifecycle({
{t("objectLifecycle.count", { - first: current + 1, - second: eventSequence.length, + first: selectedIndex + 1, + second: eventSequence?.length ?? 0, })}
@@ -567,205 +566,187 @@ export default function ObjectLifecycle({ /> )} -
- - - {eventSequence.map((item, index) => ( - - - -
-
-
- {getIconForLabel( - item.data.label, - "size-4 md:size-6 absolute left-0 top-0", - )} +
+
+
+
{ + e.stopPropagation(); + setTimeIndex(event.start_time ?? 0); + }} + role="button" + > + {getIconForLabel( + event.label, + "size-6 text-primary dark:text-white", + )} +
+ {getTranslatedLabel(event.label)} + + {formattedStart ?? ""} - {formattedEnd ?? ""} + +
+
+
+ +
+ {!eventSequence ? ( + + ) : eventSequence.length === 0 ? ( +
+ {t("detail.noObjectDetailData", { ns: "views/events" })} +
+ ) : ( +
+ {eventSequence.map((item, idx) => { + const isActive = + Math.abs((timeIndex ?? 0) - (item.timestamp ?? 0)) <= 0.5; + const formattedEventTimestamp = config + ? formatUnixTimestampToDateTime(item.timestamp ?? 0, { + timezone: config.ui.timezone, + date_format: + config.ui.time_format == "24hour" + ? t( + "time.formattedTimestampHourMinuteSecond.24hour", + { ns: "common" }, + ) + : t( + "time.formattedTimestampHourMinuteSecond.12hour", + { ns: "common" }, + ), + time_style: "medium", + date_style: "medium", + }) + : ""; + + const ratio = + Array.isArray(item.data.box) && item.data.box.length >= 4 + ? ( + aspectRatio * + (item.data.box[2] / item.data.box[3]) + ).toFixed(2) + : "N/A"; + const areaPx = + Array.isArray(item.data.box) && item.data.box.length >= 4 + ? Math.round( + (config.cameras[event.camera]?.detect?.width ?? 0) * + (config.cameras[event.camera]?.detect?.height ?? + 0) * + (item.data.box[2] * item.data.box[3]), + ) + : undefined; + const areaPct = + Array.isArray(item.data.box) && item.data.box.length >= 4 + ? (item.data.box[2] * item.data.box[3]).toFixed(4) + : undefined; + + return ( +
{ + setTimeIndex(item.timestamp ?? 0); + handleSetBox( + item.data.box ?? [], + item.data.attribute_box, + ); + setLifecycleZones(item.data.zones); + setSelectedZone(""); + }} + className={cn( + "flex cursor-pointer flex-col gap-1 rounded-md p-2 text-sm text-primary-variant", + isActive + ? "bg-secondary-highlight font-semibold text-primary outline-[1.5px] -outline-offset-[1.1px] outline-primary/40 dark:font-normal" + : "duration-500", + )} + > +
+
-
-
-
- {getLifecycleItemDescription(item)} -
-
- {formatUnixTimestampToDateTime(item.timestamp, { - timezone: config.ui.timezone, - date_format: - config.ui.time_format == "24hour" - ? t("time.formattedTimestamp2.24hour", { - ns: "common", - }) - : t("time.formattedTimestamp2.12hour", { - ns: "common", - }), - time_style: "medium", - date_style: "medium", - })} +
+
{getLifecycleItemDescription(item)}
+
+ {formattedEventTimestamp} +
-
-
-
-
-

- {t( - "objectLifecycle.lifecycleItemDesc.header.zones", + +

+
+
+ + {t( + "objectLifecycle.lifecycleItemDesc.header.ratio", + )} + + + {ratio} + +
+ +
+ + {t( + "objectLifecycle.lifecycleItemDesc.header.area", + )} + + {areaPx !== undefined && areaPct !== undefined ? ( + + px: {areaPx} ยท %: {areaPct} + + ) : ( + N/A )} -

- {item.class_type === "entered_zone" - ? item.data.zones.map((zone, index) => ( -
- {true && ( +
+ {item.class_type === "entered_zone" && ( +
+ + {t( + "objectLifecycle.lifecycleItemDesc.header.zones", + )} + +
+ {item.data.zones.map((zone, zidx) => ( +
{ + e.stopPropagation(); + setSelectedZone(zone); + }} + >
- )} -
setSelectedZone(zone)} - > - {zone.replaceAll("_", " ")} + + {zone.replaceAll("_", " ")} +
-
- )) - : "-"} -
-
-
-
-

- {t( - "objectLifecycle.lifecycleItemDesc.header.ratio", - )} -

- {Array.isArray(item.data.box) && - item.data.box.length >= 4 - ? ( - aspectRatio * - (item.data.box[2] / item.data.box[3]) - ).toFixed(2) - : "N/A"} -
-
-
-
-

- {t("objectLifecycle.lifecycleItemDesc.header.area")} -

- {Array.isArray(item.data.box) && - item.data.box.length >= 4 ? ( - <> -
- px:{" "} - {Math.round( - detectArea * - (item.data.box[2] * item.data.box[3]), - )} + ))}
-
- %:{" "} - {( - (detectArea * - (item.data.box[2] * item.data.box[3])) / - detectArea - ).toFixed(4)} -
- - ) : ( - "N/A" +
)}
- - - - ))} - - -
-
- - 4 ? "justify-start" : "justify-center", + ); + })} +
)} - > - {eventSequence.map((item, index) => ( - handleThumbnailClick(index)} - > -
- - - - - - - - - {getLifecycleItemDescription(item)} - - - - - -
- - ))} - - handleThumbnailNavigation("previous")} - /> - handleThumbnailNavigation("next")} - /> - +
+
); diff --git a/web/src/components/player/HlsVideoPlayer.tsx b/web/src/components/player/HlsVideoPlayer.tsx index 89900c266..a41d31db2 100644 --- a/web/src/components/player/HlsVideoPlayer.tsx +++ b/web/src/components/player/HlsVideoPlayer.tsx @@ -20,7 +20,7 @@ import { cn } from "@/lib/utils"; import { ASPECT_VERTICAL_LAYOUT, RecordingPlayerError } from "@/types/record"; import { useTranslation } from "react-i18next"; import ObjectTrackOverlay from "@/components/overlay/ObjectTrackOverlay"; -import { useDetailStream } from "@/context/detail-stream-context"; +import { DetailStreamContextType } from "@/context/detail-stream-context"; // Android native hls does not seek correctly const USE_NATIVE_HLS = !isAndroid; @@ -54,6 +54,7 @@ type HlsVideoPlayerProps = { onUploadFrame?: (playTime: number) => Promise | undefined; toggleFullscreen?: () => void; onError?: (error: RecordingPlayerError) => void; + detail?: Partial; }; export default function HlsVideoPlayer({ videoRef, @@ -74,16 +75,17 @@ export default function HlsVideoPlayer({ onUploadFrame, toggleFullscreen, onError, + detail, }: HlsVideoPlayerProps) { const { t } = useTranslation("components/player"); const { data: config } = useSWR("config"); - const { - selectedObjectId, - selectedObjectTimeline, - currentTime, - camera, - isDetailMode, - } = useDetailStream(); + + // for detail stream context in History + const selectedObjectId = detail?.selectedObjectId; + const selectedObjectTimeline = detail?.selectedObjectTimeline; + const currentTime = detail?.currentTime; + const camera = detail?.camera; + const isDetailMode = detail?.isDetailMode ?? false; // playback diff --git a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx index 80f8e6dbf..1b7689804 100644 --- a/web/src/components/player/dynamic/DynamicVideoPlayer.tsx +++ b/web/src/components/player/dynamic/DynamicVideoPlayer.tsx @@ -7,6 +7,7 @@ import { Preview } from "@/types/preview"; import PreviewPlayer, { PreviewController } from "../PreviewPlayer"; import { DynamicVideoController } from "./DynamicVideoController"; import HlsVideoPlayer, { HlsSource } from "../HlsVideoPlayer"; +import { useDetailStream } from "@/context/detail-stream-context"; import { TimeRange } from "@/types/timeline"; import ActivityIndicator from "@/components/indicators/activity-indicator"; import { VideoResolutionType } from "@/types/live"; @@ -59,6 +60,9 @@ export default function DynamicVideoPlayer({ const apiHost = useApiHost(); const { data: config } = useSWR("config"); + // for detail stream context in History + const detail = useDetailStream(); + // controlling playback const playerRef = useRef(null); @@ -291,6 +295,7 @@ export default function DynamicVideoPlayer({ setIsBuffering(true); } }} + detail={detail} /> 0; + const { data: fetchedEvents } = useSWR( - review?.data?.detections?.length + shouldFetchEvents ? ["event_ids", { ids: review.data.detections.join(",") }] : null, ); - const rawIconLabels: string[] = fetchedEvents - ? fetchedEvents.map((e) => e.label) - : (review.data?.objects ?? []); + const rawIconLabels: string[] = [ + ...(fetchedEvents + ? fetchedEvents.map((e) => e.label) + : (review.data?.objects ?? [])), + ...(review.data?.audio ?? []), + ]; // limit to 5 icons const seen = new Set(); @@ -310,10 +315,10 @@ function ReviewGroup({ {isActive && (
- {!fetchedEvents ? ( + {shouldFetchEvents && !fetchedEvents ? ( ) : ( - fetchedEvents.map((event) => { + (fetchedEvents || []).map((event) => { return ( 0 && ( +
+ {review.data.audio.map((audioLabel) => ( +
+
+ {getIconForLabel( + audioLabel, + "size-4 text-primary dark:text-white", + )} + {getTranslatedLabel(audioLabel)} +
+
+ ))} +
+ )}
)}
@@ -384,7 +407,7 @@ function EventCollapsible({ // Clear selectedObjectId when effectiveTime has passed this event's end_time useEffect(() => { if (selectedObjectId === event.id && effectiveTime && event.end_time) { - if (effectiveTime > event.end_time) { + if (effectiveTime >= event.end_time) { setSelectedObjectId(undefined); } } @@ -405,8 +428,9 @@ function EventCollapsible({ ? "shadow-selected outline-selected" : "outline-transparent duration-500", event.id != selectedObjectId && - (effectiveTime ?? 0) >= (event.start_time ?? 0) && - (effectiveTime ?? 0) <= (event.end_time ?? event.start_time ?? 0) && + (effectiveTime ?? 0) >= (event.start_time ?? 0) - 0.5 && + (effectiveTime ?? 0) <= + (event.end_time ?? event.start_time ?? 0) + 0.5 && "bg-secondary-highlight outline-[1.5px] -outline-offset-[1.1px] outline-primary/40", )} > diff --git a/web/src/components/timeline/EventMenu.tsx b/web/src/components/timeline/EventMenu.tsx index 36b6f0a16..4723d2b13 100644 --- a/web/src/components/timeline/EventMenu.tsx +++ b/web/src/components/timeline/EventMenu.tsx @@ -41,6 +41,13 @@ export default function EventMenu({ + { + navigate(`/explore?event_id=${event.id}`); + }} + > + {t("details.item.button.viewInExplore")} +