Detail Stream tweaks (#20553)
Some checks are pending
CI / AMD64 Build (push) Waiting to run
CI / ARM Build (push) Waiting to run
CI / Jetson Jetpack 6 (push) Waiting to run
CI / AMD64 Extra Build (push) Blocked by required conditions
CI / ARM Extra Build (push) Blocked by required conditions
CI / Synaptics Build (push) Blocked by required conditions
CI / Assemble and push default build (push) Blocked by required conditions

* show audio events in detail stream

* refactor object lifecycle to look similar to detail stream

* pass detail stream as prop to avoid context error

* fix highlighting timing

* add view in explore to menu
This commit is contained in:
Josh Hawkins 2025-10-18 13:19:21 -05:00 committed by GitHub
parent a8bcc109a9
commit a2396db2aa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 310 additions and 291 deletions

View File

@ -2,14 +2,6 @@ import useSWR from "swr";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { Event } from "@/types/event";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import {
Carousel,
CarouselApi,
CarouselContent,
CarouselItem,
CarouselNext,
CarouselPrevious,
} from "@/components/ui/carousel";
import { Button } from "@/components/ui/button";
import { ObjectLifecycleSequence } from "@/types/timeline";
import Heading from "@/components/ui/heading";
@ -33,7 +25,6 @@ import {
MdOutlinePictureInPictureAlt,
} from "react-icons/md";
import { cn } from "@/lib/utils";
import { Card, CardContent } from "@/components/ui/card";
import { useApiHost } from "@/api";
import { isDesktop, isIOS, isSafari } from "react-device-detect";
import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator";
@ -55,6 +46,7 @@ import { ObjectPath } from "./ObjectPath";
import { getLifecycleItemDescription } from "@/utils/lifecycleUtil";
import { IoPlayCircleOutline } from "react-icons/io5";
import { useTranslation } from "react-i18next";
import { getTranslatedLabel } from "@/utils/i18n";
type ObjectLifecycleProps = {
className?: string;
@ -166,16 +158,6 @@ export default function ObjectLifecycle({
configAnnotationOffset,
);
const detectArea = useMemo(() => {
if (!config) {
return 0;
}
return (
config.cameras[event.camera]?.detect?.width *
config.cameras[event.camera]?.detect?.height
);
}, [config, event.camera]);
const savedPathPoints = useMemo(() => {
return (
event.data.path_data?.map(([coords, timestamp]: [number[], number]) => ({
@ -272,91 +254,108 @@ export default function ObjectLifecycle({
// carousels
const [mainApi, setMainApi] = useState<CarouselApi>();
const [thumbnailApi, setThumbnailApi] = useState<CarouselApi>();
const [current, setCurrent] = useState(0);
const handleThumbnailClick = (index: number) => {
if (!mainApi || !thumbnailApi) {
return;
}
mainApi.scrollTo(index);
setCurrent(index);
};
const handleThumbnailNavigation = useCallback(
(direction: "next" | "previous") => {
if (!mainApi || !thumbnailApi || !eventSequence) return;
const newIndex =
direction === "next"
? Math.min(current + 1, eventSequence.length - 1)
: Math.max(current - 1, 0);
mainApi.scrollTo(newIndex);
thumbnailApi.scrollTo(newIndex);
setCurrent(newIndex);
},
[mainApi, thumbnailApi, current, eventSequence],
);
useEffect(() => {
if (eventSequence && eventSequence.length > 0) {
if (current == -1) {
// normal path point
setBoxStyle(null);
setLifecycleZones([]);
} else {
// lifecycle point
setTimeIndex(eventSequence?.[current].timestamp);
handleSetBox(
eventSequence?.[current].data.box ?? [],
eventSequence?.[current].data?.attribute_box,
);
setLifecycleZones(eventSequence?.[current].data.zones);
}
setSelectedZone("");
}
}, [current, imgLoaded, handleSetBox, eventSequence]);
useEffect(() => {
if (!mainApi || !thumbnailApi || !eventSequence || !event) {
return;
}
const handleTopSelect = () => {
const selected = mainApi.selectedScrollSnap();
setCurrent(selected);
thumbnailApi.scrollTo(selected);
};
mainApi.on("select", handleTopSelect).on("reInit", handleTopSelect);
return () => {
mainApi.off("select", handleTopSelect);
};
// we know that these deps are correct
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [mainApi, thumbnailApi]);
// Selected lifecycle item index; -1 when viewing a path-only point
const handlePathPointClick = useCallback(
(index: number) => {
if (!mainApi || !thumbnailApi || !eventSequence) return;
if (!eventSequence) return;
const sequenceIndex = eventSequence.findIndex(
(item) => item.timestamp === pathPoints[index].timestamp,
);
if (sequenceIndex !== -1) {
mainApi.scrollTo(sequenceIndex);
thumbnailApi.scrollTo(sequenceIndex);
setCurrent(sequenceIndex);
setTimeIndex(eventSequence[sequenceIndex].timestamp);
handleSetBox(
eventSequence[sequenceIndex]?.data.box ?? [],
eventSequence[sequenceIndex]?.data?.attribute_box,
);
setLifecycleZones(eventSequence[sequenceIndex]?.data.zones);
} else {
// click on a normal path point, not a lifecycle point
setCurrent(-1);
setTimeIndex(pathPoints[index].timestamp);
setBoxStyle(null);
setLifecycleZones([]);
}
},
[mainApi, thumbnailApi, eventSequence, pathPoints],
[eventSequence, pathPoints, handleSetBox],
);
if (!event.id || !eventSequence || !config || !timeIndex) {
const formattedStart = config
? formatUnixTimestampToDateTime(event.start_time ?? 0, {
timezone: config.ui.timezone,
date_format:
config.ui.time_format == "24hour"
? t("time.formattedTimestampHourMinuteSecond.24hour", {
ns: "common",
})
: t("time.formattedTimestampHourMinuteSecond.12hour", {
ns: "common",
}),
time_style: "medium",
date_style: "medium",
})
: "";
const formattedEnd = config
? formatUnixTimestampToDateTime(event.end_time ?? 0, {
timezone: config.ui.timezone,
date_format:
config.ui.time_format == "24hour"
? t("time.formattedTimestampHourMinuteSecond.24hour", {
ns: "common",
})
: t("time.formattedTimestampHourMinuteSecond.12hour", {
ns: "common",
}),
time_style: "medium",
date_style: "medium",
})
: "";
useEffect(() => {
if (!eventSequence || eventSequence.length === 0) return;
// If timeIndex hasn't been set to a non-zero value, prefer the first lifecycle timestamp
if (!timeIndex) {
setTimeIndex(eventSequence[0].timestamp);
handleSetBox(
eventSequence[0]?.data.box ?? [],
eventSequence[0]?.data?.attribute_box,
);
setLifecycleZones(eventSequence[0]?.data.zones);
}
}, [eventSequence, timeIndex, handleSetBox]);
// When timeIndex changes or image finishes loading, sync the box/zones to matching lifecycle, else clear
useEffect(() => {
if (!eventSequence || timeIndex == null) return;
const idx = eventSequence.findIndex((i) => i.timestamp === timeIndex);
if (idx !== -1) {
if (imgLoaded) {
handleSetBox(
eventSequence[idx]?.data.box ?? [],
eventSequence[idx]?.data?.attribute_box,
);
}
setLifecycleZones(eventSequence[idx]?.data.zones);
} else {
// Non-lifecycle point (e.g., saved path point)
setBoxStyle(null);
setLifecycleZones([]);
}
}, [timeIndex, imgLoaded, eventSequence, handleSetBox]);
const selectedLifecycle = useMemo(() => {
if (!eventSequence || eventSequence.length === 0) return undefined;
const idx = eventSequence.findIndex((i) => i.timestamp === timeIndex);
return idx !== -1 ? eventSequence[idx] : eventSequence[0];
}, [eventSequence, timeIndex]);
const selectedIndex = useMemo(() => {
if (!eventSequence || eventSequence.length === 0) return 0;
const idx = eventSequence.findIndex((i) => i.timestamp === timeIndex);
return idx === -1 ? 0 : idx;
}, [eventSequence, timeIndex]);
if (!config) {
return <ActivityIndicator />;
}
@ -502,7 +501,7 @@ export default function ObjectLifecycle({
className="flex w-full cursor-pointer items-center justify-start gap-2 p-2"
onClick={() =>
navigate(
`/settings?page=masksAndZones&camera=${event.camera}&object_mask=${eventSequence?.[current].data.box}`,
`/settings?page=masksAndZones&camera=${event.camera}&object_mask=${selectedLifecycle?.data.box}`,
)
}
>
@ -547,8 +546,8 @@ export default function ObjectLifecycle({
</div>
<div className="min-w-20 text-right text-sm text-muted-foreground">
{t("objectLifecycle.count", {
first: current + 1,
second: eventSequence.length,
first: selectedIndex + 1,
second: eventSequence?.length ?? 0,
})}
</div>
</div>
@ -567,205 +566,187 @@ export default function ObjectLifecycle({
/>
)}
<div className="relative flex flex-col items-center justify-center">
<Carousel className="m-0 w-full" setApi={setMainApi}>
<CarouselContent>
{eventSequence.map((item, index) => (
<CarouselItem key={index}>
<Card className="p-1 text-sm md:p-2" key={index}>
<CardContent className="flex flex-row items-center gap-3 p-1 md:p-2">
<div className="flex flex-1 flex-row items-center justify-start p-3 pl-1">
<div className="mt-4">
<div
className="rounded-lg p-2"
style={{
backgroundColor: "rgb(110,110,110)",
}}
className={cn(
"rounded-md bg-secondary p-2 outline outline-[3px] -outline-offset-[2.8px] outline-transparent duration-500",
)}
>
<div className="flex w-full items-center justify-between">
<div
key={item.data.label}
className="relative flex aspect-square size-4 flex-row items-center md:size-8"
className="flex items-center gap-2 font-medium"
onClick={(e) => {
e.stopPropagation();
setTimeIndex(event.start_time ?? 0);
}}
role="button"
>
{getIconForLabel(
item.data.label,
"size-4 md:size-6 absolute left-0 top-0",
event.label,
"size-6 text-primary dark:text-white",
)}
<LifecycleIcon
className="absolute bottom-0 right-0 size-2 md:size-4"
lifecycleItem={item}
/>
<div className="flex items-end gap-2">
<span>{getTranslatedLabel(event.label)}</span>
<span className="text-secondary-foreground">
{formattedStart ?? ""} - {formattedEnd ?? ""}
</span>
</div>
</div>
<div className="mx-3 text-lg">
<div className="flex flex-row items-center text-primary smart-capitalize">
{getLifecycleItemDescription(item)}
</div>
<div className="text-sm text-primary-variant">
{formatUnixTimestampToDateTime(item.timestamp, {
<div className="mt-2">
{!eventSequence ? (
<ActivityIndicator className="size-2" size={2} />
) : eventSequence.length === 0 ? (
<div className="py-2 text-muted-foreground">
{t("detail.noObjectDetailData", { ns: "views/events" })}
</div>
) : (
<div className="mx-2 mt-4 space-y-2">
{eventSequence.map((item, idx) => {
const isActive =
Math.abs((timeIndex ?? 0) - (item.timestamp ?? 0)) <= 0.5;
const formattedEventTimestamp = config
? formatUnixTimestampToDateTime(item.timestamp ?? 0, {
timezone: config.ui.timezone,
date_format:
config.ui.time_format == "24hour"
? t("time.formattedTimestamp2.24hour", {
ns: "common",
})
: t("time.formattedTimestamp2.12hour", {
ns: "common",
}),
? t(
"time.formattedTimestampHourMinuteSecond.24hour",
{ ns: "common" },
)
: t(
"time.formattedTimestampHourMinuteSecond.12hour",
{ ns: "common" },
),
time_style: "medium",
date_style: "medium",
})}
</div>
</div>
</div>
<div className="flex w-5/12 flex-row items-start justify-start">
<div className="text-md mr-2 w-1/3">
<div className="flex flex-col items-end justify-start">
<p className="mb-1.5 text-sm text-primary-variant">
{t(
"objectLifecycle.lifecycleItemDesc.header.zones",
)}
</p>
{item.class_type === "entered_zone"
? item.data.zones.map((zone, index) => (
<div
key={index}
className="flex flex-row items-center gap-1"
>
{true && (
<div
className="size-3 rounded-lg"
style={{
backgroundColor: `rgb(${getZoneColor(zone)})`,
}}
/>
)}
<div
key={index}
className="cursor-pointer smart-capitalize"
onClick={() => setSelectedZone(zone)}
>
{zone.replaceAll("_", " ")}
</div>
</div>
))
: "-"}
</div>
</div>
<div className="text-md mr-2 w-1/3">
<div className="flex flex-col items-end justify-start">
<p className="mb-1.5 text-sm text-primary-variant">
{t(
"objectLifecycle.lifecycleItemDesc.header.ratio",
)}
</p>
{Array.isArray(item.data.box) &&
item.data.box.length >= 4
})
: "";
const ratio =
Array.isArray(item.data.box) && item.data.box.length >= 4
? (
aspectRatio *
(item.data.box[2] / item.data.box[3])
).toFixed(2)
: "N/A"}
</div>
</div>
<div className="text-md mr-2 w-1/3">
<div className="flex flex-col items-end justify-start">
<p className="mb-1.5 text-sm text-primary-variant">
{t("objectLifecycle.lifecycleItemDesc.header.area")}
</p>
{Array.isArray(item.data.box) &&
item.data.box.length >= 4 ? (
<>
<div className="flex flex-col text-xs">
px:{" "}
{Math.round(
detectArea *
: "N/A";
const areaPx =
Array.isArray(item.data.box) && item.data.box.length >= 4
? Math.round(
(config.cameras[event.camera]?.detect?.width ?? 0) *
(config.cameras[event.camera]?.detect?.height ??
0) *
(item.data.box[2] * item.data.box[3]),
)}
</div>
<div className="flex flex-col text-xs">
%:{" "}
{(
(detectArea *
(item.data.box[2] * item.data.box[3])) /
detectArea
).toFixed(4)}
</div>
</>
) : (
"N/A"
)}
</div>
</div>
</div>
</CardContent>
</Card>
</CarouselItem>
))}
</CarouselContent>
</Carousel>
</div>
<div className="relative mt-4 flex flex-col items-center justify-center">
<Carousel
opts={{
align: "center",
containScroll: "keepSnaps",
dragFree: true,
)
: undefined;
const areaPct =
Array.isArray(item.data.box) && item.data.box.length >= 4
? (item.data.box[2] * item.data.box[3]).toFixed(4)
: undefined;
return (
<div
key={`${item.timestamp}-${item.source_id ?? ""}-${idx}`}
role="button"
onClick={() => {
setTimeIndex(item.timestamp ?? 0);
handleSetBox(
item.data.box ?? [],
item.data.attribute_box,
);
setLifecycleZones(item.data.zones);
setSelectedZone("");
}}
className="max-w-[72%] md:max-w-[85%]"
setApi={setThumbnailApi}
>
<CarouselContent
className={cn(
"-ml-1 flex select-none flex-row",
eventSequence.length > 4 ? "justify-start" : "justify-center",
"flex cursor-pointer flex-col gap-1 rounded-md p-2 text-sm text-primary-variant",
isActive
? "bg-secondary-highlight font-semibold text-primary outline-[1.5px] -outline-offset-[1.1px] outline-primary/40 dark:font-normal"
: "duration-500",
)}
>
{eventSequence.map((item, index) => (
<CarouselItem
key={index}
className={cn("basis-auto cursor-pointer pl-1")}
onClick={() => handleThumbnailClick(index)}
>
<div className="p-1">
<Card>
<CardContent
className={cn(
"flex aspect-square items-center justify-center rounded-md p-2",
index === current && "bg-selected",
)}
>
<Tooltip>
<TooltipTrigger>
<div className="flex items-center gap-2">
<div className="flex size-7 items-center justify-center">
<LifecycleIcon
className={cn(
"size-8",
index === current
? "bg-selected text-white"
: "text-muted-foreground",
)}
lifecycleItem={item}
className="size-5"
/>
</TooltipTrigger>
<TooltipPortal>
<TooltipContent className="smart-capitalize">
{getLifecycleItemDescription(item)}
</TooltipContent>
</TooltipPortal>
</Tooltip>
</CardContent>
</Card>
</div>
</CarouselItem>
<div className="flex w-full flex-row justify-between">
<div>{getLifecycleItemDescription(item)}</div>
<div className={cn("p-1 text-sm")}>
{formattedEventTimestamp}
</div>
</div>
</div>
<div className="ml-8 mt-1 flex flex-wrap items-center gap-3 text-sm text-secondary-foreground">
<div className="flex flex-col gap-1">
<div className="flex items-center gap-1">
<span className="text-muted-foreground">
{t(
"objectLifecycle.lifecycleItemDesc.header.ratio",
)}
</span>
<span className="font-medium text-foreground">
{ratio}
</span>
</div>
<div className="flex items-center gap-1">
<span className="text-muted-foreground">
{t(
"objectLifecycle.lifecycleItemDesc.header.area",
)}
</span>
{areaPx !== undefined && areaPct !== undefined ? (
<span className="font-medium text-foreground">
px: {areaPx} · %: {areaPct}
</span>
) : (
<span>N/A</span>
)}
</div>
{item.class_type === "entered_zone" && (
<div className="flex items-center gap-2">
<span className="text-muted-foreground">
{t(
"objectLifecycle.lifecycleItemDesc.header.zones",
)}
</span>
<div className="flex flex-wrap items-center gap-2">
{item.data.zones.map((zone, zidx) => (
<div
key={`${zone}-${zidx}`}
className="flex cursor-pointer items-center gap-1"
onClick={(e) => {
e.stopPropagation();
setSelectedZone(zone);
}}
>
<div
className="size-3 rounded"
style={{
backgroundColor: `rgb(${getZoneColor(zone)})`,
}}
/>
<span className="smart-capitalize">
{zone.replaceAll("_", " ")}
</span>
</div>
))}
</CarouselContent>
<CarouselPrevious
disabled={current === 0}
onClick={() => handleThumbnailNavigation("previous")}
/>
<CarouselNext
disabled={current === eventSequence.length - 1}
onClick={() => handleThumbnailNavigation("next")}
/>
</Carousel>
</div>
</div>
)}
</div>
</div>
</div>
);
})}
</div>
)}
</div>
</div>
</div>
</div>
);

View File

@ -20,7 +20,7 @@ import { cn } from "@/lib/utils";
import { ASPECT_VERTICAL_LAYOUT, RecordingPlayerError } from "@/types/record";
import { useTranslation } from "react-i18next";
import ObjectTrackOverlay from "@/components/overlay/ObjectTrackOverlay";
import { useDetailStream } from "@/context/detail-stream-context";
import { DetailStreamContextType } from "@/context/detail-stream-context";
// Android native hls does not seek correctly
const USE_NATIVE_HLS = !isAndroid;
@ -54,6 +54,7 @@ type HlsVideoPlayerProps = {
onUploadFrame?: (playTime: number) => Promise<AxiosResponse> | undefined;
toggleFullscreen?: () => void;
onError?: (error: RecordingPlayerError) => void;
detail?: Partial<DetailStreamContextType>;
};
export default function HlsVideoPlayer({
videoRef,
@ -74,16 +75,17 @@ export default function HlsVideoPlayer({
onUploadFrame,
toggleFullscreen,
onError,
detail,
}: HlsVideoPlayerProps) {
const { t } = useTranslation("components/player");
const { data: config } = useSWR<FrigateConfig>("config");
const {
selectedObjectId,
selectedObjectTimeline,
currentTime,
camera,
isDetailMode,
} = useDetailStream();
// for detail stream context in History
const selectedObjectId = detail?.selectedObjectId;
const selectedObjectTimeline = detail?.selectedObjectTimeline;
const currentTime = detail?.currentTime;
const camera = detail?.camera;
const isDetailMode = detail?.isDetailMode ?? false;
// playback

View File

@ -7,6 +7,7 @@ import { Preview } from "@/types/preview";
import PreviewPlayer, { PreviewController } from "../PreviewPlayer";
import { DynamicVideoController } from "./DynamicVideoController";
import HlsVideoPlayer, { HlsSource } from "../HlsVideoPlayer";
import { useDetailStream } from "@/context/detail-stream-context";
import { TimeRange } from "@/types/timeline";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import { VideoResolutionType } from "@/types/live";
@ -59,6 +60,9 @@ export default function DynamicVideoPlayer({
const apiHost = useApiHost();
const { data: config } = useSWR<FrigateConfig>("config");
// for detail stream context in History
const detail = useDetailStream();
// controlling playback
const playerRef = useRef<HTMLVideoElement | null>(null);
@ -291,6 +295,7 @@ export default function DynamicVideoPlayer({
setIsBuffering(true);
}
}}
detail={detail}
/>
<PreviewPlayer
className={cn(

View File

@ -232,15 +232,20 @@ function ReviewGroup({
date_style: "medium",
});
const shouldFetchEvents = review?.data?.detections?.length > 0;
const { data: fetchedEvents } = useSWR<Event[]>(
review?.data?.detections?.length
shouldFetchEvents
? ["event_ids", { ids: review.data.detections.join(",") }]
: null,
);
const rawIconLabels: string[] = fetchedEvents
const rawIconLabels: string[] = [
...(fetchedEvents
? fetchedEvents.map((e) => e.label)
: (review.data?.objects ?? []);
: (review.data?.objects ?? [])),
...(review.data?.audio ?? []),
];
// limit to 5 icons
const seen = new Set<string>();
@ -310,10 +315,10 @@ function ReviewGroup({
{isActive && (
<div className="mt-2 space-y-2">
{!fetchedEvents ? (
{shouldFetchEvents && !fetchedEvents ? (
<ActivityIndicator />
) : (
fetchedEvents.map((event) => {
(fetchedEvents || []).map((event) => {
return (
<EventCollapsible
key={event.id}
@ -325,6 +330,24 @@ function ReviewGroup({
);
})
)}
{review.data.audio && review.data.audio.length > 0 && (
<div className="space-y-1">
{review.data.audio.map((audioLabel) => (
<div
key={audioLabel}
className="rounded-md bg-secondary p-2 outline outline-[3px] -outline-offset-[2.8px] outline-transparent duration-500"
>
<div className="flex items-center gap-2 text-sm font-medium">
{getIconForLabel(
audioLabel,
"size-4 text-primary dark:text-white",
)}
<span>{getTranslatedLabel(audioLabel)}</span>
</div>
</div>
))}
</div>
)}
</div>
)}
</div>
@ -384,7 +407,7 @@ function EventCollapsible({
// Clear selectedObjectId when effectiveTime has passed this event's end_time
useEffect(() => {
if (selectedObjectId === event.id && effectiveTime && event.end_time) {
if (effectiveTime > event.end_time) {
if (effectiveTime >= event.end_time) {
setSelectedObjectId(undefined);
}
}
@ -405,8 +428,9 @@ function EventCollapsible({
? "shadow-selected outline-selected"
: "outline-transparent duration-500",
event.id != selectedObjectId &&
(effectiveTime ?? 0) >= (event.start_time ?? 0) &&
(effectiveTime ?? 0) <= (event.end_time ?? event.start_time ?? 0) &&
(effectiveTime ?? 0) >= (event.start_time ?? 0) - 0.5 &&
(effectiveTime ?? 0) <=
(event.end_time ?? event.start_time ?? 0) + 0.5 &&
"bg-secondary-highlight outline-[1.5px] -outline-offset-[1.1px] outline-primary/40",
)}
>

View File

@ -41,6 +41,13 @@ export default function EventMenu({
</DropdownMenuTrigger>
<DropdownMenuPortal>
<DropdownMenuContent>
<DropdownMenuItem
onSelect={() => {
navigate(`/explore?event_id=${event.id}`);
}}
>
{t("details.item.button.viewInExplore")}
</DropdownMenuItem>
<DropdownMenuItem asChild>
<a
download

View File

@ -3,7 +3,7 @@ import { FrigateConfig } from "@/types/frigateConfig";
import useSWR from "swr";
import { ObjectLifecycleSequence } from "@/types/timeline";
interface DetailStreamContextType {
export interface DetailStreamContextType {
selectedObjectId: string | undefined;
selectedObjectTimeline?: ObjectLifecycleSequence[];
currentTime: number;