Add Detail stream in History view (#20525)

* new type

* activity stream panel

* use context provider for activity stream

* new activity stream panel in history view

* overlay for object tracking details in history view

* use overlay in video player

* don't refetch timeline

* fix activity stream group from being highlighted prematurely

* use annotation offset

* fix scrolling and use custom hook for interaction

* memoize to prevent unnecessary renders

* i18n and timestamp formatting

* add annotation offset slider

* bg color

* add collapsible component

* refactor

* rename activity to detail

* fix merge conflicts

* i18n

* more i18n
This commit is contained in:
Josh Hawkins 2025-10-16 08:24:14 -05:00 committed by GitHub
parent 2e7a2fd780
commit b52044aecc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 1818 additions and 326 deletions

166
web/package-lock.json generated
View File

@ -14,6 +14,7 @@
"@radix-ui/react-alert-dialog": "^1.1.6",
"@radix-ui/react-aspect-ratio": "^1.1.2",
"@radix-ui/react-checkbox": "^1.1.4",
"@radix-ui/react-collapsible": "^1.1.12",
"@radix-ui/react-context-menu": "^2.2.6",
"@radix-ui/react-dialog": "^1.1.15",
"@radix-ui/react-dropdown-menu": "^2.1.6",
@ -1380,6 +1381,171 @@
}
}
},
"node_modules/@radix-ui/react-collapsible": {
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.12.tgz",
"integrity": "sha512-Uu+mSh4agx2ib1uIGPP4/CKNULyajb3p92LsVXmH2EHVMTfZWpll88XJ0j4W0z3f8NK1eYl1+Mf/szHPmcHzyA==",
"license": "MIT",
"dependencies": {
"@radix-ui/primitive": "1.1.3",
"@radix-ui/react-compose-refs": "1.1.2",
"@radix-ui/react-context": "1.1.2",
"@radix-ui/react-id": "1.1.1",
"@radix-ui/react-presence": "1.1.5",
"@radix-ui/react-primitive": "2.1.3",
"@radix-ui/react-use-controllable-state": "1.2.2",
"@radix-ui/react-use-layout-effect": "1.1.1"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/primitive": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.3.tgz",
"integrity": "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg==",
"license": "MIT"
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/react-compose-refs": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.2.tgz",
"integrity": "sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg==",
"license": "MIT",
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/react-context": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.2.tgz",
"integrity": "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA==",
"license": "MIT",
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/react-id": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.1.tgz",
"integrity": "sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-use-layout-effect": "1.1.1"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/react-presence": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.5.tgz",
"integrity": "sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-compose-refs": "1.1.2",
"@radix-ui/react-use-layout-effect": "1.1.1"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/react-primitive": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.3.tgz",
"integrity": "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-slot": "1.2.3"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/react-use-controllable-state": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.2.tgz",
"integrity": "sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-use-effect-event": "0.0.2",
"@radix-ui/react-use-layout-effect": "1.1.1"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collapsible/node_modules/@radix-ui/react-use-layout-effect": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.1.tgz",
"integrity": "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ==",
"license": "MIT",
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-collection": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.2.tgz",

View File

@ -20,6 +20,7 @@
"@radix-ui/react-alert-dialog": "^1.1.6",
"@radix-ui/react-aspect-ratio": "^1.1.2",
"@radix-ui/react-checkbox": "^1.1.4",
"@radix-ui/react-collapsible": "^1.1.12",
"@radix-ui/react-context-menu": "^2.2.6",
"@radix-ui/react-dialog": "^1.1.15",
"@radix-ui/react-dropdown-menu": "^2.1.6",

View File

@ -18,6 +18,17 @@
"aria": "Select events",
"noFoundForTimePeriod": "No events found for this time period."
},
"detail": {
"noDataFound": "No detail data to review",
"aria": "Toggle detail view",
"trackedObject_one": "tracked object",
"trackedObject_other": "tracked objects",
"noObjectDetailData": "No object detail data available."
},
"objectTrack": {
"trackedPoint": "Tracked point",
"clickToSeek": "Click to seek to this time"
},
"documentTitle": "Review - Frigate",
"recordings": {
"documentTitle": "Recordings - Frigate"

View File

@ -0,0 +1,395 @@
import { useMemo, useCallback } from "react";
import { ObjectLifecycleSequence, LifecycleClassType } from "@/types/timeline";
import { FrigateConfig } from "@/types/frigateConfig";
import useSWR from "swr";
import { useDetailStream } from "@/context/detail-stream-context";
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { TooltipPortal } from "@radix-ui/react-tooltip";
import { cn } from "@/lib/utils";
import { useTranslation } from "react-i18next";
type ObjectTrackOverlayProps = {
camera: string;
selectedObjectId: string;
currentTime: number;
videoWidth: number;
videoHeight: number;
className?: string;
onSeekToTime?: (timestamp: number) => void;
objectTimeline?: ObjectLifecycleSequence[];
};
export default function ObjectTrackOverlay({
camera,
selectedObjectId,
currentTime,
videoWidth,
videoHeight,
className,
onSeekToTime,
objectTimeline,
}: ObjectTrackOverlayProps) {
const { t } = useTranslation("views/events");
const { data: config } = useSWR<FrigateConfig>("config");
const { annotationOffset } = useDetailStream();
const effectiveCurrentTime = currentTime - annotationOffset / 1000;
// Fetch the full event data to get saved path points
const { data: eventData } = useSWR(["event_ids", { ids: selectedObjectId }]);
const typeColorMap = useMemo(
() => ({
[LifecycleClassType.VISIBLE]: [0, 255, 0], // Green
[LifecycleClassType.GONE]: [255, 0, 0], // Red
[LifecycleClassType.ENTERED_ZONE]: [255, 165, 0], // Orange
[LifecycleClassType.ATTRIBUTE]: [128, 0, 128], // Purple
[LifecycleClassType.ACTIVE]: [255, 255, 0], // Yellow
[LifecycleClassType.STATIONARY]: [128, 128, 128], // Gray
[LifecycleClassType.HEARD]: [0, 255, 255], // Cyan
[LifecycleClassType.EXTERNAL]: [165, 42, 42], // Brown
}),
[],
);
const getObjectColor = useMemo(() => {
return (label: string) => {
const objectColor = config?.model?.colormap[label];
if (objectColor) {
const reversed = [...objectColor].reverse();
return `rgb(${reversed.join(",")})`;
}
return "rgb(255, 0, 0)"; // fallback red
};
}, [config]);
const getZoneColor = useCallback(
(zoneName: string) => {
const zoneColor = config?.cameras?.[camera]?.zones?.[zoneName]?.color;
if (zoneColor) {
const reversed = [...zoneColor].reverse();
return `rgb(${reversed.join(",")})`;
}
return "rgb(255, 0, 0)"; // fallback red
},
[config, camera],
);
const currentObjectZones = useMemo(() => {
if (!objectTimeline) return [];
// Find the most recent timeline event at or before effective current time
const relevantEvents = objectTimeline
.filter((event) => event.timestamp <= effectiveCurrentTime)
.sort((a, b) => b.timestamp - a.timestamp); // Most recent first
// Get zones from the most recent event
return relevantEvents[0]?.data?.zones || [];
}, [objectTimeline, effectiveCurrentTime]);
const zones = useMemo(() => {
if (!config?.cameras?.[camera]?.zones || !currentObjectZones.length)
return [];
return Object.entries(config.cameras[camera].zones)
.filter(([name]) => currentObjectZones.includes(name))
.map(([name, zone]) => ({
name,
coordinates: zone.coordinates,
color: getZoneColor(name),
}));
}, [config, camera, getZoneColor, currentObjectZones]);
// get saved path points from event
const savedPathPoints = useMemo(() => {
return (
eventData?.[0].data?.path_data?.map(
([coords, timestamp]: [number[], number]) => ({
x: coords[0],
y: coords[1],
timestamp,
lifecycle_item: undefined,
}),
) || []
);
}, [eventData]);
// timeline points for selected event
const eventSequencePoints = useMemo(() => {
return (
objectTimeline
?.filter((event) => event.data.box !== undefined)
.map((event) => {
const [left, top, width, height] = event.data.box!;
return {
x: left + width / 2, // Center x
y: top + height, // Bottom y
timestamp: event.timestamp,
lifecycle_item: event,
};
}) || []
);
}, [objectTimeline]);
// final object path with timeline points included
const pathPoints = useMemo(() => {
// don't display a path for autotracking cameras
if (config?.cameras[camera]?.onvif.autotracking.enabled_in_config)
return [];
const combinedPoints = [...savedPathPoints, ...eventSequencePoints].sort(
(a, b) => a.timestamp - b.timestamp,
);
// Filter points around current time (within a reasonable window)
const timeWindow = 30; // 30 seconds window
return combinedPoints.filter(
(point) =>
point.timestamp >= currentTime - timeWindow &&
point.timestamp <= currentTime + timeWindow,
);
}, [savedPathPoints, eventSequencePoints, config, camera, currentTime]);
// get absolute positions on the svg canvas for each point
const absolutePositions = useMemo(() => {
if (!pathPoints) return [];
return pathPoints.map((point) => {
// Find the corresponding timeline entry for this point
const timelineEntry = objectTimeline?.find(
(entry) => entry.timestamp == point.timestamp,
);
return {
x: point.x * videoWidth,
y: point.y * videoHeight,
timestamp: point.timestamp,
lifecycle_item:
timelineEntry ||
(point.box // normal path point
? {
timestamp: point.timestamp,
camera: camera,
source: "tracked_object",
source_id: selectedObjectId,
class_type: "visible" as LifecycleClassType,
data: {
camera: camera,
label: point.label,
sub_label: "",
box: point.box,
region: [0, 0, 0, 0], // placeholder
attribute: "",
zones: [],
},
}
: undefined),
};
});
}, [
pathPoints,
videoWidth,
videoHeight,
objectTimeline,
camera,
selectedObjectId,
]);
const generateStraightPath = useCallback(
(points: { x: number; y: number }[]) => {
if (!points || points.length < 2) return "";
let path = `M ${points[0].x} ${points[0].y}`;
for (let i = 1; i < points.length; i++) {
path += ` L ${points[i].x} ${points[i].y}`;
}
return path;
},
[],
);
const getPointColor = useCallback(
(baseColor: number[], type?: string) => {
if (type && typeColorMap[type as keyof typeof typeColorMap]) {
const typeColor = typeColorMap[type as keyof typeof typeColorMap];
if (typeColor) {
return `rgb(${typeColor.join(",")})`;
}
}
// normal path point
return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`;
},
[typeColorMap],
);
const handlePointClick = useCallback(
(timestamp: number) => {
onSeekToTime?.(timestamp);
},
[onSeekToTime],
);
// render bounding box for object at current time if we have a timeline entry
const currentBoundingBox = useMemo(() => {
if (!objectTimeline) return null;
// Find the most recent timeline event at or before effective current time with a bounding box
const relevantEvents = objectTimeline
.filter(
(event) => event.timestamp <= effectiveCurrentTime && event.data.box,
)
.sort((a, b) => b.timestamp - a.timestamp); // Most recent first
const currentEvent = relevantEvents[0];
if (!currentEvent?.data.box) return null;
const [left, top, width, height] = currentEvent.data.box;
return {
left,
top,
width,
height,
centerX: left + width / 2,
centerY: top + height,
};
}, [objectTimeline, effectiveCurrentTime]);
const objectColor = useMemo(() => {
return pathPoints[0]?.label
? getObjectColor(pathPoints[0].label)
: "rgb(255, 0, 0)";
}, [pathPoints, getObjectColor]);
const objectColorArray = useMemo(() => {
return pathPoints[0]?.label
? getObjectColor(pathPoints[0].label).match(/\d+/g)?.map(Number) || [
255, 0, 0,
]
: [255, 0, 0];
}, [pathPoints, getObjectColor]);
// render any zones for object at current time
const zonePolygons = useMemo(() => {
return zones.map((zone) => {
// Convert zone coordinates from normalized (0-1) to pixel coordinates
const points = zone.coordinates
.split(",")
.map(Number.parseFloat)
.reduce((acc: string[], value, index) => {
const isXCoordinate = index % 2 === 0;
const coordinate = isXCoordinate
? value * videoWidth
: value * videoHeight;
acc.push(coordinate.toString());
return acc;
}, [])
.join(",");
return {
key: zone.name,
points,
fill: `rgba(${zone.color.replace("rgb(", "").replace(")", "")}, 0.3)`,
stroke: zone.color,
};
});
}, [zones, videoWidth, videoHeight]);
if (!pathPoints.length || !config) {
return null;
}
return (
<svg
className={cn(className)}
viewBox={`0 0 ${videoWidth} ${videoHeight}`}
style={{
width: "100%",
height: "100%",
}}
preserveAspectRatio="xMidYMid slice"
>
{zonePolygons.map((zone) => (
<polygon
key={zone.key}
points={zone.points}
fill={zone.fill}
stroke={zone.stroke}
strokeWidth="5"
opacity="0.7"
/>
))}
{absolutePositions.length > 1 && (
<path
d={generateStraightPath(absolutePositions)}
fill="none"
stroke={objectColor}
strokeWidth="5"
strokeLinecap="round"
strokeLinejoin="round"
/>
)}
{absolutePositions.map((pos, index) => (
<Tooltip key={`point-${index}`}>
<TooltipTrigger asChild>
<circle
cx={pos.x}
cy={pos.y}
r="7"
fill={getPointColor(
objectColorArray,
pos.lifecycle_item?.class_type,
)}
stroke="white"
strokeWidth="3"
style={{ cursor: onSeekToTime ? "pointer" : "default" }}
onClick={() => handlePointClick(pos.timestamp)}
/>
</TooltipTrigger>
<TooltipPortal>
<TooltipContent side="top" className="smart-capitalize">
{pos.lifecycle_item
? `${pos.lifecycle_item.class_type.replace("_", " ")} at ${new Date(pos.timestamp * 1000).toLocaleTimeString()}`
: t("objectTrack.trackedPoint")}
{onSeekToTime && (
<div className="mt-1 text-xs text-muted-foreground">
{t("objectTrack.clickToSeek")}
</div>
)}
</TooltipContent>
</TooltipPortal>
</Tooltip>
))}
{currentBoundingBox && (
<g>
<rect
x={currentBoundingBox.left * videoWidth}
y={currentBoundingBox.top * videoHeight}
width={currentBoundingBox.width * videoWidth}
height={currentBoundingBox.height * videoHeight}
fill="none"
stroke={objectColor}
strokeWidth="5"
opacity="0.9"
/>
<circle
cx={currentBoundingBox.centerX * videoWidth}
cy={currentBoundingBox.centerY * videoHeight}
r="5"
fill="rgb(255, 255, 0)" // yellow highlight
stroke={objectColor}
strokeWidth="5"
opacity="1"
/>
</g>
)}
</svg>
);
}

View File

@ -0,0 +1,95 @@
import { useCallback, useState } from "react";
import { Slider } from "@/components/ui/slider";
import { Button } from "@/components/ui/button";
import { useDetailStream } from "@/context/detail-stream-context";
import axios from "axios";
import { useSWRConfig } from "swr";
import { toast } from "sonner";
import { useTranslation } from "react-i18next";
type Props = {
className?: string;
};
export default function AnnotationOffsetSlider({ className }: Props) {
const { annotationOffset, setAnnotationOffset, camera } = useDetailStream();
const { mutate } = useSWRConfig();
const { t } = useTranslation(["views/explore"]);
const [isSaving, setIsSaving] = useState(false);
const handleChange = useCallback(
(values: number[]) => {
if (!values || values.length === 0) return;
const valueMs = values[0];
setAnnotationOffset(valueMs);
},
[setAnnotationOffset],
);
const reset = useCallback(() => {
setAnnotationOffset(0);
}, [setAnnotationOffset]);
const save = useCallback(async () => {
setIsSaving(true);
try {
// save value in milliseconds to config
await axios.put(
`config/set?cameras.${camera}.detect.annotation_offset=${annotationOffset}`,
{ requires_restart: 0 },
);
toast.success(
t("objectLifecycle.annotationSettings.offset.toast.success", {
camera,
}),
{ position: "top-center" },
);
// refresh config
await mutate("config");
} catch (e: unknown) {
const err = e as {
response?: { data?: { message?: string } };
message?: string;
};
const errorMessage =
err?.response?.data?.message || err?.message || "Unknown error";
toast.error(t("toast.save.error.title", { errorMessage, ns: "common" }), {
position: "top-center",
});
} finally {
setIsSaving(false);
}
}, [annotationOffset, camera, mutate, t]);
return (
<div
className={`absolute bottom-0 left-0 right-0 z-30 flex items-center gap-3 bg-background p-3 ${className ?? ""}`}
style={{ pointerEvents: "auto" }}
>
<div className="w-56 text-sm">
Annotation offset (ms): {annotationOffset}
</div>
<div className="flex-1">
<Slider
value={[annotationOffset]}
min={-1500}
max={1500}
step={50}
onValueChange={handleChange}
/>
</div>
<div className="flex items-center gap-2">
<Button size="sm" variant="ghost" onClick={reset}>
Reset
</Button>
<Button size="sm" onClick={save} disabled={isSaving}>
{isSaving
? t("button.saving", { ns: "common" })
: t("button.save", { ns: "common" })}
</Button>
</div>
</div>
);
}

View File

@ -174,7 +174,7 @@ export function AnnotationSettingsPane({
{t("objectLifecycle.annotationSettings.offset.label")}
</FormLabel>
<div className="flex flex-col gap-3 md:flex-row-reverse md:gap-8">
<div className="flex flex-row items-center gap-3 rounded-lg bg-destructive/50 p-3 text-sm text-primary-variant md:my-0 md:my-5">
<div className="flex flex-row items-center gap-3 rounded-lg bg-destructive/50 p-3 text-sm text-primary-variant md:my-5">
<PiWarningCircle className="size-24" />
<div>
<Trans ns="views/explore">

View File

@ -19,6 +19,8 @@ import { usePersistence } from "@/hooks/use-persistence";
import { cn } from "@/lib/utils";
import { ASPECT_VERTICAL_LAYOUT, RecordingPlayerError } from "@/types/record";
import { useTranslation } from "react-i18next";
import ObjectTrackOverlay from "@/components/overlay/ObjectTrackOverlay";
import { useDetailStream } from "@/context/detail-stream-context";
// Android native hls does not seek correctly
const USE_NATIVE_HLS = !isAndroid;
@ -47,6 +49,7 @@ type HlsVideoPlayerProps = {
onPlayerLoaded?: () => void;
onTimeUpdate?: (time: number) => void;
onPlaying?: () => void;
onSeekToTime?: (timestamp: number) => void;
setFullResolution?: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
onUploadFrame?: (playTime: number) => Promise<AxiosResponse> | undefined;
toggleFullscreen?: () => void;
@ -66,6 +69,7 @@ export default function HlsVideoPlayer({
onPlayerLoaded,
onTimeUpdate,
onPlaying,
onSeekToTime,
setFullResolution,
onUploadFrame,
toggleFullscreen,
@ -73,6 +77,13 @@ export default function HlsVideoPlayer({
}: HlsVideoPlayerProps) {
const { t } = useTranslation("components/player");
const { data: config } = useSWR<FrigateConfig>("config");
const {
selectedObjectId,
selectedObjectTimeline,
currentTime,
camera,
isDetailMode,
} = useDetailStream();
// playback
@ -84,17 +95,19 @@ export default function HlsVideoPlayer({
const handleLoadedMetadata = useCallback(() => {
setLoadedMetadata(true);
if (videoRef.current) {
const width = videoRef.current.videoWidth;
const height = videoRef.current.videoHeight;
if (setFullResolution) {
setFullResolution({
width: videoRef.current.videoWidth,
height: videoRef.current.videoHeight,
width,
height,
});
}
setTallCamera(
videoRef.current.videoWidth / videoRef.current.videoHeight <
ASPECT_VERTICAL_LAYOUT,
);
setVideoDimensions({ width, height });
setTallCamera(width / height < ASPECT_VERTICAL_LAYOUT);
}
}, [videoRef, setFullResolution]);
@ -174,6 +187,10 @@ export default function HlsVideoPlayer({
const [controls, setControls] = useState(isMobile);
const [controlsOpen, setControlsOpen] = useState(false);
const [zoomScale, setZoomScale] = useState(1.0);
const [videoDimensions, setVideoDimensions] = useState<{
width: number;
height: number;
}>({ width: 0, height: 0 });
useEffect(() => {
if (!isDesktop) {
@ -296,6 +313,30 @@ export default function HlsVideoPlayer({
height: isMobile ? "100%" : undefined,
}}
>
{isDetailMode &&
selectedObjectId &&
camera &&
currentTime &&
videoDimensions.width > 0 &&
videoDimensions.height > 0 && (
<div className="absolute z-50 size-full">
<ObjectTrackOverlay
key={`${selectedObjectId}-${currentTime}`}
camera={camera}
selectedObjectId={selectedObjectId}
currentTime={currentTime}
videoWidth={videoDimensions.width}
videoHeight={videoDimensions.height}
className="absolute inset-0 z-10"
onSeekToTime={(timestamp) => {
if (onSeekToTime) {
onSeekToTime(timestamp);
}
}}
objectTimeline={selectedObjectTimeline}
/>
</div>
)}
<video
ref={videoRef}
className={`size-full rounded-lg bg-black md:rounded-2xl ${loadedMetadata ? "" : "invisible"} cursor-pointer`}

View File

@ -32,6 +32,7 @@ type DynamicVideoPlayerProps = {
onControllerReady: (controller: DynamicVideoController) => void;
onTimestampUpdate?: (timestamp: number) => void;
onClipEnded?: () => void;
onSeekToTime?: (timestamp: number) => void;
setFullResolution: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
toggleFullscreen: () => void;
containerRef?: React.MutableRefObject<HTMLDivElement | null>;
@ -49,6 +50,7 @@ export default function DynamicVideoPlayer({
onControllerReady,
onTimestampUpdate,
onClipEnded,
onSeekToTime,
setFullResolution,
toggleFullscreen,
containerRef,
@ -265,6 +267,7 @@ export default function DynamicVideoPlayer({
onTimeUpdate={onTimeUpdate}
onPlayerLoaded={onPlayerLoaded}
onClipEnded={onValidateClipEnd}
onSeekToTime={onSeekToTime}
onPlaying={() => {
if (isScrubbing) {
playerRef.current?.pause();

View File

@ -0,0 +1,550 @@
import { useEffect, useRef, useState } from "react";
import { ObjectLifecycleSequence } from "@/types/timeline";
import { LifecycleIcon } from "@/components/overlay/detail/ObjectLifecycle";
import { getLifecycleItemDescription } from "@/utils/lifecycleUtil";
import { useDetailStream } from "@/context/detail-stream-context";
import scrollIntoView from "scroll-into-view-if-needed";
import useUserInteraction from "@/hooks/use-user-interaction";
import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
import { useTranslation } from "react-i18next";
import AnnotationOffsetSlider from "@/components/overlay/detail/AnnotationOffsetSlider";
import { FrigateConfig } from "@/types/frigateConfig";
import useSWR from "swr";
import ActivityIndicator from "../indicators/activity-indicator";
import { Event } from "@/types/event";
import { getIconForLabel } from "@/utils/iconUtil";
import { ReviewSegment, REVIEW_PADDING } from "@/types/review";
import {
Collapsible,
CollapsibleTrigger,
CollapsibleContent,
} from "@/components/ui/collapsible";
import { LuChevronUp, LuChevronDown } from "react-icons/lu";
import { getTranslatedLabel } from "@/utils/i18n";
import EventMenu from "@/components/timeline/EventMenu";
import { FrigatePlusDialog } from "@/components/overlay/dialog/FrigatePlusDialog";
import { cn } from "@/lib/utils";
type DetailStreamProps = {
reviewItems?: ReviewSegment[];
currentTime: number;
onSeek: (timestamp: number) => void;
};
export default function DetailStream({
reviewItems,
currentTime,
onSeek,
}: DetailStreamProps) {
const { data: config } = useSWR<FrigateConfig>("config");
const { t } = useTranslation("views/events");
const { annotationOffset } = useDetailStream();
const scrollRef = useRef<HTMLDivElement>(null);
const [activeReviewId, setActiveReviewId] = useState<string | undefined>(
undefined,
);
const { userInteracting, setProgrammaticScroll } = useUserInteraction({
elementRef: scrollRef,
});
const effectiveTime = currentTime + annotationOffset / 1000;
const PAD = 0; // REVIEW_PADDING ?? 2;
const [upload, setUpload] = useState<Event | undefined>(undefined);
// Ensure we initialize the active review when reviewItems first arrive.
// This helps when the component mounts while the video is already
// playing — it guarantees the matching review is highlighted right
// away instead of waiting for a future effectiveTime change.
useEffect(() => {
if (!reviewItems || reviewItems.length === 0) return;
if (activeReviewId) return;
let target: ReviewSegment | undefined;
let closest: { r: ReviewSegment; diff: number } | undefined;
for (const r of reviewItems) {
const start = (r.start_time ?? 0) - PAD;
const end = (r.end_time ?? r.start_time ?? start) + PAD;
if (effectiveTime >= start && effectiveTime <= end) {
target = r;
break;
}
const mid = (start + end) / 2;
const diff = Math.abs(effectiveTime - mid);
if (!closest || diff < closest.diff) closest = { r, diff };
}
if (!target && closest) target = closest.r;
if (target) {
const start = (target.start_time ?? 0) - PAD;
setActiveReviewId(
`review-${target.id ?? target.start_time ?? Math.floor(start)}`,
);
}
}, [reviewItems, activeReviewId, effectiveTime, PAD]);
// Auto-scroll to current time
useEffect(() => {
if (!scrollRef.current || userInteracting) return;
// Prefer the review whose range contains the effectiveTime. If none
// contains it, pick the nearest review (by mid-point distance). This is
// robust to unordered reviewItems and avoids always picking the last
// element.
const items = reviewItems ?? [];
if (items.length === 0) return;
let target: ReviewSegment | undefined;
let closest: { r: ReviewSegment; diff: number } | undefined;
for (const r of items) {
const start = (r.start_time ?? 0) - PAD;
const end = (r.end_time ?? r.start_time ?? start) + PAD;
if (effectiveTime >= start && effectiveTime <= end) {
target = r;
break;
}
const mid = (start + end) / 2;
const diff = Math.abs(effectiveTime - mid);
if (!closest || diff < closest.diff) closest = { r, diff };
}
if (!target && closest) target = closest.r;
if (target) {
const start = (target.start_time ?? 0) - PAD;
const id = `review-${target.id ?? target.start_time ?? Math.floor(start)}`;
const element = scrollRef.current.querySelector(
`[data-review-id="${id}"]`,
) as HTMLElement;
if (element) {
setProgrammaticScroll();
scrollIntoView(element, {
scrollMode: "if-needed",
behavior: "smooth",
});
}
}
}, [
reviewItems,
effectiveTime,
annotationOffset,
userInteracting,
setProgrammaticScroll,
PAD,
]);
// Auto-select active review based on effectiveTime (if inside a review range)
useEffect(() => {
if (!reviewItems || reviewItems.length === 0) return;
for (const r of reviewItems) {
const start = (r.start_time ?? 0) - PAD;
const end = (r.end_time ?? r.start_time ?? start) + PAD;
if (effectiveTime >= start && effectiveTime <= end) {
setActiveReviewId(
`review-${r.id ?? r.start_time ?? Math.floor(start)}`,
);
return;
}
}
}, [effectiveTime, reviewItems, PAD]);
if (!config) {
return <ActivityIndicator />;
}
return (
<div className="relative">
<FrigatePlusDialog
upload={upload}
onClose={() => setUpload(undefined)}
onEventUploaded={() => setUpload(undefined)}
/>
<div
ref={scrollRef}
className="scrollbar-container h-[calc(100vh-70px)] overflow-y-auto bg-secondary"
>
<div className="space-y-2 p-4">
{reviewItems?.length === 0 ? (
<div className="py-8 text-center text-muted-foreground">
{t("detail.noDataFound")}
</div>
) : (
reviewItems?.map((review: ReviewSegment) => {
const id = `review-${review.id ?? review.start_time ?? Math.floor((review.start_time ?? 0) - PAD)}`;
return (
<ReviewGroup
key={id}
id={id}
review={review}
config={config}
onSeek={onSeek}
effectiveTime={effectiveTime}
isActive={activeReviewId == id}
onActivate={() => setActiveReviewId(id)}
onOpenUpload={(e) => setUpload(e)}
/>
);
})
)}
</div>
</div>
<AnnotationOffsetSlider />
</div>
);
}
type ReviewGroupProps = {
review: ReviewSegment;
id: string;
config: FrigateConfig;
onSeek: (timestamp: number) => void;
isActive?: boolean;
onActivate?: () => void;
onOpenUpload?: (e: Event) => void;
effectiveTime?: number;
};
function ReviewGroup({
review,
id,
config,
onSeek,
isActive = false,
onActivate,
onOpenUpload,
effectiveTime,
}: ReviewGroupProps) {
const { t } = useTranslation("views/events");
const PAD = REVIEW_PADDING ?? 2;
// derive start timestamp from the review
const start = (review.start_time ?? 0) - PAD;
// display time first in the header
const displayTime = formatUnixTimestampToDateTime(start, {
timezone: config.ui.timezone,
date_format:
config.ui.time_format == "24hour"
? t("time.formattedTimestamp.24hour", { ns: "common" })
: t("time.formattedTimestamp.12hour", { ns: "common" }),
time_style: "medium",
date_style: "medium",
});
const { data: fetchedEvents } = useSWR<Event[]>(
review?.data?.detections?.length
? ["event_ids", { ids: review.data.detections.join(",") }]
: null,
);
const rawIconLabels: string[] = fetchedEvents
? fetchedEvents.map((e) => e.label)
: (review.data?.objects ?? []);
// limit to 5 icons
const seen = new Set<string>();
const iconLabels: string[] = [];
for (const lbl of rawIconLabels) {
if (!seen.has(lbl)) {
seen.add(lbl);
iconLabels.push(lbl);
if (iconLabels.length >= 5) break;
}
}
const objectCount = fetchedEvents
? fetchedEvents.length
: (review.data.objects ?? []).length;
return (
<div
data-review-id={id}
className={`cursor-pointer rounded-lg border bg-background p-3 outline outline-[3px] -outline-offset-[2.8px] ${
isActive
? "shadow-selected outline-selected"
: "outline-transparent duration-500"
}`}
>
<div
className="flex items-center justify-between"
onClick={() => {
onActivate?.();
onSeek(start);
}}
>
<div className="flex items-center gap-2">
<div className="flex flex-col">
<div className="text-sm font-medium">{displayTime}</div>
<div className="text-xs text-muted-foreground">
{objectCount} {t("detail.trackedObject", { count: objectCount })}
</div>
</div>
</div>
<div className="flex items-center gap-2">
{iconLabels.slice(0, 5).map((lbl, idx) => (
<span key={`${lbl}-${idx}`}>
{getIconForLabel(lbl, "size-4 text-primary dark:text-white")}
</span>
))}
</div>
</div>
{isActive && (
<div className="mt-2 space-y-2">
{!fetchedEvents ? (
<ActivityIndicator />
) : (
fetchedEvents.map((event) => {
return (
<EventCollapsible
key={event.id}
event={event}
effectiveTime={effectiveTime}
onSeek={onSeek}
onOpenUpload={onOpenUpload}
/>
);
})
)}
</div>
)}
</div>
);
}
type EventCollapsibleProps = {
event: Event;
effectiveTime?: number;
onSeek: (ts: number) => void;
onOpenUpload?: (e: Event) => void;
};
function EventCollapsible({
event,
effectiveTime,
onSeek,
onOpenUpload,
}: EventCollapsibleProps) {
const [open, setOpen] = useState(false);
const { t } = useTranslation("views/events");
const { data: config } = useSWR<FrigateConfig>("config");
const { selectedObjectId, setSelectedObjectId } = useDetailStream();
const formattedStart = config
? formatUnixTimestampToDateTime(event.start_time ?? 0, {
timezone: config.ui.timezone,
date_format:
config.ui.time_format == "24hour"
? t("time.formattedTimestampHourMinuteSecond.24hour", {
ns: "common",
})
: t("time.formattedTimestampHourMinuteSecond.12hour", {
ns: "common",
}),
time_style: "medium",
date_style: "medium",
})
: "";
const formattedEnd = config
? formatUnixTimestampToDateTime(event.end_time ?? 0, {
timezone: config.ui.timezone,
date_format:
config.ui.time_format == "24hour"
? t("time.formattedTimestampHourMinuteSecond.24hour", {
ns: "common",
})
: t("time.formattedTimestampHourMinuteSecond.12hour", {
ns: "common",
}),
time_style: "medium",
date_style: "medium",
})
: "";
// Clear selectedObjectId when effectiveTime has passed this event's end_time
useEffect(() => {
if (selectedObjectId === event.id && effectiveTime && event.end_time) {
if (effectiveTime > event.end_time) {
setSelectedObjectId(undefined);
}
}
}, [
selectedObjectId,
event.id,
event.end_time,
effectiveTime,
setSelectedObjectId,
]);
return (
<Collapsible open={open} onOpenChange={(o) => setOpen(o)}>
<div
className={cn(
"rounded-md bg-secondary p-2 outline outline-[3px] -outline-offset-[2.8px]",
event.id == selectedObjectId
? "shadow-selected outline-selected"
: "outline-transparent duration-500",
event.id != selectedObjectId &&
(effectiveTime ?? 0) >= (event.start_time ?? 0) &&
(effectiveTime ?? 0) <= (event.end_time ?? event.start_time ?? 0) &&
"bg-secondary-highlight/80 outline-[1px] -outline-offset-[0.8px] outline-primary/40",
)}
>
<div className="flex w-full items-center justify-between">
<div
className="flex items-center gap-2 text-sm font-medium"
onClick={(e) => {
e.stopPropagation();
onSeek(event.start_time ?? 0);
if (event.id) setSelectedObjectId(event.id);
}}
role="button"
>
{getIconForLabel(
event.label,
"size-4 text-primary dark:text-white",
)}
<div className="flex items-end gap-2">
<span>{getTranslatedLabel(event.label)}</span>
<span className="text-xs text-secondary-foreground">
{formattedStart ?? ""} - {formattedEnd ?? ""}
</span>
</div>
</div>
<div className="flex flex-1 flex-row justify-end">
<EventMenu
event={event}
config={config}
onOpenUpload={(e) => onOpenUpload?.(e)}
/>
</div>
<div className="flex items-center gap-2">
<CollapsibleTrigger asChild>
<button
onClick={(e) => e.stopPropagation()}
className="rounded bg-muted px-2 py-1 text-xs"
aria-label={t("detail.aria")}
>
{open ? (
<LuChevronUp className="size-3" />
) : (
<LuChevronDown className="size-3" />
)}
</button>
</CollapsibleTrigger>
</div>
</div>
<CollapsibleContent>
<div className="mt-2">
<ObjectTimeline
eventId={event.id}
onSeek={(ts) => {
onSeek(ts);
}}
effectiveTime={effectiveTime}
/>
</div>
</CollapsibleContent>
</div>
</Collapsible>
);
}
type LifecycleItemProps = {
event: ObjectLifecycleSequence;
onSeek: (timestamp: number) => void;
isActive?: boolean;
};
function LifecycleItem({ event, isActive }: LifecycleItemProps) {
const { t } = useTranslation("views/events");
const { data: config } = useSWR<FrigateConfig>("config");
const formattedEventTimestamp = config
? formatUnixTimestampToDateTime(event.timestamp ?? 0, {
timezone: config.ui.timezone,
date_format:
config.ui.time_format == "24hour"
? t("time.formattedTimestampHourMinuteSecond.24hour", {
ns: "common",
})
: t("time.formattedTimestampHourMinuteSecond.12hour", {
ns: "common",
}),
time_style: "medium",
date_style: "medium",
})
: "";
return (
<div
className={cn(
"flex items-center gap-2 text-sm text-primary-variant",
isActive ? "text-white" : "duration-500",
)}
>
<div className="flex size-4 items-center justify-center">
<LifecycleIcon lifecycleItem={event} className="size-3" />
</div>
<div className="flex w-full flex-row justify-between">
<div>{getLifecycleItemDescription(event)}</div>
<div className={cn("p-1 text-xs")}>{formattedEventTimestamp}</div>
</div>
</div>
);
}
// Fetch and render timeline entries for a single event id on demand.
function ObjectTimeline({
eventId,
onSeek,
effectiveTime,
}: {
eventId: string;
onSeek: (ts: number) => void;
effectiveTime?: number;
}) {
const { t } = useTranslation("views/events");
const { data: timeline, isValidating } = useSWR<ObjectLifecycleSequence[]>([
"timeline",
{
source_id: eventId,
},
]);
if ((!timeline || timeline.length === 0) && isValidating) {
return <ActivityIndicator className="h-2 w-2" size={2} />;
}
if (!timeline || timeline.length === 0) {
return (
<div className="py-2 text-sm text-muted-foreground">
{t("detail.noObjectDetailData")}
</div>
);
}
return (
<div className="mx-2 mt-4 space-y-2">
{timeline.map((event, idx) => {
const isActive =
Math.abs((effectiveTime ?? 0) - (event.timestamp ?? 0)) <= 0.5;
return (
<div
key={`${event.timestamp}-${event.source_id ?? idx}`}
onClick={() => {
onSeek(event.timestamp);
}}
>
<LifecycleItem event={event} onSeek={onSeek} isActive={isActive} />
</div>
);
})}
</div>
);
}

View File

@ -0,0 +1,87 @@
import {
DropdownMenu,
DropdownMenuTrigger,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuPortal,
} from "@/components/ui/dropdown-menu";
import { HiDotsHorizontal } from "react-icons/hi";
import { useApiHost } from "@/api";
import { useNavigate } from "react-router-dom";
import { useTranslation } from "react-i18next";
import type { Event } from "@/types/event";
import type { FrigateConfig } from "@/types/frigateConfig";
type EventMenuProps = {
event: Event;
config?: FrigateConfig;
onOpenUpload?: (e: Event) => void;
onOpenSimilarity?: (e: Event) => void;
};
export default function EventMenu({
event,
config,
onOpenUpload,
onOpenSimilarity,
}: EventMenuProps) {
const apiHost = useApiHost();
const navigate = useNavigate();
const { t } = useTranslation("views/explore");
return (
<DropdownMenu>
<DropdownMenuTrigger>
<button
className="mr-2 rounded p-1"
aria-label={t("itemMenu.openMenu", { ns: "common" })}
>
<HiDotsHorizontal className="size-4 text-muted-foreground" />
</button>
</DropdownMenuTrigger>
<DropdownMenuPortal>
<DropdownMenuContent>
<DropdownMenuItem asChild>
<a
download
href={
event.has_snapshot
? `${apiHost}api/events/${event.id}/snapshot.jpg`
: `${apiHost}api/events/${event.id}/thumbnail.webp`
}
>
{t("itemMenu.downloadSnapshot.label")}
</a>
</DropdownMenuItem>
{event.has_snapshot &&
event.plus_id == undefined &&
event.data.type == "object" &&
config?.plus?.enabled && (
<DropdownMenuItem
onSelect={() => {
onOpenUpload?.(event);
}}
>
{t("itemMenu.submitToPlus.label")}
</DropdownMenuItem>
)}
{event.has_snapshot && config?.semantic_search?.enabled && (
<DropdownMenuItem
onSelect={() => {
if (onOpenSimilarity) onOpenSimilarity(event);
else
navigate(
`/explore?search_type=similarity&event_id=${event.id}`,
);
}}
>
{t("itemMenu.findSimilar.label")}
</DropdownMenuItem>
)}
</DropdownMenuContent>
</DropdownMenuPortal>
</DropdownMenu>
);
}

View File

@ -0,0 +1,9 @@
import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"
const Collapsible = CollapsiblePrimitive.Root
const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger
const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent
export { Collapsible, CollapsibleTrigger, CollapsibleContent }

View File

@ -0,0 +1,82 @@
import React, { createContext, useContext, useState, useEffect } from "react";
import { FrigateConfig } from "@/types/frigateConfig";
import useSWR from "swr";
import { ObjectLifecycleSequence } from "@/types/timeline";
interface DetailStreamContextType {
selectedObjectId: string | undefined;
selectedObjectTimeline?: ObjectLifecycleSequence[];
currentTime: number;
camera: string;
annotationOffset: number; // milliseconds
setAnnotationOffset: (ms: number) => void;
setSelectedObjectId: (id: string | undefined) => void;
isDetailMode: boolean;
}
const DetailStreamContext = createContext<DetailStreamContextType | undefined>(
undefined,
);
interface DetailStreamProviderProps {
children: React.ReactNode;
isDetailMode: boolean;
currentTime: number;
camera: string;
}
export function DetailStreamProvider({
children,
isDetailMode,
currentTime,
camera,
}: DetailStreamProviderProps) {
const [selectedObjectId, setSelectedObjectId] = useState<
string | undefined
>();
const { data: selectedObjectTimeline } = useSWR<ObjectLifecycleSequence[]>(
selectedObjectId ? ["timeline", { source_id: selectedObjectId }] : null,
);
const { data: config } = useSWR<FrigateConfig>("config");
const [annotationOffset, setAnnotationOffset] = useState<number>(() => {
if (!config) return 0;
return config.cameras[camera]?.detect?.annotation_offset || 0;
});
useEffect(() => {
if (!config) return;
const cfgOffset = config.cameras[camera]?.detect?.annotation_offset || 0;
setAnnotationOffset(cfgOffset);
}, [config, camera]);
const value: DetailStreamContextType = {
selectedObjectId,
selectedObjectTimeline,
currentTime,
camera,
annotationOffset,
setAnnotationOffset,
setSelectedObjectId,
isDetailMode,
};
return (
<DetailStreamContext.Provider value={value}>
{children}
</DetailStreamContext.Provider>
);
}
// eslint-disable-next-line react-refresh/only-export-components
export function useDetailStream() {
const context = useContext(DetailStreamContext);
if (context === undefined) {
throw new Error(
"useDetailStream must be used within an DetailStreamProvider",
);
}
return context;
}

View File

@ -1,10 +1,11 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { useTimelineUtils } from "./use-timeline-utils";
import { FrigateConfig } from "@/types/frigateConfig";
import useSWR from "swr";
import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
import { useDateLocale } from "./use-date-locale";
import { useTranslation } from "react-i18next";
import useUserInteraction from "./use-user-interaction";
type DraggableElementProps = {
contentRef: React.RefObject<HTMLElement>;
@ -71,9 +72,9 @@ function useDraggableElement({
// track user interaction and adjust scrolling behavior
const [userInteracting, setUserInteracting] = useState(false);
const interactionTimeout = useRef<NodeJS.Timeout>();
const isProgrammaticScroll = useRef(false);
const { userInteracting } = useUserInteraction({
elementRef: timelineRef,
});
const draggingAtTopEdge = useMemo(() => {
if (clientYPosition && timelineRef.current && scrollEdgeSize) {
@ -507,47 +508,6 @@ function useDraggableElement({
}
}, [timelineRef, segmentsRef, segments]);
useEffect(() => {
const handleUserInteraction = () => {
if (!isProgrammaticScroll.current) {
setUserInteracting(true);
if (interactionTimeout.current) {
clearTimeout(interactionTimeout.current);
}
interactionTimeout.current = setTimeout(() => {
setUserInteracting(false);
}, 3000);
} else {
isProgrammaticScroll.current = false;
}
};
const timelineElement = timelineRef.current;
if (timelineElement) {
timelineElement.addEventListener("scroll", handleUserInteraction);
timelineElement.addEventListener("mousedown", handleUserInteraction);
timelineElement.addEventListener("mouseup", handleUserInteraction);
timelineElement.addEventListener("touchstart", handleUserInteraction);
timelineElement.addEventListener("touchmove", handleUserInteraction);
timelineElement.addEventListener("touchend", handleUserInteraction);
return () => {
timelineElement.removeEventListener("scroll", handleUserInteraction);
timelineElement.removeEventListener("mousedown", handleUserInteraction);
timelineElement.removeEventListener("mouseup", handleUserInteraction);
timelineElement.removeEventListener(
"touchstart",
handleUserInteraction,
);
timelineElement.removeEventListener("touchmove", handleUserInteraction);
timelineElement.removeEventListener("touchend", handleUserInteraction);
};
}
}, [timelineRef]);
return { handleMouseDown, handleMouseUp, handleMouseMove };
}

View File

@ -0,0 +1,57 @@
import { useCallback, useEffect, useRef, useState } from "react";
type UseUserInteractionProps = {
elementRef: React.RefObject<HTMLElement>;
};
function useUserInteraction({ elementRef }: UseUserInteractionProps) {
const [userInteracting, setUserInteracting] = useState(false);
const interactionTimeout = useRef<NodeJS.Timeout>();
const isProgrammaticScroll = useRef(false);
const setProgrammaticScroll = useCallback(() => {
isProgrammaticScroll.current = true;
}, []);
useEffect(() => {
const handleUserInteraction = () => {
if (!isProgrammaticScroll.current) {
setUserInteracting(true);
if (interactionTimeout.current) {
clearTimeout(interactionTimeout.current);
}
interactionTimeout.current = setTimeout(() => {
setUserInteracting(false);
}, 3000);
} else {
isProgrammaticScroll.current = false;
}
};
const element = elementRef.current;
if (element) {
element.addEventListener("scroll", handleUserInteraction);
element.addEventListener("mousedown", handleUserInteraction);
element.addEventListener("mouseup", handleUserInteraction);
element.addEventListener("touchstart", handleUserInteraction);
element.addEventListener("touchmove", handleUserInteraction);
element.addEventListener("touchend", handleUserInteraction);
return () => {
element.removeEventListener("scroll", handleUserInteraction);
element.removeEventListener("mousedown", handleUserInteraction);
element.removeEventListener("mouseup", handleUserInteraction);
element.removeEventListener("touchstart", handleUserInteraction);
element.removeEventListener("touchmove", handleUserInteraction);
element.removeEventListener("touchend", handleUserInteraction);
};
}
}, [elementRef]);
return { userInteracting, setProgrammaticScroll };
}
export default useUserInteraction;

View File

@ -30,7 +30,7 @@ export type ObjectLifecycleSequence = {
export type TimeRange = { before: number; after: number };
export type TimelineType = "timeline" | "events";
export type TimelineType = "timeline" | "events" | "detail";
export type TimelineScrubMode = "auto" | "drag" | "hover" | "compat";

View File

@ -7,6 +7,7 @@ import PreviewPlayer, {
import { DynamicVideoController } from "@/components/player/dynamic/DynamicVideoController";
import DynamicVideoPlayer from "@/components/player/dynamic/DynamicVideoPlayer";
import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline";
import DetailStream from "@/components/timeline/DetailStream";
import { Button } from "@/components/ui/button";
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
import { useOverlayState } from "@/hooks/use-overlay-state";
@ -66,6 +67,7 @@ import {
} from "@/components/ui/tooltip";
import { CameraNameLabel } from "@/components/camera/CameraNameLabel";
import { useAllowedCameras } from "@/hooks/use-allowed-cameras";
import { DetailStreamProvider } from "@/context/detail-stream-context";
import { GenAISummaryDialog } from "@/components/overlay/chip/GenAISummaryChip";
const DATA_REFRESH_TIME = 600000; // 10 minutes
@ -159,6 +161,7 @@ export function RecordingView({
chunkedTimeRange[chunkedTimeRange.length - 1],
[selectedRangeIdx, chunkedTimeRange],
);
const reviewFilterList = useMemo(() => {
const uniqueLabels = new Set<string>();
@ -521,309 +524,335 @@ export function RecordingView({
);
return (
<div ref={contentRef} className="flex size-full flex-col pt-2">
<Toaster closeButton={true} />
<div className="relative mb-2 flex h-11 w-full items-center justify-between px-2">
{isMobile && (
<Logo className="absolute inset-x-1/2 h-8 -translate-x-1/2" />
)}
<div className={cn("flex items-center gap-2")}>
<Button
className="flex items-center gap-2.5 rounded-lg"
aria-label={t("label.back", { ns: "common" })}
size="sm"
onClick={() => navigate(-1)}
>
<IoMdArrowRoundBack className="size-5 text-secondary-foreground" />
<DetailStreamProvider
isDetailMode={timelineType === "detail"}
currentTime={currentTime}
camera={mainCamera}
>
<div ref={contentRef} className="flex size-full flex-col pt-2">
<Toaster closeButton={true} />
<div className="relative mb-2 flex h-11 w-full items-center justify-between px-2">
{isMobile && (
<Logo className="absolute inset-x-1/2 h-8 -translate-x-1/2" />
)}
<div className={cn("flex items-center gap-2")}>
<Button
className="flex items-center gap-2.5 rounded-lg"
aria-label={t("label.back", { ns: "common" })}
size="sm"
onClick={() => navigate(-1)}
>
<IoMdArrowRoundBack className="size-5 text-secondary-foreground" />
{isDesktop && (
<div className="text-primary">
{t("button.back", { ns: "common" })}
</div>
)}
</Button>
<Button
className="flex items-center gap-2.5 rounded-lg"
aria-label="Go to the main camera live view"
size="sm"
onClick={() => {
navigate(`/#${mainCamera}`);
}}
>
<FaVideo className="size-5 text-secondary-foreground" />
{isDesktop && (
<div className="text-primary">
{t("menu.live.title", { ns: "common" })}
</div>
)}
</Button>
</div>
<div className="flex items-center justify-end gap-2">
<MobileCameraDrawer
allCameras={effectiveCameras}
selected={mainCamera}
onSelectCamera={onSelectCamera}
/>
{isDesktop && (
<div className="text-primary">
{t("button.back", { ns: "common" })}
</div>
<ExportDialog
camera={mainCamera}
currentTime={currentTime}
latestTime={timeRange.before}
mode={exportMode}
range={exportRange}
showPreview={showExportPreview}
setRange={(range) => {
setExportRange(range);
if (range != undefined) {
mainControllerRef.current?.pause();
}
}}
setMode={setExportMode}
setShowPreview={setShowExportPreview}
/>
)}
</Button>
<Button
className="flex items-center gap-2.5 rounded-lg"
aria-label="Go to the main camera live view"
size="sm"
onClick={() => {
navigate(`/#${mainCamera}`);
}}
>
<FaVideo className="size-5 text-secondary-foreground" />
{isDesktop && (
<div className="text-primary">
{t("menu.live.title", { ns: "common" })}
</div>
<ReviewFilterGroup
filters={["cameras", "date", "general"]}
reviewSummary={reviewSummary}
recordingsSummary={recordingsSummary}
filter={filter}
motionOnly={false}
filterList={reviewFilterList}
showReviewed
setShowReviewed={() => {}}
mainCamera={mainCamera}
onUpdateFilter={(newFilter: ReviewFilter) => {
const updatedCameras =
newFilter.cameras === undefined
? undefined // Respect undefined as "all cameras"
: newFilter.cameras
? Array.from(
new Set([mainCamera, ...(newFilter.cameras || [])]),
) // Include mainCamera if specific cameras are selected
: [mainCamera];
const adjustedFilter: ReviewFilter = {
...newFilter,
cameras: updatedCameras,
};
updateFilter(adjustedFilter);
}}
setMotionOnly={() => {}}
/>
)}
</Button>
</div>
<div className="flex items-center justify-end gap-2">
<MobileCameraDrawer
allCameras={effectiveCameras}
selected={mainCamera}
onSelectCamera={onSelectCamera}
/>
{isDesktop && (
<ExportDialog
{isDesktop ? (
<ToggleGroup
className="*:rounded-md *:px-3 *:py-4"
type="single"
size="sm"
value={timelineType}
onValueChange={(value: TimelineType) =>
value ? setTimelineType(value, true) : null
} // don't allow the severity to be unselected
>
<ToggleGroupItem
className={`${timelineType == "timeline" ? "" : "text-muted-foreground"}`}
value="timeline"
aria-label={t("timeline.aria")}
>
<div className="">{t("timeline")}</div>
</ToggleGroupItem>
<ToggleGroupItem
className={`${timelineType == "events" ? "" : "text-muted-foreground"}`}
value="events"
aria-label={t("events.aria")}
>
<div className="">{t("events.label")}</div>
</ToggleGroupItem>
<ToggleGroupItem
className={`${timelineType == "detail" ? "" : "text-muted-foreground"}`}
value="detail"
aria-label="Detail Stream"
>
<div className="">Detail</div>
</ToggleGroupItem>
</ToggleGroup>
) : (
<MobileTimelineDrawer
selected={timelineType ?? "timeline"}
onSelect={setTimelineType}
/>
)}
<MobileReviewSettingsDrawer
camera={mainCamera}
filter={filter}
currentTime={currentTime}
latestTime={timeRange.before}
recordingsSummary={recordingsSummary}
mode={exportMode}
range={exportRange}
showPreview={showExportPreview}
setRange={(range) => {
setExportRange(range);
if (range != undefined) {
mainControllerRef.current?.pause();
}
}}
showExportPreview={showExportPreview}
allLabels={reviewFilterList.labels}
allZones={reviewFilterList.zones}
onUpdateFilter={updateFilter}
setRange={setExportRange}
setMode={setExportMode}
setShowPreview={setShowExportPreview}
setShowExportPreview={setShowExportPreview}
/>
)}
{isDesktop && (
<ReviewFilterGroup
filters={["cameras", "date", "general"]}
reviewSummary={reviewSummary}
recordingsSummary={recordingsSummary}
filter={filter}
motionOnly={false}
filterList={reviewFilterList}
showReviewed
setShowReviewed={() => {}}
mainCamera={mainCamera}
onUpdateFilter={(newFilter: ReviewFilter) => {
const updatedCameras =
newFilter.cameras === undefined
? undefined // Respect undefined as "all cameras"
: newFilter.cameras
? Array.from(
new Set([mainCamera, ...(newFilter.cameras || [])]),
) // Include mainCamera if specific cameras are selected
: [mainCamera];
const adjustedFilter: ReviewFilter = {
...newFilter,
cameras: updatedCameras,
};
updateFilter(adjustedFilter);
}}
setMotionOnly={() => {}}
/>
)}
{isDesktop ? (
<ToggleGroup
className="*:rounded-md *:px-3 *:py-4"
type="single"
size="sm"
value={timelineType}
onValueChange={(value: TimelineType) =>
value ? setTimelineType(value, true) : null
} // don't allow the severity to be unselected
>
<ToggleGroupItem
className={`${timelineType == "timeline" ? "" : "text-muted-foreground"}`}
value="timeline"
aria-label={t("timeline.aria")}
>
<div className="">{t("timeline")}</div>
</ToggleGroupItem>
<ToggleGroupItem
className={`${timelineType == "events" ? "" : "text-muted-foreground"}`}
value="events"
aria-label={t("events.aria")}
>
<div className="">{t("events.label")}</div>
</ToggleGroupItem>
</ToggleGroup>
) : (
<MobileTimelineDrawer
selected={timelineType ?? "timeline"}
onSelect={setTimelineType}
/>
)}
<MobileReviewSettingsDrawer
camera={mainCamera}
filter={filter}
currentTime={currentTime}
latestTime={timeRange.before}
recordingsSummary={recordingsSummary}
mode={exportMode}
range={exportRange}
showExportPreview={showExportPreview}
allLabels={reviewFilterList.labels}
allZones={reviewFilterList.zones}
onUpdateFilter={updateFilter}
setRange={setExportRange}
setMode={setExportMode}
setShowExportPreview={setShowExportPreview}
/>
</div>
</div>
</div>
<div
ref={mainLayoutRef}
className={cn(
"flex h-full justify-center overflow-hidden",
isDesktop ? "" : "flex-col gap-2 landscape:flex-row",
)}
>
<div
ref={cameraLayoutRef}
className={cn("flex flex-1 flex-wrap", isDesktop ? "w-[80%]" : "")}
ref={mainLayoutRef}
className={cn(
"flex h-full justify-center overflow-hidden",
isDesktop ? "" : "flex-col gap-2 landscape:flex-row",
)}
>
<div
ref={cameraLayoutRef}
className={cn(
"flex size-full items-center",
mainCameraAspect == "tall"
? "flex-row justify-evenly"
: "flex-col justify-center gap-2",
"flex flex-1 flex-wrap",
isDesktop
? timelineType === "detail"
? "w-full"
: "w-[80%]"
: "",
)}
>
<div
key={mainCamera}
className={cn(
"relative",
isDesktop
? cn(
"flex justify-center px-4",
mainCameraAspect == "tall"
? "h-[50%] md:h-[60%] lg:h-[75%] xl:h-[90%]"
: mainCameraAspect == "wide"
? "w-full"
: "",
)
: cn(
"pt-2 portrait:w-full",
isMobileOnly &&
(mainCameraAspect == "wide"
? "aspect-wide landscape:w-full"
: "aspect-video landscape:h-[94%] landscape:xl:h-[65%]"),
isTablet &&
(mainCameraAspect == "wide"
? "aspect-wide landscape:w-full"
: mainCameraAspect == "normal"
? "landscape:w-full"
: "aspect-video landscape:h-[100%]"),
),
"flex size-full items-center",
timelineType === "detail"
? "flex-col"
: mainCameraAspect == "tall"
? "flex-row justify-evenly"
: "flex-col justify-center gap-2",
)}
style={{
width: mainCameraStyle ? mainCameraStyle.width : undefined,
aspectRatio: isDesktop
? mainCameraAspect == "tall"
? getCameraAspect(mainCamera)
: undefined
: Math.max(1, getCameraAspect(mainCamera) ?? 0),
}}
>
{isDesktop && (
<GenAISummaryDialog
review={activeReviewItem}
onOpen={onAnalysisOpen}
/>
)}
<DynamicVideoPlayer
className={grow}
camera={mainCamera}
timeRange={currentTimeRange}
cameraPreviews={allPreviews ?? []}
startTimestamp={playbackStart}
hotKeys={exportMode != "select"}
fullscreen={fullscreen}
onTimestampUpdate={(timestamp) => {
setPlayerTime(timestamp);
setCurrentTime(timestamp);
Object.values(previewRefs.current ?? {}).forEach((prev) =>
prev.scrubToTimestamp(Math.floor(timestamp)),
);
}}
onClipEnded={onClipEnded}
onControllerReady={(controller) => {
mainControllerRef.current = controller;
}}
isScrubbing={scrubbing || exportMode == "timeline"}
supportsFullscreen={supportsFullScreen}
setFullResolution={setFullResolution}
toggleFullscreen={toggleFullscreen}
containerRef={mainLayoutRef}
/>
</div>
{isDesktop && effectiveCameras.length > 1 && (
<div
ref={previewRowRef}
key={mainCamera}
className={cn(
"scrollbar-container flex gap-2 overflow-auto",
mainCameraAspect == "tall"
? "h-full w-72 flex-col"
: `h-28 w-full`,
previewRowOverflows ? "" : "items-center justify-center",
"relative",
isDesktop
? cn(
"flex justify-center px-4",
mainCameraAspect == "tall"
? "h-[50%] md:h-[60%] lg:h-[75%] xl:h-[90%]"
: mainCameraAspect == "wide"
? "w-full"
: "",
)
: cn(
"pt-2 portrait:w-full",
isMobileOnly &&
(mainCameraAspect == "wide"
? "aspect-wide landscape:w-full"
: "aspect-video landscape:h-[94%] landscape:xl:h-[65%]"),
isTablet &&
(mainCameraAspect == "wide"
? "aspect-wide landscape:w-full"
: mainCameraAspect == "normal"
? "landscape:w-full"
: "aspect-video landscape:h-[100%]"),
),
)}
style={{
width: mainCameraStyle ? mainCameraStyle.width : undefined,
aspectRatio: isDesktop
? mainCameraAspect == "tall"
? getCameraAspect(mainCamera)
: undefined
: Math.max(1, getCameraAspect(mainCamera) ?? 0),
}}
>
<div className="w-2" />
{effectiveCameras.map((cam) => {
if (cam == mainCamera || cam == "birdseye") {
return;
}
{isDesktop && (
<GenAISummaryDialog
review={activeReviewItem}
onOpen={onAnalysisOpen}
/>
)}
return (
<Tooltip key={cam}>
<TooltipTrigger asChild>
<div
className={
mainCameraAspect == "tall" ? "w-full" : "h-full"
}
style={{
aspectRatio: getCameraAspect(cam),
}}
>
<PreviewPlayer
previewRef={previewRef}
className="size-full"
camera={cam}
timeRange={currentTimeRange}
cameraPreviews={allPreviews ?? []}
startTime={startTime}
isScrubbing={scrubbing}
isVisible={visiblePreviews.includes(cam)}
onControllerReady={(controller) => {
previewRefs.current[cam] = controller;
controller.scrubToTimestamp(startTime);
}}
onClick={() => onSelectCamera(cam)}
/>
</div>
</TooltipTrigger>
<TooltipContent className="smart-capitalize">
<CameraNameLabel camera={cam} />
</TooltipContent>
</Tooltip>
);
})}
<div className="w-2" />
<DynamicVideoPlayer
className={grow}
camera={mainCamera}
timeRange={currentTimeRange}
cameraPreviews={allPreviews ?? []}
startTimestamp={playbackStart}
hotKeys={exportMode != "select"}
fullscreen={fullscreen}
onTimestampUpdate={(timestamp) => {
setPlayerTime(timestamp);
setCurrentTime(timestamp);
Object.values(previewRefs.current ?? {}).forEach((prev) =>
prev.scrubToTimestamp(Math.floor(timestamp)),
);
}}
onClipEnded={onClipEnded}
onSeekToTime={manuallySetCurrentTime}
onControllerReady={(controller) => {
mainControllerRef.current = controller;
}}
isScrubbing={scrubbing || exportMode == "timeline"}
supportsFullscreen={supportsFullScreen}
setFullResolution={setFullResolution}
toggleFullscreen={toggleFullscreen}
containerRef={mainLayoutRef}
/>
</div>
)}
{isDesktop &&
effectiveCameras.length > 1 &&
timelineType !== "detail" && (
<div
ref={previewRowRef}
className={cn(
"scrollbar-container flex gap-2 overflow-auto",
mainCameraAspect == "tall"
? "h-full w-72 flex-col"
: `h-28 w-full`,
previewRowOverflows ? "" : "items-center justify-center",
)}
>
<div className="w-2" />
{effectiveCameras.map((cam) => {
if (cam == mainCamera || cam == "birdseye") {
return;
}
return (
<Tooltip key={cam}>
<TooltipTrigger asChild>
<div
className={
mainCameraAspect == "tall" ? "w-full" : "h-full"
}
style={{
aspectRatio: getCameraAspect(cam),
}}
>
<PreviewPlayer
previewRef={previewRef}
className="size-full"
camera={cam}
timeRange={currentTimeRange}
cameraPreviews={allPreviews ?? []}
startTime={startTime}
isScrubbing={scrubbing}
isVisible={visiblePreviews.includes(cam)}
onControllerReady={(controller) => {
previewRefs.current[cam] = controller;
controller.scrubToTimestamp(startTime);
}}
onClick={() => onSelectCamera(cam)}
/>
</div>
</TooltipTrigger>
<TooltipContent className="smart-capitalize">
<CameraNameLabel camera={cam} />
</TooltipContent>
</Tooltip>
);
})}
<div className="w-2" />
</div>
)}
</div>
</div>
<Timeline
contentRef={contentRef}
mainCamera={mainCamera}
timelineType={
(exportRange == undefined ? timelineType : "timeline") ??
"timeline"
}
timeRange={timeRange}
mainCameraReviewItems={mainCameraReviewItems}
activeReviewItem={activeReviewItem}
currentTime={currentTime}
exportRange={exportMode == "timeline" ? exportRange : undefined}
setCurrentTime={setCurrentTime}
manuallySetCurrentTime={manuallySetCurrentTime}
setScrubbing={setScrubbing}
setExportRange={setExportRange}
onAnalysisOpen={onAnalysisOpen}
/>
</div>
<Timeline
contentRef={contentRef}
mainCamera={mainCamera}
timelineType={
(exportRange == undefined ? timelineType : "timeline") ?? "timeline"
}
timeRange={timeRange}
mainCameraReviewItems={mainCameraReviewItems}
activeReviewItem={activeReviewItem}
currentTime={currentTime}
exportRange={exportMode == "timeline" ? exportRange : undefined}
setCurrentTime={setCurrentTime}
manuallySetCurrentTime={manuallySetCurrentTime}
setScrubbing={setScrubbing}
setExportRange={setExportRange}
onAnalysisOpen={onAnalysisOpen}
/>
</div>
</div>
</DetailStreamProvider>
);
}
@ -938,8 +967,8 @@ function Timeline({
className={cn(
"relative",
isDesktop
? `${timelineType == "timeline" ? "w-[100px]" : "w-60"} no-scrollbar overflow-y-auto`
: `overflow-hidden portrait:flex-grow ${timelineType == "timeline" ? "landscape:w-[100px]" : "landscape:w-[175px]"}`,
? `${timelineType == "timeline" ? "w-[100px]" : timelineType == "detail" ? "w-[30%]" : "w-60"} no-scrollbar overflow-y-auto`
: `overflow-hidden portrait:flex-grow ${timelineType == "timeline" ? "landscape:w-[100px]" : timelineType == "detail" ? "flex-1" : "landscape:w-[175px]"} `,
)}
>
{isMobile && (
@ -975,6 +1004,12 @@ function Timeline({
) : (
<Skeleton className="size-full" />
)
) : timelineType == "detail" ? (
<DetailStream
currentTime={currentTime}
onSeek={(timestamp) => manuallySetCurrentTime(timestamp, true)}
reviewItems={mainCameraReviewItems}
/>
) : (
<div className="scrollbar-container h-full overflow-auto bg-secondary">
<div