Improve motion review and add motion search (#22253)

* implement motion search and motion previews

* tweaks

* fix merge issue

* fix copilot instructions
This commit is contained in:
Josh Hawkins
2026-03-05 17:53:48 -06:00
committed by GitHub
parent 229436c94a
commit 2babfd2ec9
28 changed files with 6089 additions and 384 deletions
+147 -139
View File
@@ -1,5 +1,6 @@
import {
MutableRefObject,
ReactNode,
useCallback,
useEffect,
useRef,
@@ -57,6 +58,7 @@ type HlsVideoPlayerProps = {
isDetailMode?: boolean;
camera?: string;
currentTimeOverride?: number;
transformedOverlay?: ReactNode;
};
export default function HlsVideoPlayer({
@@ -81,6 +83,7 @@ export default function HlsVideoPlayer({
isDetailMode = false,
camera,
currentTimeOverride,
transformedOverlay,
}: HlsVideoPlayerProps) {
const { t } = useTranslation("components/player");
const { data: config } = useSWR<FrigateConfig>("config");
@@ -350,157 +353,162 @@ export default function HlsVideoPlayer({
height: isMobile ? "100%" : undefined,
}}
>
{isDetailMode &&
camera &&
currentTime &&
loadedMetadata &&
videoDimensions.width > 0 &&
videoDimensions.height > 0 && (
<div
className={cn(
"absolute inset-0 z-50",
isDesktop
? "size-full"
: "mx-auto flex items-center justify-center portrait:max-h-[50dvh]",
)}
style={{
aspectRatio: `${videoDimensions.width} / ${videoDimensions.height}`,
}}
>
<ObjectTrackOverlay
key={`overlay-${currentTime}`}
camera={camera}
showBoundingBoxes={!isPlaying}
currentTime={currentTime}
videoWidth={videoDimensions.width}
videoHeight={videoDimensions.height}
className="absolute inset-0 z-10"
onSeekToTime={(timestamp, play) => {
if (onSeekToTime) {
onSeekToTime(timestamp, play);
}
<div className="relative size-full">
{transformedOverlay}
{isDetailMode &&
camera &&
currentTime &&
loadedMetadata &&
videoDimensions.width > 0 &&
videoDimensions.height > 0 && (
<div
className={cn(
"absolute inset-0 z-50",
isDesktop
? "size-full"
: "mx-auto flex items-center justify-center portrait:max-h-[50dvh]",
)}
style={{
aspectRatio: `${videoDimensions.width} / ${videoDimensions.height}`,
}}
/>
</div>
)}
<video
ref={videoRef}
className={`size-full rounded-lg bg-black md:rounded-2xl ${loadedMetadata ? "" : "invisible"} cursor-pointer`}
preload="auto"
autoPlay
controls={!frigateControls}
playsInline
muted={muted}
onClick={
isDesktop
? () => {
if (zoomScale == 1.0) onPlayPause(!isPlaying);
}
: undefined
}
onVolumeChange={() => {
setVolume(videoRef.current?.volume ?? 1.0, true);
if (!frigateControls) {
setMuted(videoRef.current?.muted);
}
}}
onPlay={() => {
setIsPlaying(true);
if (isMobile) {
setControls(true);
setMobileCtrlTimeout(setTimeout(() => setControls(false), 4000));
}
}}
onPlaying={onPlaying}
onPause={() => {
setIsPlaying(false);
clearTimeout(bufferTimeout);
if (isMobile && mobileCtrlTimeout) {
clearTimeout(mobileCtrlTimeout);
}
}}
onWaiting={() => {
if (onError != undefined) {
if (videoRef.current?.paused) {
return;
}
setBufferTimeout(
setTimeout(() => {
if (
document.visibilityState === "visible" &&
videoRef.current
) {
onError("stalled");
>
<ObjectTrackOverlay
key={`overlay-${currentTime}`}
camera={camera}
showBoundingBoxes={!isPlaying}
currentTime={currentTime}
videoWidth={videoDimensions.width}
videoHeight={videoDimensions.height}
className="absolute inset-0 z-10"
onSeekToTime={(timestamp, play) => {
if (onSeekToTime) {
onSeekToTime(timestamp, play);
}
}}
/>
</div>
)}
<video
ref={videoRef}
className={`size-full rounded-lg bg-black md:rounded-2xl ${loadedMetadata ? "" : "invisible"} cursor-pointer`}
preload="auto"
autoPlay
controls={!frigateControls}
playsInline
muted={muted}
onClick={
isDesktop
? () => {
if (zoomScale == 1.0) onPlayPause(!isPlaying);
}
}, 3000),
);
: undefined
}
}}
onProgress={() => {
if (onError != undefined) {
if (videoRef.current?.paused) {
onVolumeChange={() => {
setVolume(videoRef.current?.volume ?? 1.0, true);
if (!frigateControls) {
setMuted(videoRef.current?.muted);
}
}}
onPlay={() => {
setIsPlaying(true);
if (isMobile) {
setControls(true);
setMobileCtrlTimeout(
setTimeout(() => setControls(false), 4000),
);
}
}}
onPlaying={onPlaying}
onPause={() => {
setIsPlaying(false);
clearTimeout(bufferTimeout);
if (isMobile && mobileCtrlTimeout) {
clearTimeout(mobileCtrlTimeout);
}
}}
onWaiting={() => {
if (onError != undefined) {
if (videoRef.current?.paused) {
return;
}
setBufferTimeout(
setTimeout(() => {
if (
document.visibilityState === "visible" &&
videoRef.current
) {
onError("stalled");
}
}, 3000),
);
}
}}
onProgress={() => {
if (onError != undefined) {
if (videoRef.current?.paused) {
return;
}
if (bufferTimeout) {
clearTimeout(bufferTimeout);
setBufferTimeout(undefined);
}
}
}}
onTimeUpdate={() => {
if (!onTimeUpdate) {
return;
}
if (bufferTimeout) {
clearTimeout(bufferTimeout);
setBufferTimeout(undefined);
const frameTime = getVideoTime();
if (frameTime) {
onTimeUpdate(frameTime);
}
}
}}
onTimeUpdate={() => {
if (!onTimeUpdate) {
return;
}
}}
onLoadedData={() => {
onPlayerLoaded?.();
handleLoadedMetadata();
const frameTime = getVideoTime();
if (videoRef.current) {
if (playbackRate) {
videoRef.current.playbackRate = playbackRate;
}
if (frameTime) {
onTimeUpdate(frameTime);
}
}}
onLoadedData={() => {
onPlayerLoaded?.();
handleLoadedMetadata();
if (videoRef.current) {
if (playbackRate) {
videoRef.current.playbackRate = playbackRate;
if (volume) {
videoRef.current.volume = volume;
}
}
if (volume) {
videoRef.current.volume = volume;
}}
onEnded={() => {
if (onClipEnded) {
onClipEnded(getVideoTime() ?? 0);
}
}
}}
onEnded={() => {
if (onClipEnded) {
onClipEnded(getVideoTime() ?? 0);
}
}}
onError={(e) => {
if (
!hlsRef.current &&
// @ts-expect-error code does exist
unsupportedErrorCodes.includes(e.target.error.code) &&
videoRef.current
) {
setLoadedMetadata(false);
setUseHlsCompat(true);
} else {
toast.error(
}}
onError={(e) => {
if (
!hlsRef.current &&
// @ts-expect-error code does exist
`Failed to play recordings (error ${e.target.error.code}): ${e.target.error.message}`,
{
position: "top-center",
},
);
}
}}
/>
unsupportedErrorCodes.includes(e.target.error.code) &&
videoRef.current
) {
setLoadedMetadata(false);
setUseHlsCompat(true);
} else {
toast.error(
// @ts-expect-error code does exist
`Failed to play recordings (error ${e.target.error.code}): ${e.target.error.message}`,
{
position: "top-center",
},
);
}
}}
/>
</div>
</TransformComponent>
</TransformWrapper>
);
@@ -1,4 +1,11 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
ReactNode,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { useApiHost } from "@/api";
import useSWR from "swr";
import { FrigateConfig } from "@/types/frigateConfig";
@@ -40,6 +47,7 @@ type DynamicVideoPlayerProps = {
setFullResolution: React.Dispatch<React.SetStateAction<VideoResolutionType>>;
toggleFullscreen: () => void;
containerRef?: React.MutableRefObject<HTMLDivElement | null>;
transformedOverlay?: ReactNode;
};
export default function DynamicVideoPlayer({
className,
@@ -58,6 +66,7 @@ export default function DynamicVideoPlayer({
setFullResolution,
toggleFullscreen,
containerRef,
transformedOverlay,
}: DynamicVideoPlayerProps) {
const { t } = useTranslation(["components/player"]);
const apiHost = useApiHost();
@@ -312,6 +321,7 @@ export default function DynamicVideoPlayer({
isDetailMode={isDetailMode}
camera={contextCamera || camera}
currentTimeOverride={currentTime}
transformedOverlay={transformedOverlay}
/>
)}
<PreviewPlayer
@@ -25,6 +25,7 @@ export type MotionReviewTimelineProps = {
timestampSpread: number;
timelineStart: number;
timelineEnd: number;
scrollToTime?: number;
showHandlebar?: boolean;
handlebarTime?: number;
setHandlebarTime?: React.Dispatch<React.SetStateAction<number>>;
@@ -58,6 +59,7 @@ export function MotionReviewTimeline({
timestampSpread,
timelineStart,
timelineEnd,
scrollToTime,
showHandlebar = false,
handlebarTime,
setHandlebarTime,
@@ -176,6 +178,15 @@ export function MotionReviewTimeline({
[],
);
// allow callers to request the timeline center on a specific time
useEffect(() => {
if (scrollToTime == undefined) return;
setTimeout(() => {
scrollToSegment(alignStartDateToTimeline(scrollToTime), true, "auto");
}, 0);
}, [scrollToTime, scrollToSegment, alignStartDateToTimeline]);
// keep handlebar centered when zooming
useEffect(() => {
setTimeout(() => {
@@ -343,9 +343,12 @@ export function ReviewTimeline({
useEffect(() => {
if (onHandlebarDraggingChange) {
onHandlebarDraggingChange(isDraggingHandlebar);
// Keep existing callback name but treat it as a generic dragging signal.
// This allows consumers (e.g. export-handle timelines) to correctly
// enable preview scrubbing while dragging export handles.
onHandlebarDraggingChange(isDragging);
}
}, [isDraggingHandlebar, onHandlebarDraggingChange]);
}, [isDragging, onHandlebarDraggingChange]);
const isHandlebarInNoRecordingPeriod = useMemo(() => {
if (!getRecordingAvailability || handlebarTime === undefined) return false;
+26
View File
@@ -0,0 +1,26 @@
import * as React from "react"
import * as ProgressPrimitive from "@radix-ui/react-progress"
import { cn } from "@/lib/utils"
const Progress = React.forwardRef<
React.ElementRef<typeof ProgressPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof ProgressPrimitive.Root>
>(({ className, value, ...props }, ref) => (
<ProgressPrimitive.Root
ref={ref}
className={cn(
"relative h-4 w-full overflow-hidden rounded-full bg-secondary",
className
)}
{...props}
>
<ProgressPrimitive.Indicator
className="h-full w-full flex-1 bg-primary transition-all"
style={{ transform: `translateX(-${100 - (value || 0)}%)` }}
/>
</ProgressPrimitive.Root>
))
Progress.displayName = ProgressPrimitive.Root.displayName
export { Progress }
+114 -15
View File
@@ -8,14 +8,19 @@ import {
import { CameraConfig, FrigateConfig } from "@/types/frigateConfig";
import { MotionData, ReviewSegment } from "@/types/review";
import { useCallback, useEffect, useMemo, useState } from "react";
import { useTimelineUtils } from "./use-timeline-utils";
import { AudioDetection, ObjectType } from "@/types/ws";
import { useTimelineUtils } from "./use-timeline-utils";
import useDeepMemo from "./use-deep-memo";
import { isEqual } from "lodash";
import { useAutoFrigateStats } from "./use-stats";
import useSWR from "swr";
import { getAttributeLabels } from "@/utils/iconUtil";
export type MotionOnlyRange = {
start_time: number;
end_time: number;
};
type useCameraActivityReturn = {
enabled?: boolean;
activeTracking: boolean;
@@ -204,9 +209,9 @@ export function useCameraMotionNextTimestamp(
return [];
}
const ranges = [];
let currentSegmentStart = null;
let currentSegmentEnd = null;
const ranges: [number, number][] = [];
let currentSegmentStart: number | null = null;
let currentSegmentEnd: number | null = null;
// align motion start to timeline start
const offset =
@@ -215,13 +220,19 @@ export function useCameraMotionNextTimestamp(
segmentDuration;
const startIndex = Math.abs(Math.floor(offset / 15));
const now = Date.now() / 1000;
for (
let i = startIndex;
i < motionData.length;
i = i + segmentDuration / 15
) {
const motionStart = motionData[i].start_time;
const motionStart = motionData[i]?.start_time;
if (motionStart == undefined) {
continue;
}
const motionEnd = motionStart + segmentDuration;
const segmentMotion = motionData
@@ -230,10 +241,10 @@ export function useCameraMotionNextTimestamp(
const overlappingReviewItems = reviewItems.some(
(item) =>
(item.start_time >= motionStart && item.start_time < motionEnd) ||
((item.end_time ?? Date.now() / 1000) > motionStart &&
(item.end_time ?? Date.now() / 1000) <= motionEnd) ||
((item.end_time ?? now) > motionStart &&
(item.end_time ?? now) <= motionEnd) ||
(item.start_time <= motionStart &&
(item.end_time ?? Date.now() / 1000) >= motionEnd),
(item.end_time ?? now) >= motionEnd),
);
if (!segmentMotion || overlappingReviewItems) {
@@ -241,16 +252,14 @@ export function useCameraMotionNextTimestamp(
currentSegmentStart = motionStart;
}
currentSegmentEnd = motionEnd;
} else {
if (currentSegmentStart !== null) {
ranges.push([currentSegmentStart, currentSegmentEnd]);
currentSegmentStart = null;
currentSegmentEnd = null;
}
} else if (currentSegmentStart !== null && currentSegmentEnd !== null) {
ranges.push([currentSegmentStart, currentSegmentEnd]);
currentSegmentStart = null;
currentSegmentEnd = null;
}
}
if (currentSegmentStart !== null) {
if (currentSegmentStart !== null && currentSegmentEnd !== null) {
ranges.push([currentSegmentStart, currentSegmentEnd]);
}
@@ -304,3 +313,93 @@ export function useCameraMotionNextTimestamp(
return nextTimestamp;
}
export function useCameraMotionOnlyRanges(
segmentDuration: number,
reviewItems: ReviewSegment[],
motionData: MotionData[],
) {
const motionOnlyRanges = useMemo(() => {
if (!motionData?.length || !reviewItems) {
return [];
}
const fallbackBucketDuration = Math.max(1, segmentDuration / 2);
const normalizedMotionData = Array.from(
motionData
.reduce((accumulator, item) => {
const currentMotion = accumulator.get(item.start_time) ?? 0;
accumulator.set(
item.start_time,
Math.max(currentMotion, item.motion ?? 0),
);
return accumulator;
}, new Map<number, number>())
.entries(),
)
.map(([start_time, motion]) => ({ start_time, motion }))
.sort((left, right) => left.start_time - right.start_time);
const bucketRanges: MotionOnlyRange[] = [];
const now = Date.now() / 1000;
for (let i = 0; i < normalizedMotionData.length; i++) {
const motionStart = normalizedMotionData[i].start_time;
const motionEnd = motionStart + fallbackBucketDuration;
const overlappingReviewItems = reviewItems.some(
(item) =>
(item.start_time >= motionStart && item.start_time < motionEnd) ||
((item.end_time ?? now) > motionStart &&
(item.end_time ?? now) <= motionEnd) ||
(item.start_time <= motionStart &&
(item.end_time ?? now) >= motionEnd),
);
const isMotionOnlySegment =
(normalizedMotionData[i].motion ?? 0) > 0 && !overlappingReviewItems;
if (!isMotionOnlySegment) {
continue;
}
bucketRanges.push({
start_time: motionStart,
end_time: motionEnd,
});
}
if (!bucketRanges.length) {
return [];
}
const mergedRanges = bucketRanges.reduce<MotionOnlyRange[]>(
(ranges, range) => {
if (!ranges.length) {
return [range];
}
const previousRange = ranges[ranges.length - 1];
const isContiguous =
range.start_time <= previousRange.end_time + 0.001 &&
range.start_time >= previousRange.end_time - 0.001;
if (isContiguous) {
previousRange.end_time = Math.max(
previousRange.end_time,
range.end_time,
);
return ranges;
}
ranges.push(range);
return ranges;
},
[],
);
return mergedRanges;
}, [motionData, reviewItems, segmentDuration]);
return motionOnlyRanges;
}
+94 -1
View File
@@ -1,5 +1,6 @@
import ActivityIndicator from "@/components/indicators/activity-indicator";
import useApiFilter from "@/hooks/use-api-filter";
import { useAllowedCameras } from "@/hooks/use-allowed-cameras";
import { useCameraPreviews } from "@/hooks/use-camera-previews";
import { useTimezone } from "@/hooks/use-date-utils";
import { useOverlayState, useSearchEffect } from "@/hooks/use-overlay-state";
@@ -21,6 +22,7 @@ import {
getEndOfDayTimestamp,
} from "@/utils/dateUtil";
import EventView from "@/views/events/EventView";
import MotionSearchView from "@/views/motion-search/MotionSearchView";
import { RecordingView } from "@/views/recording/RecordingView";
import axios from "axios";
import { useCallback, useEffect, useMemo, useState } from "react";
@@ -34,6 +36,7 @@ export default function Events() {
revalidateOnFocus: false,
});
const timezone = useTimezone(config);
const allowedCameras = useAllowedCameras();
// recordings viewer
@@ -52,6 +55,74 @@ export default function Events() {
undefined,
false,
);
const [motionPreviewsCamera, setMotionPreviewsCamera] = useOverlayState<
string | undefined
>("motionPreviewsCamera", undefined);
const [motionSearchCamera, setMotionSearchCamera] = useState<string | null>(
null,
);
const [motionSearchDay, setMotionSearchDay] = useState<Date | undefined>(
undefined,
);
const motionSearchCameras = useMemo(() => {
if (!config?.cameras) {
return [] as string[];
}
return Object.keys(config.cameras).filter((cam) =>
allowedCameras.includes(cam),
);
}, [allowedCameras, config?.cameras]);
const selectedMotionSearchCamera = useMemo(() => {
if (!motionSearchCamera) {
return null;
}
if (motionSearchCameras.includes(motionSearchCamera)) {
return motionSearchCamera;
}
return motionSearchCameras[0] ?? null;
}, [motionSearchCamera, motionSearchCameras]);
const motionSearchTimeRange = useMemo(() => {
if (motionSearchDay) {
return {
after: getBeginningOfDayTimestamp(new Date(motionSearchDay)),
before: getEndOfDayTimestamp(new Date(motionSearchDay)),
};
}
const now = Date.now() / 1000;
return {
after: now - 86400,
before: now,
};
}, [motionSearchDay]);
const closeMotionSearch = useCallback(() => {
setMotionSearchCamera(null);
setMotionSearchDay(undefined);
setBeforeTs(Date.now() / 1000);
}, []);
const handleMotionSearchCameraSelect = useCallback((camera: string) => {
setMotionSearchCamera(camera);
}, []);
const handleMotionSearchDaySelect = useCallback((day: Date | undefined) => {
if (day == undefined) {
setMotionSearchDay(undefined);
return;
}
const normalizedDay = new Date(day);
normalizedDay.setHours(0, 0, 0, 0);
setMotionSearchDay(normalizedDay);
}, []);
const [notificationTab, setNotificationTab] =
useState<TimelineType>("timeline");
@@ -508,7 +579,24 @@ export default function Events() {
);
}
} else {
return (
return motionSearchCamera ? (
!config || !selectedMotionSearchCamera ? (
<ActivityIndicator />
) : (
<MotionSearchView
config={config}
cameras={motionSearchCameras}
selectedCamera={selectedMotionSearchCamera}
onCameraSelect={handleMotionSearchCameraSelect}
cameraLocked={true}
selectedDay={motionSearchDay}
onDaySelect={handleMotionSearchDaySelect}
timeRange={motionSearchTimeRange}
timezone={timezone}
onBack={closeMotionSearch}
/>
)
) : (
<EventView
reviewItems={reviewItems}
currentReviewItems={currentItems}
@@ -525,6 +613,11 @@ export default function Events() {
markItemAsReviewed={markItemAsReviewed}
markAllItemsAsReviewed={markAllItemsAsReviewed}
onOpenRecording={setRecording}
motionPreviewsCamera={motionPreviewsCamera ?? null}
setMotionPreviewsCamera={(camera) =>
setMotionPreviewsCamera(camera ?? undefined)
}
setMotionSearchCamera={setMotionSearchCamera}
pullLatestData={reloadData}
updateFilter={onUpdateFilter}
/>
+112
View File
@@ -0,0 +1,112 @@
import { useEffect, useMemo, useState, useCallback } from "react";
import { useTranslation } from "react-i18next";
import useSWR from "swr";
import { FrigateConfig } from "@/types/frigateConfig";
import { useTimezone } from "@/hooks/use-date-utils";
import MotionSearchView from "@/views/motion-search/MotionSearchView";
import {
getBeginningOfDayTimestamp,
getEndOfDayTimestamp,
} from "@/utils/dateUtil";
import { useAllowedCameras } from "@/hooks/use-allowed-cameras";
import { useSearchEffect } from "@/hooks/use-overlay-state";
import ActivityIndicator from "@/components/indicators/activity-indicator";
export default function MotionSearch() {
const { t } = useTranslation(["views/motionSearch"]);
const { data: config } = useSWR<FrigateConfig>("config", {
revalidateOnFocus: false,
});
const timezone = useTimezone(config);
useEffect(() => {
document.title = t("documentTitle");
}, [t]);
// Get allowed cameras
const allowedCameras = useAllowedCameras();
const cameras = useMemo(() => {
if (!config?.cameras) return [];
return Object.keys(config.cameras).filter((cam) =>
allowedCameras.includes(cam),
);
}, [config?.cameras, allowedCameras]);
// Selected camera state
const [selectedCamera, setSelectedCamera] = useState<string | null>(null);
const [cameraLocked, setCameraLocked] = useState(false);
useSearchEffect("camera", (camera: string) => {
if (cameras.length > 0 && cameras.includes(camera)) {
setSelectedCamera(camera);
setCameraLocked(true);
}
return false;
});
// Initialize with first camera when available (only if not set by camera param)
useEffect(() => {
if (cameras.length === 0) return;
if (!selectedCamera) {
setSelectedCamera(cameras[0]);
}
}, [cameras, selectedCamera]);
// Time range state - default to last 24 hours
const [selectedDay, setSelectedDay] = useState<Date | undefined>(undefined);
const timeRange = useMemo(() => {
if (selectedDay) {
return {
after: getBeginningOfDayTimestamp(new Date(selectedDay)),
before: getEndOfDayTimestamp(new Date(selectedDay)),
};
}
// Default to last 24 hours
const now = Date.now() / 1000;
return {
after: now - 86400,
before: now,
};
}, [selectedDay]);
const handleCameraSelect = useCallback((camera: string) => {
setSelectedCamera(camera);
}, []);
const handleDaySelect = useCallback((day: Date | undefined) => {
if (day == undefined) {
setSelectedDay(undefined);
return;
}
const normalizedDay = new Date(day);
normalizedDay.setHours(0, 0, 0, 0);
setSelectedDay(normalizedDay);
}, []);
if (!config || cameras.length === 0) {
return (
<div className="flex size-full items-center justify-center">
<ActivityIndicator />
</div>
);
}
return (
<MotionSearchView
config={config}
cameras={cameras}
selectedCamera={selectedCamera ?? null}
onCameraSelect={handleCameraSelect}
cameraLocked={cameraLocked}
selectedDay={selectedDay}
onDaySelect={handleDaySelect}
timeRange={timeRange}
timezone={timezone}
/>
);
}
+46
View File
@@ -0,0 +1,46 @@
/**
* Types for the Motion Search feature
*/
export interface MotionSearchResult {
timestamp: number;
change_percentage: number;
}
export interface MotionSearchRequest {
start_time: number;
end_time: number;
polygon_points: number[][];
parallel?: boolean;
threshold?: number;
min_area?: number;
frame_skip?: number;
max_results?: number;
}
export interface MotionSearchStartResponse {
success: boolean;
message: string;
job_id: string;
}
export interface MotionSearchMetrics {
segments_scanned: number;
segments_processed: number;
metadata_inactive_segments: number;
heatmap_roi_skip_segments: number;
fallback_full_range_segments: number;
frames_decoded: number;
wall_time_seconds: number;
segments_with_errors: number;
}
export interface MotionSearchStatusResponse {
success: boolean;
message: string;
status: "queued" | "running" | "success" | "failed" | "cancelled";
results?: MotionSearchResult[];
total_frames_processed?: number;
error_message?: string;
metrics?: MotionSearchMetrics;
}
+1
View File
@@ -11,6 +11,7 @@ export type Recording = {
duration: number;
motion: number;
objects: number;
motion_heatmap?: Record<string, number> | null;
dBFS: number;
};
+601 -224
View File
@@ -1,11 +1,21 @@
import Logo from "@/components/Logo";
import NewReviewData from "@/components/dynamic/NewReviewData";
import CalendarFilterButton from "@/components/filter/CalendarFilterButton";
import ReviewActionGroup from "@/components/filter/ReviewActionGroup";
import ReviewFilterGroup from "@/components/filter/ReviewFilterGroup";
import PreviewThumbnailPlayer from "@/components/player/PreviewThumbnailPlayer";
import EventReviewTimeline from "@/components/timeline/EventReviewTimeline";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
import { VolumeSlider } from "@/components/ui/slider";
import {
Select,
SelectContent,
SelectItem,
SelectSeparator,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { useTimelineUtils } from "@/hooks/use-timeline-utils";
import { useScrollLockout } from "@/hooks/use-mouse-listener";
import { FrigateConfig } from "@/types/frigateConfig";
@@ -22,6 +32,7 @@ import {
ZoomLevel,
} from "@/types/review";
import { getChunkedTimeRange } from "@/utils/timelineUtil";
import { getEndOfDayTimestamp } from "@/utils/dateUtil";
import axios from "axios";
import {
MutableRefObject,
@@ -34,9 +45,18 @@ import {
import { isDesktop, isMobile, isMobileOnly } from "react-device-detect";
import { LuFolderCheck, LuFolderX } from "react-icons/lu";
import { MdCircle } from "react-icons/md";
import { FiMoreVertical } from "react-icons/fi";
import { IoMdArrowRoundBack } from "react-icons/io";
import useSWR from "swr";
import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline";
import { Button } from "@/components/ui/button";
import BlurredIconButton from "@/components/button/BlurredIconButton";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import PreviewPlayer, {
PreviewController,
} from "@/components/player/PreviewPlayer";
@@ -44,7 +64,10 @@ import SummaryTimeline from "@/components/timeline/SummaryTimeline";
import { RecordingStartingPoint } from "@/types/record";
import VideoControls from "@/components/player/VideoControls";
import { TimeRange } from "@/types/timeline";
import { useCameraMotionNextTimestamp } from "@/hooks/use-camera-activity";
import {
useCameraMotionNextTimestamp,
useCameraMotionOnlyRanges,
} from "@/hooks/use-camera-activity";
import useOptimisticState from "@/hooks/use-optimistic-state";
import { Skeleton } from "@/components/ui/skeleton";
import scrollIntoView from "scroll-into-view-if-needed";
@@ -56,6 +79,10 @@ import { GiSoundWaves } from "react-icons/gi";
import useKeyboardListener from "@/hooks/use-keyboard-listener";
import { useTimelineZoom } from "@/hooks/use-timeline-zoom";
import { useTranslation } from "react-i18next";
import { FaCog } from "react-icons/fa";
import ReviewActivityCalendar from "@/components/overlay/ReviewActivityCalendar";
import PlatformAwareDialog from "@/components/overlay/dialog/PlatformAwareDialog";
import MotionPreviewsPane from "./MotionPreviewsPane";
import { EmptyCard } from "@/components/card/EmptyCard";
import { EmptyCardData } from "@/types/card";
@@ -75,6 +102,9 @@ type EventViewProps = {
markItemAsReviewed: (review: ReviewSegment) => void;
markAllItemsAsReviewed: (currentItems: ReviewSegment[]) => void;
onOpenRecording: (recordingInfo: RecordingStartingPoint) => void;
motionPreviewsCamera: string | null;
setMotionPreviewsCamera: (camera: string | null) => void;
setMotionSearchCamera: (camera: string) => void;
pullLatestData: () => void;
updateFilter: (filter: ReviewFilter) => void;
};
@@ -94,6 +124,9 @@ export default function EventView({
markItemAsReviewed,
markAllItemsAsReviewed,
onOpenRecording,
motionPreviewsCamera,
setMotionPreviewsCamera,
setMotionSearchCamera,
pullLatestData,
updateFilter,
}: EventViewProps) {
@@ -274,6 +307,15 @@ export default function EventView({
100,
);
const motionPreviewsOpen =
severity === "significant_motion" && motionPreviewsCamera != null;
useEffect(() => {
if (severity !== "significant_motion") {
setMotionPreviewsCamera(null);
}
}, [setMotionPreviewsCamera, severity]);
// review filter info
const reviewFilterList = useMemo<FilterList>(() => {
@@ -301,124 +343,136 @@ export default function EventView({
return (
<div className="flex size-full flex-col pt-2 md:py-2">
<Toaster closeButton={true} />
<div className="relative mb-2 flex h-11 items-center justify-between pl-2 pr-2 md:pl-3">
{isMobile && (
<Logo className="absolute inset-x-1/2 h-8 -translate-x-1/2" />
)}
<ToggleGroup
className="*:rounded-md *:px-3 *:py-4"
type="single"
size="sm"
value={severityToggle}
onValueChange={(value: ReviewSeverity) =>
value ? setSeverityToggle(value) : null
} // don't allow the severity to be unselected
>
<ToggleGroupItem
className={cn(severityToggle != "alert" && "text-muted-foreground")}
value="alert"
aria-label={t("alerts")}
{!motionPreviewsOpen && (
<div className="relative mb-2 flex h-11 items-center justify-between pl-2 pr-2 md:pl-3">
{isMobile && (
<Logo className="absolute inset-x-1/2 h-8 -translate-x-1/2" />
)}
<ToggleGroup
className="*:rounded-md *:px-3 *:py-4"
type="single"
size="sm"
value={severityToggle}
onValueChange={(value: ReviewSeverity) =>
value ? setSeverityToggle(value) : null
} // don't allow the severity to be unselected
>
<div
<ToggleGroupItem
className={cn(
"flex size-6 items-center justify-center rounded text-severity_alert sm:hidden",
severityToggle == "alert" ? "font-semibold" : "font-medium",
severityToggle != "alert" && "text-muted-foreground",
)}
value="alert"
aria-label={t("alerts")}
>
{reviewCounts.alert > -1 ? (
reviewCounts.alert
) : (
<ActivityIndicator className="size-4" />
)}
</div>
<div className="hidden items-center sm:flex">
<MdCircle className="size-2 text-severity_alert md:mr-[10px]" />
<div className="hidden md:flex md:flex-row md:items-center">
{t("alerts")}
<div
className={cn(
"flex size-6 items-center justify-center rounded text-severity_alert sm:hidden",
severityToggle == "alert" ? "font-semibold" : "font-medium",
)}
>
{reviewCounts.alert > -1 ? (
`${reviewCounts.alert}`
reviewCounts.alert
) : (
<ActivityIndicator className="ml-2 size-4" />
<ActivityIndicator className="size-4" />
)}
</div>
</div>
</ToggleGroupItem>
<ToggleGroupItem
className={cn(
severityToggle != "detection" && "text-muted-foreground",
)}
value="detection"
aria-label={t("detections")}
>
<div
<div className="hidden items-center sm:flex">
<MdCircle className="size-2 text-severity_alert md:mr-[10px]" />
<div className="hidden md:flex md:flex-row md:items-center">
{t("alerts")}
{reviewCounts.alert > -1 ? (
`${reviewCounts.alert}`
) : (
<ActivityIndicator className="ml-2 size-4" />
)}
</div>
</div>
</ToggleGroupItem>
<ToggleGroupItem
className={cn(
"flex size-6 items-center justify-center rounded text-severity_detection sm:hidden",
severityToggle == "detection" ? "font-semibold" : "font-medium",
severityToggle != "detection" && "text-muted-foreground",
)}
value="detection"
aria-label={t("detections")}
>
{reviewCounts.detection > -1 ? (
reviewCounts.detection
) : (
<ActivityIndicator className="size-4" />
)}
</div>
<div className="hidden items-center sm:flex">
<MdCircle className="size-2 text-severity_detection md:mr-[10px]" />
<div className="hidden md:flex md:flex-row md:items-center">
{t("detections")}
<div
className={cn(
"flex size-6 items-center justify-center rounded text-severity_detection sm:hidden",
severityToggle == "detection"
? "font-semibold"
: "font-medium",
)}
>
{reviewCounts.detection > -1 ? (
`${reviewCounts.detection}`
reviewCounts.detection
) : (
<ActivityIndicator className="ml-2 size-4" />
<ActivityIndicator className="size-4" />
)}
</div>
</div>
</ToggleGroupItem>
<ToggleGroupItem
className={cn(
"rounded-lg px-3 py-4",
severityToggle != "significant_motion" && "text-muted-foreground",
)}
value="significant_motion"
aria-label={t("motion.label")}
>
<GiSoundWaves className="size-6 rotate-90 text-severity_significant_motion sm:hidden" />
<div className="hidden items-center sm:flex">
<MdCircle className="size-2 text-severity_significant_motion md:mr-[10px]" />
<div className="hidden md:block">{t("motion.label")}</div>
</div>
</ToggleGroupItem>
</ToggleGroup>
<div className="hidden items-center sm:flex">
<MdCircle className="size-2 text-severity_detection md:mr-[10px]" />
<div className="hidden md:flex md:flex-row md:items-center">
{t("detections")}
{reviewCounts.detection > -1 ? (
`${reviewCounts.detection}`
) : (
<ActivityIndicator className="ml-2 size-4" />
)}
</div>
</div>
</ToggleGroupItem>
<ToggleGroupItem
className={cn(
"rounded-lg px-3 py-4",
severityToggle != "significant_motion" &&
"text-muted-foreground",
)}
value="significant_motion"
aria-label={t("motion.label")}
>
<GiSoundWaves className="size-6 rotate-90 text-severity_significant_motion sm:hidden" />
<div className="hidden items-center sm:flex">
<MdCircle className="size-2 text-severity_significant_motion md:mr-[10px]" />
<div className="hidden md:block">{t("motion.label")}</div>
</div>
</ToggleGroupItem>
</ToggleGroup>
{selectedReviews.length <= 0 ? (
<ReviewFilterGroup
filters={
severity == "significant_motion"
? ["cameras", "date", "motionOnly"]
: ["cameras", "reviewed", "date", "general"]
}
currentSeverity={severityToggle}
reviewSummary={reviewSummary}
recordingsSummary={recordingsSummary}
filter={filter}
motionOnly={motionOnly}
filterList={reviewFilterList}
showReviewed={showReviewed}
setShowReviewed={setShowReviewed}
onUpdateFilter={updateFilter}
setMotionOnly={setMotionOnly}
/>
) : (
<ReviewActionGroup
selectedReviews={selectedReviews}
setSelectedReviews={setSelectedReviews}
onExport={exportReview}
pullLatestData={pullLatestData}
/>
{selectedReviews.length <= 0 ? (
<ReviewFilterGroup
filters={
severity == "significant_motion"
? ["cameras", "date", "motionOnly"]
: ["cameras", "reviewed", "date", "general"]
}
currentSeverity={severityToggle}
reviewSummary={reviewSummary}
recordingsSummary={recordingsSummary}
filter={filter}
motionOnly={motionOnly}
filterList={reviewFilterList}
showReviewed={showReviewed}
setShowReviewed={setShowReviewed}
onUpdateFilter={updateFilter}
setMotionOnly={setMotionOnly}
/>
) : (
<ReviewActionGroup
selectedReviews={selectedReviews}
setSelectedReviews={setSelectedReviews}
onExport={exportReview}
pullLatestData={pullLatestData}
/>
)}
</div>
)}
<div
className={cn(
"h-full min-h-0 overflow-hidden",
motionPreviewsOpen ? "flex flex-col" : "flex",
)}
</div>
<div className="flex h-full overflow-hidden">
>
{severity != "significant_motion" && (
<DetectionReview
contentRef={contentRef}
@@ -447,10 +501,16 @@ export default function EventView({
contentRef={contentRef}
reviewItems={reviewItems}
relevantPreviews={relevantPreviews}
reviewSummary={reviewSummary}
recordingsSummary={recordingsSummary}
timeRange={timeRange}
startTime={startTime}
filter={filter}
motionOnly={motionOnly}
updateFilter={updateFilter}
motionPreviewsCamera={motionPreviewsCamera}
setMotionPreviewsCamera={setMotionPreviewsCamera}
setMotionSearchCamera={setMotionSearchCamera}
emptyCardData={emptyCardData}
onOpenRecording={onOpenRecording}
/>
@@ -898,10 +958,16 @@ type MotionReviewProps = {
significant_motion: ReviewSegment[];
};
relevantPreviews?: Preview[];
reviewSummary?: ReviewSummary;
recordingsSummary?: RecordingsSummary;
timeRange: TimeRange;
startTime?: number;
filter?: ReviewFilter;
motionOnly?: boolean;
updateFilter: (filter: ReviewFilter) => void;
motionPreviewsCamera: string | null;
setMotionPreviewsCamera: (camera: string | null) => void;
setMotionSearchCamera: (camera: string) => void;
emptyCardData: EmptyCardData;
onOpenRecording: (data: RecordingStartingPoint) => void;
};
@@ -909,13 +975,20 @@ function MotionReview({
contentRef,
reviewItems,
relevantPreviews,
reviewSummary,
recordingsSummary,
timeRange,
startTime,
filter,
motionOnly = false,
updateFilter,
motionPreviewsCamera,
setMotionPreviewsCamera,
setMotionSearchCamera,
emptyCardData,
onOpenRecording,
}: MotionReviewProps) {
const { t } = useTranslation(["views/events", "common"]);
const segmentDuration = 30;
const { data: config } = useSWR<FrigateConfig>("config");
@@ -961,6 +1034,15 @@ function MotionReview({
},
]);
const { data: overlapReviewSegments } = useSWR<ReviewSegment[]>([
"review",
{
before: alignedBefore,
after: alignedAfter,
cameras: filter?.cameras?.join(",") ?? null,
},
]);
// timeline time
const timeRangeSegments = useMemo(
@@ -973,19 +1055,29 @@ function MotionReview({
return timeRangeSegments.ranges.length - 1;
}
return timeRangeSegments.ranges.findIndex(
const index = timeRangeSegments.ranges.findIndex(
(seg) => seg.after <= startTime && seg.before >= startTime,
);
if (index === -1) {
return timeRangeSegments.ranges.length - 1;
}
return index;
// only render once
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const [selectedRangeIdx, setSelectedRangeIdx] = useState(initialIndex);
const [currentTime, setCurrentTime] = useState<number>(
startTime ?? timeRangeSegments.ranges[selectedRangeIdx]?.before,
startTime ??
timeRangeSegments.ranges[selectedRangeIdx]?.before ??
timeRangeSegments.end,
);
const currentTimeRange = useMemo(
() => timeRangeSegments.ranges[selectedRangeIdx],
() =>
timeRangeSegments.ranges[selectedRangeIdx] ??
timeRangeSegments.ranges[timeRangeSegments.ranges.length - 1],
[selectedRangeIdx, timeRangeSegments],
);
@@ -1023,18 +1115,86 @@ function MotionReview({
const [playbackRate, setPlaybackRate] = useState(8);
const [controlsOpen, setControlsOpen] = useState(false);
const [dimStrength, setDimStrength] = useState(82);
const [isPreviewSettingsOpen, setIsPreviewSettingsOpen] = useState(false);
const objectReviewItems = useMemo(
() =>
(overlapReviewSegments ?? []).filter(
(item) =>
item.severity === "alert" ||
item.severity === "detection" ||
(item.data.detections?.length ?? 0) > 0 ||
(item.data.objects?.length ?? 0) > 0,
),
[overlapReviewSegments],
);
const nextTimestamp = useCameraMotionNextTimestamp(
timeRangeSegments.end,
segmentDuration,
motionOnly,
reviewItems?.all ?? [],
objectReviewItems,
motionData ?? [],
currentTime,
);
const timeoutIdRef = useRef<NodeJS.Timeout | null>(null);
const selectedMotionPreviewCamera = useMemo(
() =>
reviewCameras.find((camera) => camera.name === motionPreviewsCamera) ??
null,
[motionPreviewsCamera, reviewCameras],
);
const onUpdateSelectedDay = useCallback(
(day?: Date) => {
updateFilter({
...filter,
after: day == undefined ? undefined : day.getTime() / 1000,
before: day == undefined ? undefined : getEndOfDayTimestamp(day),
});
},
[filter, updateFilter],
);
const selectedCameraMotionData = useMemo(() => {
if (!motionPreviewsCamera) {
return [];
}
return (motionData ?? []).filter((item) => {
const cameras = item.camera.split(",").map((camera) => camera.trim());
return cameras.includes(motionPreviewsCamera);
});
}, [motionData, motionPreviewsCamera]);
const selectedCameraReviewItems = useMemo(() => {
if (!motionPreviewsCamera) {
return [];
}
return objectReviewItems.filter(
(item) => item.camera === motionPreviewsCamera,
);
}, [motionPreviewsCamera, objectReviewItems]);
const motionPreviewRanges = useCameraMotionOnlyRanges(
segmentDuration,
selectedCameraReviewItems,
selectedCameraMotionData,
);
useEffect(() => {
if (
motionPreviewsCamera &&
!reviewCameras.some((camera) => camera.name === motionPreviewsCamera)
) {
setMotionPreviewsCamera(null);
}
}, [motionPreviewsCamera, reviewCameras, setMotionPreviewsCamera]);
useEffect(() => {
if (nextTimestamp) {
if (!playing && timeoutIdRef.current != null) {
@@ -1124,132 +1284,349 @@ function MotionReview({
return (
<>
<div className="no-scrollbar flex flex-1 flex-wrap content-start gap-2 overflow-y-auto md:gap-4">
<div
ref={contentRef}
className={cn(
"no-scrollbar grid w-full grid-cols-1",
isMobile && "landscape:grid-cols-2",
reviewCameras.length > 3 &&
isMobile &&
"portrait:md:grid-cols-2 landscape:md:grid-cols-3",
isDesktop && "grid-cols-2 lg:grid-cols-3",
"gap-2 overflow-auto px-1 md:mx-2 md:gap-4 xl:grid-cols-3 3xl:grid-cols-4",
)}
>
{reviewCameras.map((camera) => {
let grow;
let spans;
const aspectRatio = camera.detect.width / camera.detect.height;
if (aspectRatio > 2) {
grow = "aspect-wide";
spans = "sm:col-span-2";
} else if (aspectRatio < 1) {
grow = "h-full aspect-tall";
spans = "md:row-span-2";
} else {
grow = "aspect-video";
}
const detectionType = getDetectionType(camera.name);
return (
<div key={camera.name} className={`relative ${spans}`}>
{motionData ? (
<>
<PreviewPlayer
className={`rounded-lg md:rounded-2xl ${spans} ${grow}`}
camera={camera.name}
timeRange={currentTimeRange}
startTime={previewStart}
cameraPreviews={relevantPreviews}
isScrubbing={scrubbing}
onControllerReady={(controller) => {
videoPlayersRef.current[camera.name] = controller;
}}
onClick={() =>
onOpenRecording({
camera: camera.name,
startTime: Math.min(
currentTime,
Date.now() / 1000 - 30,
),
severity: "significant_motion",
})
}
/>
<div
className={`review-item-ring pointer-events-none absolute inset-0 z-20 size-full rounded-lg outline outline-[3px] -outline-offset-[2.8px] ${detectionType ? `outline-severity_${detectionType} shadow-severity_${detectionType}` : "outline-transparent duration-500"}`}
/>
</>
) : (
<Skeleton
className={`size-full rounded-lg md:rounded-2xl ${spans} ${grow}`}
/>
{motionPreviewsCamera && selectedMotionPreviewCamera ? (
<>
<div className="relative mb-2 flex h-11 items-center justify-between pl-2 pr-2 md:px-3">
<Button
className="flex items-center gap-2.5 rounded-lg"
aria-label={t("label.back", { ns: "common" })}
size="sm"
onClick={() => setMotionPreviewsCamera(null)}
>
<IoMdArrowRoundBack className="size-5 text-secondary-foreground" />
{isDesktop && (
<div className="text-primary">
{t("button.back", { ns: "common" })}
</div>
)}
</Button>
<div className="flex items-center gap-2">
{isDesktop && (
<CalendarFilterButton
reviewSummary={reviewSummary}
recordingsSummary={recordingsSummary}
day={
filter?.after == undefined
? undefined
: new Date(filter.after * 1000)
}
updateSelectedDay={onUpdateSelectedDay}
/>
)}
<PlatformAwareDialog
trigger={
<Button
className={cn(
isDesktop ? "flex items-center gap-2" : "rounded-lg",
)}
size="sm"
aria-label={
isDesktop
? t("motionPreviews.mobileSettingsTitle")
: t("filters", { ns: "views/recording" })
}
>
<FaCog className="text-secondary-foreground" />
{isDesktop && t("motionPreviews.mobileSettingsTitle")}
</Button>
}
content={
<div className="space-y-4 py-2">
{!isDesktop && (
<div className="space-y-1">
<div className="text-md">
{t("motionPreviews.mobileSettingsTitle")}
</div>
<div className="text-xs text-muted-foreground">
{t("motionPreviews.mobileSettingsDesc")}
</div>
</div>
)}
<div className="space-y-3">
<div className="space-y-0.5">
<div className="text-md">
{t("motionPreviews.speed")}
</div>
<div className="text-xs text-muted-foreground">
{t("motionPreviews.speedDesc")}
</div>
</div>
<Select
value={String(playbackRate)}
onValueChange={(value) =>
setPlaybackRate(Number(value))
}
>
<SelectTrigger
className="h-10 w-full"
aria-label={t("motionPreviews.speedAria")}
>
<SelectValue />
</SelectTrigger>
<SelectContent>
{[4, 8, 12, 16].map((speed) => (
<SelectItem key={speed} value={String(speed)}>
{speed}x
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="space-y-3">
<div className="space-y-0.5">
<div className="text-md">{t("motionPreviews.dim")}</div>
<div className="text-xs text-muted-foreground">
{t("motionPreviews.dimDesc")}
</div>
</div>
<div className="px-1 py-2">
<VolumeSlider
className="w-full"
min={25}
max={95}
step={1}
value={[dimStrength]}
aria-label={t("motionPreviews.dimAria")}
onValueChange={(values) => {
const nextValue = values[0];
if (nextValue == undefined) {
return;
}
setDimStrength(nextValue);
}}
/>
</div>
</div>
{!isDesktop && (
<>
<SelectSeparator />
<div className="flex w-full flex-row justify-center">
<ReviewActivityCalendar
recordingsSummary={recordingsSummary}
selectedDay={
filter?.after == undefined
? undefined
: new Date(filter.after * 1000)
}
onSelect={(day) => {
onUpdateSelectedDay(day);
setIsPreviewSettingsOpen(false);
}}
/>
</div>
<div className="flex items-center justify-center p-2">
<Button
aria-label={t("button.reset", { ns: "common" })}
onClick={() => {
onUpdateSelectedDay(undefined);
setIsPreviewSettingsOpen(false);
}}
>
{t("button.reset", { ns: "common" })}
</Button>
</div>
</>
)}
</div>
}
contentClassName={cn(
isDesktop
? "w-80"
: "scrollbar-container max-h-[75dvh] overflow-y-auto overflow-x-hidden px-4",
)}
</div>
);
})}
</div>
</div>
<div className="no-scrollbar w-[55px] overflow-y-auto md:w-[100px]">
{motionData ? (
<MotionReviewTimeline
segmentDuration={segmentDuration}
timestampSpread={15}
timelineStart={timeRangeSegments.end}
timelineEnd={timeRangeSegments.start}
motionOnly={motionOnly}
showHandlebar
handlebarTime={currentTime}
setHandlebarTime={setCurrentTime}
events={reviewItems?.all ?? []}
motion_events={motionData ?? []}
open={isPreviewSettingsOpen}
onOpenChange={setIsPreviewSettingsOpen}
/>
</div>
</div>
<MotionPreviewsPane
camera={selectedMotionPreviewCamera}
contentRef={contentRef}
onHandlebarDraggingChange={(scrubbing) => {
if (playing && scrubbing) {
setPlaying(false);
}
setScrubbing(scrubbing);
cameraPreviews={relevantPreviews}
motionRanges={motionPreviewRanges}
isLoadingMotionRanges={
motionData == undefined || overlapReviewSegments == undefined
}
playbackRate={playbackRate}
nonMotionAlpha={dimStrength / 100}
onSeek={(timestamp) => {
onOpenRecording({
camera: selectedMotionPreviewCamera.name,
startTime: timestamp,
severity: "significant_motion",
});
}}
dense={isMobileOnly}
isZooming={false}
zoomDirection={null}
alwaysShowMotionLine={true}
/>
) : (
<Skeleton className="size-full" />
)}
</div>
</>
) : (
<div className="no-scrollbar flex min-w-0 flex-1 flex-wrap content-start gap-2 overflow-y-auto md:gap-4">
<div
ref={contentRef}
className={cn(
"no-scrollbar grid w-full grid-cols-1",
isMobile && "landscape:grid-cols-2",
reviewCameras.length > 3 &&
isMobile &&
"portrait:md:grid-cols-2 landscape:md:grid-cols-3",
isDesktop && "grid-cols-2 lg:grid-cols-3",
"gap-2 overflow-auto px-1 md:mx-2 md:gap-4 xl:grid-cols-3 3xl:grid-cols-4",
)}
>
{reviewCameras.map((camera) => {
let grow;
let spans;
const aspectRatio = camera.detect.width / camera.detect.height;
if (aspectRatio > 2) {
grow = "aspect-wide";
spans = "sm:col-span-2";
} else if (aspectRatio < 1) {
grow = "h-full aspect-tall";
spans = "md:row-span-2";
} else {
grow = "aspect-video";
}
const detectionType = getDetectionType(camera.name);
return (
<div key={camera.name} className={`relative ${spans}`}>
{motionData ? (
<>
<PreviewPlayer
className={`rounded-lg md:rounded-2xl ${spans} ${grow}`}
camera={camera.name}
timeRange={currentTimeRange}
startTime={previewStart}
cameraPreviews={relevantPreviews}
isScrubbing={scrubbing}
onControllerReady={(controller) => {
videoPlayersRef.current[camera.name] = controller;
}}
onClick={() =>
onOpenRecording({
camera: camera.name,
startTime: Math.min(
currentTime,
Date.now() / 1000 - 30,
),
severity: "significant_motion",
})
}
/>
<div
className={`review-item-ring pointer-events-none absolute inset-0 z-20 size-full rounded-lg outline outline-[3px] -outline-offset-[2.8px] ${detectionType ? `outline-severity_${detectionType} shadow-severity_${detectionType}` : "outline-transparent duration-500"}`}
/>
<div className="absolute bottom-2 right-2 z-30">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<BlurredIconButton
aria-label={t("motionSearch.openMenu")}
onClick={(e) => e.stopPropagation()}
>
<FiMoreVertical className="size-5" />
</BlurredIconButton>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
setMotionPreviewsCamera(camera.name);
}}
>
{t("motionPreviews.menuItem")}
</DropdownMenuItem>
<DropdownMenuItem
onClick={(e) => {
e.stopPropagation();
setMotionSearchCamera(camera.name);
}}
>
{t("motionSearch.menuItem")}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
</>
) : (
<Skeleton
className={`size-full rounded-lg md:rounded-2xl ${spans} ${grow}`}
/>
)}
</div>
);
})}
</div>
</div>
)}
{!selectedMotionPreviewCamera && (
<div className="no-scrollbar w-[55px] overflow-y-auto md:w-[100px]">
{motionData ? (
<MotionReviewTimeline
segmentDuration={segmentDuration}
timestampSpread={15}
timelineStart={timeRangeSegments.end}
timelineEnd={timeRangeSegments.start}
motionOnly={motionOnly}
showHandlebar
handlebarTime={currentTime}
setHandlebarTime={setCurrentTime}
events={reviewItems?.all ?? []}
motion_events={motionData ?? []}
contentRef={contentRef}
onHandlebarDraggingChange={(scrubbing) => {
if (playing && scrubbing) {
setPlaying(false);
}
<VideoControls
className="absolute bottom-16 left-1/2 -translate-x-1/2 bg-secondary"
features={{
volume: false,
seek: true,
playbackRate: true,
fullscreen: false,
}}
isPlaying={playing}
show={!scrubbing || controlsOpen}
playbackRates={[4, 8, 12, 16]}
playbackRate={playbackRate}
setControlsOpen={setControlsOpen}
onPlayPause={setPlaying}
onSeek={(diff) => {
const wasPlaying = playing;
setScrubbing(scrubbing);
}}
dense={isMobileOnly}
isZooming={false}
zoomDirection={null}
alwaysShowMotionLine={true}
/>
) : (
<Skeleton className="size-full" />
)}
</div>
)}
if (wasPlaying) {
setPlaying(false);
}
{!selectedMotionPreviewCamera && (
<VideoControls
className="absolute bottom-16 left-1/2 -translate-x-1/2 bg-secondary"
features={{
volume: false,
seek: true,
playbackRate: true,
fullscreen: false,
}}
isPlaying={playing}
show={!scrubbing || controlsOpen}
playbackRates={[4, 8, 12, 16]}
playbackRate={playbackRate}
setControlsOpen={setControlsOpen}
onPlayPause={setPlaying}
onSeek={(diff) => {
const wasPlaying = playing;
setCurrentTime(currentTime + diff);
if (wasPlaying) {
setPlaying(false);
}
if (wasPlaying) {
setTimeout(() => setPlaying(true), 100);
}
}}
onSetPlaybackRate={setPlaybackRate}
/>
setCurrentTime(currentTime + diff);
if (wasPlaying) {
setTimeout(() => setPlaying(true), 100);
}
}}
onSetPlaybackRate={setPlaybackRate}
/>
)}
</>
);
}
+898
View File
@@ -0,0 +1,898 @@
import { MotionOnlyRange } from "@/hooks/use-camera-activity";
import { Preview } from "@/types/preview";
import {
MutableRefObject,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { isCurrentHour } from "@/utils/dateUtil";
import { useTranslation } from "react-i18next";
import { CameraConfig } from "@/types/frigateConfig";
import useSWR from "swr";
import { baseUrl } from "@/api/baseUrl";
import { Recording } from "@/types/record";
import { useResizeObserver } from "@/hooks/resize-observer";
import { Skeleton } from "@/components/ui/skeleton";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import TimeAgo from "@/components/dynamic/TimeAgo";
import { useFormattedTimestamp } from "@/hooks/use-date-utils";
import { FrigateConfig } from "@/types/frigateConfig";
const MOTION_HEATMAP_GRID_SIZE = 16;
const MIN_MOTION_CELL_ALPHA = 0.06;
function getPreviewForMotionRange(
cameraPreviews: Preview[],
cameraName: string,
range: MotionOnlyRange,
) {
const matchingPreviews = cameraPreviews.filter(
(preview) =>
preview.camera === cameraName &&
preview.end > range.start_time &&
preview.start < range.end_time,
);
if (!matchingPreviews.length) {
return;
}
const getOverlap = (preview: Preview) =>
Math.max(
0,
Math.min(preview.end, range.end_time) -
Math.max(preview.start, range.start_time),
);
return matchingPreviews.reduce((best, current) => {
return getOverlap(current) > getOverlap(best) ? current : best;
});
}
function getRangeOverlapSeconds(
rangeStart: number,
rangeEnd: number,
recordingStart: number,
recordingEnd: number,
) {
return Math.max(
0,
Math.min(rangeEnd, recordingEnd) - Math.max(rangeStart, recordingStart),
);
}
function getMotionHeatmapForRange(
recordings: Recording[],
range: MotionOnlyRange,
) {
const weightedHeatmap = new Map<number, number>();
let totalWeight = 0;
recordings.forEach((recording) => {
const overlapSeconds = getRangeOverlapSeconds(
range.start_time,
range.end_time,
recording.start_time,
recording.end_time,
);
if (overlapSeconds <= 0) {
return;
}
totalWeight += overlapSeconds;
if (!recording.motion_heatmap) {
return;
}
Object.entries(recording.motion_heatmap).forEach(
([cellIndex, intensity]) => {
const index = Number(cellIndex);
const level = Number(intensity);
if (Number.isNaN(index) || Number.isNaN(level) || level <= 0) {
return;
}
const existingWeight = weightedHeatmap.get(index) ?? 0;
weightedHeatmap.set(index, existingWeight + level * overlapSeconds);
},
);
});
if (!totalWeight || weightedHeatmap.size === 0) {
return null;
}
const mergedHeatmap: Record<string, number> = {};
weightedHeatmap.forEach((weightedLevel, index) => {
const normalizedLevel = Math.max(
0,
Math.min(255, Math.round(weightedLevel / totalWeight)),
);
if (normalizedLevel > 0) {
mergedHeatmap[index.toString()] = normalizedLevel;
}
});
return Object.keys(mergedHeatmap).length > 0 ? mergedHeatmap : null;
}
type MotionPreviewClipProps = {
cameraName: string;
range: MotionOnlyRange;
playbackRate: number;
preview?: Preview;
fallbackFrameTimes?: number[];
motionHeatmap?: Record<string, number> | null;
nonMotionAlpha: number;
isVisible: boolean;
onSeek: (timestamp: number) => void;
};
function MotionPreviewClip({
cameraName,
range,
playbackRate,
preview,
fallbackFrameTimes,
motionHeatmap,
nonMotionAlpha,
isVisible,
onSeek,
}: MotionPreviewClipProps) {
const { t } = useTranslation(["views/events", "common"]);
const { data: config } = useSWR<FrigateConfig>("config");
const videoRef = useRef<HTMLVideoElement | null>(null);
const dimOverlayCanvasRef = useRef<HTMLCanvasElement | null>(null);
const overlayContainerRef = useRef<HTMLDivElement | null>(null);
const [{ width: overlayWidth, height: overlayHeight }] =
useResizeObserver(overlayContainerRef);
const [videoLoaded, setVideoLoaded] = useState(false);
const [videoPlaying, setVideoPlaying] = useState(false);
const [fallbackImageLoaded, setFallbackImageLoaded] = useState(false);
const [mediaDimensions, setMediaDimensions] = useState<{
width: number;
height: number;
} | null>(null);
const [fallbackFrameIndex, setFallbackFrameIndex] = useState(0);
const [fallbackFramesReady, setFallbackFramesReady] = useState(false);
const formattedDate = useFormattedTimestamp(
range.start_time,
config?.ui.time_format == "24hour"
? t("time.formattedTimestampMonthDayHourMinute.24hour", {
ns: "common",
})
: t("time.formattedTimestampMonthDayHourMinute.12hour", {
ns: "common",
}),
config?.ui.timezone,
);
const fallbackFrameSrcs = useMemo(() => {
if (!fallbackFrameTimes || fallbackFrameTimes.length === 0) {
return [] as string[];
}
return fallbackFrameTimes.map(
(frameTime) =>
`${baseUrl}api/preview/preview_${cameraName}-${frameTime}.webp/thumbnail.webp`,
);
}, [cameraName, fallbackFrameTimes]);
useEffect(() => {
setFallbackFrameIndex(0);
setFallbackFramesReady(false);
}, [range.start_time, range.end_time, fallbackFrameTimes]);
useEffect(() => {
if (fallbackFrameSrcs.length === 0) {
setFallbackFramesReady(false);
return;
}
let cancelled = false;
const preloadFrames = async () => {
await Promise.allSettled(
fallbackFrameSrcs.map(
(src) =>
new Promise<void>((resolve) => {
const image = new Image();
image.onload = () => resolve();
image.onerror = () => resolve();
image.src = src;
}),
),
);
if (!cancelled) {
setFallbackFramesReady(true);
}
};
void preloadFrames();
return () => {
cancelled = true;
};
}, [fallbackFrameSrcs]);
useEffect(() => {
if (!fallbackFramesReady || fallbackFrameSrcs.length <= 1 || !isVisible) {
return;
}
const intervalMs = Math.max(
50,
Math.round(1000 / Math.max(1, playbackRate)),
);
const intervalId = window.setInterval(() => {
setFallbackFrameIndex((previous) => {
return (previous + 1) % fallbackFrameSrcs.length;
});
}, intervalMs);
return () => {
window.clearInterval(intervalId);
};
}, [fallbackFrameSrcs.length, fallbackFramesReady, isVisible, playbackRate]);
const fallbackFrameSrc = useMemo(() => {
if (fallbackFrameSrcs.length === 0) {
return undefined;
}
return fallbackFrameSrcs[fallbackFrameIndex] ?? fallbackFrameSrcs[0];
}, [fallbackFrameIndex, fallbackFrameSrcs]);
useEffect(() => {
setVideoLoaded(false);
setVideoPlaying(false);
setMediaDimensions(null);
}, [preview?.src]);
useEffect(() => {
if (!preview || !isVisible || videoLoaded || !videoRef.current) {
return;
}
if (videoRef.current.currentSrc || videoRef.current.error) {
setVideoLoaded(true);
}
}, [isVisible, preview, videoLoaded]);
useEffect(() => {
setFallbackImageLoaded(false);
setMediaDimensions(null);
}, [fallbackFrameSrcs]);
useEffect(() => {
if (!fallbackFrameSrc || !isVisible || !fallbackFramesReady) {
return;
}
setFallbackImageLoaded(true);
}, [fallbackFrameSrc, fallbackFramesReady, isVisible]);
const showLoadingIndicator =
(preview != undefined && isVisible && !videoPlaying) ||
(fallbackFrameSrc != undefined && isVisible && !fallbackImageLoaded);
const clipStart = useMemo(() => {
if (!preview) {
return 0;
}
return Math.max(0, range.start_time - preview.start);
}, [preview, range.start_time]);
const clipEnd = useMemo(() => {
if (!preview) {
return 0;
}
const previewDuration = preview.end - preview.start;
return Math.min(
previewDuration,
Math.max(clipStart + 0.1, range.end_time - preview.start),
);
}, [clipStart, preview, range.end_time]);
const resetPlayback = useCallback(() => {
if (!videoRef.current || !preview) {
return;
}
videoRef.current.currentTime = clipStart;
videoRef.current.playbackRate = playbackRate;
}, [clipStart, playbackRate, preview]);
useEffect(() => {
if (!videoRef.current || !preview) {
return;
}
if (!isVisible) {
videoRef.current.pause();
videoRef.current.currentTime = clipStart;
return;
}
if (videoRef.current.readyState >= 2) {
resetPlayback();
void videoRef.current.play().catch(() => undefined);
}
}, [clipStart, isVisible, preview, resetPlayback]);
const drawDimOverlay = useCallback(() => {
if (!dimOverlayCanvasRef.current) {
return;
}
const canvas = dimOverlayCanvasRef.current;
const context = canvas.getContext("2d");
if (!context) {
return;
}
if (overlayWidth <= 0 || overlayHeight <= 0) {
return;
}
const width = Math.max(1, overlayWidth);
const height = Math.max(1, overlayHeight);
const dpr = window.devicePixelRatio || 1;
const pixelWidth = Math.max(1, Math.round(width * dpr));
const pixelHeight = Math.max(1, Math.round(height * dpr));
if (canvas.width !== pixelWidth || canvas.height !== pixelHeight) {
canvas.width = pixelWidth;
canvas.height = pixelHeight;
}
canvas.style.width = `${width}px`;
canvas.style.height = `${height}px`;
context.setTransform(dpr, 0, 0, dpr, 0, 0);
context.clearRect(0, 0, width, height);
if (!motionHeatmap) {
return;
}
// Calculate the actual rendered media area (object-contain letterboxing)
let drawX = 0;
let drawY = 0;
let drawWidth = width;
let drawHeight = height;
if (
mediaDimensions &&
mediaDimensions.width > 0 &&
mediaDimensions.height > 0
) {
const containerAspect = width / height;
const mediaAspect = mediaDimensions.width / mediaDimensions.height;
if (mediaAspect < containerAspect) {
// Portrait / tall: constrained by height, bars on left and right
drawHeight = height;
drawWidth = height * mediaAspect;
drawX = (width - drawWidth) / 2;
drawY = 0;
} else {
// Wide / landscape: constrained by width, bars on top and bottom
drawWidth = width;
drawHeight = width / mediaAspect;
drawX = 0;
drawY = (height - drawHeight) / 2;
}
}
const heatmapLevels = Object.values(motionHeatmap)
.map((value) => Number(value))
.filter((value) => Number.isFinite(value) && value > 0);
const maxHeatmapLevel =
heatmapLevels.length > 0 ? Math.max(...heatmapLevels) : 0;
const maskCanvas = document.createElement("canvas");
maskCanvas.width = MOTION_HEATMAP_GRID_SIZE;
maskCanvas.height = MOTION_HEATMAP_GRID_SIZE;
const maskContext = maskCanvas.getContext("2d");
if (!maskContext) {
return;
}
const imageData = maskContext.createImageData(
MOTION_HEATMAP_GRID_SIZE,
MOTION_HEATMAP_GRID_SIZE,
);
for (let index = 0; index < MOTION_HEATMAP_GRID_SIZE ** 2; index++) {
const level = Number(motionHeatmap[index.toString()] ?? 0);
const normalizedLevel =
maxHeatmapLevel > 0
? Math.min(1, Math.max(0, level / maxHeatmapLevel))
: 0;
const boostedLevel = Math.sqrt(normalizedLevel);
const alpha =
nonMotionAlpha -
boostedLevel * (nonMotionAlpha - MIN_MOTION_CELL_ALPHA);
const pixelOffset = index * 4;
imageData.data[pixelOffset] = 0;
imageData.data[pixelOffset + 1] = 0;
imageData.data[pixelOffset + 2] = 0;
imageData.data[pixelOffset + 3] = Math.round(
Math.max(0, Math.min(1, alpha)) * 255,
);
}
maskContext.putImageData(imageData, 0, 0);
context.imageSmoothingEnabled = true;
context.imageSmoothingQuality = "high";
context.drawImage(maskCanvas, drawX, drawY, drawWidth, drawHeight);
}, [
motionHeatmap,
nonMotionAlpha,
overlayHeight,
overlayWidth,
mediaDimensions,
]);
useEffect(() => {
drawDimOverlay();
}, [drawDimOverlay]);
return (
<div
ref={overlayContainerRef}
className="relative aspect-video size-full cursor-pointer overflow-hidden rounded-lg bg-black md:rounded-2xl"
onClick={() => onSeek(range.start_time)}
>
{showLoadingIndicator && (
<Skeleton className="absolute inset-0 z-10 rounded-lg md:rounded-2xl" />
)}
{preview ? (
<>
<video
ref={videoRef}
className="size-full bg-black object-contain"
playsInline
preload={isVisible ? "metadata" : "none"}
muted
autoPlay={isVisible}
onLoadedMetadata={() => {
setVideoLoaded(true);
if (videoRef.current) {
setMediaDimensions({
width: videoRef.current.videoWidth,
height: videoRef.current.videoHeight,
});
}
if (!isVisible) {
return;
}
resetPlayback();
if (videoRef.current) {
void videoRef.current.play().catch(() => undefined);
}
}}
onCanPlay={() => {
setVideoLoaded(true);
if (!isVisible) {
return;
}
if (videoRef.current) {
void videoRef.current.play().catch(() => undefined);
}
}}
onPlay={() => setVideoPlaying(true)}
onLoadedData={() => setVideoLoaded(true)}
onError={() => {
setVideoLoaded(true);
setVideoPlaying(true);
}}
onTimeUpdate={() => {
if (!videoRef.current || !preview || !isVisible) {
return;
}
if (videoRef.current.currentTime >= clipEnd) {
videoRef.current.currentTime = clipStart;
}
}}
>
{isVisible && (
<source
src={`${baseUrl}${preview.src.substring(1)}`}
type={preview.type}
/>
)}
</video>
{motionHeatmap && (
<canvas
ref={dimOverlayCanvasRef}
className="pointer-events-none absolute inset-0"
aria-hidden="true"
/>
)}
</>
) : fallbackFrameSrc ? (
<>
<img
src={fallbackFrameSrc}
className="size-full bg-black object-contain"
loading="lazy"
alt=""
onLoad={(e) => {
setFallbackImageLoaded(true);
const img = e.currentTarget;
if (img.naturalWidth > 0 && img.naturalHeight > 0) {
setMediaDimensions({
width: img.naturalWidth,
height: img.naturalHeight,
});
}
}}
onError={() => setFallbackImageLoaded(true)}
/>
{motionHeatmap && (
<canvas
ref={dimOverlayCanvasRef}
className="pointer-events-none absolute inset-0"
aria-hidden="true"
/>
)}
</>
) : (
<div className="flex size-full items-center justify-center text-sm text-muted-foreground">
{t("motionPreviews.noPreview")}
</div>
)}
<div className="pointer-events-none absolute bottom-0 left-0 right-0 z-30 p-2">
<div className="flex flex-col items-start text-xs text-white/90 drop-shadow-lg">
{range.end_time ? (
<TimeAgo time={range.start_time * 1000} dense />
) : (
<ActivityIndicator size={14} />
)}
{formattedDate}
</div>
</div>
</div>
);
}
type MotionPreviewsPaneProps = {
camera: CameraConfig;
contentRef: MutableRefObject<HTMLDivElement | null>;
cameraPreviews: Preview[];
motionRanges: MotionOnlyRange[];
isLoadingMotionRanges?: boolean;
playbackRate: number;
nonMotionAlpha: number;
onSeek: (timestamp: number) => void;
};
export default function MotionPreviewsPane({
camera,
contentRef,
cameraPreviews,
motionRanges,
isLoadingMotionRanges = false,
playbackRate,
nonMotionAlpha,
onSeek,
}: MotionPreviewsPaneProps) {
const { t } = useTranslation(["views/events"]);
const [scrollContainer, setScrollContainer] = useState<HTMLDivElement | null>(
null,
);
const [windowVisible, setWindowVisible] = useState(true);
useEffect(() => {
const visibilityListener = () => {
setWindowVisible(document.visibilityState == "visible");
};
addEventListener("visibilitychange", visibilityListener);
return () => {
removeEventListener("visibilitychange", visibilityListener);
};
}, []);
const [visibleClips, setVisibleClips] = useState<string[]>([]);
const [hasVisibilityData, setHasVisibilityData] = useState(false);
const clipObserver = useRef<IntersectionObserver | null>(null);
const recordingTimeRange = useMemo(() => {
if (!motionRanges.length) {
return null;
}
return motionRanges.reduce(
(bounds, range) => ({
after: Math.min(bounds.after, range.start_time),
before: Math.max(bounds.before, range.end_time),
}),
{
after: motionRanges[0].start_time,
before: motionRanges[0].end_time,
},
);
}, [motionRanges]);
const { data: cameraRecordings } = useSWR<Recording[]>(
recordingTimeRange
? [
`${camera.name}/recordings`,
{
after: Math.floor(recordingTimeRange.after),
before: Math.ceil(recordingTimeRange.before),
},
]
: null,
{
revalidateOnFocus: false,
revalidateOnReconnect: false,
},
);
const { data: previewFrames } = useSWR<string[]>(
recordingTimeRange
? `preview/${camera.name}/start/${Math.floor(recordingTimeRange.after)}/end/${Math.ceil(recordingTimeRange.before)}/frames`
: null,
{
revalidateOnFocus: false,
revalidateOnReconnect: false,
},
);
const previewFrameTimes = useMemo(() => {
if (!previewFrames) {
return [] as number[];
}
return previewFrames
.map((frame) => {
const timestampPart = frame.split("-").at(-1)?.replace(".webp", "");
return timestampPart ? Number(timestampPart) : NaN;
})
.filter((value) => Number.isFinite(value))
.sort((a, b) => a - b);
}, [previewFrames]);
const getFallbackFrameTimesForRange = useCallback(
(range: MotionOnlyRange) => {
if (!isCurrentHour(range.end_time) || previewFrameTimes.length === 0) {
return [] as number[];
}
const inRangeFrames = previewFrameTimes.filter(
(frameTime) =>
frameTime >= range.start_time && frameTime <= range.end_time,
);
// Use all in-range frames when enough data exists for natural animation
if (inRangeFrames.length > 1) {
return inRangeFrames;
}
// If sparse, keep the single in-range frame and add only the next 2 frames
if (inRangeFrames.length === 1) {
const inRangeFrame = inRangeFrames[0];
const nextFrames = previewFrameTimes
.filter((frameTime) => frameTime > inRangeFrame)
.slice(0, 2);
return [inRangeFrame, ...nextFrames];
}
const nextFramesFromStart = previewFrameTimes
.filter((frameTime) => frameTime >= range.start_time)
.slice(0, 3);
// If no in-range frame exists, take up to 3 frames starting at clip start
if (nextFramesFromStart.length > 0) {
return nextFramesFromStart;
}
const lastFrame = previewFrameTimes.at(-1);
return lastFrame != undefined ? [lastFrame] : [];
},
[previewFrameTimes],
);
const setContentNode = useCallback(
(node: HTMLDivElement | null) => {
contentRef.current = node;
setScrollContainer(node);
},
[contentRef],
);
useEffect(() => {
if (!scrollContainer) {
return;
}
const visibleClipIds = new Set<string>();
clipObserver.current = new IntersectionObserver(
(entries) => {
setHasVisibilityData(true);
entries.forEach((entry) => {
const clipId = (entry.target as HTMLElement).dataset.clipId;
if (!clipId) {
return;
}
if (entry.isIntersecting) {
visibleClipIds.add(clipId);
} else {
visibleClipIds.delete(clipId);
}
});
const rootRect = scrollContainer.getBoundingClientRect();
const prunedVisibleClipIds = [...visibleClipIds].filter((clipId) => {
const clipElement = scrollContainer.querySelector<HTMLElement>(
`[data-clip-id="${clipId}"]`,
);
if (!clipElement) {
return false;
}
const clipRect = clipElement.getBoundingClientRect();
return (
clipRect.bottom > rootRect.top && clipRect.top < rootRect.bottom
);
});
setVisibleClips(prunedVisibleClipIds);
},
{
root: scrollContainer,
threshold: 0,
},
);
scrollContainer
.querySelectorAll<HTMLElement>("[data-clip-id]")
.forEach((node) => {
clipObserver.current?.observe(node);
});
return () => {
clipObserver.current?.disconnect();
};
}, [scrollContainer]);
const clipRef = useCallback((node: HTMLElement | null) => {
if (!clipObserver.current) {
return;
}
try {
if (node) {
clipObserver.current.observe(node);
}
} catch {
// no op
}
}, []);
const clipData = useMemo(
() =>
motionRanges
.filter((range) => range.end_time > range.start_time)
.sort((left, right) => right.start_time - left.start_time)
.map((range) => {
const preview = getPreviewForMotionRange(
cameraPreviews,
camera.name,
range,
);
return {
range,
preview,
fallbackFrameTimes: !preview
? getFallbackFrameTimesForRange(range)
: undefined,
motionHeatmap: getMotionHeatmapForRange(
cameraRecordings ?? [],
range,
),
};
}),
[
cameraPreviews,
camera.name,
cameraRecordings,
getFallbackFrameTimesForRange,
motionRanges,
],
);
const hasCurrentHourRanges = useMemo(
() => motionRanges.some((range) => isCurrentHour(range.end_time)),
[motionRanges],
);
const isLoadingPane =
isLoadingMotionRanges ||
(motionRanges.length > 0 && cameraRecordings == undefined) ||
(hasCurrentHourRanges && previewFrames == undefined);
if (isLoadingPane) {
return (
<ActivityIndicator className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2" />
);
}
return (
<div className="flex min-h-0 flex-1 flex-col gap-3 overflow-hidden px-1 md:mx-2 md:gap-4">
<div
ref={setContentNode}
className="no-scrollbar min-h-0 flex-1 overflow-y-auto"
>
{clipData.length === 0 ? (
<div className="flex h-full items-center justify-center text-lg text-primary">
{t("motionPreviews.empty")}
</div>
) : (
<div className="grid grid-cols-1 gap-2 pb-2 sm:grid-cols-2 md:gap-4 xl:grid-cols-4">
{clipData.map(
({ range, preview, fallbackFrameTimes, motionHeatmap }, idx) => (
<div
key={`${camera.name}-${range.start_time}-${range.end_time}-${preview?.src ?? "none"}-${idx}`}
data-clip-id={`${camera.name}-${range.start_time}-${range.end_time}-${idx}`}
ref={clipRef}
>
<MotionPreviewClip
cameraName={camera.name}
range={range}
playbackRate={playbackRate}
preview={preview}
fallbackFrameTimes={fallbackFrameTimes}
motionHeatmap={motionHeatmap}
nonMotionAlpha={nonMotionAlpha}
isVisible={
windowVisible &&
(visibleClips.includes(
`${camera.name}-${range.start_time}-${range.end_time}-${idx}`,
) ||
(!hasVisibilityData && idx < 8))
}
onSeek={onSeek}
/>
</div>
),
)}
</div>
)}
</div>
</div>
);
}
@@ -0,0 +1,708 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import { isDesktop, isIOS, isMobile } from "react-device-detect";
import { FaArrowRight, FaCalendarAlt, FaCheckCircle } from "react-icons/fa";
import { MdOutlineRestartAlt, MdUndo } from "react-icons/md";
import { FrigateConfig } from "@/types/frigateConfig";
import { TimeRange } from "@/types/timeline";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { Drawer, DrawerContent } from "@/components/ui/drawer";
import { Label } from "@/components/ui/label";
import { Slider } from "@/components/ui/slider";
import { Switch } from "@/components/ui/switch";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import {
Popover,
PopoverContent,
PopoverTrigger,
} from "@/components/ui/popover";
import { SelectSeparator } from "@/components/ui/select";
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import { CameraNameLabel } from "@/components/camera/FriendlyNameLabel";
import { TimezoneAwareCalendar } from "@/components/overlay/ReviewActivityCalendar";
import { useApiHost } from "@/api";
import { useResizeObserver } from "@/hooks/resize-observer";
import { useFormattedTimestamp } from "@/hooks/use-date-utils";
import { getUTCOffset } from "@/utils/dateUtil";
import { cn } from "@/lib/utils";
import MotionSearchROICanvas from "./MotionSearchROICanvas";
import { TransformComponent, TransformWrapper } from "react-zoom-pan-pinch";
type MotionSearchDialogProps = {
open: boolean;
onOpenChange: (open: boolean) => void;
config: FrigateConfig;
cameras: string[];
selectedCamera: string | null;
onCameraSelect: (camera: string) => void;
cameraLocked?: boolean;
polygonPoints: number[][];
setPolygonPoints: React.Dispatch<React.SetStateAction<number[][]>>;
isDrawingROI: boolean;
setIsDrawingROI: React.Dispatch<React.SetStateAction<boolean>>;
parallelMode: boolean;
setParallelMode: React.Dispatch<React.SetStateAction<boolean>>;
threshold: number;
setThreshold: React.Dispatch<React.SetStateAction<number>>;
minArea: number;
setMinArea: React.Dispatch<React.SetStateAction<number>>;
frameSkip: number;
setFrameSkip: React.Dispatch<React.SetStateAction<number>>;
maxResults: number;
setMaxResults: React.Dispatch<React.SetStateAction<number>>;
searchRange?: TimeRange;
setSearchRange: React.Dispatch<React.SetStateAction<TimeRange | undefined>>;
defaultRange: TimeRange;
isSearching: boolean;
canStartSearch: boolean;
onStartSearch: () => void;
timezone?: string;
};
export default function MotionSearchDialog({
open,
onOpenChange,
config,
cameras,
selectedCamera,
onCameraSelect,
cameraLocked = false,
polygonPoints,
setPolygonPoints,
isDrawingROI,
setIsDrawingROI,
parallelMode,
setParallelMode,
threshold,
setThreshold,
minArea,
setMinArea,
frameSkip,
setFrameSkip,
maxResults,
setMaxResults,
searchRange,
setSearchRange,
defaultRange,
isSearching,
canStartSearch,
onStartSearch,
timezone,
}: MotionSearchDialogProps) {
const { t } = useTranslation(["views/motionSearch", "common"]);
const apiHost = useApiHost();
const containerRef = useRef<HTMLDivElement>(null);
const [{ width: containerWidth, height: containerHeight }] =
useResizeObserver(containerRef);
const [imageLoaded, setImageLoaded] = useState(false);
const cameraConfig = useMemo(() => {
if (!selectedCamera) return undefined;
return config.cameras[selectedCamera];
}, [config, selectedCamera]);
const polygonClosed = useMemo(
() => !isDrawingROI && polygonPoints.length >= 3,
[isDrawingROI, polygonPoints.length],
);
const undoPolygonPoint = useCallback(() => {
if (polygonPoints.length === 0 || isSearching) {
return;
}
setPolygonPoints((prev) => prev.slice(0, -1));
setIsDrawingROI(true);
}, [isSearching, setIsDrawingROI, setPolygonPoints, polygonPoints.length]);
const resetPolygon = useCallback(() => {
if (polygonPoints.length === 0 || isSearching) {
return;
}
setPolygonPoints([]);
setIsDrawingROI(true);
}, [isSearching, polygonPoints.length, setIsDrawingROI, setPolygonPoints]);
const imageSize = useMemo(() => {
if (!containerWidth || !containerHeight || !cameraConfig) {
return { width: 0, height: 0 };
}
const cameraAspectRatio =
cameraConfig.detect.width / cameraConfig.detect.height;
const availableAspectRatio = containerWidth / containerHeight;
if (availableAspectRatio >= cameraAspectRatio) {
return {
width: containerHeight * cameraAspectRatio,
height: containerHeight,
};
}
return {
width: containerWidth,
height: containerWidth / cameraAspectRatio,
};
}, [containerWidth, containerHeight, cameraConfig]);
useEffect(() => {
setImageLoaded(false);
}, [selectedCamera]);
const Overlay = isDesktop ? Dialog : Drawer;
const Content = isDesktop ? DialogContent : DrawerContent;
return (
<Overlay open={open} onOpenChange={onOpenChange}>
<Content
{...(isDesktop
? {
onOpenAutoFocus: (event: Event) => event.preventDefault(),
}
: {})}
className={cn(
isDesktop
? "scrollbar-container max-h-[90dvh] overflow-y-auto sm:max-w-[75%]"
: "flex max-h-[90dvh] flex-col overflow-hidden rounded-lg pb-4",
)}
>
<div
className={cn(
!isDesktop &&
"scrollbar-container flex min-h-0 w-full flex-col gap-4 overflow-y-auto overflow-x-hidden px-4",
)}
>
<DialogHeader>
<DialogTitle className="mt-4 md:mt-auto">
{t("dialog.title")}
</DialogTitle>
<p className="my-1 text-sm text-muted-foreground">
{t("description")}
</p>
</DialogHeader>
<DialogDescription className="hidden" />
<div
className={cn(
"flex gap-4",
isDesktop ? "mt-4 flex-row" : "flex-col landscape:flex-row",
)}
>
<div
className={cn("flex flex-1 flex-col", !isDesktop && "min-w-0")}
>
{(!cameraLocked || !selectedCamera) && (
<div className="flex items-end justify-between gap-2">
<div className="mt-2 md:min-w-64">
<div className="grid gap-2">
<Label htmlFor="motion-search-camera">
{t("dialog.cameraLabel")}
</Label>
<Select
value={selectedCamera ?? undefined}
onValueChange={(value) => onCameraSelect(value)}
>
<SelectTrigger id="motion-search-camera">
<SelectValue placeholder={t("selectCamera")} />
</SelectTrigger>
<SelectContent>
{cameras.map((camera) => (
<SelectItem
key={camera}
value={camera}
className="cursor-pointer hover:bg-accent hover:text-accent-foreground"
>
<CameraNameLabel camera={camera} />
</SelectItem>
))}
</SelectContent>
</Select>
</div>
</div>
</div>
)}
<TransformWrapper minScale={1.0} wheel={{ smoothStep: 0.005 }}>
<div className="flex flex-col gap-2">
<TransformComponent
wrapperStyle={{
width: "100%",
height: isDesktop ? "100%" : "auto",
}}
contentStyle={{
position: "relative",
width: "100%",
height: "100%",
}}
>
<div
ref={containerRef}
className="relative flex w-full items-center justify-center overflow-hidden rounded-lg border bg-secondary"
style={{ aspectRatio: "16 / 9" }}
>
{selectedCamera && cameraConfig && imageSize.width > 0 ? (
<div
className="relative"
style={{
width: imageSize.width,
height: imageSize.height,
}}
>
<img
alt={t("dialog.previewAlt", {
camera: selectedCamera,
})}
src={`${apiHost}api/${selectedCamera}/latest.jpg?h=500`}
className="h-full w-full object-contain"
onLoad={() => setImageLoaded(true)}
/>
{!imageLoaded && (
<div className="absolute inset-0 flex items-center justify-center">
<ActivityIndicator className="h-8 w-8" />
</div>
)}
<MotionSearchROICanvas
camera={selectedCamera}
width={cameraConfig.detect.width}
height={cameraConfig.detect.height}
polygonPoints={polygonPoints}
setPolygonPoints={setPolygonPoints}
isDrawing={isDrawingROI}
setIsDrawing={setIsDrawingROI}
isInteractive={true}
/>
</div>
) : (
<div className="flex h-full w-full items-center justify-center text-sm text-muted-foreground">
{t("selectCamera")}
</div>
)}
</div>
</TransformComponent>
</div>
</TransformWrapper>
{selectedCamera && (
<div className="my-2 flex w-full flex-row justify-between rounded-md bg-background_alt p-2 text-sm">
<div className="my-1 inline-flex items-center">
{t("polygonControls.points", {
count: polygonPoints.length,
})}
{polygonClosed && <FaCheckCircle className="ml-2 size-5" />}
</div>
<div className="flex flex-row justify-center gap-2">
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="default"
className="size-6 rounded-md p-1"
aria-label={t("polygonControls.undo")}
disabled={polygonPoints.length === 0 || isSearching}
onClick={undoPolygonPoint}
>
<MdUndo className="text-secondary-foreground" />
</Button>
</TooltipTrigger>
<TooltipContent>
{t("polygonControls.undo")}
</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="default"
className="size-6 rounded-md p-1"
aria-label={t("polygonControls.reset")}
disabled={polygonPoints.length === 0 || isSearching}
onClick={resetPolygon}
>
<MdOutlineRestartAlt className="text-secondary-foreground" />
</Button>
</TooltipTrigger>
<TooltipContent>
{t("polygonControls.reset")}
</TooltipContent>
</Tooltip>
</div>
</div>
)}
</div>
<div
className={cn(
"flex w-full flex-col gap-4 space-y-4 lg:w-[340px]",
isMobile && "landscape:w-[40%] landscape:flex-shrink-0",
)}
>
<div className="grid gap-3">
<h4 className="mb-4 font-medium leading-none">
{t("settings.title")}
</h4>
<div className="grid gap-4 space-y-2">
<div className="grid gap-2">
<Label htmlFor="threshold">{t("settings.threshold")}</Label>
<div className="flex items-center gap-2">
<Slider
id="threshold"
min={1}
max={255}
step={1}
value={[threshold]}
onValueChange={([value]) => setThreshold(value)}
/>
<span className="w-12 text-sm">{threshold}</span>
</div>
<p className="text-xs text-muted-foreground">
{t("settings.thresholdDesc")}
</p>
</div>
<div className="grid gap-2">
<Label htmlFor="minArea">{t("settings.minArea")}</Label>
<div className="flex items-center gap-2">
<Slider
id="minArea"
min={1}
max={100}
step={1}
value={[minArea]}
onValueChange={([value]) => setMinArea(value)}
/>
<span className="w-12 text-sm">{minArea}%</span>
</div>
<p className="text-xs text-muted-foreground">
{t("settings.minAreaDesc")}
</p>
</div>
<div className="grid gap-2">
<Label htmlFor="frameSkip">{t("settings.frameSkip")}</Label>
<div className="flex items-center gap-2">
<Slider
id="frameSkip"
min={1}
max={60}
step={1}
value={[frameSkip]}
onValueChange={([value]) => setFrameSkip(value)}
/>
<span className="w-12 text-sm">{frameSkip}</span>
</div>
<p className="text-xs text-muted-foreground">
{t("settings.frameSkipDesc")}
</p>
</div>
<div className="grid gap-2">
<div className="flex items-center justify-between gap-2">
<Label htmlFor="parallelMode">
{t("settings.parallelMode")}
</Label>
<Switch
id="parallelMode"
checked={parallelMode}
onCheckedChange={setParallelMode}
/>
</div>
<p className="text-xs text-muted-foreground">
{t("settings.parallelModeDesc")}
</p>
</div>
<div className="grid gap-2">
<Label htmlFor="maxResults">
{t("settings.maxResults")}
</Label>
<div className="flex items-center gap-2">
<Slider
id="maxResults"
min={1}
max={200}
step={1}
value={[maxResults]}
onValueChange={([value]) => setMaxResults(value)}
/>
<span className="w-12 text-sm">{maxResults}</span>
</div>
<p className="text-xs text-muted-foreground">
{t("settings.maxResultsDesc")}
</p>
</div>
</div>
</div>
<SearchRangeSelector
range={searchRange}
setRange={setSearchRange}
defaultRange={defaultRange}
timeFormat={config.ui?.time_format}
timezone={timezone}
/>
<Button
className="w-full"
variant="select"
onClick={onStartSearch}
disabled={!canStartSearch || isSearching}
>
{t("startSearch")}
</Button>
</div>
</div>
</div>
</Content>
</Overlay>
);
}
type SearchRangeSelectorProps = {
range?: TimeRange;
setRange: React.Dispatch<React.SetStateAction<TimeRange | undefined>>;
defaultRange: TimeRange;
timeFormat?: "browser" | "12hour" | "24hour";
timezone?: string;
};
function SearchRangeSelector({
range,
setRange,
defaultRange,
timeFormat,
timezone,
}: SearchRangeSelectorProps) {
const { t } = useTranslation(["views/motionSearch", "common"]);
const [startOpen, setStartOpen] = useState(false);
const [endOpen, setEndOpen] = useState(false);
const timezoneOffset = useMemo(
() =>
timezone ? Math.round(getUTCOffset(new Date(), timezone)) : undefined,
[timezone],
);
const localTimeOffset = useMemo(
() =>
Math.round(
getUTCOffset(
new Date(),
Intl.DateTimeFormat().resolvedOptions().timeZone,
),
),
[],
);
const startTime = useMemo(() => {
let time = range?.after ?? defaultRange.after;
if (timezoneOffset !== undefined) {
time = time + (timezoneOffset - localTimeOffset) * 60;
}
return time;
}, [range, defaultRange, timezoneOffset, localTimeOffset]);
const endTime = useMemo(() => {
let time = range?.before ?? defaultRange.before;
if (timezoneOffset !== undefined) {
time = time + (timezoneOffset - localTimeOffset) * 60;
}
return time;
}, [range, defaultRange, timezoneOffset, localTimeOffset]);
const formattedStart = useFormattedTimestamp(
startTime,
timeFormat === "24hour"
? t("time.formattedTimestamp.24hour", { ns: "common" })
: t("time.formattedTimestamp.12hour", { ns: "common" }),
);
const formattedEnd = useFormattedTimestamp(
endTime,
timeFormat === "24hour"
? t("time.formattedTimestamp.24hour", { ns: "common" })
: t("time.formattedTimestamp.12hour", { ns: "common" }),
);
const startClock = useMemo(() => {
const date = new Date(startTime * 1000);
return `${date.getHours().toString().padStart(2, "0")}:${date
.getMinutes()
.toString()
.padStart(2, "0")}:${date.getSeconds().toString().padStart(2, "0")}`;
}, [startTime]);
const endClock = useMemo(() => {
const date = new Date(endTime * 1000);
return `${date.getHours().toString().padStart(2, "0")}:${date
.getMinutes()
.toString()
.padStart(2, "0")}:${date.getSeconds().toString().padStart(2, "0")}`;
}, [endTime]);
return (
<div className="grid gap-2">
<Label>{t("timeRange.title")}</Label>
<div className="flex items-center rounded-lg bg-secondary px-2 py-1 text-secondary-foreground">
<FaCalendarAlt />
<div className="flex flex-wrap items-center">
<Popover
open={startOpen}
onOpenChange={(open) => {
if (!open) {
setStartOpen(false);
}
}}
modal={false}
>
<PopoverTrigger asChild>
<Button
className="text-primary"
aria-label={t("timeRange.start")}
variant={startOpen ? "select" : "default"}
size="sm"
onClick={() => {
setStartOpen(true);
setEndOpen(false);
}}
>
{formattedStart}
</Button>
</PopoverTrigger>
<PopoverContent
disablePortal
className="flex flex-col items-center"
>
<TimezoneAwareCalendar
timezone={timezone}
selectedDay={new Date(startTime * 1000)}
onSelect={(day) => {
if (!day) {
return;
}
setRange({
before: endTime,
after: day.getTime() / 1000 + 1,
});
}}
/>
<SelectSeparator className="bg-secondary" />
<input
className="text-md mx-4 w-full border border-input bg-background p-1 text-secondary-foreground hover:bg-accent hover:text-accent-foreground dark:[color-scheme:dark]"
id="startTime"
type="time"
value={startClock}
step={isIOS ? "60" : "1"}
onChange={(e) => {
const clock = e.target.value;
const [hour, minute, second] = isIOS
? [...clock.split(":"), "00"]
: clock.split(":");
const start = new Date(startTime * 1000);
start.setHours(
parseInt(hour),
parseInt(minute),
parseInt(second ?? 0),
0,
);
setRange({
before: endTime,
after: start.getTime() / 1000,
});
}}
/>
</PopoverContent>
</Popover>
<FaArrowRight className="size-4 text-primary" />
<Popover
open={endOpen}
onOpenChange={(open) => {
if (!open) {
setEndOpen(false);
}
}}
modal={false}
>
<PopoverTrigger asChild>
<Button
className="text-primary"
aria-label={t("timeRange.end")}
variant={endOpen ? "select" : "default"}
size="sm"
onClick={() => {
setEndOpen(true);
setStartOpen(false);
}}
>
{formattedEnd}
</Button>
</PopoverTrigger>
<PopoverContent
disablePortal
className="flex flex-col items-center"
>
<TimezoneAwareCalendar
timezone={timezone}
selectedDay={new Date(endTime * 1000)}
onSelect={(day) => {
if (!day) {
return;
}
setRange({
after: startTime,
before: day.getTime() / 1000,
});
}}
/>
<SelectSeparator className="bg-secondary" />
<input
className="text-md mx-4 w-full border border-input bg-background p-1 text-secondary-foreground hover:bg-accent hover:text-accent-foreground dark:[color-scheme:dark]"
id="endTime"
type="time"
value={endClock}
step={isIOS ? "60" : "1"}
onChange={(e) => {
const clock = e.target.value;
const [hour, minute, second] = isIOS
? [...clock.split(":"), "00"]
: clock.split(":");
const end = new Date(endTime * 1000);
end.setHours(
parseInt(hour),
parseInt(minute),
parseInt(second ?? 0),
0,
);
setRange({
before: end.getTime() / 1000,
after: startTime,
});
}}
/>
</PopoverContent>
</Popover>
</div>
</div>
</div>
);
}
@@ -0,0 +1,398 @@
import { useCallback, useMemo, useRef } from "react";
import { Stage, Layer, Line, Circle, Image } from "react-konva";
import Konva from "konva";
import type { KonvaEventObject } from "konva/lib/Node";
import { flattenPoints } from "@/utils/canvasUtil";
import { cn } from "@/lib/utils";
import { useResizeObserver } from "@/hooks/resize-observer";
type MotionSearchROICanvasProps = {
camera: string;
width: number;
height: number;
polygonPoints: number[][];
setPolygonPoints: React.Dispatch<React.SetStateAction<number[][]>>;
isDrawing: boolean;
setIsDrawing: React.Dispatch<React.SetStateAction<boolean>>;
isInteractive?: boolean;
motionHeatmap?: Record<string, number> | null;
showMotionHeatmap?: boolean;
};
export default function MotionSearchROICanvas({
width,
height,
polygonPoints,
setPolygonPoints,
isDrawing,
setIsDrawing,
isInteractive = true,
motionHeatmap,
showMotionHeatmap = false,
}: MotionSearchROICanvasProps) {
const containerRef = useRef<HTMLDivElement>(null);
const stageRef = useRef<Konva.Stage>(null);
const [{ width: containerWidth, height: containerHeight }] =
useResizeObserver(containerRef);
const stageSize = useMemo(
() => ({
width: containerWidth > 0 ? Math.ceil(containerWidth) : 0,
height: containerHeight > 0 ? Math.ceil(containerHeight) : 0,
}),
[containerHeight, containerWidth],
);
const videoRect = useMemo(() => {
const stageWidth = stageSize.width;
const stageHeight = stageSize.height;
const sourceWidth = width > 0 ? width : 1;
const sourceHeight = height > 0 ? height : 1;
if (stageWidth <= 0 || stageHeight <= 0) {
return { x: 0, y: 0, width: 0, height: 0 };
}
const stageAspect = stageWidth / stageHeight;
const sourceAspect = sourceWidth / sourceHeight;
if (stageAspect > sourceAspect) {
const fittedHeight = stageHeight;
const fittedWidth = fittedHeight * sourceAspect;
return {
x: (stageWidth - fittedWidth) / 2,
y: 0,
width: fittedWidth,
height: fittedHeight,
};
}
const fittedWidth = stageWidth;
const fittedHeight = fittedWidth / sourceAspect;
return {
x: 0,
y: (stageHeight - fittedHeight) / 2,
width: fittedWidth,
height: fittedHeight,
};
}, [height, stageSize.height, stageSize.width, width]);
// Convert normalized points to stage coordinates
const scaledPoints = useMemo(() => {
return polygonPoints.map((point) => [
videoRect.x + point[0] * videoRect.width,
videoRect.y + point[1] * videoRect.height,
]);
}, [
polygonPoints,
videoRect.height,
videoRect.width,
videoRect.x,
videoRect.y,
]);
const flattenedPoints = useMemo(
() => flattenPoints(scaledPoints),
[scaledPoints],
);
const heatmapOverlayCanvas = useMemo(() => {
if (
!showMotionHeatmap ||
!motionHeatmap ||
videoRect.width === 0 ||
videoRect.height === 0
) {
return null;
}
const gridSize = 16;
const heatmapLevels = Object.values(motionHeatmap)
.map((value) => Number(value))
.filter((value) => Number.isFinite(value) && value > 0);
const maxHeatmapLevel =
heatmapLevels.length > 0 ? Math.max(...heatmapLevels) : 0;
if (maxHeatmapLevel <= 0) {
return null;
}
const maskCanvas = document.createElement("canvas");
maskCanvas.width = gridSize;
maskCanvas.height = gridSize;
const maskContext = maskCanvas.getContext("2d");
if (!maskContext) {
return null;
}
const imageData = maskContext.createImageData(gridSize, gridSize);
const heatmapStops = [
{ t: 0, r: 0, g: 0, b: 255 },
{ t: 0.25, r: 0, g: 255, b: 255 },
{ t: 0.5, r: 0, g: 255, b: 0 },
{ t: 0.75, r: 255, g: 255, b: 0 },
{ t: 1, r: 255, g: 0, b: 0 },
];
const getHeatmapColor = (value: number) => {
const clampedValue = Math.min(1, Math.max(0, value));
const upperIndex = heatmapStops.findIndex(
(stop) => stop.t >= clampedValue,
);
if (upperIndex <= 0) {
return heatmapStops[0];
}
const lower = heatmapStops[upperIndex - 1];
const upper = heatmapStops[upperIndex];
const range = upper.t - lower.t;
const blend = range > 0 ? (clampedValue - lower.t) / range : 0;
return {
r: Math.round(lower.r + (upper.r - lower.r) * blend),
g: Math.round(lower.g + (upper.g - lower.g) * blend),
b: Math.round(lower.b + (upper.b - lower.b) * blend),
};
};
for (let index = 0; index < gridSize ** 2; index++) {
const level = Number(motionHeatmap[index.toString()] ?? 0);
const normalizedLevel =
level > 0 ? Math.min(1, Math.max(0, level / maxHeatmapLevel)) : 0;
const alpha =
level > 0
? Math.min(0.95, Math.max(0.1, 0.15 + normalizedLevel * 0.5))
: 0;
const color = getHeatmapColor(normalizedLevel);
const pixelOffset = index * 4;
imageData.data[pixelOffset] = color.r;
imageData.data[pixelOffset + 1] = color.g;
imageData.data[pixelOffset + 2] = color.b;
imageData.data[pixelOffset + 3] = Math.round(alpha * 255);
}
maskContext.putImageData(imageData, 0, 0);
return maskCanvas;
}, [motionHeatmap, showMotionHeatmap, videoRect.height, videoRect.width]);
// Handle mouse click to add point
const handleMouseDown = useCallback(
(e: KonvaEventObject<MouseEvent | TouchEvent>) => {
if (!isInteractive || !isDrawing) return;
if (videoRect.width <= 0 || videoRect.height <= 0) return;
const stage = e.target.getStage();
if (!stage) return;
const mousePos = stage.getPointerPosition();
if (!mousePos) return;
const intersection = stage.getIntersection(mousePos);
// If clicking on first point and we have at least 3 points, close the polygon
if (polygonPoints.length >= 3 && intersection?.name() === "point-0") {
setIsDrawing(false);
return;
}
// Only add point if not clicking on an existing point
if (intersection?.getClassName() !== "Circle") {
const clampedX = Math.min(
Math.max(mousePos.x, videoRect.x),
videoRect.x + videoRect.width,
);
const clampedY = Math.min(
Math.max(mousePos.y, videoRect.y),
videoRect.y + videoRect.height,
);
// Convert to normalized coordinates (0-1)
const normalizedX = (clampedX - videoRect.x) / videoRect.width;
const normalizedY = (clampedY - videoRect.y) / videoRect.height;
setPolygonPoints([...polygonPoints, [normalizedX, normalizedY]]);
}
},
[
isDrawing,
polygonPoints,
setPolygonPoints,
setIsDrawing,
isInteractive,
videoRect.height,
videoRect.width,
videoRect.x,
videoRect.y,
],
);
// Handle point drag
const handlePointDragMove = useCallback(
(e: KonvaEventObject<MouseEvent | TouchEvent>, index: number) => {
if (!isInteractive) return;
const stage = e.target.getStage();
if (!stage) return;
const pos = { x: e.target.x(), y: e.target.y() };
// Constrain to fitted video boundaries
pos.x = Math.max(
videoRect.x,
Math.min(pos.x, videoRect.x + videoRect.width),
);
pos.y = Math.max(
videoRect.y,
Math.min(pos.y, videoRect.y + videoRect.height),
);
// Convert to normalized coordinates
const normalizedX = (pos.x - videoRect.x) / videoRect.width;
const normalizedY = (pos.y - videoRect.y) / videoRect.height;
const newPoints = [...polygonPoints];
newPoints[index] = [normalizedX, normalizedY];
setPolygonPoints(newPoints);
},
[
polygonPoints,
setPolygonPoints,
isInteractive,
videoRect.height,
videoRect.width,
videoRect.x,
videoRect.y,
],
);
// Handle right-click to delete point
const handleContextMenu = useCallback(
(e: KonvaEventObject<MouseEvent>, index: number) => {
if (!isInteractive) return;
e.evt.preventDefault();
if (polygonPoints.length <= 3 && !isDrawing) {
// Don't delete if we have a closed polygon with minimum points
return;
}
const newPoints = polygonPoints.filter((_, i) => i !== index);
setPolygonPoints(newPoints);
// If we deleted enough points, go back to drawing mode
if (newPoints.length < 3) {
setIsDrawing(true);
}
},
[polygonPoints, isDrawing, setPolygonPoints, setIsDrawing, isInteractive],
);
// Handle mouse hover on first point
const handleMouseOverPoint = useCallback(
(e: KonvaEventObject<MouseEvent | TouchEvent>, index: number) => {
if (!isInteractive) return;
if (!isDrawing || polygonPoints.length < 3 || index !== 0) return;
e.target.scale({ x: 2, y: 2 });
},
[isDrawing, isInteractive, polygonPoints.length],
);
const handleMouseOutPoint = useCallback(
(e: KonvaEventObject<MouseEvent | TouchEvent>, index: number) => {
if (!isInteractive) return;
if (index === 0) {
e.target.scale({ x: 1, y: 1 });
}
},
[isInteractive],
);
const vertexRadius = 6;
const polygonColorString = "rgba(66, 135, 245, 0.8)";
const polygonFillColor = "rgba(66, 135, 245, 0.2)";
return (
<div
ref={containerRef}
className={cn(
"absolute inset-0 z-10",
isInteractive ? "pointer-events-auto" : "pointer-events-none",
)}
style={{ cursor: isDrawing ? "crosshair" : "default" }}
>
{stageSize.width > 0 && stageSize.height > 0 && (
<Stage
ref={stageRef}
width={stageSize.width}
height={stageSize.height}
onMouseDown={handleMouseDown}
onTouchStart={handleMouseDown}
onContextMenu={(e) => e.evt.preventDefault()}
className="absolute inset-0"
>
<Layer>
{/* Segment heatmap overlay */}
{heatmapOverlayCanvas && (
<Image
image={heatmapOverlayCanvas}
x={videoRect.x}
y={videoRect.y}
width={videoRect.width}
height={videoRect.height}
listening={false}
/>
)}
{/* Polygon outline */}
{scaledPoints.length > 0 && (
<Line
points={flattenedPoints}
stroke={polygonColorString}
strokeWidth={2}
closed={!isDrawing && scaledPoints.length >= 3}
fill={
!isDrawing && scaledPoints.length >= 3
? polygonFillColor
: undefined
}
/>
)}
{/* Draw line from last point to cursor when drawing */}
{isDrawing && scaledPoints.length > 0 && (
<Line
points={flattenedPoints}
stroke={polygonColorString}
strokeWidth={2}
dash={[5, 5]}
/>
)}
{/* Vertex points */}
{scaledPoints.map((point, index) => (
<Circle
key={index}
name={`point-${index}`}
x={point[0]}
y={point[1]}
radius={vertexRadius}
fill={polygonColorString}
stroke="white"
strokeWidth={2}
draggable={!isDrawing && isInteractive}
onDragMove={(e) => handlePointDragMove(e, index)}
onMouseOver={(e) => handleMouseOverPoint(e, index)}
onMouseOut={(e) => handleMouseOutPoint(e, index)}
onContextMenu={(e) => handleContextMenu(e, index)}
/>
))}
</Layer>
</Stage>
)}
</div>
);
}
File diff suppressed because it is too large Load Diff