Rewrite desktop timeline to use separate dynamic video player component

This commit is contained in:
Nick Mowen 2024-01-25 12:10:00 -07:00
parent 11133cb05a
commit b1d4ea9fa8
7 changed files with 448 additions and 311 deletions

View File

@ -0,0 +1,351 @@
import { MutableRefObject, useEffect, useMemo, useRef, useState } from "react";
import VideoPlayer from "./VideoPlayer";
import Player from "video.js/dist/types/player";
import TimelineEventOverlay from "../overlay/TimelineDataOverlay";
import { useApiHost } from "@/api";
import useSWR from "swr";
import { FrigateConfig } from "@/types/frigateConfig";
import ActivityIndicator from "../ui/activity-indicator";
/**
* Dynamically switches between video playback and scrubbing preview player.
*/
type DynamicVideoPlayerProps = {
className?: string;
camera: string;
timeRange: { start: number; end: number };
cameraPreviews: Preview[];
onControllerReady?: (controller: DynamicVideoController) => void;
};
export default function DynamicVideoPlayer({
className,
camera,
timeRange,
cameraPreviews,
onControllerReady,
}: DynamicVideoPlayerProps) {
const apiHost = useApiHost();
const { data: config } = useSWR<FrigateConfig>("config");
const timezone = useMemo(
() =>
config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone,
[config]
);
// initial state
const initialPlaybackSource = useMemo(() => {
const date = new Date(timeRange.start * 1000);
return {
src: `${apiHost}vod/${date.getFullYear()}-${
date.getMonth() + 1
}/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll(
"/",
","
)}/master.m3u8`,
type: "application/vnd.apple.mpegurl",
};
}, []);
const initialPreviewSource = useMemo(() => {
const source = cameraPreviews.find(
(preview) =>
Math.round(preview.start) >= timeRange.start &&
Math.floor(preview.end) <= timeRange.end
)?.src;
if (source) {
return {
src: source,
type: "video/mp4",
};
} else {
return undefined;
}
}, []);
// controlling playback
const playerRef = useRef<Player | undefined>(undefined);
const previewRef = useRef<Player | undefined>(undefined);
const [isScrubbing, setIsScrubbing] = useState(false);
const [focusedItem, setFocusedItem] = useState<Timeline | undefined>(
undefined
);
const controller = useMemo(() => {
if (!config) {
return undefined;
}
return new DynamicVideoController(
playerRef,
previewRef,
(config.cameras[camera]?.detect?.annotation_offset || 0) / 1000,
setIsScrubbing,
setFocusedItem
);
}, [config]);
// state of playback player
const recordingParams = useMemo(() => {
return {
before: timeRange.end,
after: timeRange.start,
};
}, [timeRange]);
const { data: recordings } = useSWR<Recording[]>(
[`${camera}/recordings`, recordingParams],
{ revalidateOnFocus: false }
);
useEffect(() => {
if (!controller || !recordings || recordings.length == 0) {
return;
}
const date = new Date(timeRange.start * 1000);
const playbackUri = `${apiHost}vod/${date.getFullYear()}-${
date.getMonth() + 1
}/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll(
"/",
","
)}/master.m3u8`;
controller.newPlayback({
recordings,
playbackUri,
preview: cameraPreviews.find(
(preview) =>
Math.round(preview.start) >= timeRange.start &&
Math.floor(preview.end) <= timeRange.end
),
});
}, [controller, recordings]);
const hasPreview = true;
if (!controller) {
return <ActivityIndicator />;
}
return (
<div className={className}>
<div
className={`w-full relative ${
hasPreview && isScrubbing ? "hidden" : "visible"
}`}
>
<VideoPlayer
options={{
preload: "auto",
autoplay: true,
sources: [initialPlaybackSource],
controlBar: {
remainingTimeDisplay: false,
progressControl: {
seekBar: false,
},
},
}}
seekOptions={{ forward: 10, backward: 5 }}
onReady={(player) => {
playerRef.current = player;
player.on("playing", () => setFocusedItem(undefined));
player.on("timeupdate", () => {
controller.updateProgress(player.currentTime() || 0);
});
}}
onDispose={() => {
playerRef.current = undefined;
}}
>
{config && focusedItem && (
<TimelineEventOverlay
timeline={focusedItem}
cameraConfig={config.cameras[camera]}
/>
)}
</VideoPlayer>
</div>
<div className={`w-full ${isScrubbing ? "visible" : "hidden"}`}>
<VideoPlayer
options={{
preload: "auto",
autoplay: false,
controls: false,
muted: true,
loadingSpinner: false,
sources: [initialPreviewSource],
}}
seekOptions={{}}
onReady={(player) => {
previewRef.current = player;
player.on("seeked", () => controller.finishedSeeking());
if (onControllerReady) {
onControllerReady(controller);
}
}}
onDispose={() => {
previewRef.current = undefined;
}}
/>
</div>
</div>
);
}
export class DynamicVideoController {
// main state
private playerRef: MutableRefObject<Player | undefined>;
private previewRef: MutableRefObject<Player | undefined>;
private setScrubbing: (isScrubbing: boolean) => void;
private setFocusedItem: (timeline: Timeline) => void;
private playerMode: "playback" | "scrubbing" = "playback";
// playback
private recordings: Recording[] = [];
private onPlaybackTimestamp: ((time: number) => void) | undefined = undefined;
private annotationOffset: number;
private timeToStart: number | undefined = undefined;
// preview
private preview: Preview | undefined = undefined;
private timeToSeek: number | undefined = undefined;
private seeking = false;
constructor(
playerRef: MutableRefObject<Player | undefined>,
previewRef: MutableRefObject<Player | undefined>,
annotationOffset: number,
setScrubbing: (isScrubbing: boolean) => void,
setFocusedItem: (timeline: Timeline) => void
) {
this.playerRef = playerRef;
this.previewRef = previewRef;
this.annotationOffset = annotationOffset;
this.setScrubbing = setScrubbing;
this.setFocusedItem = setFocusedItem;
}
newPlayback(newPlayback: DynamicPlayback) {
this.recordings = newPlayback.recordings;
this.playerRef.current?.src({
src: newPlayback.playbackUri,
type: "application/vnd.apple.mpegurl",
});
if (this.timeToStart) {
this.seekToTimestamp(this.timeToStart);
this.timeToStart = undefined;
}
this.preview = newPlayback.preview;
if (this.preview && this.previewRef.current) {
this.previewRef.current.src({
src: this.preview.src,
type: this.preview.type,
});
}
}
seekToTimestamp(time: number, play: boolean = false) {
if (this.playerMode != "playback") {
this.playerMode = "playback";
this.setScrubbing(false);
this.timeToSeek = undefined;
this.seeking = false;
}
if (this.recordings.length == 0) {
this.timeToStart = time;
}
let seekSeconds = 0;
(this.recordings || []).every((segment) => {
// if the next segment is past the desired time, stop calculating
if (segment.start_time > time) {
return false;
}
if (segment.end_time < time) {
seekSeconds += segment.end_time - segment.start_time;
return true;
}
seekSeconds +=
segment.end_time - segment.start_time - (segment.end_time - time);
return true;
});
this.playerRef.current?.currentTime(seekSeconds);
if (play) {
this.playerRef.current?.play();
}
}
seekToTimelineItem(timeline: Timeline) {
this.playerRef.current?.pause();
this.seekToTimestamp(timeline.timestamp + this.annotationOffset);
this.setFocusedItem(timeline);
}
updateProgress(playerTime: number) {
if (this.onPlaybackTimestamp) {
// take a player time in seconds and convert to timestamp in timeline
let timestamp = 0;
let totalTime = 0;
(this.recordings || []).every((segment) => {
if (totalTime + segment.duration > playerTime) {
// segment is here
timestamp = segment.start_time + (playerTime - totalTime);
return false;
} else {
totalTime += segment.duration;
return true;
}
});
this.onPlaybackTimestamp(timestamp);
}
}
onPlayerTimeUpdate(listener: (timestamp: number) => void) {
this.onPlaybackTimestamp = listener;
}
scrubToTimestamp(time: number) {
if (this.playerMode != "scrubbing") {
this.playerMode = "scrubbing";
this.playerRef.current?.pause();
this.setScrubbing(true);
}
if (this.preview) {
if (this.seeking) {
this.timeToSeek = time;
} else {
this.previewRef.current?.currentTime(time - this.preview.start);
this.seeking = true;
}
}
}
finishedSeeking() {
if (!this.preview || this.playerMode == "playback") {
return;
}
if (
this.timeToSeek &&
this.timeToSeek != this.previewRef.current?.currentTime()
) {
this.previewRef.current?.currentTime(
this.timeToSeek - this.preview.start
);
} else {
this.seeking = false;
}
}
}

View File

@ -21,38 +21,6 @@ type Preview = {
end: number;
};
type Timeline = {
camera: string;
timestamp: number;
data: {
camera: string;
label: string;
sub_label: string;
box?: [number, number, number, number];
region: [number, number, number, number];
attribute: string;
zones: string[];
};
class_type:
| "visible"
| "gone"
| "entered_zone"
| "attribute"
| "active"
| "stationary"
| "heard"
| "external";
source_id: string;
source: string;
};
type HourlyTimeline = {
start: number;
end: number;
count: number;
hours: { [key: string]: Timeline[] };
};
interface HistoryFilter extends FilterType {
cameras: string[];
labels: string[];

View File

@ -0,0 +1,5 @@
type DynamicPlayback = {
recordings: Recording[];
playbackUri: string;
preview: Preview | undefined;
};

31
web/src/types/timeline.ts Normal file
View File

@ -0,0 +1,31 @@
type Timeline = {
camera: string;
timestamp: number;
data: {
camera: string;
label: string;
sub_label: string;
box?: [number, number, number, number];
region: [number, number, number, number];
attribute: string;
zones: string[];
};
class_type:
| "visible"
| "gone"
| "entered_zone"
| "attribute"
| "active"
| "stationary"
| "heard"
| "external";
source_id: string;
source: string;
};
type HourlyTimeline = {
start: number;
end: number;
count: number;
hours: { [key: string]: Timeline[] };
};

View File

@ -117,7 +117,7 @@ export function getHourlyTimelineData(
export function getTimelineHoursForDay(
camera: string,
cards: CardsData,
allPreviews: Preview[],
cameraPreviews: Preview[],
timestamp: number
): HistoryTimeline {
const endOfThisHour = new Date();
@ -131,14 +131,6 @@ export function getTimelineHoursForDay(
let start = startDay.getTime() / 1000;
let end = 0;
const relevantPreviews = allPreviews.filter((preview) => {
return (
preview.camera == camera &&
preview.start >= start &&
Math.floor(preview.end - 1) <= dayEnd
);
});
const dayIdx = Object.keys(cards).find((day) => {
if (parseInt(day) > start) {
return false;
@ -178,7 +170,7 @@ export function getTimelineHoursForDay(
return [];
})
: [];
const relevantPreview = relevantPreviews.find(
const relevantPreview = cameraPreviews.find(
(preview) =>
Math.round(preview.start) >= start && Math.floor(preview.end) <= end
);

View File

@ -1,18 +1,17 @@
import { useApiHost } from "@/api";
import TimelineEventOverlay from "@/components/overlay/TimelineDataOverlay";
import VideoPlayer from "@/components/player/VideoPlayer";
import ActivityScrubber from "@/components/scrubber/ActivityScrubber";
import ActivityIndicator from "@/components/ui/activity-indicator";
import { FrigateConfig } from "@/types/frigateConfig";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useEffect, useMemo, useRef, useState } from "react";
import useSWR from "swr";
import Player from "video.js/dist/types/player";
import TimelineItemCard from "@/components/card/TimelineItemCard";
import { getTimelineHoursForDay } from "@/utils/historyUtil";
import { GraphDataPoint } from "@/types/graph";
import TimelineGraph from "@/components/graph/TimelineGraph";
import TimelineBar from "@/components/bar/TimelineBar";
import DynamicVideoPlayer, {
DynamicVideoController,
} from "@/components/player/DynamicVideoPlayer";
type DesktopTimelineViewProps = {
timelineData: CardsData;
@ -25,124 +24,19 @@ export default function DesktopTimelineView({
allPreviews,
initialPlayback,
}: DesktopTimelineViewProps) {
const apiHost = useApiHost();
const { data: config } = useSWR<FrigateConfig>("config");
const timezone = useMemo(
() =>
config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone,
[config]
);
const controllerRef = useRef<DynamicVideoController | undefined>(undefined);
const [selectedPlayback, setSelectedPlayback] = useState(initialPlayback);
const playerRef = useRef<Player | undefined>(undefined);
const previewRef = useRef<Player | undefined>(undefined);
const initialScrollRef = useRef<HTMLDivElement | null>(null);
const [scrubbing, setScrubbing] = useState(false);
const [focusedItem, setFocusedItem] = useState<Timeline | undefined>(
undefined
);
const [seeking, setSeeking] = useState(false);
const [timeToSeek, setTimeToSeek] = useState<number | undefined>(undefined);
const [playerTime, setPlayerTime] = useState(
initialPlayback.timelineItems.length > 0
? initialPlayback.timelineItems[0].timestamp - initialPlayback.range.start
: 0
);
const annotationOffset = useMemo(() => {
if (!config) {
return 0;
}
return (
(config.cameras[initialPlayback.camera]?.detect?.annotation_offset || 0) /
1000
);
}, [config]);
const recordingParams = useMemo(() => {
return {
before: selectedPlayback.range.end,
after: selectedPlayback.range.start,
};
}, [selectedPlayback]);
const { data: recordings } = useSWR<Recording[]>(
selectedPlayback
? [`${selectedPlayback.camera}/recordings`, recordingParams]
: null,
{ revalidateOnFocus: false }
);
const playbackUri = useMemo(() => {
if (!selectedPlayback) {
return "";
}
const date = new Date(selectedPlayback.range.start * 1000);
return `${apiHost}vod/${date.getFullYear()}-${
date.getMonth() + 1
}/${date.getDate()}/${date.getHours()}/${
selectedPlayback.camera
}/${timezone.replaceAll("/", ",")}/master.m3u8`;
}, [selectedPlayback]);
const onSelectItem = useCallback(
(timeline: Timeline | undefined) => {
if (timeline) {
setFocusedItem(timeline);
const selected = timeline.timestamp;
playerRef.current?.pause();
let seekSeconds = 0;
(recordings || []).every((segment) => {
// if the next segment is past the desired time, stop calculating
if (segment.start_time > selected) {
return false;
}
if (segment.end_time < selected) {
seekSeconds += segment.end_time - segment.start_time;
return true;
}
seekSeconds +=
segment.end_time -
segment.start_time -
(segment.end_time - selected);
return true;
});
playerRef.current?.currentTime(seekSeconds);
} else {
setFocusedItem(undefined);
}
},
[annotationOffset, recordings, playerRef]
);
const timelineTime = useMemo(() => {
if (scrubbing) {
return selectedPlayback.range.start + playerTime;
} else {
// take a player time in seconds and convert to timestamp in timeline
let timestamp = 0;
let totalTime = 0;
(recordings || []).every((segment) => {
if (totalTime + segment.duration > playerTime) {
// segment is here
timestamp = segment.start_time + (playerTime - totalTime);
return false;
} else {
totalTime += segment.duration;
return true;
}
});
return timestamp;
}
}, [playerTime]);
const [timelineTime, setTimelineTime] = useState(0);
// handle scrolling to initial timeline item
useEffect(() => {
@ -151,55 +45,18 @@ export default function DesktopTimelineView({
}
}, [initialScrollRef]);
// handle seeking to next frame when seek is finished
useEffect(() => {
if (seeking) {
return;
}
if (timeToSeek && !scrubbing) {
setScrubbing(true);
playerRef.current?.pause();
}
if (timeToSeek && timeToSeek != previewRef.current?.currentTime()) {
setSeeking(true);
previewRef.current?.currentTime(timeToSeek);
}
}, [timeToSeek, seeking]);
// handle loading main / preview playback when selected hour changes
useEffect(() => {
if (!playerRef.current) {
return;
}
setPlayerTime(
selectedPlayback.timelineItems.length > 0
? selectedPlayback.timelineItems[0].timestamp -
selectedPlayback.range.start
: 0
);
playerRef.current.src({
src: playbackUri,
type: "application/vnd.apple.mpegurl",
const cameraPreviews = useMemo(() => {
return allPreviews.filter((preview) => {
return preview.camera == initialPlayback.camera;
});
if (selectedPlayback.relevantPreview && previewRef.current) {
previewRef.current.src({
src: selectedPlayback.relevantPreview.src,
type: selectedPlayback.relevantPreview.type,
});
}
}, [playerRef, previewRef, playbackUri]);
}, []);
const timelineStack = useMemo(
() =>
getTimelineHoursForDay(
selectedPlayback.camera,
timelineData,
allPreviews,
cameraPreviews,
selectedPlayback.range.start + 60
),
[]
@ -254,88 +111,25 @@ export default function DesktopTimelineView({
<div className="w-full">
<div className="flex">
<>
<div className="w-2/3 bg-black flex justify-center items-center">
<div
className={`w-full relative ${
selectedPlayback.relevantPreview != undefined && scrubbing
? "hidden"
: "visible"
}`}
>
<VideoPlayer
options={{
preload: "auto",
autoplay: true,
sources: [
{
src: playbackUri,
type: "application/vnd.apple.mpegurl",
},
],
controlBar: {
remainingTimeDisplay: false,
progressControl: {
seekBar: false,
},
},
}}
seekOptions={{ forward: 10, backward: 5 }}
onReady={(player) => {
playerRef.current = player;
<DynamicVideoPlayer
className="w-2/3 bg-black flex justify-center items-center"
camera={initialPlayback.camera}
timeRange={selectedPlayback.range}
cameraPreviews={cameraPreviews}
onControllerReady={(controller) => {
controllerRef.current = controller;
controllerRef.current.onPlayerTimeUpdate((timestamp: number) => {
setTimelineTime(timestamp);
});
if (selectedPlayback.timelineItems.length > 0) {
player.currentTime(
selectedPlayback.timelineItems[0].timestamp -
selectedPlayback.range.start
);
} else {
player.currentTime(0);
}
player.on("playing", () => onSelectItem(undefined));
player.on("timeupdate", () => {
setPlayerTime(Math.floor(player.currentTime() || 0));
});
}}
onDispose={() => {
playerRef.current = undefined;
}}
>
{focusedItem && (
<TimelineEventOverlay
timeline={focusedItem}
cameraConfig={config.cameras[selectedPlayback.camera]}
/>
)}
</VideoPlayer>
</div>
{selectedPlayback.relevantPreview && (
<div className={`w-full ${scrubbing ? "visible" : "hidden"}`}>
<VideoPlayer
options={{
preload: "auto",
autoplay: false,
controls: false,
muted: true,
loadingSpinner: false,
sources: [
{
src: `${selectedPlayback.relevantPreview?.src}`,
type: "video/mp4",
},
],
}}
seekOptions={{}}
onReady={(player) => {
previewRef.current = player;
player.on("seeked", () => setSeeking(false));
}}
onDispose={() => {
previewRef.current = undefined;
}}
/>
</div>
)}
</div>
if (initialPlayback.timelineItems.length > 0) {
controllerRef.current?.seekToTimestamp(
selectedPlayback.timelineItems[0].timestamp,
true
);
}
}}
/>
</>
<div className="px-2 h-[608px] w-1/3 overflow-auto">
{selectedPlayback.timelineItems.map((timeline) => {
@ -344,7 +138,9 @@ export default function DesktopTimelineView({
key={timeline.timestamp}
timeline={timeline}
relevantPreview={selectedPlayback.relevantPreview}
onSelect={() => onSelectItem(timeline)}
onSelect={() => {
controllerRef.current?.seekToTimelineItem(timeline);
}}
/>
);
})}
@ -371,7 +167,10 @@ export default function DesktopTimelineView({
isSelected
? [
{
time: new Date(timelineTime * 1000),
time: new Date(
Math.max(timeline.range.start, timelineTime) *
1000
),
id: "playback",
},
]
@ -384,33 +183,16 @@ export default function DesktopTimelineView({
zoomable: false,
}}
timechangeHandler={(data) => {
if (!timeline.relevantPreview) {
playerRef.current?.pause();
return;
}
const seekTimestamp = data.time.getTime() / 1000;
const seekTime =
seekTimestamp - timeline.relevantPreview.start;
setPlayerTime(seekTimestamp - timeline.range.start);
setTimeToSeek(Math.round(seekTime));
controllerRef.current?.scrubToTimestamp(
data.time.getTime() / 1000
);
setTimelineTime(data.time.getTime() / 1000);
}}
timechangedHandler={(data) => {
setScrubbing(false);
const playbackTime =
data.time.getTime() / 1000 - timeline.range.start;
playerRef.current?.currentTime(playbackTime);
playerRef.current?.play();
}}
selectHandler={(data) => {
if (data.items.length > 0) {
const selected = data.items[0];
onSelectItem(
selectedPlayback.timelineItems.find(
(timeline) => timeline.timestamp == selected
)
);
}
controllerRef.current?.seekToTimestamp(
data.time.getTime() / 1000,
true
);
}}
/>
{isSelected && graphData && (
@ -435,8 +217,16 @@ export default function DesktopTimelineView({
startTime={timeline.range.start}
graphData={graphData}
onClick={() => {
setScrubbing(false);
setSelectedPlayback(timeline);
let startTs;
if (timeline.timelineItems.length > 0) {
startTs = selectedPlayback.timelineItems[0].timestamp;
} else {
startTs = timeline.range.start;
}
controllerRef.current?.seekToTimestamp(startTs, true);
}}
/>
)}

View File

@ -12,24 +12,24 @@ export default defineConfig({
server: {
proxy: {
'/api': {
target: 'http://localhost:5000',
target: 'http://192.168.50.106:5000',
ws: true,
},
'/vod': {
target: 'http://localhost:5000'
target: 'http://192.168.50.106:5000'
},
'/clips': {
target: 'http://localhost:5000'
target: 'http://192.168.50.106:5000'
},
'/exports': {
target: 'http://localhost:5000'
target: 'http://192.168.50.106:5000'
},
'/ws': {
target: 'ws://localhost:5000',
target: 'ws://192.168.50.106:5000',
ws: true,
},
'/live': {
target: 'ws://localhost:5000',
target: 'ws://192.168.50.106:5000',
changeOrigin: true,
ws: true,
},