diff --git a/web/src/components/player/DynamicVideoPlayer.tsx b/web/src/components/player/DynamicVideoPlayer.tsx new file mode 100644 index 000000000..20a83f959 --- /dev/null +++ b/web/src/components/player/DynamicVideoPlayer.tsx @@ -0,0 +1,351 @@ +import { MutableRefObject, useEffect, useMemo, useRef, useState } from "react"; +import VideoPlayer from "./VideoPlayer"; +import Player from "video.js/dist/types/player"; +import TimelineEventOverlay from "../overlay/TimelineDataOverlay"; +import { useApiHost } from "@/api"; +import useSWR from "swr"; +import { FrigateConfig } from "@/types/frigateConfig"; +import ActivityIndicator from "../ui/activity-indicator"; + +/** + * Dynamically switches between video playback and scrubbing preview player. + */ +type DynamicVideoPlayerProps = { + className?: string; + camera: string; + timeRange: { start: number; end: number }; + cameraPreviews: Preview[]; + onControllerReady?: (controller: DynamicVideoController) => void; +}; +export default function DynamicVideoPlayer({ + className, + camera, + timeRange, + cameraPreviews, + onControllerReady, +}: DynamicVideoPlayerProps) { + const apiHost = useApiHost(); + const { data: config } = useSWR("config"); + const timezone = useMemo( + () => + config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone, + [config] + ); + + // initial state + + const initialPlaybackSource = useMemo(() => { + const date = new Date(timeRange.start * 1000); + return { + src: `${apiHost}vod/${date.getFullYear()}-${ + date.getMonth() + 1 + }/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll( + "/", + "," + )}/master.m3u8`, + type: "application/vnd.apple.mpegurl", + }; + }, []); + const initialPreviewSource = useMemo(() => { + const source = cameraPreviews.find( + (preview) => + Math.round(preview.start) >= timeRange.start && + Math.floor(preview.end) <= timeRange.end + )?.src; + + if (source) { + return { + src: source, + type: "video/mp4", + }; + } else { + return undefined; + } + }, []); + + // controlling playback + + const playerRef = useRef(undefined); + const previewRef = useRef(undefined); + const [isScrubbing, setIsScrubbing] = useState(false); + const [focusedItem, setFocusedItem] = useState( + undefined + ); + const controller = useMemo(() => { + if (!config) { + return undefined; + } + + return new DynamicVideoController( + playerRef, + previewRef, + (config.cameras[camera]?.detect?.annotation_offset || 0) / 1000, + setIsScrubbing, + setFocusedItem + ); + }, [config]); + + // state of playback player + + const recordingParams = useMemo(() => { + return { + before: timeRange.end, + after: timeRange.start, + }; + }, [timeRange]); + const { data: recordings } = useSWR( + [`${camera}/recordings`, recordingParams], + { revalidateOnFocus: false } + ); + + useEffect(() => { + if (!controller || !recordings || recordings.length == 0) { + return; + } + + const date = new Date(timeRange.start * 1000); + const playbackUri = `${apiHost}vod/${date.getFullYear()}-${ + date.getMonth() + 1 + }/${date.getDate()}/${date.getHours()}/${camera}/${timezone.replaceAll( + "/", + "," + )}/master.m3u8`; + + controller.newPlayback({ + recordings, + playbackUri, + preview: cameraPreviews.find( + (preview) => + Math.round(preview.start) >= timeRange.start && + Math.floor(preview.end) <= timeRange.end + ), + }); + }, [controller, recordings]); + + const hasPreview = true; + + if (!controller) { + return ; + } + + return ( +
+
+ { + playerRef.current = player; + player.on("playing", () => setFocusedItem(undefined)); + player.on("timeupdate", () => { + controller.updateProgress(player.currentTime() || 0); + }); + }} + onDispose={() => { + playerRef.current = undefined; + }} + > + {config && focusedItem && ( + + )} + +
+
+ { + previewRef.current = player; + player.on("seeked", () => controller.finishedSeeking()); + + if (onControllerReady) { + onControllerReady(controller); + } + }} + onDispose={() => { + previewRef.current = undefined; + }} + /> +
+
+ ); +} + +export class DynamicVideoController { + // main state + private playerRef: MutableRefObject; + private previewRef: MutableRefObject; + private setScrubbing: (isScrubbing: boolean) => void; + private setFocusedItem: (timeline: Timeline) => void; + private playerMode: "playback" | "scrubbing" = "playback"; + + // playback + private recordings: Recording[] = []; + private onPlaybackTimestamp: ((time: number) => void) | undefined = undefined; + private annotationOffset: number; + private timeToStart: number | undefined = undefined; + + // preview + private preview: Preview | undefined = undefined; + private timeToSeek: number | undefined = undefined; + private seeking = false; + + constructor( + playerRef: MutableRefObject, + previewRef: MutableRefObject, + annotationOffset: number, + setScrubbing: (isScrubbing: boolean) => void, + setFocusedItem: (timeline: Timeline) => void + ) { + this.playerRef = playerRef; + this.previewRef = previewRef; + this.annotationOffset = annotationOffset; + this.setScrubbing = setScrubbing; + this.setFocusedItem = setFocusedItem; + } + + newPlayback(newPlayback: DynamicPlayback) { + this.recordings = newPlayback.recordings; + + this.playerRef.current?.src({ + src: newPlayback.playbackUri, + type: "application/vnd.apple.mpegurl", + }); + + if (this.timeToStart) { + this.seekToTimestamp(this.timeToStart); + this.timeToStart = undefined; + } + + this.preview = newPlayback.preview; + if (this.preview && this.previewRef.current) { + this.previewRef.current.src({ + src: this.preview.src, + type: this.preview.type, + }); + } + } + + seekToTimestamp(time: number, play: boolean = false) { + if (this.playerMode != "playback") { + this.playerMode = "playback"; + this.setScrubbing(false); + this.timeToSeek = undefined; + this.seeking = false; + } + + if (this.recordings.length == 0) { + this.timeToStart = time; + } + + let seekSeconds = 0; + (this.recordings || []).every((segment) => { + // if the next segment is past the desired time, stop calculating + if (segment.start_time > time) { + return false; + } + + if (segment.end_time < time) { + seekSeconds += segment.end_time - segment.start_time; + return true; + } + + seekSeconds += + segment.end_time - segment.start_time - (segment.end_time - time); + return true; + }); + this.playerRef.current?.currentTime(seekSeconds); + + if (play) { + this.playerRef.current?.play(); + } + } + + seekToTimelineItem(timeline: Timeline) { + this.playerRef.current?.pause(); + this.seekToTimestamp(timeline.timestamp + this.annotationOffset); + this.setFocusedItem(timeline); + } + + updateProgress(playerTime: number) { + if (this.onPlaybackTimestamp) { + // take a player time in seconds and convert to timestamp in timeline + let timestamp = 0; + let totalTime = 0; + (this.recordings || []).every((segment) => { + if (totalTime + segment.duration > playerTime) { + // segment is here + timestamp = segment.start_time + (playerTime - totalTime); + return false; + } else { + totalTime += segment.duration; + return true; + } + }); + + this.onPlaybackTimestamp(timestamp); + } + } + + onPlayerTimeUpdate(listener: (timestamp: number) => void) { + this.onPlaybackTimestamp = listener; + } + + scrubToTimestamp(time: number) { + if (this.playerMode != "scrubbing") { + this.playerMode = "scrubbing"; + this.playerRef.current?.pause(); + this.setScrubbing(true); + } + + if (this.preview) { + if (this.seeking) { + this.timeToSeek = time; + } else { + this.previewRef.current?.currentTime(time - this.preview.start); + this.seeking = true; + } + } + } + + finishedSeeking() { + if (!this.preview || this.playerMode == "playback") { + return; + } + + if ( + this.timeToSeek && + this.timeToSeek != this.previewRef.current?.currentTime() + ) { + this.previewRef.current?.currentTime( + this.timeToSeek - this.preview.start + ); + } else { + this.seeking = false; + } + } +} diff --git a/web/src/types/history.ts b/web/src/types/history.ts index 189a8d0b9..23e30c48c 100644 --- a/web/src/types/history.ts +++ b/web/src/types/history.ts @@ -21,38 +21,6 @@ type Preview = { end: number; }; -type Timeline = { - camera: string; - timestamp: number; - data: { - camera: string; - label: string; - sub_label: string; - box?: [number, number, number, number]; - region: [number, number, number, number]; - attribute: string; - zones: string[]; - }; - class_type: - | "visible" - | "gone" - | "entered_zone" - | "attribute" - | "active" - | "stationary" - | "heard" - | "external"; - source_id: string; - source: string; -}; - -type HourlyTimeline = { - start: number; - end: number; - count: number; - hours: { [key: string]: Timeline[] }; -}; - interface HistoryFilter extends FilterType { cameras: string[]; labels: string[]; diff --git a/web/src/types/playback.ts b/web/src/types/playback.ts new file mode 100644 index 000000000..07867b0f9 --- /dev/null +++ b/web/src/types/playback.ts @@ -0,0 +1,5 @@ +type DynamicPlayback = { + recordings: Recording[]; + playbackUri: string; + preview: Preview | undefined; +}; diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts new file mode 100644 index 000000000..3698a6ebe --- /dev/null +++ b/web/src/types/timeline.ts @@ -0,0 +1,31 @@ +type Timeline = { + camera: string; + timestamp: number; + data: { + camera: string; + label: string; + sub_label: string; + box?: [number, number, number, number]; + region: [number, number, number, number]; + attribute: string; + zones: string[]; + }; + class_type: + | "visible" + | "gone" + | "entered_zone" + | "attribute" + | "active" + | "stationary" + | "heard" + | "external"; + source_id: string; + source: string; + }; + + type HourlyTimeline = { + start: number; + end: number; + count: number; + hours: { [key: string]: Timeline[] }; + }; \ No newline at end of file diff --git a/web/src/utils/historyUtil.ts b/web/src/utils/historyUtil.ts index 858c2b1fc..14c6a8b34 100644 --- a/web/src/utils/historyUtil.ts +++ b/web/src/utils/historyUtil.ts @@ -117,7 +117,7 @@ export function getHourlyTimelineData( export function getTimelineHoursForDay( camera: string, cards: CardsData, - allPreviews: Preview[], + cameraPreviews: Preview[], timestamp: number ): HistoryTimeline { const endOfThisHour = new Date(); @@ -131,14 +131,6 @@ export function getTimelineHoursForDay( let start = startDay.getTime() / 1000; let end = 0; - const relevantPreviews = allPreviews.filter((preview) => { - return ( - preview.camera == camera && - preview.start >= start && - Math.floor(preview.end - 1) <= dayEnd - ); - }); - const dayIdx = Object.keys(cards).find((day) => { if (parseInt(day) > start) { return false; @@ -178,7 +170,7 @@ export function getTimelineHoursForDay( return []; }) : []; - const relevantPreview = relevantPreviews.find( + const relevantPreview = cameraPreviews.find( (preview) => Math.round(preview.start) >= start && Math.floor(preview.end) <= end ); diff --git a/web/src/views/history/DesktopTimelineView.tsx b/web/src/views/history/DesktopTimelineView.tsx index e2ca1674a..9b875a6a6 100644 --- a/web/src/views/history/DesktopTimelineView.tsx +++ b/web/src/views/history/DesktopTimelineView.tsx @@ -1,18 +1,17 @@ -import { useApiHost } from "@/api"; -import TimelineEventOverlay from "@/components/overlay/TimelineDataOverlay"; -import VideoPlayer from "@/components/player/VideoPlayer"; import ActivityScrubber from "@/components/scrubber/ActivityScrubber"; import ActivityIndicator from "@/components/ui/activity-indicator"; import { FrigateConfig } from "@/types/frigateConfig"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import useSWR from "swr"; -import Player from "video.js/dist/types/player"; import TimelineItemCard from "@/components/card/TimelineItemCard"; import { getTimelineHoursForDay } from "@/utils/historyUtil"; import { GraphDataPoint } from "@/types/graph"; import TimelineGraph from "@/components/graph/TimelineGraph"; import TimelineBar from "@/components/bar/TimelineBar"; +import DynamicVideoPlayer, { + DynamicVideoController, +} from "@/components/player/DynamicVideoPlayer"; type DesktopTimelineViewProps = { timelineData: CardsData; @@ -25,124 +24,19 @@ export default function DesktopTimelineView({ allPreviews, initialPlayback, }: DesktopTimelineViewProps) { - const apiHost = useApiHost(); const { data: config } = useSWR("config"); const timezone = useMemo( () => config?.ui?.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone, [config] ); + const controllerRef = useRef(undefined); const [selectedPlayback, setSelectedPlayback] = useState(initialPlayback); - const playerRef = useRef(undefined); - const previewRef = useRef(undefined); const initialScrollRef = useRef(null); - const [scrubbing, setScrubbing] = useState(false); - const [focusedItem, setFocusedItem] = useState( - undefined - ); - - const [seeking, setSeeking] = useState(false); - const [timeToSeek, setTimeToSeek] = useState(undefined); - const [playerTime, setPlayerTime] = useState( - initialPlayback.timelineItems.length > 0 - ? initialPlayback.timelineItems[0].timestamp - initialPlayback.range.start - : 0 - ); - - const annotationOffset = useMemo(() => { - if (!config) { - return 0; - } - - return ( - (config.cameras[initialPlayback.camera]?.detect?.annotation_offset || 0) / - 1000 - ); - }, [config]); - - const recordingParams = useMemo(() => { - return { - before: selectedPlayback.range.end, - after: selectedPlayback.range.start, - }; - }, [selectedPlayback]); - const { data: recordings } = useSWR( - selectedPlayback - ? [`${selectedPlayback.camera}/recordings`, recordingParams] - : null, - { revalidateOnFocus: false } - ); - - const playbackUri = useMemo(() => { - if (!selectedPlayback) { - return ""; - } - - const date = new Date(selectedPlayback.range.start * 1000); - return `${apiHost}vod/${date.getFullYear()}-${ - date.getMonth() + 1 - }/${date.getDate()}/${date.getHours()}/${ - selectedPlayback.camera - }/${timezone.replaceAll("/", ",")}/master.m3u8`; - }, [selectedPlayback]); - - const onSelectItem = useCallback( - (timeline: Timeline | undefined) => { - if (timeline) { - setFocusedItem(timeline); - const selected = timeline.timestamp; - playerRef.current?.pause(); - - let seekSeconds = 0; - (recordings || []).every((segment) => { - // if the next segment is past the desired time, stop calculating - if (segment.start_time > selected) { - return false; - } - - if (segment.end_time < selected) { - seekSeconds += segment.end_time - segment.start_time; - return true; - } - - seekSeconds += - segment.end_time - - segment.start_time - - (segment.end_time - selected); - return true; - }); - playerRef.current?.currentTime(seekSeconds); - } else { - setFocusedItem(undefined); - } - }, - [annotationOffset, recordings, playerRef] - ); - - const timelineTime = useMemo(() => { - if (scrubbing) { - return selectedPlayback.range.start + playerTime; - } else { - // take a player time in seconds and convert to timestamp in timeline - let timestamp = 0; - let totalTime = 0; - (recordings || []).every((segment) => { - if (totalTime + segment.duration > playerTime) { - // segment is here - timestamp = segment.start_time + (playerTime - totalTime); - return false; - } else { - totalTime += segment.duration; - return true; - } - }); - - return timestamp; - } - }, [playerTime]); + const [timelineTime, setTimelineTime] = useState(0); // handle scrolling to initial timeline item useEffect(() => { @@ -151,55 +45,18 @@ export default function DesktopTimelineView({ } }, [initialScrollRef]); - // handle seeking to next frame when seek is finished - useEffect(() => { - if (seeking) { - return; - } - - if (timeToSeek && !scrubbing) { - setScrubbing(true); - playerRef.current?.pause(); - } - - if (timeToSeek && timeToSeek != previewRef.current?.currentTime()) { - setSeeking(true); - previewRef.current?.currentTime(timeToSeek); - } - }, [timeToSeek, seeking]); - - // handle loading main / preview playback when selected hour changes - useEffect(() => { - if (!playerRef.current) { - return; - } - - setPlayerTime( - selectedPlayback.timelineItems.length > 0 - ? selectedPlayback.timelineItems[0].timestamp - - selectedPlayback.range.start - : 0 - ); - - playerRef.current.src({ - src: playbackUri, - type: "application/vnd.apple.mpegurl", + const cameraPreviews = useMemo(() => { + return allPreviews.filter((preview) => { + return preview.camera == initialPlayback.camera; }); - - if (selectedPlayback.relevantPreview && previewRef.current) { - previewRef.current.src({ - src: selectedPlayback.relevantPreview.src, - type: selectedPlayback.relevantPreview.type, - }); - } - }, [playerRef, previewRef, playbackUri]); + }, []); const timelineStack = useMemo( () => getTimelineHoursForDay( selectedPlayback.camera, timelineData, - allPreviews, + cameraPreviews, selectedPlayback.range.start + 60 ), [] @@ -254,88 +111,25 @@ export default function DesktopTimelineView({
<> -
-
- { - playerRef.current = player; + { + controllerRef.current = controller; + controllerRef.current.onPlayerTimeUpdate((timestamp: number) => { + setTimelineTime(timestamp); + }); - if (selectedPlayback.timelineItems.length > 0) { - player.currentTime( - selectedPlayback.timelineItems[0].timestamp - - selectedPlayback.range.start - ); - } else { - player.currentTime(0); - } - player.on("playing", () => onSelectItem(undefined)); - player.on("timeupdate", () => { - setPlayerTime(Math.floor(player.currentTime() || 0)); - }); - }} - onDispose={() => { - playerRef.current = undefined; - }} - > - {focusedItem && ( - - )} - -
- {selectedPlayback.relevantPreview && ( -
- { - previewRef.current = player; - player.on("seeked", () => setSeeking(false)); - }} - onDispose={() => { - previewRef.current = undefined; - }} - /> -
- )} -
+ if (initialPlayback.timelineItems.length > 0) { + controllerRef.current?.seekToTimestamp( + selectedPlayback.timelineItems[0].timestamp, + true + ); + } + }} + />
{selectedPlayback.timelineItems.map((timeline) => { @@ -344,7 +138,9 @@ export default function DesktopTimelineView({ key={timeline.timestamp} timeline={timeline} relevantPreview={selectedPlayback.relevantPreview} - onSelect={() => onSelectItem(timeline)} + onSelect={() => { + controllerRef.current?.seekToTimelineItem(timeline); + }} /> ); })} @@ -371,7 +167,10 @@ export default function DesktopTimelineView({ isSelected ? [ { - time: new Date(timelineTime * 1000), + time: new Date( + Math.max(timeline.range.start, timelineTime) * + 1000 + ), id: "playback", }, ] @@ -384,33 +183,16 @@ export default function DesktopTimelineView({ zoomable: false, }} timechangeHandler={(data) => { - if (!timeline.relevantPreview) { - playerRef.current?.pause(); - return; - } - - const seekTimestamp = data.time.getTime() / 1000; - const seekTime = - seekTimestamp - timeline.relevantPreview.start; - setPlayerTime(seekTimestamp - timeline.range.start); - setTimeToSeek(Math.round(seekTime)); + controllerRef.current?.scrubToTimestamp( + data.time.getTime() / 1000 + ); + setTimelineTime(data.time.getTime() / 1000); }} timechangedHandler={(data) => { - setScrubbing(false); - const playbackTime = - data.time.getTime() / 1000 - timeline.range.start; - playerRef.current?.currentTime(playbackTime); - playerRef.current?.play(); - }} - selectHandler={(data) => { - if (data.items.length > 0) { - const selected = data.items[0]; - onSelectItem( - selectedPlayback.timelineItems.find( - (timeline) => timeline.timestamp == selected - ) - ); - } + controllerRef.current?.seekToTimestamp( + data.time.getTime() / 1000, + true + ); }} /> {isSelected && graphData && ( @@ -435,8 +217,16 @@ export default function DesktopTimelineView({ startTime={timeline.range.start} graphData={graphData} onClick={() => { - setScrubbing(false); setSelectedPlayback(timeline); + + let startTs; + if (timeline.timelineItems.length > 0) { + startTs = selectedPlayback.timelineItems[0].timestamp; + } else { + startTs = timeline.range.start; + } + + controllerRef.current?.seekToTimestamp(startTs, true); }} /> )} diff --git a/web/vite.config.ts b/web/vite.config.ts index a97dbd014..5d5bf8207 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -12,24 +12,24 @@ export default defineConfig({ server: { proxy: { '/api': { - target: 'http://localhost:5000', + target: 'http://192.168.50.106:5000', ws: true, }, '/vod': { - target: 'http://localhost:5000' + target: 'http://192.168.50.106:5000' }, '/clips': { - target: 'http://localhost:5000' + target: 'http://192.168.50.106:5000' }, '/exports': { - target: 'http://localhost:5000' + target: 'http://192.168.50.106:5000' }, '/ws': { - target: 'ws://localhost:5000', + target: 'ws://192.168.50.106:5000', ws: true, }, '/live': { - target: 'ws://localhost:5000', + target: 'ws://192.168.50.106:5000', changeOrigin: true, ws: true, },