From 241d53eca6fd4ae0ad94c1b876ace20a46244147 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 21 Feb 2024 07:46:23 -0700 Subject: [PATCH] autoplay first video on mobile --- web/package-lock.json | 35 +++++ web/package.json | 1 + .../player/PreviewThumbnailPlayer.tsx | 129 ++++-------------- web/src/pages/Events.tsx | 6 +- web/src/pages/Live.tsx | 5 +- web/src/utils/browserUtil.ts | 7 - web/src/views/events/DesktopEventView.tsx | 3 +- web/src/views/events/MobileEventView.tsx | 70 +++++++++- 8 files changed, 131 insertions(+), 125 deletions(-) delete mode 100644 web/src/utils/browserUtil.ts diff --git a/web/package-lock.json b/web/package-lock.json index c85bb8743..aa997a927 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -40,6 +40,7 @@ "react": "^18.2.0", "react-apexcharts": "^1.4.1", "react-day-picker": "^8.9.1", + "react-device-detect": "^2.2.3", "react-dom": "^18.2.0", "react-hook-form": "^7.48.2", "react-icons": "^4.12.0", @@ -6804,6 +6805,18 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/react-device-detect": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-device-detect/-/react-device-detect-2.2.3.tgz", + "integrity": "sha512-buYY3qrCnQVlIFHrC5UcUoAj7iANs/+srdkwsnNjI7anr3Tt7UY6MqNxtMLlr0tMBied0O49UZVK8XKs3ZIiPw==", + "dependencies": { + "ua-parser-js": "^1.0.33" + }, + "peerDependencies": { + "react": ">= 0.14.0", + "react-dom": ">= 0.14.0" + } + }, "node_modules/react-dom": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", @@ -8118,6 +8131,28 @@ "node": ">=14.17" } }, + "node_modules/ua-parser-js": { + "version": "1.0.37", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.37.tgz", + "integrity": "sha512-bhTyI94tZofjo+Dn8SN6Zv8nBDvyXTymAdM3LDI/0IboIUwTu1rEhW7v2TfiVsoYWgkQ4kOVqnI8APUFbIQIFQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + }, + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + } + ], + "engines": { + "node": "*" + } + }, "node_modules/ufo": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.3.2.tgz", diff --git a/web/package.json b/web/package.json index ef6b9eb13..61ef26429 100644 --- a/web/package.json +++ b/web/package.json @@ -45,6 +45,7 @@ "react": "^18.2.0", "react-apexcharts": "^1.4.1", "react-day-picker": "^8.9.1", + "react-device-detect": "^2.2.3", "react-dom": "^18.2.0", "react-hook-form": "^7.48.2", "react-icons": "^4.12.0", diff --git a/web/src/components/player/PreviewThumbnailPlayer.tsx b/web/src/components/player/PreviewThumbnailPlayer.tsx index 844abd366..6011e1ee3 100644 --- a/web/src/components/player/PreviewThumbnailPlayer.tsx +++ b/web/src/components/player/PreviewThumbnailPlayer.tsx @@ -1,22 +1,21 @@ import VideoPlayer from "./VideoPlayer"; -import React, { useCallback, useEffect, useRef, useState } from "react"; +import React, { useCallback, useMemo, useRef, useState } from "react"; import { useApiHost } from "@/api"; import Player from "video.js/dist/types/player"; import { formatUnixTimestampToDateTime } from "@/utils/dateUtil"; -import { isSafari } from "@/utils/browserUtil"; import { ReviewSegment } from "@/types/review"; import { Slider } from "../ui/slider"; import { getIconForLabel, getIconForSubLabel } from "@/utils/iconUtil"; import TimeAgo from "../dynamic/TimeAgo"; import useSWR from "swr"; import { FrigateConfig } from "@/types/frigateConfig"; +import { isMobile, isSafari } from "react-device-detect"; type PreviewPlayerProps = { review: ReviewSegment; relevantPreview?: Preview; - isMobile: boolean; + autoPlayback?: boolean; setReviewed?: () => void; - onClick?: () => void; }; type Preview = { @@ -30,19 +29,22 @@ type Preview = { export default function PreviewThumbnailPlayer({ review, relevantPreview, - isMobile, + autoPlayback = false, setReviewed, - onClick, }: PreviewPlayerProps) { const apiHost = useApiHost(); const { data: config } = useSWR("config"); const playerRef = useRef(null); - const [visible, setVisible] = useState(false); const [hoverTimeout, setHoverTimeout] = useState(); const [hover, setHover] = useState(false); const [progress, setProgress] = useState(0); + const playingBack = useMemo( + () => relevantPreview && (hover || autoPlayback), + [hover, autoPlayback, relevantPreview] + ); + const onPlayback = useCallback( (isHovered: Boolean) => { if (!relevantPreview) { @@ -75,72 +77,20 @@ export default function PreviewThumbnailPlayer({ [hoverTimeout, relevantPreview, review, playerRef] ); - const autoPlayObserver = useRef(); - const preloadObserver = useRef(); - const inViewRef = useCallback( - (node: HTMLElement | null) => { - if (!preloadObserver.current) { - try { - preloadObserver.current = new IntersectionObserver( - (entries) => { - const [{ isIntersecting }] = entries; - setVisible(isIntersecting); - }, - { - threshold: 0, - root: document.getElementById("pageRoot"), - rootMargin: "10% 0px 25% 0px", - } - ); - if (node) preloadObserver.current.observe(node); - } catch (e) { - // no op - } - } - - if (isMobile && !autoPlayObserver.current) { - try { - autoPlayObserver.current = new IntersectionObserver( - (entries) => { - const [{ isIntersecting }] = entries; - if (isIntersecting) { - onPlayback(true); - } else { - onPlayback(false); - } - }, - { - threshold: 1.0, - root: document.getElementById("pageRoot"), - rootMargin: "-10% 0px -25% 0px", - } - ); - if (node) autoPlayObserver.current.observe(node); - } catch (e) { - // no op - } - } - }, - [preloadObserver, autoPlayObserver, onPlayback] - ); - return (
onPlayback(true)} - onMouseLeave={() => onPlayback(false)} + onMouseEnter={isMobile ? undefined : () => onPlayback(true)} + onMouseLeave={isMobile ? undefined : () => onPlayback(false)} > - {hover ? ( + {playingBack ? ( ) : ( )} - {!hover && + {!playingBack && (review.severity == "alert" || review.severity == "detection") && (
{review.data.objects.map((object) => { @@ -162,7 +112,7 @@ export default function PreviewThumbnailPlayer({ })}
)} - {!hover && ( + {!playingBack && (
{config && @@ -176,7 +126,7 @@ export default function PreviewThumbnailPlayer({ )}
- {hover && ( + {playingBack && ( )} - {!hover && review.has_been_reviewed && ( + {!playingBack && review.has_been_reviewed && (
)}
@@ -196,49 +146,19 @@ type PreviewContentProps = { playerRef: React.MutableRefObject; review: ReviewSegment; relevantPreview: Preview | undefined; - isVisible: boolean; - isMobile: boolean; + playback: boolean; setProgress?: (progress: number) => void; setReviewed?: () => void; - onClick?: () => void; }; function PreviewContent({ playerRef, review, relevantPreview, - isVisible, - isMobile, + playback, setProgress, setReviewed, - onClick, }: PreviewContentProps) { - const slowPlayack = isSafari(); - - // handle touchstart -> touchend as click - const [touchStart, setTouchStart] = useState(0); - const handleTouchStart = useCallback(() => { - setTouchStart(new Date().getTime()); - }, []); - useEffect(() => { - if (!isMobile || !playerRef.current || !onClick) { - return; - } - - playerRef.current.on("touchend", () => { - if (!onClick) { - return; - } - - const touchEnd = new Date().getTime(); - - // consider tap less than 100 ms - if (touchEnd - touchStart < 100) { - onClick(); - } - }); - }, [playerRef, touchStart]); - - if (relevantPreview && isVisible) { + if (relevantPreview && playback) { return ( { if (!setProgress || playerRef.current?.paused()) { @@ -281,7 +201,9 @@ function PreviewContent({ const playerProgress = (player.currentTime() || 0) - playerStartTime; - const playerDuration = review.end_time - review.start_time; + + // end with a bit of padding + const playerDuration = (review.end_time - review.start_time) + 8; const playerPercent = (playerProgress / playerDuration) * 100; if ( @@ -299,9 +221,6 @@ function PreviewContent({ setProgress(playerPercent); } }); - if (isMobile && onClick) { - player.on("touchstart", handleTouchStart); - } }} onDispose={() => { playerRef.current = null; diff --git a/web/src/pages/Events.tsx b/web/src/pages/Events.tsx index dd8a3a52b..0a465afe8 100644 --- a/web/src/pages/Events.tsx +++ b/web/src/pages/Events.tsx @@ -1,12 +1,8 @@ import DesktopEventView from "@/views/events/DesktopEventView"; import MobileEventView from "@/views/events/MobileEventView"; -import { useMemo } from "react"; +import { isMobile } from 'react-device-detect'; export default function Events() { - const isMobile = useMemo(() => { - return window.innerWidth < 768; - }, []); - if (isMobile) { return ; } diff --git a/web/src/pages/Live.tsx b/web/src/pages/Live.tsx index 7042d48fd..d7d98c68c 100644 --- a/web/src/pages/Live.tsx +++ b/web/src/pages/Live.tsx @@ -5,8 +5,8 @@ import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; import { TooltipProvider } from "@/components/ui/tooltip"; import { Event as FrigateEvent } from "@/types/event"; import { FrigateConfig } from "@/types/frigateConfig"; -import { isSafari } from "@/utils/browserUtil"; import { useCallback, useEffect, useMemo, useState } from "react"; +import { isSafari } from "react-device-detect"; import useSWR from "swr"; function Live() { @@ -65,7 +65,6 @@ function Live() { .sort((aConf, bConf) => aConf.ui.order - bConf.ui.order); }, [config]); - const safari = isSafari(); const [windowVisible, setWindowVisible] = useState(true); const visibilityListener = useCallback(() => { setWindowVisible(document.visibilityState == "visible"); @@ -111,7 +110,7 @@ function Live() { className={`mb-2 md:mb-0 rounded-2xl bg-black ${grow}`} windowVisible={windowVisible} cameraConfig={camera} - preferredLiveMode={safari ? "webrtc" : "mse"} + preferredLiveMode={isSafari ? "webrtc" : "mse"} /> ); })} diff --git a/web/src/utils/browserUtil.ts b/web/src/utils/browserUtil.ts deleted file mode 100644 index ca788309d..000000000 --- a/web/src/utils/browserUtil.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { useMemo } from "react"; - -export function isSafari() { - return useMemo(() => { - return /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - }, []); -} diff --git a/web/src/views/events/DesktopEventView.tsx b/web/src/views/events/DesktopEventView.tsx index 213910783..6b8c61dca 100644 --- a/web/src/views/events/DesktopEventView.tsx +++ b/web/src/views/events/DesktopEventView.tsx @@ -148,7 +148,7 @@ export default function DesktopEventView() { setMinimap([...visibleTimestamps]); }); }, - { root: contentRef.current } + { root: contentRef.current, threshold: 0.5 } ); return () => { @@ -305,7 +305,6 @@ export default function DesktopEventView() { setReviewed(value.id)} />
diff --git a/web/src/views/events/MobileEventView.tsx b/web/src/views/events/MobileEventView.tsx index 70ba0635e..e4fd1f29b 100644 --- a/web/src/views/events/MobileEventView.tsx +++ b/web/src/views/events/MobileEventView.tsx @@ -4,7 +4,7 @@ import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; import { FrigateConfig } from "@/types/frigateConfig"; import { ReviewSegment, ReviewSeverity } from "@/types/review"; import axios from "axios"; -import { useCallback, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { MdCircle } from "react-icons/md"; import useSWR from "swr"; import useSWRInfinite from "swr/infinite"; @@ -112,6 +112,70 @@ export default function MobileEventView() { [isValidating, isDone] ); + const [minimap, setMinimap] = useState([]); + const minimapObserver = useRef(); + useEffect(() => { + if (!contentRef.current) { + return; + } + + const visibleTimestamps = new Set(); + minimapObserver.current = new IntersectionObserver( + (entries) => { + entries.forEach((entry) => { + const start = (entry.target as HTMLElement).dataset.start; + console.log(`${start} has been updated as intersect ${entry.isIntersecting}`) + + if (!start) { + return; + } + + if (entry.isIntersecting) { + visibleTimestamps.add(start); + } else { + visibleTimestamps.delete(start); + } + + setMinimap([...visibleTimestamps]); + }); + }, { threshold: 0.5 } + ); + + return () => { + minimapObserver.current?.disconnect(); + }; + }, [contentRef]); + const minimapRef = useCallback( + (node: HTMLElement | null) => { + if (!minimapObserver.current) { + return; + } + + try { + if (node) minimapObserver.current.observe(node); + } catch (e) { + // no op + } + }, + [minimapObserver.current] + ); + const minimapBounds = useMemo(() => { + const data = { + start: Math.floor(Date.now() / 1000) - 35 * 60, + end: Math.floor(Date.now() / 1000) - 21 * 60, + }; + const list = minimap.sort(); + + if (list.length > 0) { + data.end = parseFloat(list.at(-1)!!); + data.start = parseFloat(list[0]); + } + + console.log("the new times are " + JSON.stringify(data)) + + return data; + }, [minimap]); + // review status const setReviewed = useCallback( @@ -213,14 +277,14 @@ export default function MobileEventView() { return (
setReviewed(value.id)} />