From f01bc1819605cb63d8ea98112b885777fe982c1f Mon Sep 17 00:00:00 2001
From: Nicolas Mowen <nickmowen213@gmail.com>
Date: Mon, 11 Mar 2024 13:56:44 -0600
Subject: [PATCH] Update to get preview player working

---
 frigate/output/preview.py                     |   8 +-
 .../components/player/DynamicVideoPlayer.tsx  |  14 +-
 web/src/components/player/PreviewPlayer.tsx   | 458 ++++++++++++++++++
 web/src/views/events/EventView.tsx            |  21 +-
 4 files changed, 477 insertions(+), 24 deletions(-)
 create mode 100644 web/src/components/player/PreviewPlayer.tsx

diff --git a/frigate/output/preview.py b/frigate/output/preview.py
index 370ac6cce..47b17b188 100644
--- a/frigate/output/preview.py
+++ b/frigate/output/preview.py
@@ -186,7 +186,7 @@ class PreviewRecorder:
                 os.unlink(os.path.join(PREVIEW_CACHE_DIR, file))
                 continue
 
-            ts = float(file.split("-")[1][:-4])
+            ts = float(file.split("-")[1][: -(len(PREVIEW_FRAME_TYPE) + 1)])
 
             if self.start_time == 0:
                 self.start_time = ts
@@ -242,7 +242,11 @@ class PreviewRecorder:
             small_frame,
             cv2.COLOR_YUV2BGR_I420,
         )
-        cv2.imwrite(get_cache_image_name(self.config.name, frame_time), small_frame, [int(cv2.IMWRITE_WEBP_QUALITY), 80])
+        cv2.imwrite(
+            get_cache_image_name(self.config.name, frame_time),
+            small_frame,
+            [int(cv2.IMWRITE_WEBP_QUALITY), 80],
+        )
 
     def write_data(
         self,
diff --git a/web/src/components/player/DynamicVideoPlayer.tsx b/web/src/components/player/DynamicVideoPlayer.tsx
index 77eb51ee8..c1bfbaddc 100644
--- a/web/src/components/player/DynamicVideoPlayer.tsx
+++ b/web/src/components/player/DynamicVideoPlayer.tsx
@@ -9,9 +9,7 @@ import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { Recording } from "@/types/record";
 import { Preview } from "@/types/preview";
 import { DynamicPlayback } from "@/types/playback";
-import PreviewVideoPlayer, {
-  PreviewVideoController,
-} from "./PreviewVideoPlayer";
+import PreviewPlayer, { PreviewController } from "./PreviewPlayer";
 
 type PlayerMode = "playback" | "scrubbing";
 
@@ -63,7 +61,7 @@ export default function DynamicVideoPlayer({
 
   const [playerRef, setPlayerRef] = useState<Player | null>(null);
   const [previewController, setPreviewController] =
-    useState<PreviewVideoController | null>(null);
+    useState<PreviewController | null>(null);
   const [isScrubbing, setIsScrubbing] = useState(previewOnly);
   const [focusedItem, setFocusedItem] = useState<Timeline | undefined>(
     undefined,
@@ -280,7 +278,7 @@ export default function DynamicVideoPlayer({
           )}
         </VideoPlayer>
       </div>
-      <PreviewVideoPlayer
+      <PreviewPlayer
         className={`${isScrubbing ? "visible" : "hidden"} ${className ?? ""}`}
         camera={camera}
         timeRange={timeRange}
@@ -298,7 +296,7 @@ export class DynamicVideoController {
   // main state
   public camera = "";
   private playerController: Player;
-  private previewController: PreviewVideoController;
+  private previewController: PreviewController;
   private setScrubbing: (isScrubbing: boolean) => void;
   private setFocusedItem: (timeline: Timeline) => void;
   private playerMode: PlayerMode = "playback";
@@ -315,7 +313,7 @@ export class DynamicVideoController {
   constructor(
     camera: string,
     playerController: Player,
-    previewController: PreviewVideoController,
+    previewController: PreviewController,
     annotationOffset: number,
     defaultMode: PlayerMode,
     setScrubbing: (isScrubbing: boolean) => void,
@@ -438,7 +436,7 @@ export class DynamicVideoController {
     const scrubResult = this.previewController.scrubToTimestamp(time);
 
     if (!scrubResult && saveIfNotReady) {
-      this.previewController.setNewPreviewStartTime(time);
+      //this.previewController.setNewPreviewStartTime(time);
     }
 
     if (scrubResult && this.playerMode != "scrubbing") {
diff --git a/web/src/components/player/PreviewPlayer.tsx b/web/src/components/player/PreviewPlayer.tsx
new file mode 100644
index 000000000..34ee725fb
--- /dev/null
+++ b/web/src/components/player/PreviewPlayer.tsx
@@ -0,0 +1,458 @@
+import {
+  MutableRefObject,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+import useSWR from "swr";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { Preview } from "@/types/preview";
+import { PreviewPlayback } from "@/types/playback";
+import { isCurrentHour } from "@/utils/dateUtil";
+import { baseUrl } from "@/api/baseUrl";
+
+type PreviewPlayerProps = {
+  className?: string;
+  camera: string;
+  timeRange: { start: number; end: number };
+  cameraPreviews: Preview[];
+  startTime?: number;
+  onControllerReady: (controller: PreviewController) => void;
+  onClick?: () => void;
+};
+export default function PreviewPlayer({
+  className,
+  camera,
+  timeRange,
+  cameraPreviews,
+  startTime,
+  onControllerReady,
+  onClick,
+}: PreviewPlayerProps) {
+  if (isCurrentHour(timeRange.end)) {
+    return (
+      <PreviewFramesPlayer
+        className={className}
+        camera={camera}
+        timeRange={timeRange}
+        startTime={startTime}
+        onControllerReady={onControllerReady}
+        onClick={onClick}
+      />
+    );
+  }
+
+  return (
+    <PreviewVideoPlayer
+      className={className}
+      camera={camera}
+      timeRange={timeRange}
+      cameraPreviews={cameraPreviews}
+      startTime={startTime}
+      onControllerReady={onControllerReady}
+      onClick={onClick}
+    />
+  );
+}
+
+export abstract class PreviewController {
+  public camera = "";
+
+  constructor(camera: string) {
+    this.camera = camera;
+  }
+
+  abstract scrubToTimestamp(time: number): boolean;
+
+  abstract finishedSeeking(): void;
+
+  abstract setNewPreviewStartTime(time: number): void;
+}
+
+type PreviewVideoPlayerProps = {
+  className?: string;
+  camera: string;
+  timeRange: { start: number; end: number };
+  cameraPreviews: Preview[];
+  startTime?: number;
+  onControllerReady: (controller: PreviewVideoController) => void;
+  onClick?: () => void;
+};
+function PreviewVideoPlayer({
+  className,
+  camera,
+  timeRange,
+  cameraPreviews,
+  startTime,
+  onControllerReady,
+  onClick,
+}: PreviewVideoPlayerProps) {
+  const { data: config } = useSWR<FrigateConfig>("config");
+
+  // controlling playback
+
+  const previewRef = useRef<HTMLVideoElement | null>(null);
+  const controller = useMemo(() => {
+    if (!config || !previewRef.current) {
+      return undefined;
+    }
+
+    return new PreviewVideoController(camera, previewRef);
+    // we only care when preview is ready
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [camera, config, previewRef.current]);
+
+  useEffect(() => {
+    if (!controller) {
+      return;
+    }
+
+    if (controller) {
+      onControllerReady(controller);
+    }
+    // we only want to fire once when players are ready
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [controller]);
+
+  // initial state
+
+  const initialPreview = useMemo(() => {
+    return cameraPreviews.find(
+      (preview) =>
+        preview.camera == camera &&
+        Math.round(preview.start) >= timeRange.start &&
+        Math.floor(preview.end) <= timeRange.end,
+    );
+
+    // we only want to calculate this once
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const [currentPreview, setCurrentPreview] = useState(initialPreview);
+
+  const onPreviewSeeked = useCallback(() => {
+    if (!controller) {
+      return;
+    }
+
+    controller.finishedSeeking();
+  }, [controller]);
+
+  useEffect(() => {
+    if (!controller) {
+      return;
+    }
+
+    const preview = cameraPreviews.find(
+      (preview) =>
+        preview.camera == camera &&
+        Math.round(preview.start) >= timeRange.start &&
+        Math.floor(preview.end) <= timeRange.end,
+    );
+    setCurrentPreview(preview);
+
+    controller.newPlayback({
+      preview,
+      timeRange,
+    });
+
+    // we only want this to change when recordings update
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [controller, timeRange]);
+
+  useEffect(() => {
+    if (!currentPreview || !previewRef.current) {
+      return;
+    }
+
+    previewRef.current.load();
+  }, [currentPreview, previewRef]);
+
+  return (
+    <div
+      className={`relative w-full ${className ?? ""} ${onClick ? "cursor-pointer" : ""}`}
+      onClick={onClick}
+    >
+      <video
+        ref={previewRef}
+        className={`size-full rounded-2xl bg-black`}
+        preload="auto"
+        autoPlay
+        playsInline
+        muted
+        disableRemotePlayback
+        onSeeked={onPreviewSeeked}
+        onLoadedData={() => {
+          if (controller) {
+            controller.previewReady();
+          } else {
+            previewRef.current?.pause();
+          }
+
+          if (previewRef.current && startTime && currentPreview) {
+            previewRef.current.currentTime = startTime - currentPreview.start;
+          }
+        }}
+      >
+        {currentPreview != undefined && (
+          <source src={currentPreview.src} type={currentPreview.type} />
+        )}
+      </video>
+      {cameraPreviews && !currentPreview && (
+        <div className="absolute inset-x-0 top-1/2 -y-translate-1/2 bg-black text-white rounded-2xl align-center text-center">
+          No Preview Found
+        </div>
+      )}
+    </div>
+  );
+}
+
+class PreviewVideoController extends PreviewController {
+  // main state
+  private previewRef: MutableRefObject<HTMLVideoElement | null>;
+  private timeRange: { start: number; end: number } | undefined = undefined;
+
+  // preview
+  private preview: Preview | undefined = undefined;
+  private timeToSeek: number | undefined = undefined;
+  private seeking = false;
+
+  constructor(
+    camera: string,
+    previewRef: MutableRefObject<HTMLVideoElement | null>,
+  ) {
+    super(camera);
+    this.previewRef = previewRef;
+  }
+
+  newPlayback(newPlayback: PreviewPlayback) {
+    this.preview = newPlayback.preview;
+    this.seeking = false;
+
+    this.timeRange = newPlayback.timeRange;
+  }
+
+  override scrubToTimestamp(time: number): boolean {
+    if (!this.preview || !this.timeRange) {
+      return false;
+    }
+
+    if (time < this.preview.start || time > this.preview.end) {
+      return false;
+    }
+
+    if (this.seeking) {
+      this.timeToSeek = time;
+    } else {
+      if (this.previewRef.current) {
+        this.previewRef.current.currentTime = Math.max(
+          0,
+          time - this.preview.start,
+        );
+        this.seeking = true;
+      }
+    }
+
+    return true;
+  }
+
+  override finishedSeeking() {
+    if (!this.previewRef.current || !this.preview) {
+      return;
+    }
+
+    if (
+      this.timeToSeek &&
+      this.timeToSeek != this.previewRef.current?.currentTime
+    ) {
+      this.previewRef.current.currentTime =
+        this.timeToSeek - this.preview.start;
+    } else {
+      this.seeking = false;
+    }
+  }
+
+  override setNewPreviewStartTime(time: number) {
+    this.timeToSeek = time;
+  }
+
+  previewReady() {
+    this.seeking = false;
+    this.previewRef.current?.pause();
+
+    if (this.timeToSeek) {
+      this.finishedSeeking();
+    }
+  }
+}
+
+type PreviewFramesPlayerProps = {
+  className?: string;
+  camera: string;
+  timeRange: { start: number; end: number };
+  startTime?: number;
+  onControllerReady: (controller: PreviewController) => void;
+  onClick?: () => void;
+};
+function PreviewFramesPlayer({
+  className,
+  camera,
+  timeRange,
+  startTime,
+  onControllerReady,
+  onClick,
+}: PreviewFramesPlayerProps) {
+  // frames data
+
+  const { data: previewFrames } = useSWR<string[]>(
+    `preview/${camera}/start/${Math.floor(timeRange.start)}/end/${Math.ceil(
+      timeRange.end,
+    )}/frames`,
+    { revalidateOnFocus: false },
+  );
+  const frameTimes = useMemo(() => {
+    if (!previewFrames) {
+      return undefined;
+    }
+
+    return previewFrames.map((frame) =>
+      parseFloat(frame.split("-")[1].slice(undefined, -5)),
+    );
+  }, [previewFrames]);
+
+  // controlling frames
+
+  const imgRef = useRef<HTMLImageElement | null>(null);
+  const controller = useMemo(() => {
+    if (!frameTimes || !imgRef.current) {
+      return undefined;
+    }
+
+    return new PreviewFramesController(camera, imgRef, frameTimes);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [imgRef, frameTimes, imgRef.current]);
+
+  // initial state
+
+  useEffect(() => {
+    if (!controller) {
+      return;
+    }
+
+    if (controller) {
+      onControllerReady(controller);
+    }
+
+    // we only want to fire once when players are ready
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [controller]);
+
+  const onImageLoaded = useCallback(() => {
+    if (!controller) {
+      return;
+    }
+
+    controller.finishedSeeking();
+  }, [controller]);
+
+  useEffect(() => {
+    if (!controller) {
+      return;
+    }
+
+    if (!startTime) {
+      controller.scrubToTimestamp(timeRange.start);
+    } else {
+      controller.scrubToTimestamp(startTime);
+    }
+    // we only want to calculate this once
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [controller]);
+
+  return (
+    <div
+      className={`relative w-full ${className ?? ""} ${onClick ? "cursor-pointer" : ""}`}
+      onClick={onClick}
+    >
+      <img
+        ref={imgRef}
+        className={`size-full object-contain rounded-2xl bg-black`}
+        onLoad={onImageLoaded}
+      />
+      {previewFrames?.length === 0 && (
+        <div className="absolute inset-x-0 top-1/2 -y-translate-1/2 bg-black text-white rounded-2xl align-center text-center">
+          No Preview Found
+        </div>
+      )}
+    </div>
+  );
+}
+
+class PreviewFramesController extends PreviewController {
+  imgController: MutableRefObject<HTMLImageElement | null>;
+  frameTimes: number[];
+  seeking: boolean = false;
+  private timeToSeek: number | undefined = undefined;
+
+  constructor(
+    camera: string,
+    imgController: MutableRefObject<HTMLImageElement | null>,
+    frameTimes: number[],
+  ) {
+    super(camera);
+    this.imgController = imgController;
+    this.frameTimes = frameTimes;
+  }
+
+  override scrubToTimestamp(time: number): boolean {
+    if (!this.imgController.current) {
+      return false;
+    }
+
+    const frame = this.frameTimes.find((p) => {
+      return time < p;
+    });
+
+    if (!frame) {
+      return false;
+    }
+
+    if (this.seeking) {
+      this.timeToSeek = frame;
+    } else {
+      const newSrc = `${baseUrl}api/preview/preview_${this.camera}-${frame}.webp/thumbnail.webp`;
+
+      if (this.imgController.current.src != newSrc) {
+        this.imgController.current.src = newSrc;
+        this.seeking = true;
+      }
+    }
+
+    return true;
+  }
+
+  override finishedSeeking() {
+    if (!this.imgController.current) {
+      return false;
+    }
+
+    if (this.timeToSeek) {
+      const newSrc = `${baseUrl}api/preview/preview_${this.camera}-${this.timeToSeek}.webp/thumbnail.webp`;
+
+      if (this.imgController.current.src != newSrc) {
+        this.imgController.current.src = newSrc;
+      } else {
+        this.timeToSeek = undefined;
+        this.seeking = false;
+      }
+    } else {
+      this.seeking = false;
+    }
+  }
+
+  override setNewPreviewStartTime(time: number) {
+    this.timeToSeek = time;
+  }
+}
diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx
index b78a45703..c65cd6965 100644
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@@ -33,9 +33,9 @@ import { MdCircle } from "react-icons/md";
 import useSWR from "swr";
 import MotionReviewTimeline from "@/components/timeline/MotionReviewTimeline";
 import { Button } from "@/components/ui/button";
-import PreviewVideoPlayer, {
-  PreviewVideoController,
-} from "@/components/player/PreviewVideoPlayer";
+import PreviewPlayer, {
+  PreviewController,
+} from "@/components/player/PreviewPlayer";
 
 type EventViewProps = {
   reviews?: ReviewSegment[];
@@ -578,9 +578,7 @@ function MotionReview({
     return cameras.sort((a, b) => a.ui.order - b.ui.order);
   }, [config, filter]);
 
-  const videoPlayersRef = useRef<{ [camera: string]: PreviewVideoController }>(
-    {},
-  );
+  const videoPlayersRef = useRef<{ [camera: string]: PreviewController }>({});
 
   // motion data
 
@@ -596,14 +594,9 @@ function MotionReview({
 
   // timeline time
 
-  const lastFullHour = useMemo(() => {
-    const end = new Date(timeRange.before * 1000);
-    end.setMinutes(0, 0, 0);
-    return end.getTime() / 1000;
-  }, [timeRange]);
   const timeRangeSegments = useMemo(
-    () => getChunkedTimeRange(timeRange.after, lastFullHour),
-    [lastFullHour, timeRange],
+    () => getChunkedTimeRange(timeRange.after, timeRange.before),
+    [timeRange],
   );
 
   const initialIndex = useMemo(() => {
@@ -674,7 +667,7 @@ function MotionReview({
               grow = "aspect-video";
             }
             return (
-              <PreviewVideoPlayer
+              <PreviewPlayer
                 key={camera.name}
                 className={`${grow}`}
                 camera={camera.name}