fix scrolling and use chunked log download

frigate/api/app.py

@@ -8,7 +8,7 @@ import os
 import traceback
 from datetime import datetime, timedelta
 from functools import reduce
-from typing import Any, Optional
+from typing import Any, List, Optional, Tuple
 
 import aiofiles
 import requests
@@ -399,7 +399,7 @@ def process_logs(
     service: Optional[str] = None,
     start: Optional[int] = None,
     end: Optional[int] = None,
-) -> list:
+) -> Tuple[int, List[str]]:
     log_lines = []
     key_length = 0
     date_end = 0
@@ -436,7 +436,7 @@ def process_logs(
             current_line = clean_line
 
     log_lines.append(current_line)
-    return log_lines[start:end]
+    return len(log_lines), log_lines[start:end]
 
 
 @router.get("/logs/{service}", tags=[Tags.logs])
@@ -474,7 +474,7 @@ async def logs(
                         buffer += line
                         # Process logs only when there are enough lines in the buffer
                         if "\n" in buffer:
-                            processed_lines = process_logs(buffer, service)
+                            _, processed_lines = process_logs(buffer, service)
                             buffer = ""
                             for processed_line in processed_lines:
                                 yield f"{processed_line}\n"
@@ -507,9 +507,9 @@ async def logs(
         async with aiofiles.open(service_location, "r") as file:
             contents = await file.read()
 
-        log_lines = process_logs(contents, service, start, end)
+        total_lines, log_lines = process_logs(contents, service, start, end)
         return JSONResponse(
-            content={"totalLines": len(log_lines), "lines": log_lines},
+            content={"totalLines": total_lines, "lines": log_lines},
             status_code=200,
         )
     except FileNotFoundError as e:

web/src/components/dynamic/EnhancedScrollFollow.tsx

@@ -1,9 +1,14 @@
-import { useRef, useCallback, ReactNode } from "react";
+import { useRef, useCallback, useEffect, type ReactNode } from "react";
 import { ScrollFollow } from "@melloware/react-logviewer";
 
 export type ScrollFollowProps = {
   startFollowing?: boolean;
   render: (renderProps: ScrollFollowRenderProps) => ReactNode;
+  onCustomScroll?: (
+    scrollTop: number,
+    scrollHeight: number,
+    clientHeight: number,
+  ) => void;
 };
 
 export type ScrollFollowRenderProps = {
@@ -15,12 +20,22 @@ export type ScrollFollowRenderProps = {
   }) => void;
   startFollowing: () => void;
   stopFollowing: () => void;
+  onCustomScroll?: (
+    scrollTop: number,
+    scrollHeight: number,
+    clientHeight: number,
+  ) => void;
 };
 
-const SCROLL_BUFFER = 5; // Additional buffer for scroll checks
+const SCROLL_BUFFER = 5;
 
 export default function EnhancedScrollFollow(props: ScrollFollowProps) {
   const followRef = useRef(props.startFollowing || false);
+  const prevScrollTopRef = useRef<number | undefined>(undefined);
+
+  useEffect(() => {
+    prevScrollTopRef.current = undefined;
+  }, []);
 
   const wrappedRender = useCallback(
     (renderProps: ScrollFollowRenderProps) => {
@@ -29,6 +44,17 @@ export default function EnhancedScrollFollow(props: ScrollFollowProps) {
         scrollHeight: number;
         clientHeight: number;
       }) => {
+        // Check if scrolling up and immediately stop following
+        if (
+          prevScrollTopRef.current !== undefined &&
+          args.scrollTop < prevScrollTopRef.current
+        ) {
+          if (followRef.current) {
+            renderProps.stopFollowing();
+            followRef.current = false;
+          }
+        }
+
         const bottomThreshold =
           args.scrollHeight - args.clientHeight - SCROLL_BUFFER;
         const isNearBottom = args.scrollTop >= bottomThreshold;
@@ -41,24 +67,20 @@ export default function EnhancedScrollFollow(props: ScrollFollowProps) {
           followRef.current = false;
         }
 
+        prevScrollTopRef.current = args.scrollTop;
         renderProps.onScroll(args);
-      };
-
-      const wrappedStartFollowing = () => {
-        renderProps.startFollowing();
-        followRef.current = true;
-      };
-
-      const wrappedStopFollowing = () => {
-        renderProps.stopFollowing();
-        followRef.current = false;
+        if (props.onCustomScroll) {
+          props.onCustomScroll(
+            args.scrollTop,
+            args.scrollHeight,
+            args.clientHeight,
+          );
+        }
       };
 
       return props.render({
         ...renderProps,
         onScroll: wrappedOnScroll,
-        startFollowing: wrappedStartFollowing,
-        stopFollowing: wrappedStopFollowing,
         follow: followRef.current,
       });
     },

web/src/pages/Logs.tsx

@@ -2,7 +2,7 @@ import { Button } from "@/components/ui/button";
 import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
 import { LogLine, LogSeverity, LogType, logTypes } from "@/types/log";
 import copy from "copy-to-clipboard";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import axios from "axios";
 import LogInfoDialog from "@/components/overlay/LogInfoDialog";
 import { LogChip } from "@/components/indicators/Chip";
@@ -24,6 +24,7 @@ import {
   TooltipContent,
   TooltipTrigger,
 } from "@/components/ui/tooltip";
+import { debounce } from "lodash";
 
 function Logs() {
   const [logService, setLogService] = useState<LogType>("frigate");
@@ -34,6 +35,7 @@ function Logs() {
   const [selectedLog, setSelectedLog] = useState<LogLine>();
   const lazyLogRef = useRef<LazyLog>(null);
   const [isLoading, setIsLoading] = useState(true);
+  const lastFetchedIndexRef = useRef(-1);
 
   useEffect(() => {
     document.title = `${logService[0].toUpperCase()}${logService.substring(1)} Logs - Frigate`;
@@ -53,6 +55,17 @@ function Logs() {
     }
   }, [tabsRef, logService]);
 
+  // scrolling data
+
+  const pageSize = useMemo(() => {
+    const startIndex =
+      lazyLogRef.current?.listRef.current?.findStartIndex() ?? 0;
+    const endIndex = lazyLogRef.current?.listRef.current?.findEndIndex() ?? 0;
+    return endIndex - startIndex;
+    // recalculate page size when new logs load too
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [lazyLogRef, logs]);
+
   // filter
 
   const filterLines = useCallback(
@@ -69,10 +82,38 @@ function Logs() {
 
   // fetchers
 
+  const fetchLogRange = useCallback(
+    async (start: number, end: number) => {
+      try {
+        const response = await axios.get(`logs/${logService}`, {
+          params: { start, end },
+        });
+        if (
+          response.status === 200 &&
+          response.data &&
+          Array.isArray(response.data.lines)
+        ) {
+          const filteredLines = filterLines(response.data.lines);
+          return filteredLines;
+        }
+      } catch (error) {
+        const errorMessage =
+          error instanceof Error ? error.message : "An unknown error occurred";
+        toast.error(`Error fetching logs: ${errorMessage}`, {
+          position: "top-center",
+        });
+      }
+      return [];
+    },
+    [logService, filterLines],
+  );
+
   const fetchInitialLogs = useCallback(async () => {
     setIsLoading(true);
     try {
-      const response = await axios.get(`logs/${logService}`);
+      const response = await axios.get(`logs/${logService}`, {
+        params: { start: -100 },
+      });
       if (
         response.status === 200 &&
         response.data &&
@@ -80,6 +121,8 @@ function Logs() {
       ) {
         const filteredLines = filterLines(response.data.lines);
         setLogs(filteredLines);
+        lastFetchedIndexRef.current =
+          response.data.totalLines - filteredLines.length;
       }
     } catch (error) {
       const errorMessage =
@@ -92,7 +135,6 @@ function Logs() {
     }
   }, [logService, filterLines]);
 
-  const [isStreaming, setIsStreaming] = useState(false);
   const abortControllerRef = useRef<AbortController | null>(null);
 
   const fetchLogsStream = useCallback(() => {
@@ -157,7 +199,6 @@ function Logs() {
               ? error.message
               : "An unknown error occurred";
           toast.error(`Error while streaming logs: ${errorMessage}`);
-          setIsStreaming(false);
         }
       });
   }, [logService, filterSeverity]);
@@ -165,20 +206,56 @@ function Logs() {
   useEffect(() => {
     setIsLoading(true);
     setLogs([]);
+    lastFetchedIndexRef.current = -1;
     fetchInitialLogs().then(() => {
       // Start streaming after initial load
-      setIsStreaming(true);
       fetchLogsStream();
     });
 
     return () => {
       abortControllerRef.current?.abort();
-      setIsStreaming(false);
     };
-  }, [fetchInitialLogs, fetchLogsStream]);
+    // we know that these deps are correct
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [logService, filterSeverity]);
 
   // handlers
 
+  // debounced 200ms
+  const handleScroll = useMemo(
+    () =>
+      debounce((scrollTop: number) => {
+        const scrollThreshold = 0;
+        if (
+          scrollTop <= scrollThreshold &&
+          lastFetchedIndexRef.current > 0 &&
+          !isLoading
+        ) {
+          const savedStart =
+            lazyLogRef.current?.listRef.current?.findStartIndex() ?? 0;
+          const savedEnd =
+            lazyLogRef.current?.listRef.current?.findEndIndex() ?? 0;
+          const nextEnd = lastFetchedIndexRef.current;
+          const nextStart = Math.max(0, nextEnd - (pageSize || 100));
+          setIsLoading(true);
+
+          fetchLogRange(nextStart, nextEnd).then((newLines) => {
+            if (newLines.length > 0) {
+              setLogs((prev) => [...newLines, ...prev]);
+              lastFetchedIndexRef.current = nextStart;
+
+              lazyLogRef.current?.listRef.current?.scrollTo(
+                savedEnd + (pageSize - savedStart),
+              );
+            }
+          });
+
+          setIsLoading(false);
+        }
+      }, 200),
+    [fetchLogRange, isLoading, pageSize],
+  );
+
   const handleCopyLogs = useCallback(() => {
     if (logs.length) {
       fetchInitialLogs()
@@ -444,18 +521,6 @@ function Logs() {
             )}
           >
             <div className="flex flex-1">Message</div>
-            {isStreaming && (
-              <div className="flex flex-row justify-end">
-                <Tooltip>
-                  <TooltipTrigger>
-                    <MdCircle className="mr-2 size-2 animate-pulse cursor-default text-selected shadow-selected drop-shadow-md" />
-                  </TooltipTrigger>
-                  <TooltipContent>
-                    Logs are streaming from the server
-                  </TooltipContent>
-                </Tooltip>
-              </div>
-            )}
           </div>
         </div>
 
@@ -464,21 +529,37 @@ function Logs() {
             <ActivityIndicator className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2" />
           ) : (
             <EnhancedScrollFollow
-              startFollowing={true}
+              startFollowing={!isLoading}
+              onCustomScroll={handleScroll}
               render={({ follow, onScroll }) => (
-                <LazyLog
-                  ref={lazyLogRef}
-                  enableLineNumbers={false}
-                  selectableLines
-                  lineClassName="text-primary bg-background"
-                  highlightLineClassName="bg-primary/20"
-                  onRowClick={handleRowClick}
-                  formatPart={formatPart}
-                  text={logs.join("\n")}
-                  follow={follow}
-                  onScroll={onScroll}
-                  loadingComponent={<ActivityIndicator />}
-                />
+                <>
+                  {follow && (
+                    <div className="absolute right-1 top-3">
+                      <Tooltip>
+                        <TooltipTrigger>
+                          <MdCircle className="mr-2 size-2 animate-pulse cursor-default text-selected shadow-selected drop-shadow-md" />
+                        </TooltipTrigger>
+                        <TooltipContent>
+                          Logs are streaming from the server
+                        </TooltipContent>
+                      </Tooltip>
+                    </div>
+                  )}
+                  <LazyLog
+                    ref={lazyLogRef}
+                    enableLineNumbers={false}
+                    selectableLines
+                    lineClassName="text-primary bg-background"
+                    highlightLineClassName="bg-primary/20"
+                    onRowClick={handleRowClick}
+                    formatPart={formatPart}
+                    text={logs.join("\n")}
+                    follow={follow}
+                    onScroll={onScroll}
+                    loadingComponent={<ActivityIndicator />}
+                    loading={isLoading}
+                  />
+                </>
               )}
             />
           )}
@@ -506,7 +587,7 @@ function LogLineData({
   return (
     <div
       className={cn(
-        "grid w-full cursor-pointer grid-cols-5 gap-2 border-t border-secondary py-2 hover:bg-muted md:grid-cols-12 md:py-0",
+        "grid w-full cursor-pointer grid-cols-5 gap-2 border-t border-secondary bg-background_alt py-2 hover:bg-muted md:grid-cols-12 md:py-0",
         className,
         "*:text-xs",
       )}