From 75da1f8452b4e24143e91b3fcbb3e772962375e4 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 31 Jan 2025 07:41:07 -0600 Subject: [PATCH] fix scrolling and use chunked log download --- frigate/api/app.py | 12 +- .../dynamic/EnhancedScrollFollow.tsx | 50 ++++-- web/src/pages/Logs.tsx | 149 ++++++++++++++---- 3 files changed, 157 insertions(+), 54 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 11ee37665..b6efaeb5e 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -8,7 +8,7 @@ import os import traceback from datetime import datetime, timedelta from functools import reduce -from typing import Any, Optional +from typing import Any, List, Optional, Tuple import aiofiles import requests @@ -399,7 +399,7 @@ def process_logs( service: Optional[str] = None, start: Optional[int] = None, end: Optional[int] = None, -) -> list: +) -> Tuple[int, List[str]]: log_lines = [] key_length = 0 date_end = 0 @@ -436,7 +436,7 @@ def process_logs( current_line = clean_line log_lines.append(current_line) - return log_lines[start:end] + return len(log_lines), log_lines[start:end] @router.get("/logs/{service}", tags=[Tags.logs]) @@ -474,7 +474,7 @@ async def logs( buffer += line # Process logs only when there are enough lines in the buffer if "\n" in buffer: - processed_lines = process_logs(buffer, service) + _, processed_lines = process_logs(buffer, service) buffer = "" for processed_line in processed_lines: yield f"{processed_line}\n" @@ -507,9 +507,9 @@ async def logs( async with aiofiles.open(service_location, "r") as file: contents = await file.read() - log_lines = process_logs(contents, service, start, end) + total_lines, log_lines = process_logs(contents, service, start, end) return JSONResponse( - content={"totalLines": len(log_lines), "lines": log_lines}, + content={"totalLines": total_lines, "lines": log_lines}, status_code=200, ) except FileNotFoundError as e: diff --git a/web/src/components/dynamic/EnhancedScrollFollow.tsx b/web/src/components/dynamic/EnhancedScrollFollow.tsx index 2cfed3bdf..35673c80e 100644 --- a/web/src/components/dynamic/EnhancedScrollFollow.tsx +++ b/web/src/components/dynamic/EnhancedScrollFollow.tsx @@ -1,9 +1,14 @@ -import { useRef, useCallback, ReactNode } from "react"; +import { useRef, useCallback, useEffect, type ReactNode } from "react"; import { ScrollFollow } from "@melloware/react-logviewer"; export type ScrollFollowProps = { startFollowing?: boolean; render: (renderProps: ScrollFollowRenderProps) => ReactNode; + onCustomScroll?: ( + scrollTop: number, + scrollHeight: number, + clientHeight: number, + ) => void; }; export type ScrollFollowRenderProps = { @@ -15,12 +20,22 @@ export type ScrollFollowRenderProps = { }) => void; startFollowing: () => void; stopFollowing: () => void; + onCustomScroll?: ( + scrollTop: number, + scrollHeight: number, + clientHeight: number, + ) => void; }; -const SCROLL_BUFFER = 5; // Additional buffer for scroll checks +const SCROLL_BUFFER = 5; export default function EnhancedScrollFollow(props: ScrollFollowProps) { const followRef = useRef(props.startFollowing || false); + const prevScrollTopRef = useRef(undefined); + + useEffect(() => { + prevScrollTopRef.current = undefined; + }, []); const wrappedRender = useCallback( (renderProps: ScrollFollowRenderProps) => { @@ -29,6 +44,17 @@ export default function EnhancedScrollFollow(props: ScrollFollowProps) { scrollHeight: number; clientHeight: number; }) => { + // Check if scrolling up and immediately stop following + if ( + prevScrollTopRef.current !== undefined && + args.scrollTop < prevScrollTopRef.current + ) { + if (followRef.current) { + renderProps.stopFollowing(); + followRef.current = false; + } + } + const bottomThreshold = args.scrollHeight - args.clientHeight - SCROLL_BUFFER; const isNearBottom = args.scrollTop >= bottomThreshold; @@ -41,24 +67,20 @@ export default function EnhancedScrollFollow(props: ScrollFollowProps) { followRef.current = false; } + prevScrollTopRef.current = args.scrollTop; renderProps.onScroll(args); - }; - - const wrappedStartFollowing = () => { - renderProps.startFollowing(); - followRef.current = true; - }; - - const wrappedStopFollowing = () => { - renderProps.stopFollowing(); - followRef.current = false; + if (props.onCustomScroll) { + props.onCustomScroll( + args.scrollTop, + args.scrollHeight, + args.clientHeight, + ); + } }; return props.render({ ...renderProps, onScroll: wrappedOnScroll, - startFollowing: wrappedStartFollowing, - stopFollowing: wrappedStopFollowing, follow: followRef.current, }); }, diff --git a/web/src/pages/Logs.tsx b/web/src/pages/Logs.tsx index 7e0a4353d..0b1553f0b 100644 --- a/web/src/pages/Logs.tsx +++ b/web/src/pages/Logs.tsx @@ -2,7 +2,7 @@ import { Button } from "@/components/ui/button"; import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; import { LogLine, LogSeverity, LogType, logTypes } from "@/types/log"; import copy from "copy-to-clipboard"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import axios from "axios"; import LogInfoDialog from "@/components/overlay/LogInfoDialog"; import { LogChip } from "@/components/indicators/Chip"; @@ -24,6 +24,7 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { debounce } from "lodash"; function Logs() { const [logService, setLogService] = useState("frigate"); @@ -34,6 +35,7 @@ function Logs() { const [selectedLog, setSelectedLog] = useState(); const lazyLogRef = useRef(null); const [isLoading, setIsLoading] = useState(true); + const lastFetchedIndexRef = useRef(-1); useEffect(() => { document.title = `${logService[0].toUpperCase()}${logService.substring(1)} Logs - Frigate`; @@ -53,6 +55,17 @@ function Logs() { } }, [tabsRef, logService]); + // scrolling data + + const pageSize = useMemo(() => { + const startIndex = + lazyLogRef.current?.listRef.current?.findStartIndex() ?? 0; + const endIndex = lazyLogRef.current?.listRef.current?.findEndIndex() ?? 0; + return endIndex - startIndex; + // recalculate page size when new logs load too + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [lazyLogRef, logs]); + // filter const filterLines = useCallback( @@ -69,10 +82,38 @@ function Logs() { // fetchers + const fetchLogRange = useCallback( + async (start: number, end: number) => { + try { + const response = await axios.get(`logs/${logService}`, { + params: { start, end }, + }); + if ( + response.status === 200 && + response.data && + Array.isArray(response.data.lines) + ) { + const filteredLines = filterLines(response.data.lines); + return filteredLines; + } + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "An unknown error occurred"; + toast.error(`Error fetching logs: ${errorMessage}`, { + position: "top-center", + }); + } + return []; + }, + [logService, filterLines], + ); + const fetchInitialLogs = useCallback(async () => { setIsLoading(true); try { - const response = await axios.get(`logs/${logService}`); + const response = await axios.get(`logs/${logService}`, { + params: { start: -100 }, + }); if ( response.status === 200 && response.data && @@ -80,6 +121,8 @@ function Logs() { ) { const filteredLines = filterLines(response.data.lines); setLogs(filteredLines); + lastFetchedIndexRef.current = + response.data.totalLines - filteredLines.length; } } catch (error) { const errorMessage = @@ -92,7 +135,6 @@ function Logs() { } }, [logService, filterLines]); - const [isStreaming, setIsStreaming] = useState(false); const abortControllerRef = useRef(null); const fetchLogsStream = useCallback(() => { @@ -157,7 +199,6 @@ function Logs() { ? error.message : "An unknown error occurred"; toast.error(`Error while streaming logs: ${errorMessage}`); - setIsStreaming(false); } }); }, [logService, filterSeverity]); @@ -165,20 +206,56 @@ function Logs() { useEffect(() => { setIsLoading(true); setLogs([]); + lastFetchedIndexRef.current = -1; fetchInitialLogs().then(() => { // Start streaming after initial load - setIsStreaming(true); fetchLogsStream(); }); return () => { abortControllerRef.current?.abort(); - setIsStreaming(false); }; - }, [fetchInitialLogs, fetchLogsStream]); + // we know that these deps are correct + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [logService, filterSeverity]); // handlers + // debounced 200ms + const handleScroll = useMemo( + () => + debounce((scrollTop: number) => { + const scrollThreshold = 0; + if ( + scrollTop <= scrollThreshold && + lastFetchedIndexRef.current > 0 && + !isLoading + ) { + const savedStart = + lazyLogRef.current?.listRef.current?.findStartIndex() ?? 0; + const savedEnd = + lazyLogRef.current?.listRef.current?.findEndIndex() ?? 0; + const nextEnd = lastFetchedIndexRef.current; + const nextStart = Math.max(0, nextEnd - (pageSize || 100)); + setIsLoading(true); + + fetchLogRange(nextStart, nextEnd).then((newLines) => { + if (newLines.length > 0) { + setLogs((prev) => [...newLines, ...prev]); + lastFetchedIndexRef.current = nextStart; + + lazyLogRef.current?.listRef.current?.scrollTo( + savedEnd + (pageSize - savedStart), + ); + } + }); + + setIsLoading(false); + } + }, 200), + [fetchLogRange, isLoading, pageSize], + ); + const handleCopyLogs = useCallback(() => { if (logs.length) { fetchInitialLogs() @@ -444,18 +521,6 @@ function Logs() { )} >
Message
- {isStreaming && ( -
- - - - - - Logs are streaming from the server - - -
- )} @@ -464,21 +529,37 @@ function Logs() { ) : ( ( - } - /> + <> + {follow && ( +
+ + + + + + Logs are streaming from the server + + +
+ )} + } + loading={isLoading} + /> + )} /> )} @@ -506,7 +587,7 @@ function LogLineData({ return (