use react-logviewer and backend streaming

docker/main/requirements-wheels.txt

@@ -1,3 +1,4 @@
+aiofiles == 24.1.*
 click == 8.1.*
 # FastAPI
 aiohttp == 3.11.2

frigate/api/app.py

@@ -1,5 +1,6 @@
 """Main api runner."""
 
+import asyncio
 import copy
 import json
 import logging
@@ -9,11 +10,12 @@ from datetime import datetime, timedelta
 from functools import reduce
 from typing import Any, Optional
 
+import aiofiles
 import requests
 from fastapi import APIRouter, Body, Path, Request, Response
 from fastapi.encoders import jsonable_encoder
 from fastapi.params import Depends
-from fastapi.responses import JSONResponse, PlainTextResponse
+from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse
 from markupsafe import escape
 from peewee import operator
 
@@ -392,10 +394,53 @@ def nvinfo():
     return JSONResponse(content=get_nvidia_driver_info())
 
 
+def process_logs(
+    contents: str,
+    service: Optional[str] = None,
+    start: Optional[int] = None,
+    end: Optional[int] = None,
+) -> list:
+    log_lines = []
+    key_length = 0
+    date_end = 0
+    current_key = ""
+    current_line = ""
+
+    for raw_line in contents.splitlines():
+        clean_line = raw_line.strip()
+
+        if len(clean_line) < 10:
+            continue
+
+        # handle cases where S6 does not include date in log line
+        if "  " not in clean_line:
+            clean_line = f"{datetime.now()}  {clean_line}"
+
+        if date_end == 0:
+            date_end = clean_line.index("  ")
+            key_length = date_end - (6 if service == "frigate" else 0)
+
+        new_key = clean_line[:key_length]
+
+        if new_key == current_key:
+            current_line += f"\n{clean_line[date_end:].strip()}"
+            continue
+        else:
+            if current_line:
+                log_lines.append(current_line)
+
+            current_key = new_key
+            current_line = clean_line
+
+    log_lines.append(current_line)
+    return log_lines[start:end]
+
+
 @router.get("/logs/{service}", tags=[Tags.logs])
-def logs(
+async def logs(
     service: str = Path(enum=["frigate", "nginx", "go2rtc"]),
     download: Optional[str] = None,
+    stream: Optional[bool] = False,
     start: Optional[int] = 0,
     end: Optional[int] = None,
 ):
@@ -414,6 +459,27 @@ def logs(
                 status_code=500,
             )
 
+    async def stream_logs(file_path: str):
+        """Asynchronously stream log lines."""
+        buffer = ""
+        try:
+            async with aiofiles.open(file_path, "r") as file:
+                await file.seek(0, 2)
+                while True:
+                    line = await file.readline()
+                    if line:
+                        buffer += line
+                        # Process logs only when there are enough lines in the buffer
+                        if "\n" in buffer:
+                            processed_lines = process_logs(buffer, service)
+                            buffer = ""
+                            for processed_line in processed_lines:
+                                yield f"{processed_line}\n"
+                    else:
+                        await asyncio.sleep(0.1)
+        except FileNotFoundError:
+            yield "Log file not found.\n"
+
     log_locations = {
         "frigate": "/dev/shm/logs/frigate/current",
         "go2rtc": "/dev/shm/logs/go2rtc/current",
@@ -430,48 +496,17 @@ def logs(
     if download:
         return download_logs(service_location)
 
+    if stream:
+        return StreamingResponse(stream_logs(service_location), media_type="text/plain")
+
+    # For full logs initially
     try:
-        file = open(service_location, "r")
+        async with aiofiles.open(service_location, "r") as file:
-        contents = file.read()
+            contents = await file.read()
-        file.close()
-
-        # use the start timestamp to group logs together``
-        logLines = []
-        keyLength = 0
-        dateEnd = 0
-        currentKey = ""
-        currentLine = ""
-
-        for rawLine in contents.splitlines():
-            cleanLine = rawLine.strip()
-
-            if len(cleanLine) < 10:
-                continue
-
-            # handle cases where S6 does not include date in log line
-            if "  " not in cleanLine:
-                cleanLine = f"{datetime.now()}  {cleanLine}"
-
-            if dateEnd == 0:
-                dateEnd = cleanLine.index("  ")
-                keyLength = dateEnd - (6 if service_location == "frigate" else 0)
-
-            newKey = cleanLine[0:keyLength]
-
-            if newKey == currentKey:
-                currentLine += f"\n{cleanLine[dateEnd:].strip()}"
-                continue
-            else:
-                if len(currentLine) > 0:
-                    logLines.append(currentLine)
-
-                currentKey = newKey
-                currentLine = cleanLine
-
-        logLines.append(currentLine)
 
+        log_lines = process_logs(contents, service, start, end)
         return JSONResponse(
-            content={"totalLines": len(logLines), "lines": logLines[start:end]},
+            content={"totalLines": len(log_lines), "lines": log_lines},
             status_code=200,
         )
     except FileNotFoundError as e:

web/package-lock.json

@@ -10,6 +10,7 @@
       "dependencies": {
         "@cycjimmy/jsmpeg-player": "^6.1.1",
         "@hookform/resolvers": "^3.9.0",
+        "@melloware/react-logviewer": "^6.1.1",
         "@radix-ui/react-alert-dialog": "^1.1.2",
         "@radix-ui/react-aspect-ratio": "^1.1.0",
         "@radix-ui/react-checkbox": "^1.1.2",
@@ -1002,6 +1003,22 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@melloware/react-logviewer": {
+      "version": "6.1.1",
+      "resolved": "https://registry.npmjs.org/@melloware/react-logviewer/-/react-logviewer-6.1.1.tgz",
+      "integrity": "sha512-bbjHvTAVDyQj1yE/56Y90FGH94bxOOQo0e/J2jTZTprAM11gchpcp5tTJUV5ZidmVmF4uUSUs/NtJxceA3Wrig==",
+      "license": "MPL-2.0",
+      "dependencies": {
+        "hotkeys-js": "3.13.9",
+        "mitt": "3.0.1",
+        "react-string-replace": "1.1.1",
+        "virtua": "0.39.2"
+      },
+      "peerDependencies": {
+        "react": ">=17.0.0",
+        "react-dom": ">=17.0.0"
+      }
+    },
     "node_modules/@mswjs/interceptors": {
       "version": "0.29.1",
       "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.29.1.tgz",
@@ -5511,6 +5528,15 @@
       "integrity": "sha512-wA66nnYFvQa1o4DO/BFgLNRKnBTVXpNeldGRBJ2Y0SvFtdwvFKCbqa9zhHoZLoxHhZ+jYsj3aIBkWQQCPNOhMw==",
       "license": "Apache-2.0"
     },
+    "node_modules/hotkeys-js": {
+      "version": "3.13.9",
+      "resolved": "https://registry.npmjs.org/hotkeys-js/-/hotkeys-js-3.13.9.tgz",
+      "integrity": "sha512-3TRCj9u9KUH6cKo25w4KIdBfdBfNRjfUwrljCLDC2XhmPDG0SjAZFcFZekpUZFmXzfYoGhFDcdx2gX/vUVtztQ==",
+      "license": "MIT",
+      "funding": {
+        "url": "https://jaywcjlove.github.io/#/sponsor"
+      }
+    },
     "node_modules/html-encoding-sniffer": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz",
@@ -6273,6 +6299,12 @@
         "node": ">=16 || 14 >=14.17"
       }
     },
+    "node_modules/mitt": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz",
+      "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
+      "license": "MIT"
+    },
     "node_modules/mock-socket": {
       "version": "9.3.1",
       "resolved": "https://registry.npmjs.org/mock-socket/-/mock-socket-9.3.1.tgz",
@@ -7425,6 +7457,15 @@
         "react-dom": ">=16.8"
       }
     },
+    "node_modules/react-string-replace": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/react-string-replace/-/react-string-replace-1.1.1.tgz",
+      "integrity": "sha512-26TUbLzLfHQ5jO5N7y3Mx88eeKo0Ml0UjCQuX4BMfOd/JX+enQqlKpL1CZnmjeBRvQE8TR+ds9j1rqx9CxhKHQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.12.0"
+      }
+    },
     "node_modules/react-style-singleton": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
@@ -8766,6 +8807,36 @@
         "react-dom": "^16.8 || ^17.0 || ^18.0"
       }
     },
+    "node_modules/virtua": {
+      "version": "0.39.2",
+      "resolved": "https://registry.npmjs.org/virtua/-/virtua-0.39.2.tgz",
+      "integrity": "sha512-KhDYmfDe36L1W5ir1b5+jeV40+u4bb5bRiZggWwGinreGXKnxxcLGdk+yVZlO5dNdBq/nxj4v4w6yxuwgLXSBg==",
+      "license": "MIT",
+      "peerDependencies": {
+        "react": ">=16.14.0",
+        "react-dom": ">=16.14.0",
+        "solid-js": ">=1.0",
+        "svelte": ">=5.0",
+        "vue": ">=3.2"
+      },
+      "peerDependenciesMeta": {
+        "react": {
+          "optional": true
+        },
+        "react-dom": {
+          "optional": true
+        },
+        "solid-js": {
+          "optional": true
+        },
+        "svelte": {
+          "optional": true
+        },
+        "vue": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/vite": {
       "version": "5.4.0",
       "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz",

web/package.json

@@ -16,6 +16,7 @@
   "dependencies": {
     "@cycjimmy/jsmpeg-player": "^6.1.1",
     "@hookform/resolvers": "^3.9.0",
+    "@melloware/react-logviewer": "^6.1.1",
     "@radix-ui/react-alert-dialog": "^1.1.2",
     "@radix-ui/react-aspect-ratio": "^1.1.0",
     "@radix-ui/react-checkbox": "^1.1.2",

web/src/components/dynamic/EnhancedScrollFollow.tsx

@@ -0,0 +1,69 @@
+import { useRef, useCallback, ReactNode } from "react";
+import { ScrollFollow } from "@melloware/react-logviewer";
+
+export type ScrollFollowProps = {
+  startFollowing?: boolean;
+  render: (renderProps: ScrollFollowRenderProps) => ReactNode;
+};
+
+export type ScrollFollowRenderProps = {
+  follow: boolean;
+  onScroll: (args: {
+    scrollTop: number;
+    scrollHeight: number;
+    clientHeight: number;
+  }) => void;
+  startFollowing: () => void;
+  stopFollowing: () => void;
+};
+
+const SCROLL_BUFFER = 5; // Additional buffer for scroll checks
+
+export default function EnhancedScrollFollow(props: ScrollFollowProps) {
+  const followRef = useRef(props.startFollowing || false);
+
+  const wrappedRender = useCallback(
+    (renderProps: ScrollFollowRenderProps) => {
+      const wrappedOnScroll = (args: {
+        scrollTop: number;
+        scrollHeight: number;
+        clientHeight: number;
+      }) => {
+        const bottomThreshold =
+          args.scrollHeight - args.clientHeight - SCROLL_BUFFER;
+        const isNearBottom = args.scrollTop >= bottomThreshold;
+
+        if (isNearBottom && !followRef.current) {
+          renderProps.startFollowing();
+          followRef.current = true;
+        } else if (!isNearBottom && followRef.current) {
+          renderProps.stopFollowing();
+          followRef.current = false;
+        }
+
+        renderProps.onScroll(args);
+      };
+
+      const wrappedStartFollowing = () => {
+        renderProps.startFollowing();
+        followRef.current = true;
+      };
+
+      const wrappedStopFollowing = () => {
+        renderProps.stopFollowing();
+        followRef.current = false;
+      };
+
+      return props.render({
+        ...renderProps,
+        onScroll: wrappedOnScroll,
+        startFollowing: wrappedStartFollowing,
+        stopFollowing: wrappedStopFollowing,
+        follow: followRef.current,
+      });
+    },
+    [props],
+  );
+
+  return <ScrollFollow {...props} render={wrappedRender} />;
+}

web/src/index.css

@@ -179,3 +179,8 @@ html {
   border: 3px solid #a00000 !important;
   opacity: 0.5 !important;
 }
+
+.react-lazylog,
+.react-lazylog-searchbar {
+  background-color: transparent !important;
+}

web/src/pages/Logs.tsx

@@ -1,35 +1,33 @@
 import { Button } from "@/components/ui/button";
 import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group";
-import { LogData, LogLine, LogSeverity, LogType, logTypes } from "@/types/log";
+import { LogLine, LogSeverity, LogType, logTypes } from "@/types/log";
 import copy from "copy-to-clipboard";
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import axios from "axios";
 import LogInfoDialog from "@/components/overlay/LogInfoDialog";
 import { LogChip } from "@/components/indicators/Chip";
 import { LogLevelFilterButton } from "@/components/filter/LogLevelFilter";
-import { FaCopy } from "react-icons/fa6";
+import { FaCopy, FaDownload } from "react-icons/fa";
 import { Toaster } from "@/components/ui/sonner";
 import { toast } from "sonner";
-import {
-  isDesktop,
-  isMobile,
-  isMobileOnly,
-  isTablet,
-} from "react-device-detect";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
 import { cn } from "@/lib/utils";
-import { MdVerticalAlignBottom } from "react-icons/md";
 import { parseLogLines } from "@/utils/logUtil";
-import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
 import scrollIntoView from "scroll-into-view-if-needed";
-import { FaDownload } from "react-icons/fa";
+import { LazyLog } from "@melloware/react-logviewer";
-
+import useKeyboardListener from "@/hooks/use-keyboard-listener";
-type LogRange = { start: number; end: number };
+import EnhancedScrollFollow from "@/components/dynamic/EnhancedScrollFollow";
+import { MdCircle } from "react-icons/md";
 
 function Logs() {
   const [logService, setLogService] = useState<LogType>("frigate");
   const tabsRef = useRef<HTMLDivElement | null>(null);
+  const lazyLogWrapperRef = useRef<HTMLDivElement>(null);
+  const [logs, setLogs] = useState<string[]>([]);
+  const [filterSeverity, setFilterSeverity] = useState<LogSeverity[]>();
+  const [selectedLog, setSelectedLog] = useState<LogLine>();
+  const lazyLogRef = useRef<LazyLog>(null);
 
   useEffect(() => {
     document.title = `${logService[0].toUpperCase()}${logService.substring(1)} Logs - Frigate`;
@@ -49,96 +47,20 @@ function Logs() {
     }
   }, [tabsRef, logService]);
 
-  // log data handling
+  // handlers
-
-  const logPageSize = useMemo(() => {
-    if (isMobileOnly) {
-      return 15;
-    }
-
-    if (isTablet) {
-      return 25;
-    }
-
-    return 40;
-  }, []);
-
-  const [logRange, setLogRange] = useState<LogRange>({ start: 0, end: 0 });
-  const [logs, setLogs] = useState<string[]>([]);
-  const [logLines, setLogLines] = useState<LogLine[]>([]);
-
-  useEffect(() => {
-    axios
-      .get(`logs/${logService}?start=-${logPageSize}`)
-      .then((resp) => {
-        if (resp.status == 200) {
-          const data = resp.data as LogData;
-          setLogRange({
-            start: Math.max(0, data.totalLines - logPageSize),
-            end: data.totalLines,
-          });
-          setLogs(data.lines);
-          setLogLines(parseLogLines(logService, data.lines));
-        }
-      })
-      .catch(() => {});
-  }, [logPageSize, logService]);
-
-  useEffect(() => {
-    if (!logs || logs.length == 0) {
-      return;
-    }
-
-    const id = setTimeout(() => {
-      axios
-        .get(`logs/${logService}?start=${logRange.end}`)
-        .then((resp) => {
-          if (resp.status == 200) {
-            const data = resp.data as LogData;
-
-            if (data.lines.length > 0) {
-              setLogRange({
-                start: logRange.start,
-                end: data.totalLines,
-              });
-              setLogs([...logs, ...data.lines]);
-              setLogLines([
-                ...logLines,
-                ...parseLogLines(logService, data.lines),
-              ]);
-            }
-          }
-        })
-        .catch(() => {});
-    }, 5000);
-
-    return () => {
-      if (id) {
-        clearTimeout(id);
-      }
-    };
-    // we need to listen on the current range of visible items
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [logLines, logService, logRange]);
-
-  // convert to log data
 
   const handleCopyLogs = useCallback(() => {
-    if (logs) {
+    if (logs.length) {
       copy(logs.join("\n"));
-      toast.success(
+      toast.success("Copied logs to clipboard");
-        logRange.start == 0
-          ? "Copied logs to clipboard"
-          : "Copied visible logs to clipboard",
-      );
     } else {
       toast.error("Could not copy logs to clipboard");
     }
-  }, [logs, logRange]);
+  }, [logs]);
 
   const handleDownloadLogs = useCallback(() => {
     axios
-      .get(`logs/${logService}?download=true`)
+      .get(`api/logs/${logService}?download=true`)
       .then((resp) => {
         const element = document.createElement("a");
         element.setAttribute(
@@ -157,226 +79,192 @@ function Logs() {
       .catch(() => {});
   }, [logService]);
 
-  // scroll to bottom
+  const handleRowClick = useCallback(
-
+    (rowInfo: { lineNumber: number; rowIndex: number }) => {
-  const [initialScroll, setInitialScroll] = useState(false);
+      const clickedLine = parseLogLines(logService, [
-
+        logs[rowInfo.rowIndex],
-  const contentRef = useRef<HTMLDivElement | null>(null);
+      ])[0];
-  const [endVisible, setEndVisible] = useState(true);
+      setSelectedLog(clickedLine);
-  const endObserver = useRef<IntersectionObserver | null>(null);
-  const endLogRef = useCallback(
-    (node: HTMLElement | null) => {
-      if (endObserver.current) endObserver.current.disconnect();
-      try {
-        endObserver.current = new IntersectionObserver((entries) => {
-          setEndVisible(entries[0].isIntersecting);
-        });
-        if (node) endObserver.current.observe(node);
-      } catch (e) {
-        // no op
-      }
     },
-    [setEndVisible],
+    [logs, logService],
   );
-  const startObserver = useRef<IntersectionObserver | null>(null);
-  const startLogRef = useCallback(
-    (node: HTMLElement | null) => {
-      if (startObserver.current) startObserver.current.disconnect();
 
-      if (logs.length == 0 || !initialScroll) {
+  // filter
-        return;
-      }
 
-      try {
+  const filterLines = useCallback(
-        startObserver.current = new IntersectionObserver(
+    (lines: string[]) => {
-          (entries) => {
+      // console.log(lines);
-            if (entries[0].isIntersecting && logRange.start > 0) {
+      if (!filterSeverity?.length) return lines;
-              const start = Math.max(0, logRange.start - logPageSize);
 
-              axios
+      return lines.filter((line) => {
-                .get(`logs/${logService}?start=${start}&end=${logRange.start}`)
+        // console.log(line);
-                .then((resp) => {
+        const parsedLine = parseLogLines(logService, [line])[0];
-                  if (resp.status == 200) {
+        return filterSeverity.includes(parsedLine.severity);
-                    const data = resp.data as LogData;
+      });
-
-                    if (data.lines.length > 0) {
-                      setLogRange({
-                        start: start,
-                        end: logRange.end,
-                      });
-                      setLogs([...data.lines, ...logs]);
-                      setLogLines([
-                        ...parseLogLines(logService, data.lines),
-                        ...logLines,
-                      ]);
-                    }
-                  }
-                })
-                .catch(() => {});
-              contentRef.current?.scrollBy({
-                top: 10,
-              });
-            }
-          },
-          { rootMargin: `${10 * (isMobile ? 64 : 48)}px 0px 0px 0px` },
-        );
-        if (node) startObserver.current.observe(node);
-      } catch (e) {
-        // no op
-      }
     },
-    // we need to listen on the current range of visible items
+    [filterSeverity, logService],
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    [logRange, initialScroll],
   );
 
+  // fetchers
+
+  const fetchInitialLogs = useCallback(async () => {
+    try {
+      const response = await axios.get(`logs/${logService}`);
+      if (
+        response.status === 200 &&
+        response.data &&
+        Array.isArray(response.data.lines)
+      ) {
+        const filteredLines = filterLines(response.data.lines);
+        setLogs(filteredLines);
+      }
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : "An unknown error occurred";
+      toast.error(`Error fetching logs: ${errorMessage}`, {
+        position: "top-center",
+      });
+    }
+  }, [logService, filterLines]);
+
+  const [isStreaming, setIsStreaming] = useState(false);
+  const abortControllerRef = useRef<AbortController | null>(null);
+
+  const fetchLogsStream = useCallback(() => {
+    // Cancel any existing stream
+    abortControllerRef.current?.abort();
+    const abortController = new AbortController();
+    abortControllerRef.current = abortController;
+    let buffer = "";
+    const decoder = new TextDecoder();
+
+    const processStreamChunk = (
+      reader: ReadableStreamDefaultReader<Uint8Array>,
+    ): Promise<void> => {
+      return reader.read().then(({ done, value }) => {
+        if (done) return;
+
+        // Decode the chunk and add it to our buffer
+        buffer += decoder.decode(value, { stream: true });
+
+        // Split on newlines, keeping any partial line in the buffer
+        const lines = buffer.split("\n");
+
+        // Keep the last partial line
+        buffer = lines.pop() || "";
+
+        // Filter and append complete lines
+        if (lines.length > 0) {
+          const filteredLines = filterSeverity?.length
+            ? lines.filter((line) => {
+                const parsedLine = parseLogLines(logService, [line])[0];
+                return filterSeverity.includes(parsedLine.severity);
+              })
+            : lines;
+          if (filteredLines.length > 0) {
+            lazyLogRef.current?.appendLines(filteredLines);
+          }
+        }
+        // Process next chunk
+        return processStreamChunk(reader);
+      });
+    };
+
+    fetch(`api/logs/${logService}?stream=true`, {
+      signal: abortController.signal,
+    })
+      .then((response): Promise<void> => {
+        if (!response.ok) {
+          throw new Error(
+            `Error while fetching log stream, status: ${response.status}`,
+          );
+        }
+        const reader = response.body?.getReader();
+        if (!reader) {
+          throw new Error("No reader available");
+        }
+        return processStreamChunk(reader);
+      })
+      .catch((error) => {
+        if (error.name !== "AbortError") {
+          const errorMessage =
+            error instanceof Error
+              ? error.message
+              : "An unknown error occurred";
+          toast.error(`Error while streaming logs: ${errorMessage}`);
+          setIsStreaming(false);
+        }
+      });
+  }, [logService, filterSeverity]);
+
   useEffect(() => {
-    if (logLines.length == 0) {
+    fetchInitialLogs().then(() => {
-      setInitialScroll(false);
+      // Start streaming after initial load
-      return;
+      setIsStreaming(true);
-    }
+      fetchLogsStream();
-
-    if (initialScroll) {
-      return;
-    }
-
-    if (!contentRef.current) {
-      return;
-    }
-
-    if (contentRef.current.scrollHeight <= contentRef.current.clientHeight) {
-      setInitialScroll(true);
-      return;
-    }
-
-    contentRef.current?.scrollTo({
-      top: contentRef.current?.scrollHeight,
-      behavior: "instant",
     });
-    setTimeout(() => setInitialScroll(true), 300);
-    // we need to listen on the current range of visible items
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [logLines, logService]);
 
-  // log filtering
+    return () => {
+      abortControllerRef.current?.abort();
+      setIsStreaming(false);
+    };
+  }, [fetchInitialLogs, fetchLogsStream]);
 
-  const [filterSeverity, setFilterSeverity] = useState<LogSeverity[]>();
+  // keyboard listener
-
-  // log selection
-
-  const [selectedLog, setSelectedLog] = useState<LogLine>();
-
-  // interaction
 
   useKeyboardListener(
     ["PageDown", "PageUp", "ArrowDown", "ArrowUp"],
     (key, modifiers) => {
-      if (!modifiers.down) {
+      if (!key || !modifiers.down || !lazyLogWrapperRef.current) {
         return;
       }
 
-      switch (key) {
+      const container =
-        case "PageDown":
+        lazyLogWrapperRef.current.querySelector(".react-lazylog");
-          contentRef.current?.scrollBy({
+
-            top: 480,
+      const logLineHeight = container?.querySelector(".log-line")?.clientHeight;
-          });
+
-          break;
+      if (!logLineHeight) {
-        case "PageUp":
+        return;
-          contentRef.current?.scrollBy({
-            top: -480,
-          });
-          break;
-        case "ArrowDown":
-          contentRef.current?.scrollBy({
-            top: 48,
-          });
-          break;
-        case "ArrowUp":
-          contentRef.current?.scrollBy({
-            top: -48,
-          });
-          break;
       }
+
+      const scrollAmount = key.includes("Page")
+        ? logLineHeight * 10
+        : logLineHeight;
+      const direction = key.includes("Down") ? 1 : -1;
+      container?.scrollBy({ top: scrollAmount * direction });
     },
   );
 
-  useEffect(() => {
+  // format lines
-    const handleCopy = (e: ClipboardEvent) => {
-      e.preventDefault();
-      if (!contentRef.current) return;
 
-      const selection = window.getSelection();
+  const lineBufferRef = useRef<string>("");
-      if (!selection) return;
 
-      const range = selection.getRangeAt(0);
+  const formatPart = useCallback(
-      const fragment = range.cloneContents();
+    (text: string) => {
+      lineBufferRef.current += text;
 
-      const extractLogData = (element: Element) => {
+      if (text.endsWith("\n")) {
-        const severity =
+        const completeLine = lineBufferRef.current.trim();
-          element.querySelector(".log-severity")?.textContent?.trim() || "";
+        lineBufferRef.current = "";
-        const dateStamp =
-          element.querySelector(".log-timestamp")?.textContent?.trim() || "";
-        const section =
-          element.querySelector(".log-section")?.textContent?.trim() || "";
-        const content =
-          element.querySelector(".log-content")?.textContent?.trim() || "";
 
-        return { severity, dateStamp, section, content };
+        if (completeLine) {
-      };
+          const parsedLine = parseLogLines(logService, [completeLine])[0];
-
+          return (
-      let copyData: {
+            <LogLineData
-        severity: string;
+              line={parsedLine}
-        dateStamp: string;
+              logService={logService}
-        section: string;
+              onClickSeverity={() => setFilterSeverity([parsedLine.severity])}
-        content: string;
+              onSelect={() => setSelectedLog(parsedLine)}
-      }[] = [];
+            />
-
+          );
-      if (fragment.querySelectorAll(".grid").length > 0) {
-        // Multiple grid elements
-        copyData = Array.from(fragment.querySelectorAll(".grid")).map(
-          extractLogData,
-        );
-      } else {
-        // Try to find the closest grid element or use the first child element
-        const gridElement =
-          fragment.querySelector(".grid") || (fragment.firstChild as Element);
-
-        if (gridElement) {
-          const data = extractLogData(gridElement);
-          if (data.severity || data.dateStamp || data.section || data.content) {
-            copyData.push(data);
-          }
         }
       }
 
-      if (copyData.length === 0) return; // No valid data to copy
+      return null;
-
+    },
-      // Calculate maximum widths for each column
+    [logService, setFilterSeverity, setSelectedLog],
-      const maxWidths = {
+  );
-        severity: Math.max(...copyData.map((d) => d.severity.length)),
-        dateStamp: Math.max(...copyData.map((d) => d.dateStamp.length)),
-        section: Math.max(...copyData.map((d) => d.section.length)),
-      };
-
-      const pad = (str: string, length: number) => str.padEnd(length, " ");
-
-      // Create the formatted copy text
-      const copyText = copyData
-        .map(
-          (d) =>
-            `${pad(d.severity, maxWidths.severity)} | ${pad(d.dateStamp, maxWidths.dateStamp)} | ${pad(d.section, maxWidths.section)} | ${d.content}`,
-        )
-        .join("\n");
-
-      e.clipboardData?.setData("text/plain", copyText);
-    };
-
-    const content = contentRef.current;
-    content?.addEventListener("copy", handleCopy);
-    return () => {
-      content?.removeEventListener("copy", handleCopy);
-    };
-  }, []);
 
   return (
     <div className="flex size-full flex-col p-2">
@@ -393,11 +281,10 @@ function Logs() {
               onValueChange={(value: LogType) => {
                 if (value) {
                   setLogs([]);
-                  setLogLines([]);
                   setFilterSeverity(undefined);
                   setLogService(value);
                 }
-              }} // don't allow the severity to be unselected
+              }}
             >
               {Object.values(logTypes).map((item) => (
                 <ToggleGroupItem
@@ -438,125 +325,111 @@ function Logs() {
           <LogLevelFilterButton
             selectedLabels={filterSeverity}
             updateLabelFilter={setFilterSeverity}
+            // setStreaming={setStreaming}
           />
         </div>
       </div>
 
-      {initialScroll && !endVisible && (
+      <div className="font-mono relative my-2 flex size-full flex-col overflow-hidden whitespace-pre-wrap rounded-md border border-secondary bg-background_alt text-xs sm:p-1">
-        <Button
+        <div className="grid grid-cols-5 *:px-0 *:py-3 *:text-sm *:text-primary/40 sm:grid-cols-8 md:grid-cols-12">
-          className="absolute bottom-8 left-[50%] z-20 flex -translate-x-[50%] items-center gap-1 rounded-md p-2"
+          <div className="ml-1 flex items-center p-1 capitalize">Type</div>
-          aria-label="Jump to bottom of logs"
-          onClick={() =>
-            contentRef.current?.scrollTo({
-              top: contentRef.current?.scrollHeight,
-              behavior: "smooth",
-            })
-          }
-        >
-          <MdVerticalAlignBottom />
-          Jump to Bottom
-        </Button>
-      )}
-
-      <div className="font-mono relative my-2 flex size-full flex-col overflow-hidden whitespace-pre-wrap rounded-md border border-secondary bg-background_alt text-sm sm:p-2">
-        <div className="grid grid-cols-5 *:px-2 *:py-3 *:text-sm *:text-primary/40 sm:grid-cols-8 md:grid-cols-12">
-          <div className="flex items-center p-1 capitalize">Type</div>
           <div className="col-span-2 flex items-center sm:col-span-1">
             Timestamp
           </div>
-          <div className="col-span-2 flex items-center">Tag</div>
+          <div
-          <div className="col-span-5 flex items-center sm:col-span-4 md:col-span-8">
+            className={cn(
-            Message
+              "flex items-center",
+              logService == "frigate" ? "col-span-2" : "col-span-1",
+            )}
+          >
+            Tag
+          </div>
+          <div
+            className={cn(
+              "col-span-4 flex items-center",
+              logService == "frigate" ? "md:col-span-8" : "md:col-span-9",
+            )}
+          >
+            <div className="flex flex-1">Message</div>
+            {isStreaming && (
+              <div className="flex flex-row justify-end">
+                <MdCircle className="mr-2 size-2 animate-pulse text-selected shadow-selected drop-shadow-md" />
+              </div>
+            )}
           </div>
         </div>
-        <div
-          ref={contentRef}
-          className="no-scrollbar flex w-full flex-col overflow-y-auto overscroll-contain"
-        >
-          {logLines.length > 0 &&
-            [...Array(logRange.end).keys()].map((idx) => {
-              const logLine =
-                idx >= logRange.start
-                  ? logLines[idx - logRange.start]
-                  : undefined;
 
-              if (logLine) {
+        <div ref={lazyLogWrapperRef} className="size-full">
-                const line = logLines[idx - logRange.start];
+          <EnhancedScrollFollow
-                if (filterSeverity && !filterSeverity.includes(line.severity)) {
+            startFollowing={true}
-                  return (
+            render={({ follow, onScroll }) => (
-                    <div
+              <LazyLog
-                      ref={idx == logRange.start + 10 ? startLogRef : undefined}
+                ref={lazyLogRef}
-                    />
+                enableLineNumbers={false}
-                  );
+                selectableLines
-                }
+                lineClassName="text-primary bg-background"
-
+                highlightLineClassName="bg-primary/20"
-                return (
+                onRowClick={handleRowClick}
-                  <LogLineData
+                formatPart={formatPart}
-                    key={`${idx}-${logService}`}
+                text={logs.join("\n")}
-                    startRef={
+                follow={follow}
-                      idx == logRange.start + 10 ? startLogRef : undefined
+                onScroll={onScroll}
-                    }
+                loadingComponent={<ActivityIndicator />}
-                    className={initialScroll ? "" : "invisible"}
+              />
-                    line={line}
+            )}
-                    onClickSeverity={() => setFilterSeverity([line.severity])}
+          />
-                    onSelect={() => setSelectedLog(line)}
-                  />
-                );
-              }
-
-              return (
-                <div
-                  key={`${idx}-${logService}`}
-                  className={isDesktop ? "h-12" : "h-16"}
-                />
-              );
-            })}
-          {logLines.length > 0 && <div id="page-bottom" ref={endLogRef} />}
         </div>
-        {logLines.length == 0 && (
-          <ActivityIndicator className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2" />
-        )}
       </div>
     </div>
   );
 }
 
 type LogLineDataProps = {
-  startRef?: (node: HTMLDivElement | null) => void;
+  className?: string;
-  className: string;
   line: LogLine;
+  logService: string;
   onClickSeverity: () => void;
   onSelect: () => void;
 };
+
 function LogLineData({
-  startRef,
   className,
   line,
+  logService,
   onClickSeverity,
   onSelect,
 }: LogLineDataProps) {
   return (
     <div
-      ref={startRef}
       className={cn(
-        "grid w-full cursor-pointer grid-cols-5 gap-2 border-t border-secondary py-2 hover:bg-muted sm:grid-cols-8 md:grid-cols-12",
+        "grid w-full cursor-pointer grid-cols-8 gap-2 border-t border-secondary py-0 hover:bg-muted md:grid-cols-12",
         className,
-        "*:text-sm",
+        "*:text-xs",
       )}
       onClick={onSelect}
     >
       <div className="log-severity flex h-full items-center gap-2 p-1">
         <LogChip severity={line.severity} onClickSeverity={onClickSeverity} />
       </div>
-      <div className="log-timestamp col-span-2 flex h-full items-center sm:col-span-1">
+      <div className="log-timestamp col-span-1 flex h-full items-center">
         {line.dateStamp}
       </div>
-      <div className="log-section col-span-2 flex size-full items-center pr-2">
+      <div
+        className={cn(
+          "log-section flex size-full items-center pr-2",
+          logService == "frigate" ? "col-span-2" : "col-span-1",
+        )}
+      >
         <div className="w-full overflow-hidden text-ellipsis whitespace-nowrap">
           {line.section}
         </div>
       </div>
-      <div className="log-content col-span-5 flex size-full items-center justify-between pl-2 pr-2 sm:col-span-4 sm:pl-0 md:col-span-8">
+      <div
+        className={cn(
+          "log-content col-span-4 flex size-full items-center justify-between pr-2",
+          logService == "frigate" ? "md:col-span-8" : "md:col-span-9",
+        )}
+      >
         <div className="w-full overflow-hidden text-ellipsis whitespace-nowrap">
           {line.content}
         </div>

web/src/utils/logUtil.ts

@@ -116,18 +116,63 @@ export function parseLogLines(logService: LogType, logs: string[]) {
   } else if (logService == "nginx") {
     return logs
       .map((line) => {
-        if (line.length == 0) {
+        if (line.trim().length === 0) return null;
-          return null;
-        }
 
-        return {
+        // Match full timestamp including nanoseconds
-          dateStamp: line.substring(0, 19),
+        const timestampRegex = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+/;
-          severity: "info",
+        const timestampMatch = timestampRegex.exec(line);
-          section: httpMethods.exec(line)?.at(0)?.toString() ?? "META",
+        const fullTimestamp = timestampMatch ? timestampMatch[0] : "";
-          content: line.substring(line.indexOf(" ", 20)).trim(),
+        // Remove nanoseconds from the final output
-        };
+        const dateStamp = fullTimestamp.split(".")[0];
+
+        // Handle different types of lines
+        if (line.includes("[INFO]")) {
+          // Info log
+          return {
+            dateStamp,
+            severity: "info",
+            section: "startup",
+            content: line.slice(fullTimestamp.length).trim(),
+          };
+        } else if (line.includes("[error]")) {
+          // Error log
+          const errorMatch = line.match(/(\[error\].*?,.*request: "[^"]*")/);
+          const content = errorMatch ? errorMatch[1] : line;
+          return {
+            dateStamp,
+            severity: "error",
+            section: "error",
+            content,
+          };
+        } else if (
+          line.includes("GET") ||
+          line.includes("POST") ||
+          line.includes("HTTP")
+        ) {
+          // HTTP request log
+          const httpMethodMatch = httpMethods.exec(line);
+          const section = httpMethodMatch ? httpMethodMatch[0] : "META";
+          const contentStart = line.indexOf('"', fullTimestamp.length);
+          const content =
+            contentStart !== -1 ? line.slice(contentStart).trim() : line;
+
+          return {
+            dateStamp,
+            severity: "info",
+            section,
+            content,
+          };
+        } else {
+          // Fallback: unknown format
+          return {
+            dateStamp,
+            severity: "unknown",
+            section: "unknown",
+            content: line.slice(fullTimestamp.length).trim(),
+          };
+        }
       })
-      .filter((value) => value != null) as LogLine[];
+      .filter((value) => value !== null) as LogLine[];
   }
 
   return [];