basic deduplication

frigate/api/app.py

@@ -401,10 +401,9 @@ def process_logs(
     end: Optional[int] = None,
 ) -> Tuple[int, List[str]]:
     log_lines = []
-    key_length = 0
+    last_message = None
-    date_end = 0
+    last_timestamp = None
-    current_key = ""
+    repeat_count = 0
-    current_line = ""
 
     for raw_line in contents.splitlines():
         clean_line = raw_line.strip()
@@ -412,28 +411,37 @@ def process_logs(
         if len(clean_line) < 10:
             continue
 
-        # handle cases where S6 does not include date in log line
+        # Handle cases where S6 does not include date in log line
         if "  " not in clean_line:
             clean_line = f"{datetime.now()}  {clean_line}"
 
-        if date_end == 0:
+        # Find the position of the first double space to extract timestamp and message
         date_end = clean_line.index("  ")
-            key_length = date_end
+        timestamp = clean_line[:date_end]
+        message_part = clean_line[date_end:].strip()
 
-        new_key = clean_line[:key_length]
+        if message_part == last_message:
-
+            repeat_count += 1
-        if new_key == current_key:
-            # use zero-width space character to delineate that this is a continuation
-            current_line += f"\u200b{clean_line[date_end:].strip()}"
             continue
         else:
-            if current_line:
+            if repeat_count > 0:
-                log_lines.append(current_line)
+                # Insert a deduplication message formatted the same way as logs
+                dedup_message = f"{last_timestamp}  [LOGGING] Last message repeated {repeat_count} times"
+                log_lines.append(dedup_message)
+                repeat_count = 0
 
-            current_key = new_key
+            log_lines.append(clean_line)
-            current_line = clean_line
+            last_timestamp = timestamp
+
+        last_message = message_part
+
+    # If there were repeated messages at the end, log the count
+    if repeat_count > 0:
+        dedup_message = (
+            f"{last_timestamp}  [LOGGING] Last message repeated {repeat_count} times"
+        )
+        log_lines.append(dedup_message)
 
-    log_lines.append(current_line)
     return len(log_lines), log_lines[start:end]
 
 

web/src/utils/logUtil.ts

@@ -18,6 +18,19 @@ export function parseLogLines(logService: LogType, logs: string[]) {
 
         if (!match) {
           const infoIndex = line.indexOf("[INFO]");
+          const loggingIndex = line.indexOf("[LOGGING]");
+
+          if (loggingIndex != -1) {
+            return {
+              dateStamp: line.substring(0, 19),
+              severity: "info",
+              section: "logging",
+              content: line
+                .substring(loggingIndex + 9)
+                .trim()
+                .replace(/\u200b/g, "\n"),
+            };
+          }
 
           if (infoIndex != -1) {
             return {
@@ -50,7 +63,7 @@ export function parseLogLines(logService: LogType, logs: string[]) {
           return null;
         }
 
-        const logLine = {
+        return {
           dateStamp: match.toString().slice(1, -1),
           severity: pythonSeverity
             .exec(line)
@@ -63,8 +76,6 @@ export function parseLogLines(logService: LogType, logs: string[]) {
             .trim()
             .replace(/\u200b/g, "\n"),
         };
-
-        return logLine;
       })
       .filter((value) => value != null) as LogLine[];
   } else if (logService == "go2rtc") {
@@ -95,6 +106,15 @@ export function parseLogLines(logService: LogType, logs: string[]) {
           contentStart = line.indexOf(section) + section.length + 2;
         }
 
+        if (line.includes("[LOGGING]")) {
+          return {
+            dateStamp: line.substring(0, 19),
+            severity: "info",
+            section: "logging",
+            content: line.substring(line.indexOf("[LOGGING]") + 9).trim(),
+          };
+        }
+
         let severityCat: LogSeverity;
         switch (severity?.at(0)?.toString().trim()) {
           case "INF":
@@ -134,9 +154,14 @@ export function parseLogLines(logService: LogType, logs: string[]) {
         // Remove nanoseconds from the final output
         const dateStamp = fullTimestamp.split(".")[0];
 
-        // Handle different types of lines
+        if (line.includes("[LOGGING]")) {
-        if (line.includes("[INFO]")) {
+          return {
-          // Info log
+            dateStamp,
+            severity: "info",
+            section: "logging",
+            content: line.slice(line.indexOf("[LOGGING]") + 9).trim(),
+          };
+        } else if (line.includes("[INFO]")) {
           return {
             dateStamp,
             severity: "info",