frigate/web/src/utils/logUtil.ts
Josh Hawkins 24ac9f3e5a
Use sqlite-vec extension instead of chromadb for embeddings (#14163)
* swap sqlite_vec for chroma in requirements

* load sqlite_vec in embeddings manager

* remove chroma and revamp Embeddings class for sqlite_vec

* manual minilm onnx inference

* remove chroma in clip model

* migrate api from chroma to sqlite_vec

* migrate event cleanup from chroma to sqlite_vec

* migrate embedding maintainer from chroma to sqlite_vec

* genai description for sqlite_vec

* load sqlite_vec in main thread db

* extend the SqliteQueueDatabase class and use peewee db.execute_sql

* search with Event type for similarity

* fix similarity search

* install and add comment about transformers

* fix normalization

* add id filter

* clean up

* clean up

* fully remove chroma and add transformers env var

* readd uvicorn for fastapi

* readd tokenizer parallelism env var

* remove chroma from docs

* remove chroma from UI

* try removing custom pysqlite3 build

* hard code limit

* optimize queries

* revert explore query

* fix query

* keep building pysqlite3

* single pass fetch and process

* remove unnecessary re-embed

* update deps

* move SqliteVecQueueDatabase to db directory

* make search thumbnail take up full size of results box

* improve typing

* improve model downloading and add status screen

* daemon downloading thread

* catch case when semantic search is disabled

* fix typing

* build sqlite_vec from source

* resolve conflict

* file permissions

* try build deps

* remove sources

* sources

* fix thread start

* include git in build

* reorder embeddings after detectors are started

* build with sqlite amalgamation

* non-platform specific

* use wget instead of curl

* remove unzip -d

* remove sqlite_vec from requirements and load the compiled version

* fix build

* avoid race in db connection

* add scale_factor and bias to description zscore normalization
2024-10-07 14:30:45 -06:00

135 lines
3.5 KiB
TypeScript

import { LogLine, LogSeverity, LogType } from "@/types/log";
const pythonSeverity = /(DEBUG)|(INFO)|(WARNING)|(ERROR)/;
const frigateDateStamp = /\[[\d\s-:]*]/;
const frigateSection = /[\w.]*/;
const goSeverity = /(DEB )|(INF )|(WRN )|(ERR )/;
const goSection = /\[[\w]*]/;
const httpMethods = /(GET)|(POST)|(PUT)|(PATCH)|(DELETE)/;
export function parseLogLines(logService: LogType, logs: string[]) {
if (logService == "frigate") {
return logs
.map((line) => {
const match = frigateDateStamp.exec(line);
if (!match) {
const infoIndex = line.indexOf("[INFO]");
if (infoIndex != -1) {
return {
dateStamp: line.substring(0, 19),
severity: "info",
section: "startup",
content: line.substring(infoIndex + 6).trim(),
};
}
return {
dateStamp: line.substring(0, 19),
severity: "unknown",
section: "unknown",
content: line.substring(30).trim(),
};
}
const sectionMatch = frigateSection.exec(
line.substring(match.index + match[0].length).trim(),
);
if (!sectionMatch) {
return null;
}
return {
dateStamp: match.toString().slice(1, -1),
severity: pythonSeverity
.exec(line)
?.at(0)
?.toString()
?.toLowerCase() as LogSeverity,
section: sectionMatch.toString(),
content: line
.substring(line.indexOf(":", match.index + match[0].length) + 2)
.trim(),
};
})
.filter((value) => value != null) as LogLine[];
} else if (logService == "go2rtc") {
return logs
.map((line) => {
if (line.length == 0) {
return null;
}
const severity = goSeverity.exec(line);
let section =
goSection.exec(line)?.toString()?.slice(1, -1) ?? "startup";
if (pythonSeverity.exec(section)) {
section = "startup";
}
let contentStart;
if (section == "startup") {
if (severity) {
contentStart = severity.index + severity[0].length;
} else {
contentStart = line.lastIndexOf("]") + 1;
}
} else {
contentStart = line.indexOf(section) + section.length + 2;
}
let severityCat: LogSeverity;
switch (severity?.at(0)?.toString().trim()) {
case "INF":
severityCat = "info";
break;
case "WRN":
severityCat = "warning";
break;
case "ERR":
severityCat = "error";
break;
case "DBG":
case "TRC":
severityCat = "debug";
break;
default:
severityCat = "info";
}
return {
dateStamp: line.substring(0, 19),
severity: severityCat,
section: section,
content: line.substring(contentStart).trim(),
};
})
.filter((value) => value != null) as LogLine[];
} else if (logService == "nginx") {
return logs
.map((line) => {
if (line.length == 0) {
return null;
}
return {
dateStamp: line.substring(0, 19),
severity: "info",
section: httpMethods.exec(line)?.at(0)?.toString() ?? "META",
content: line.substring(line.indexOf(" ", 20)).trim(),
};
})
.filter((value) => value != null) as LogLine[];
}
return [];
}