From e24eb676a98ffae3157a0427b918d6af46826edb Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 4 Apr 2026 07:54:51 -0500 Subject: [PATCH] GenAI tweaks (#22756) * add DictAsYamlField for genai provider and runtime options * regenerate config translations * chat tweaks - add page title - scroll if near bottom - add tool call group that dynamically updates as tool calls are made - add bouncing loading indicator and other UI polish * tool call grouping --- frigate/config/camera/genai.py | 4 +- web/package-lock.json | 17 +- web/package.json | 2 + web/public/locales/en/config/global.json | 2 +- web/public/locales/en/views/chat.json | 1 + web/src/components/chat/ChatMessage.tsx | 78 ++++---- web/src/components/chat/ChatStartingState.tsx | 2 +- web/src/components/chat/ToolCallBubble.tsx | 8 +- web/src/components/chat/ToolCallsGroup.tsx | 103 ++++++++++ .../config-form/section-configs/genai.ts | 9 +- .../theme/fields/DictAsYamlField.tsx | 112 +++++++++++ .../config-form/theme/frigateTheme.ts | 2 + web/src/pages/Chat.tsx | 178 ++++++++++-------- 13 files changed, 386 insertions(+), 132 deletions(-) create mode 100644 web/src/components/chat/ToolCallsGroup.tsx create mode 100644 web/src/components/config-form/theme/fields/DictAsYamlField.tsx diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index 2adf223c1..721eeb60d 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -59,11 +59,11 @@ class GenAIConfig(FrigateBaseModel): default={}, title="Provider options", description="Additional provider-specific options to pass to the GenAI client.", - json_schema_extra={"additionalProperties": {"type": "string"}}, + json_schema_extra={"additionalProperties": {}}, ) runtime_options: dict[str, Any] = Field( default={}, title="Runtime options", description="Runtime options passed to the provider for each inference call.", - json_schema_extra={"additionalProperties": {"type": "string"}}, + json_schema_extra={"additionalProperties": {}}, ) diff --git a/web/package-lock.json b/web/package-lock.json index f910cb4ad..494498f30 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -52,6 +52,7 @@ "i18next-http-backend": "^3.0.1", "idb-keyval": "^6.2.1", "immer": "^10.1.1", + "js-yaml": "^4.1.1", "konva": "^10.2.3", "lodash": "^4.17.23", "lucide-react": "^0.577.0", @@ -90,6 +91,7 @@ "devDependencies": { "@tailwindcss/forms": "^0.5.9", "@testing-library/jest-dom": "^6.6.2", + "@types/js-yaml": "^4.0.9", "@types/lodash": "^4.17.12", "@types/node": "^20.14.10", "@types/react": "^19.2.14", @@ -5494,6 +5496,13 @@ "@types/unist": "*" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -6132,7 +6141,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, "license": "Python-2.0" }, "node_modules/aria-hidden": { @@ -9178,10 +9186,9 @@ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", "license": "MIT", "dependencies": { "argparse": "^2.0.1" diff --git a/web/package.json b/web/package.json index ed522b827..960b556ff 100644 --- a/web/package.json +++ b/web/package.json @@ -61,6 +61,7 @@ "i18next-http-backend": "^3.0.1", "idb-keyval": "^6.2.1", "immer": "^10.1.1", + "js-yaml": "^4.1.1", "konva": "^10.2.3", "lodash": "^4.17.23", "lucide-react": "^0.577.0", @@ -99,6 +100,7 @@ "devDependencies": { "@tailwindcss/forms": "^0.5.9", "@testing-library/jest-dom": "^6.6.2", + "@types/js-yaml": "^4.0.9", "@types/lodash": "^4.17.12", "@types/node": "^20.14.10", "@types/react": "^19.2.14", diff --git a/web/public/locales/en/config/global.json b/web/public/locales/en/config/global.json index 318d3a8c9..69c77fad1 100644 --- a/web/public/locales/en/config/global.json +++ b/web/public/locales/en/config/global.json @@ -527,7 +527,7 @@ }, "roles": { "label": "Roles", - "description": "GenAI roles (tools, vision, embeddings); one provider per role." + "description": "GenAI roles (chat, descriptions, embeddings); one provider per role." }, "provider_options": { "label": "Provider options", diff --git a/web/public/locales/en/views/chat.json b/web/public/locales/en/views/chat.json index bafe488a6..ca0520d88 100644 --- a/web/public/locales/en/views/chat.json +++ b/web/public/locales/en/views/chat.json @@ -1,4 +1,5 @@ { + "documentTitle": "Chat - Frigate", "title": "Frigate Chat", "subtitle": "Your AI assistant for camera management and insights", "placeholder": "Ask anything...", diff --git a/web/src/components/chat/ChatMessage.tsx b/web/src/components/chat/ChatMessage.tsx index a644a9d7d..b21fae435 100644 --- a/web/src/components/chat/ChatMessage.tsx +++ b/web/src/components/chat/ChatMessage.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef } from "react"; +import { useState, useEffect, useRef, useCallback } from "react"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; import { useTranslation } from "react-i18next"; @@ -6,6 +6,7 @@ import copy from "copy-to-clipboard"; import { toast } from "sonner"; import { FaCopy, FaPencilAlt } from "react-icons/fa"; import { FaArrowUpLong } from "react-icons/fa6"; +import { LuCheck } from "react-icons/lu"; import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; import { @@ -50,13 +51,17 @@ export function MessageBubble({ } }, [isEditing]); - const handleCopy = () => { + const [copied, setCopied] = useState(false); + + const handleCopy = useCallback(() => { const text = content?.trim() || ""; if (!text) return; if (copy(text)) { + setCopied(true); toast.success(t("button.copiedToClipboard", { ns: "common" })); + setTimeout(() => setCopied(false), 2000); } - }; + }, [content, t]); const handleEditClick = () => { setDraftContent(content); @@ -93,7 +98,7 @@ export function MessageBubble({ value={draftContent} onChange={(e) => setDraftContent(e.target.value)} onKeyDown={handleEditKeyDown} - className="min-h-[80px] w-full resize-y rounded-lg bg-primary px-3 py-2 text-primary-foreground placeholder:text-primary-foreground/60" + className="min-h-[80px] w-full resize-y rounded-2xl bg-primary px-4 py-3 text-primary-foreground placeholder:text-primary-foreground/60" placeholder={t("placeholder")} rows={3} /> @@ -124,44 +129,49 @@ export function MessageBubble({ return (
{isUser ? ( content ) : ( - ( - - ), - th: ({ node: _n, ...props }) => ( -
- ), - td: ({ node: _n, ...props }) => ( - - ), - }} - > - {content} - + <> + ( + + ), + th: ({ node: _n, ...props }) => ( +
+ ), + td: ({ node: _n, ...props }) => ( + + ), + }} + > + {content} + + {!isComplete && ( + + )} + )}
@@ -194,7 +204,11 @@ export function MessageBubble({ disabled={!content?.trim()} aria-label={t("button.copy", { ns: "common" })} > - + {copied ? ( + + ) : ( + + )} diff --git a/web/src/components/chat/ChatStartingState.tsx b/web/src/components/chat/ChatStartingState.tsx index d2309223f..e6b611bf9 100644 --- a/web/src/components/chat/ChatStartingState.tsx +++ b/web/src/components/chat/ChatStartingState.tsx @@ -75,7 +75,7 @@ export function ChatStartingState({ onSendMessage }: ChatStartingStateProps) {
-
+
{open ? ( - + ) : ( - + )} {isLeft ? t("call") : t("result")} {normalizedName} diff --git a/web/src/components/chat/ToolCallsGroup.tsx b/web/src/components/chat/ToolCallsGroup.tsx new file mode 100644 index 000000000..7cb20a1ad --- /dev/null +++ b/web/src/components/chat/ToolCallsGroup.tsx @@ -0,0 +1,103 @@ +import { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; +import { LuChevronsUpDown } from "react-icons/lu"; +import type { ToolCall } from "@/types/chat"; + +type ToolCallsGroupProps = { + toolCalls: ToolCall[]; +}; + +function normalizeName(name: string): string { + return name + .replace(/_/g, " ") + .split(" ") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) + .join(" "); +} + +export function ToolCallsGroup({ toolCalls }: ToolCallsGroupProps) { + const grouped = useMemo(() => { + const map = new Map(); + for (const tc of toolCalls) { + const existing = map.get(tc.name); + if (existing) { + existing.push(tc); + } else { + map.set(tc.name, [tc]); + } + } + return map; + }, [toolCalls]); + + if (toolCalls.length === 0) return null; + + return ( +
+ {[...grouped.entries()].map(([name, calls]) => ( + + ))} +
+ ); +} + +type ToolCallRowProps = { + name: string; + calls: ToolCall[]; +}; + +function ToolCallRow({ name, calls }: ToolCallRowProps) { + const { t } = useTranslation(["views/chat"]); + const [open, setOpen] = useState(false); + const displayName = normalizeName(name); + const label = + calls.length > 1 ? `${displayName} (\u00d7${calls.length})` : displayName; + + return ( + + + {label} + + + +
+ {calls.map((tc, idx) => ( +
1 + ? "space-y-1 border-l-2 border-border pl-3" + : "space-y-1" + } + > + {tc.arguments && Object.keys(tc.arguments).length > 0 && ( +
+
+ {t("arguments")} +
+ 
+                    {JSON.stringify(tc.arguments, null, 2)}
+
+
+ )} + {tc.response && tc.response !== "" && ( +
+
+ {t("response")} +
+ 
+                    {tc.response}
+
+
+ )} +
+ ))} +
+ + + ); +} diff --git a/web/src/components/config-form/section-configs/genai.ts b/web/src/components/config-form/section-configs/genai.ts index e5a298460..a5f1cd8a3 100644 --- a/web/src/components/config-form/section-configs/genai.ts +++ b/web/src/components/config-form/section-configs/genai.ts @@ -5,6 +5,7 @@ const genai: SectionConfigOverrides = { sectionDocs: "/configuration/genai/config", advancedFields: ["*.base_url", "*.provider_options", "*.runtime_options"], hiddenFields: ["genai.enabled_in_config"], + restartRequired: [], uiSchema: { "ui:options": { disableNestedCard: true }, "*": { @@ -36,14 +37,10 @@ const genai: SectionConfigOverrides = { "ui:options": { size: "xs" }, }, "*.provider_options": { - additionalProperties: { - "ui:options": { size: "lg" }, - }, + "ui:field": "DictAsYamlField", }, "*.runtime_options": { - additionalProperties: { - "ui:options": { size: "lg" }, - }, + "ui:field": "DictAsYamlField", }, }, }, diff --git a/web/src/components/config-form/theme/fields/DictAsYamlField.tsx b/web/src/components/config-form/theme/fields/DictAsYamlField.tsx new file mode 100644 index 000000000..0b21f9b1a --- /dev/null +++ b/web/src/components/config-form/theme/fields/DictAsYamlField.tsx @@ -0,0 +1,112 @@ +import type { FieldPathList, FieldProps } from "@rjsf/utils"; +import yaml from "js-yaml"; +import { Textarea } from "@/components/ui/textarea"; +import { cn } from "@/lib/utils"; +import { useCallback, useEffect, useMemo, useState } from "react"; + +function formatYaml(value: unknown): string { + if ( + value == null || + (typeof value === "object" && + !Array.isArray(value) && + Object.keys(value as Record).length === 0) + ) { + return ""; + } + try { + return yaml.dump(value, { indent: 2, lineWidth: -1 }).trimEnd(); + } catch { + return ""; + } +} + +function parseYaml(text: string): { + value: Record | undefined; + error: string | undefined; +} { + const trimmed = text.trim(); + if (trimmed === "") { + return { value: {}, error: undefined }; + } + try { + const parsed = yaml.load(trimmed); + if ( + typeof parsed !== "object" || + parsed === null || + Array.isArray(parsed) + ) { + return { value: undefined, error: "Must be a YAML mapping" }; + } + return { value: parsed as Record, error: undefined }; + } catch (e) { + const msg = e instanceof yaml.YAMLException ? e.reason : "Invalid YAML"; + return { value: undefined, error: msg }; + } +} + +export function DictAsYamlField(props: FieldProps) { + const { formData, onChange, readonly, disabled, idSchema, schema } = props; + + const emptyPath = useMemo(() => [] as FieldPathList, []); + const fieldPath = + (props as { fieldPathId?: { path?: FieldPathList } }).fieldPathId?.path ?? + emptyPath; + + const [text, setText] = useState(() => formatYaml(formData)); + const [error, setError] = useState(); + + useEffect(() => { + setText(formatYaml(formData)); + setError(undefined); + }, [formData]); + + const handleChange = useCallback( + (e: React.ChangeEvent) => { + const raw = e.target.value; + setText(raw); + const { value, error: parseError } = parseYaml(raw); + setError(parseError); + if (value !== undefined) { + onChange(value, fieldPath); + } + }, + [onChange, fieldPath], + ); + + const handleBlur = useCallback( + (_e: React.FocusEvent) => { + // Reformat on blur if valid + const { value } = parseYaml(text); + if (value !== undefined) { + setText(formatYaml(value)); + } + }, + [text], + ); + + const id = idSchema?.$id ?? props.name; + + return ( +
+ {schema.title && ( + + )} +