From d5aa2a0341d8e1dad4779d6cda76102c870e3130 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 12 Feb 2026 19:35:40 -0700 Subject: [PATCH] Improvements --- frigate/api/chat.py | 70 +++++++++++++++++--- frigate/api/defs/response/chat_response.py | 23 ++++++- web/public/locales/en/views/chat.json | 5 +- web/src/components/chat/AssistantMessage.tsx | 66 ++++++++++++++++++ web/src/pages/Chat.tsx | 23 +++++-- 5 files changed, 171 insertions(+), 16 deletions(-) create mode 100644 web/src/components/chat/AssistantMessage.tsx diff --git a/frigate/api/chat.py b/frigate/api/chat.py index 415f422da..9e491c55e 100644 --- a/frigate/api/chat.py +++ b/frigate/api/chat.py @@ -3,7 +3,7 @@ import base64 import json import logging -from datetime import datetime, timezone +from datetime import datetime from typing import Any, Dict, List, Optional import cv2 @@ -20,6 +20,7 @@ from frigate.api.defs.request.chat_body import ChatCompletionRequest from frigate.api.defs.response.chat_response import ( ChatCompletionResponse, ChatMessageResponse, + ToolCall, ) from frigate.api.defs.tags import Tags from frigate.api.event import events @@ -29,6 +30,29 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=[Tags.chat]) +def _format_events_with_local_time(events_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Add human-readable local start/end times to each event for the LLM.""" + result = [] + for evt in events_list: + if not isinstance(evt, dict): + result.append(evt) + continue + copy_evt = dict(evt) + try: + start_ts = evt.get("start_time") + end_ts = evt.get("end_time") + if start_ts is not None: + dt_start = datetime.fromtimestamp(start_ts) + copy_evt["start_time_local"] = dt_start.strftime("%Y-%m-%d %H:%M:%S %Z") + if end_ts is not None: + dt_end = datetime.fromtimestamp(end_ts) + copy_evt["end_time_local"] = dt_end.strftime("%Y-%m-%d %H:%M:%S %Z") + except (TypeError, ValueError, OSError): + pass + result.append(copy_evt) + return result + + class ToolExecuteRequest(BaseModel): """Request model for tool execution.""" @@ -394,7 +418,7 @@ async def chat_completion( tools = get_tool_definitions() conversation = [] - current_datetime = datetime.now(timezone.utc) + current_datetime = datetime.now() current_date_str = current_datetime.strftime("%Y-%m-%d") current_time_str = current_datetime.strftime("%H:%M:%S %Z") @@ -429,9 +453,10 @@ async def chat_completion( system_prompt = f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events. -Current date and time: {current_date_str} at {current_time_str} (UTC) +Current server local date and time: {current_date_str} at {current_time_str} -When users ask questions about "today", "yesterday", "this week", etc., use the current date above as reference. +Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields. +When users ask about "today", "yesterday", "this week", etc., use the current date above as reference. When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today). Always be accurate with time calculations based on the current date provided.{cameras_section}{live_image_note}""" @@ -471,6 +496,7 @@ Always be accurate with time calculations based on the current date provided.{ca conversation.append(msg_dict) tool_iterations = 0 + tool_calls: List[ToolCall] = [] max_iterations = body.max_tool_iterations logger.debug( @@ -517,8 +543,8 @@ Always be accurate with time calculations based on the current date provided.{ca ] conversation.append(assistant_message) - tool_calls = response.get("tool_calls") - if not tool_calls: + pending_tool_calls = response.get("tool_calls") + if not pending_tool_calls: logger.debug( f"Chat completion finished with final answer (iterations: {tool_iterations})" ) @@ -531,6 +557,7 @@ Always be accurate with time calculations based on the current date provided.{ca ), finish_reason=response.get("finish_reason", "stop"), tool_iterations=tool_iterations, + tool_calls=tool_calls, ).model_dump(), ) @@ -538,11 +565,11 @@ Always be accurate with time calculations based on the current date provided.{ca tool_iterations += 1 logger.debug( f"Tool calls detected (iteration {tool_iterations}/{max_iterations}): " - f"{len(tool_calls)} tool(s) to execute" + f"{len(pending_tool_calls)} tool(s) to execute" ) tool_results = [] - for tool_call in tool_calls: + for tool_call in pending_tool_calls: tool_name = tool_call["name"] tool_args = tool_call["arguments"] tool_call_id = tool_call["id"] @@ -556,6 +583,12 @@ Always be accurate with time calculations based on the current date provided.{ca tool_name, tool_args, request, allowed_cameras ) + # Add local time fields to search_objects results so the LLM doesn't hallucinate timestamps + if tool_name == "search_objects" and isinstance( + tool_result, list + ): + tool_result = _format_events_with_local_time(tool_result) + if isinstance(tool_result, dict): result_content = json.dumps(tool_result) result_summary = tool_result @@ -573,6 +606,12 @@ Always be accurate with time calculations based on the current date provided.{ca f"Tool {tool_name} (id: {tool_call_id}) completed successfully. " f"Result: {json.dumps(result_summary, indent=2)}" ) + elif isinstance(tool_result, list): + result_content = json.dumps(tool_result) + logger.debug( + f"Tool {tool_name} (id: {tool_call_id}) completed successfully. " + f"Result: {len(tool_result)} item(s)" + ) elif isinstance(tool_result, str): result_content = tool_result logger.debug( @@ -586,6 +625,13 @@ Always be accurate with time calculations based on the current date provided.{ca f"Result type: {type(tool_result).__name__}" ) + tool_calls.append( + ToolCall( + name=tool_name, + arguments=tool_args or {}, + response=result_content, + ) + ) tool_results.append( { "role": "tool", @@ -599,6 +645,13 @@ Always be accurate with time calculations based on the current date provided.{ca exc_info=True, ) error_content = json.dumps({"error": "Tool execution failed"}) + tool_calls.append( + ToolCall( + name=tool_name, + arguments=tool_args or {}, + response=error_content, + ) + ) tool_results.append( { "role": "tool", @@ -628,6 +681,7 @@ Always be accurate with time calculations based on the current date provided.{ca ), finish_reason="length", tool_iterations=tool_iterations, + tool_calls=tool_calls, ).model_dump(), ) diff --git a/frigate/api/defs/response/chat_response.py b/frigate/api/defs/response/chat_response.py index f1cc9194b..0bc864ba6 100644 --- a/frigate/api/defs/response/chat_response.py +++ b/frigate/api/defs/response/chat_response.py @@ -5,8 +5,8 @@ from typing import Any, Optional from pydantic import BaseModel, Field -class ToolCall(BaseModel): - """A tool call from the LLM.""" +class ToolCallInvocation(BaseModel): + """A tool call requested by the LLM (before execution).""" id: str = Field(description="Unique identifier for this tool call") name: str = Field(description="Tool name to call") @@ -20,11 +20,24 @@ class ChatMessageResponse(BaseModel): content: Optional[str] = Field( default=None, description="Message content (None if tool calls present)" ) - tool_calls: Optional[list[ToolCall]] = Field( + tool_calls: Optional[list[ToolCallInvocation]] = Field( default=None, description="Tool calls if LLM wants to call tools" ) +class ToolCall(BaseModel): + """A tool that was executed during the completion, with its response.""" + + name: str = Field(description="Tool name that was called") + arguments: dict[str, Any] = Field( + default_factory=dict, description="Arguments passed to the tool" + ) + response: str = Field( + default="", + description="The response or result returned from the tool execution", + ) + + class ChatCompletionResponse(BaseModel): """Response from chat completion.""" @@ -35,3 +48,7 @@ class ChatCompletionResponse(BaseModel): tool_iterations: int = Field( default=0, description="Number of tool call iterations performed" ) + tool_calls: list[ToolCall] = Field( + default_factory=list, + description="List of tool calls that were executed during this completion", + ) diff --git a/web/public/locales/en/views/chat.json b/web/public/locales/en/views/chat.json index 29078ff54..1fe688fdd 100644 --- a/web/public/locales/en/views/chat.json +++ b/web/public/locales/en/views/chat.json @@ -1,5 +1,8 @@ { "placeholder": "Ask anything...", "error": "Something went wrong. Please try again.", - "processing": "Processing..." + "processing": "Processing...", + "toolsUsed": "Used: {{tools}}", + "showTools": "Show tools ({{count}})", + "hideTools": "Hide tools" } diff --git a/web/src/components/chat/AssistantMessage.tsx b/web/src/components/chat/AssistantMessage.tsx new file mode 100644 index 000000000..73008527f --- /dev/null +++ b/web/src/components/chat/AssistantMessage.tsx @@ -0,0 +1,66 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import ReactMarkdown from "react-markdown"; +import { Button } from "@/components/ui/button"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; + +export type ToolCall = { + name: string; + arguments?: Record; + response?: string; +}; + +type AssistantMessageProps = { + content: string; + toolCalls?: ToolCall[]; +}; + +export function AssistantMessage({ + content, + toolCalls, +}: AssistantMessageProps) { + const { t } = useTranslation(["views/chat"]); + const [open, setOpen] = useState(false); + const hasToolCalls = toolCalls && toolCalls.length > 0; + + return ( +
+ {content} + {hasToolCalls && ( + + + + + +
    + {toolCalls.map((tc, idx) => ( +
  • + + {tc.name} + + {tc.response != null && tc.response !== "" && ( + 
    +                      {tc.response}
+
    + )} +
    • + ))} +
+ + + )} +
+ ); +} diff --git a/web/src/pages/Chat.tsx b/web/src/pages/Chat.tsx index b487b3a4f..8c49119c1 100644 --- a/web/src/pages/Chat.tsx +++ b/web/src/pages/Chat.tsx @@ -4,9 +4,16 @@ import { FaArrowUpLong } from "react-icons/fa6"; import { useTranslation } from "react-i18next"; import { useState, useCallback } from "react"; import axios from "axios"; -import ReactMarkdown from "react-markdown"; +import { + AssistantMessage, + type ToolCall, +} from "@/components/chat/AssistantMessage"; -type ChatMessage = { role: "user" | "assistant"; content: string }; +type ChatMessage = { + role: "user" | "assistant"; + content: string; + toolCalls?: ToolCall[]; +}; export default function ChatPage() { const { t } = useTranslation(["views/chat"]); @@ -32,12 +39,17 @@ export default function ChatPage() { })); const { data } = await axios.post<{ message: { role: string; content: string | null }; + tool_calls?: ToolCall[]; }>("chat/completion", { messages: apiMessages }); const content = data.message?.content ?? ""; setMessages((prev) => [ ...prev, - { role: "assistant", content: content || " " }, + { + role: "assistant", + content: content || " ", + toolCalls: data.tool_calls?.length ? data.tool_calls : undefined, + }, ]); } catch { setError(t("error")); @@ -59,7 +71,10 @@ export default function ChatPage() { } > {msg.role === "assistant" ? ( - {msg.content} + ) : ( msg.content )}