Add GenAI Backend Streaming and Chat (#22152)

* Add basic chat page with entry

* Add chat history

* processing

* Add markdown

* Improvements

* Adjust timing format

* Reduce fields in response

* More time parsing improvements

* Show tool calls separately from message

* Add title

* Improve UI handling

* Support streaming

* Full streaming support

* Fix tool calling

* Add copy button

* Improvements to UI

* Improve default behavior

* Implement message editing

* Add sub label to event tool filtering

* Cleanup

* Cleanup UI and prompt

* Cleanup UI bubbles

* Fix loading

* Add support for markdown tables

* Add thumbnail images to object results

* Add a starting state for chat

* Clenaup

frigate/genai/llama_cpp.py

@@ -5,10 +5,12 @@ import json
 import logging
 from typing import Any, Optional
 
+import httpx
 import requests
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
+from frigate.genai.utils import parse_tool_calls_from_message
 
 logger = logging.getLogger(__name__)
 
@@ -100,7 +102,79 @@ class LlamaCppClient(GenAIClient):
 
     def get_context_size(self) -> int:
         """Get the context window size for llama.cpp."""
-        return self.genai_config.provider_options.get("context_size", 4096)
+        return self.provider_options.get("context_size", 4096)
+
+    def _build_payload(
+        self,
+        messages: list[dict[str, Any]],
+        tools: Optional[list[dict[str, Any]]],
+        tool_choice: Optional[str],
+        stream: bool = False,
+    ) -> dict[str, Any]:
+        """Build request payload for chat completions (sync or stream)."""
+        openai_tool_choice = None
+        if tool_choice:
+            if tool_choice == "none":
+                openai_tool_choice = "none"
+            elif tool_choice == "auto":
+                openai_tool_choice = "auto"
+            elif tool_choice == "required":
+                openai_tool_choice = "required"
+
+        payload: dict[str, Any] = {
+            "messages": messages,
+            "model": self.genai_config.model,
+        }
+        if stream:
+            payload["stream"] = True
+        if tools:
+            payload["tools"] = tools
+            if openai_tool_choice is not None:
+                payload["tool_choice"] = openai_tool_choice
+        provider_opts = {
+            k: v for k, v in self.provider_options.items() if k != "context_size"
+        }
+        payload.update(provider_opts)
+        return payload
+
+    def _message_from_choice(self, choice: dict[str, Any]) -> dict[str, Any]:
+        """Parse OpenAI-style choice into {content, tool_calls, finish_reason}."""
+        message = choice.get("message", {})
+        content = message.get("content")
+        content = content.strip() if content else None
+        tool_calls = parse_tool_calls_from_message(message)
+        finish_reason = choice.get("finish_reason") or (
+            "tool_calls" if tool_calls else "stop" if content else "error"
+        )
+        return {
+            "content": content,
+            "tool_calls": tool_calls,
+            "finish_reason": finish_reason,
+        }
+
+    @staticmethod
+    def _streamed_tool_calls_to_list(
+        tool_calls_by_index: dict[int, dict[str, Any]],
+    ) -> Optional[list[dict[str, Any]]]:
+        """Convert streamed tool_calls index map to list of {id, name, arguments}."""
+        if not tool_calls_by_index:
+            return None
+        result = []
+        for idx in sorted(tool_calls_by_index.keys()):
+            t = tool_calls_by_index[idx]
+            args_str = t.get("arguments") or "{}"
+            try:
+                arguments = json.loads(args_str)
+            except json.JSONDecodeError:
+                arguments = {}
+            result.append(
+                {
+                    "id": t.get("id", ""),
+                    "name": t.get("name", ""),
+                    "arguments": arguments,
+                }
+            )
+        return result if result else None
 
     def chat_with_tools(
         self,
@@ -123,32 +197,8 @@ class LlamaCppClient(GenAIClient):
                 "tool_calls": None,
                 "finish_reason": "error",
             }
-
         try:
-            openai_tool_choice = None
-            if tool_choice:
-                if tool_choice == "none":
-                    openai_tool_choice = "none"
-                elif tool_choice == "auto":
-                    openai_tool_choice = "auto"
-                elif tool_choice == "required":
-                    openai_tool_choice = "required"
-
-            payload = {
-                "model": self.genai_config.model,
-                "messages": messages,
-            }
-
-            if tools:
-                payload["tools"] = tools
-                if openai_tool_choice is not None:
-                    payload["tool_choice"] = openai_tool_choice
-
-            provider_opts = {
-                k: v for k, v in self.provider_options.items() if k != "context_size"
-            }
-            payload.update(provider_opts)
-
+            payload = self._build_payload(messages, tools, tool_choice, stream=False)
             response = requests.post(
                 f"{self.provider}/v1/chat/completions",
                 json=payload,
@@ -156,60 +206,13 @@ class LlamaCppClient(GenAIClient):
             )
             response.raise_for_status()
             result = response.json()
-
             if result is None or "choices" not in result or len(result["choices"]) == 0:
                 return {
                     "content": None,
                     "tool_calls": None,
                     "finish_reason": "error",
                 }
-
-            choice = result["choices"][0]
-            message = choice.get("message", {})
-
-            content = message.get("content")
-            if content:
-                content = content.strip()
-            else:
-                content = None
-
-            tool_calls = None
-            if "tool_calls" in message and message["tool_calls"]:
-                tool_calls = []
-                for tool_call in message["tool_calls"]:
-                    try:
-                        function_data = tool_call.get("function", {})
-                        arguments_str = function_data.get("arguments", "{}")
-                        arguments = json.loads(arguments_str)
-                    except (json.JSONDecodeError, KeyError, TypeError) as e:
-                        logger.warning(
-                            f"Failed to parse tool call arguments: {e}, "
-                            f"tool: {function_data.get('name', 'unknown')}"
-                        )
-                        arguments = {}
-
-                    tool_calls.append(
-                        {
-                            "id": tool_call.get("id", ""),
-                            "name": function_data.get("name", ""),
-                            "arguments": arguments,
-                        }
-                    )
-
-            finish_reason = "error"
-            if "finish_reason" in choice and choice["finish_reason"]:
-                finish_reason = choice["finish_reason"]
-            elif tool_calls:
-                finish_reason = "tool_calls"
-            elif content:
-                finish_reason = "stop"
-
-            return {
-                "content": content,
-                "tool_calls": tool_calls,
-                "finish_reason": finish_reason,
-            }
-
+            return self._message_from_choice(result["choices"][0])
         except requests.exceptions.Timeout as e:
             logger.warning("llama.cpp request timed out: %s", str(e))
             return {
@@ -221,8 +224,7 @@ class LlamaCppClient(GenAIClient):
             error_detail = str(e)
             if hasattr(e, "response") and e.response is not None:
                 try:
-                    error_body = e.response.text
-                    error_detail = f"{str(e)} - Response: {error_body[:500]}"
+                    error_detail = f"{str(e)} - Response: {e.response.text[:500]}"
                 except Exception:
                     pass
             logger.warning("llama.cpp returned an error: %s", error_detail)
@@ -238,3 +240,111 @@ class LlamaCppClient(GenAIClient):
                 "tool_calls": None,
                 "finish_reason": "error",
             }
+
+    async def chat_with_tools_stream(
+        self,
+        messages: list[dict[str, Any]],
+        tools: Optional[list[dict[str, Any]]] = None,
+        tool_choice: Optional[str] = "auto",
+    ):
+        """Stream chat with tools via OpenAI-compatible streaming API."""
+        if self.provider is None:
+            logger.warning(
+                "llama.cpp provider has not been initialized. Check your llama.cpp configuration."
+            )
+            yield (
+                "message",
+                {
+                    "content": None,
+                    "tool_calls": None,
+                    "finish_reason": "error",
+                },
+            )
+            return
+        try:
+            payload = self._build_payload(messages, tools, tool_choice, stream=True)
+            content_parts: list[str] = []
+            tool_calls_by_index: dict[int, dict[str, Any]] = {}
+            finish_reason = "stop"
+
+            async with httpx.AsyncClient(timeout=float(self.timeout)) as client:
+                async with client.stream(
+                    "POST",
+                    f"{self.provider}/v1/chat/completions",
+                    json=payload,
+                ) as response:
+                    response.raise_for_status()
+                    async for line in response.aiter_lines():
+                        if not line.startswith("data: "):
+                            continue
+                        data_str = line[6:].strip()
+                        if data_str == "[DONE]":
+                            break
+                        try:
+                            data = json.loads(data_str)
+                        except json.JSONDecodeError:
+                            continue
+                        choices = data.get("choices") or []
+                        if not choices:
+                            continue
+                        delta = choices[0].get("delta", {})
+                        if choices[0].get("finish_reason"):
+                            finish_reason = choices[0]["finish_reason"]
+                        if delta.get("content"):
+                            content_parts.append(delta["content"])
+                            yield ("content_delta", delta["content"])
+                        for tc in delta.get("tool_calls") or []:
+                            idx = tc.get("index", 0)
+                            fn = tc.get("function") or {}
+                            if idx not in tool_calls_by_index:
+                                tool_calls_by_index[idx] = {
+                                    "id": tc.get("id", ""),
+                                    "name": tc.get("name") or fn.get("name", ""),
+                                    "arguments": "",
+                                }
+                            t = tool_calls_by_index[idx]
+                            if tc.get("id"):
+                                t["id"] = tc["id"]
+                            name = tc.get("name") or fn.get("name")
+                            if name:
+                                t["name"] = name
+                            arg = tc.get("arguments") or fn.get("arguments")
+                            if arg is not None:
+                                t["arguments"] += (
+                                    arg if isinstance(arg, str) else json.dumps(arg)
+                                )
+
+            full_content = "".join(content_parts).strip() or None
+            tool_calls_list = self._streamed_tool_calls_to_list(tool_calls_by_index)
+            if tool_calls_list:
+                finish_reason = "tool_calls"
+            yield (
+                "message",
+                {
+                    "content": full_content,
+                    "tool_calls": tool_calls_list,
+                    "finish_reason": finish_reason,
+                },
+            )
+        except httpx.HTTPStatusError as e:
+            logger.warning("llama.cpp streaming HTTP error: %s", e)
+            yield (
+                "message",
+                {
+                    "content": None,
+                    "tool_calls": None,
+                    "finish_reason": "error",
+                },
+            )
+        except Exception as e:
+            logger.warning(
+                "Unexpected error in llama.cpp chat_with_tools_stream: %s", str(e)
+            )
+            yield (
+                "message",
+                {
+                    "content": None,
+                    "tool_calls": None,
+                    "finish_reason": "error",
+                },
+            )

frigate/genai/ollama.py

@@ -1,15 +1,16 @@
 """Ollama Provider for Frigate AI."""
 
-import json
 import logging
 from typing import Any, Optional
 
 from httpx import RemoteProtocolError, TimeoutException
+from ollama import AsyncClient as OllamaAsyncClient
 from ollama import Client as ApiClient
 from ollama import ResponseError
 
 from frigate.config import GenAIProviderEnum
 from frigate.genai import GenAIClient, register_genai_provider
+from frigate.genai.utils import parse_tool_calls_from_message
 
 logger = logging.getLogger(__name__)
 
@@ -88,6 +89,73 @@ class OllamaClient(GenAIClient):
             "num_ctx", 4096
         )
 
+    def _build_request_params(
+        self,
+        messages: list[dict[str, Any]],
+        tools: Optional[list[dict[str, Any]]],
+        tool_choice: Optional[str],
+        stream: bool = False,
+    ) -> dict[str, Any]:
+        """Build request_messages and params for chat (sync or stream)."""
+        request_messages = []
+        for msg in messages:
+            msg_dict = {
+                "role": msg.get("role"),
+                "content": msg.get("content", ""),
+            }
+            if msg.get("tool_call_id"):
+                msg_dict["tool_call_id"] = msg["tool_call_id"]
+            if msg.get("name"):
+                msg_dict["name"] = msg["name"]
+            if msg.get("tool_calls"):
+                msg_dict["tool_calls"] = msg["tool_calls"]
+            request_messages.append(msg_dict)
+
+        request_params: dict[str, Any] = {
+            "model": self.genai_config.model,
+            "messages": request_messages,
+            **self.provider_options,
+        }
+        if stream:
+            request_params["stream"] = True
+        if tools:
+            request_params["tools"] = tools
+            if tool_choice:
+                request_params["tool_choice"] = (
+                    "none"
+                    if tool_choice == "none"
+                    else "required"
+                    if tool_choice == "required"
+                    else "auto"
+                )
+        return request_params
+
+    def _message_from_response(self, response: dict[str, Any]) -> dict[str, Any]:
+        """Parse Ollama chat response into {content, tool_calls, finish_reason}."""
+        if not response or "message" not in response:
+            return {
+                "content": None,
+                "tool_calls": None,
+                "finish_reason": "error",
+            }
+        message = response["message"]
+        content = message.get("content", "").strip() if message.get("content") else None
+        tool_calls = parse_tool_calls_from_message(message)
+        finish_reason = "error"
+        if response.get("done"):
+            finish_reason = (
+                "tool_calls" if tool_calls else "stop" if content else "error"
+            )
+        elif tool_calls:
+            finish_reason = "tool_calls"
+        elif content:
+            finish_reason = "stop"
+        return {
+            "content": content,
+            "tool_calls": tool_calls,
+            "finish_reason": finish_reason,
+        }
+
     def chat_with_tools(
         self,
         messages: list[dict[str, Any]],
@@ -103,93 +171,12 @@ class OllamaClient(GenAIClient):
                 "tool_calls": None,
                 "finish_reason": "error",
             }
-
         try:
-            request_messages = []
-            for msg in messages:
-                msg_dict = {
-                    "role": msg.get("role"),
-                    "content": msg.get("content", ""),
-                }
-                if msg.get("tool_call_id"):
-                    msg_dict["tool_call_id"] = msg["tool_call_id"]
-                if msg.get("name"):
-                    msg_dict["name"] = msg["name"]
-                if msg.get("tool_calls"):
-                    msg_dict["tool_calls"] = msg["tool_calls"]
-                request_messages.append(msg_dict)
-
-            request_params = {
-                "model": self.genai_config.model,
-                "messages": request_messages,
-            }
-
-            if tools:
-                request_params["tools"] = tools
-                if tool_choice:
-                    if tool_choice == "none":
-                        request_params["tool_choice"] = "none"
-                    elif tool_choice == "required":
-                        request_params["tool_choice"] = "required"
-                    elif tool_choice == "auto":
-                        request_params["tool_choice"] = "auto"
-
-            request_params.update(self.provider_options)
-
-            response = self.provider.chat(**request_params)
-
-            if not response or "message" not in response:
-                return {
-                    "content": None,
-                    "tool_calls": None,
-                    "finish_reason": "error",
-                }
-
-            message = response["message"]
-            content = (
-                message.get("content", "").strip() if message.get("content") else None
+            request_params = self._build_request_params(
+                messages, tools, tool_choice, stream=False
             )
-
-            tool_calls = None
-            if "tool_calls" in message and message["tool_calls"]:
-                tool_calls = []
-                for tool_call in message["tool_calls"]:
-                    try:
-                        function_data = tool_call.get("function", {})
-                        arguments_str = function_data.get("arguments", "{}")
-                        arguments = json.loads(arguments_str)
-                    except (json.JSONDecodeError, KeyError, TypeError) as e:
-                        logger.warning(
-                            f"Failed to parse tool call arguments: {e}, "
-                            f"tool: {function_data.get('name', 'unknown')}"
-                        )
-                        arguments = {}
-
-                    tool_calls.append(
-                        {
-                            "id": tool_call.get("id", ""),
-                            "name": function_data.get("name", ""),
-                            "arguments": arguments,
-                        }
-                    )
-
-            finish_reason = "error"
-            if "done" in response and response["done"]:
-                if tool_calls:
-                    finish_reason = "tool_calls"
-                elif content:
-                    finish_reason = "stop"
-            elif tool_calls:
-                finish_reason = "tool_calls"
-            elif content:
-                finish_reason = "stop"
-
-            return {
-                "content": content,
-                "tool_calls": tool_calls,
-                "finish_reason": finish_reason,
-            }
-
+            response = self.provider.chat(**request_params)
+            return self._message_from_response(response)
         except (TimeoutException, ResponseError, ConnectionError) as e:
             logger.warning("Ollama returned an error: %s", str(e))
             return {
@@ -204,3 +191,89 @@ class OllamaClient(GenAIClient):
                 "tool_calls": None,
                 "finish_reason": "error",
             }
+
+    async def chat_with_tools_stream(
+        self,
+        messages: list[dict[str, Any]],
+        tools: Optional[list[dict[str, Any]]] = None,
+        tool_choice: Optional[str] = "auto",
+    ):
+        """Stream chat with tools; yields content deltas then final message."""
+        if self.provider is None:
+            logger.warning(
+                "Ollama provider has not been initialized. Check your Ollama configuration."
+            )
+            yield (
+                "message",
+                {
+                    "content": None,
+                    "tool_calls": None,
+                    "finish_reason": "error",
+                },
+            )
+            return
+        try:
+            request_params = self._build_request_params(
+                messages, tools, tool_choice, stream=True
+            )
+            async_client = OllamaAsyncClient(
+                host=self.genai_config.base_url,
+                timeout=self.timeout,
+            )
+            content_parts: list[str] = []
+            final_message: dict[str, Any] | None = None
+            try:
+                stream = await async_client.chat(**request_params)
+                async for chunk in stream:
+                    if not chunk or "message" not in chunk:
+                        continue
+                    msg = chunk.get("message", {})
+                    delta = msg.get("content") or ""
+                    if delta:
+                        content_parts.append(delta)
+                        yield ("content_delta", delta)
+                    if chunk.get("done"):
+                        full_content = "".join(content_parts).strip() or None
+                        tool_calls = parse_tool_calls_from_message(msg)
+                        final_message = {
+                            "content": full_content,
+                            "tool_calls": tool_calls,
+                            "finish_reason": "tool_calls" if tool_calls else "stop",
+                        }
+                        break
+            finally:
+                await async_client.close()
+
+            if final_message is not None:
+                yield ("message", final_message)
+            else:
+                yield (
+                    "message",
+                    {
+                        "content": "".join(content_parts).strip() or None,
+                        "tool_calls": None,
+                        "finish_reason": "stop",
+                    },
+                )
+        except (TimeoutException, ResponseError, ConnectionError) as e:
+            logger.warning("Ollama streaming error: %s", str(e))
+            yield (
+                "message",
+                {
+                    "content": None,
+                    "tool_calls": None,
+                    "finish_reason": "error",
+                },
+            )
+        except Exception as e:
+            logger.warning(
+                "Unexpected error in Ollama chat_with_tools_stream: %s", str(e)
+            )
+            yield (
+                "message",
+                {
+                    "content": None,
+                    "tool_calls": None,
+                    "finish_reason": "error",
+                },
+            )

frigate/genai/utils.py

@@ -0,0 +1,70 @@
+"""Shared helpers for GenAI providers and chat (OpenAI-style messages, tool call parsing)."""
+
+import json
+import logging
+from typing import Any, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def parse_tool_calls_from_message(
+    message: dict[str, Any],
+) -> Optional[list[dict[str, Any]]]:
+    """
+    Parse tool_calls from an OpenAI-style message dict.
+
+    Message may have "tool_calls" as a list of:
+      {"id": str, "function": {"name": str, "arguments": str}, ...}
+
+    Returns a list of {"id", "name", "arguments"} with arguments parsed as dict,
+    or None if no tool_calls. Used by Ollama and LlamaCpp (non-stream) responses.
+    """
+    raw = message.get("tool_calls")
+    if not raw or not isinstance(raw, list):
+        return None
+    result = []
+    for tool_call in raw:
+        function_data = tool_call.get("function") or {}
+        try:
+            arguments_str = function_data.get("arguments") or "{}"
+            arguments = json.loads(arguments_str)
+        except (json.JSONDecodeError, KeyError, TypeError) as e:
+            logger.warning(
+                "Failed to parse tool call arguments: %s, tool: %s",
+                e,
+                function_data.get("name", "unknown"),
+            )
+            arguments = {}
+        result.append(
+            {
+                "id": tool_call.get("id", ""),
+                "name": function_data.get("name", ""),
+                "arguments": arguments,
+            }
+        )
+    return result if result else None
+
+
+def build_assistant_message_for_conversation(
+    content: Any,
+    tool_calls_raw: Optional[List[dict[str, Any]]],
+) -> dict[str, Any]:
+    """
+    Build the assistant message dict in OpenAI format for appending to a conversation.
+
+    tool_calls_raw: list of {"id", "name", "arguments"} (arguments as dict), or None.
+    """
+    msg: dict[str, Any] = {"role": "assistant", "content": content}
+    if tool_calls_raw:
+        msg["tool_calls"] = [
+            {
+                "id": tc["id"],
+                "type": "function",
+                "function": {
+                    "name": tc["name"],
+                    "arguments": json.dumps(tc.get("arguments") or {}),
+                },
+            }
+            for tc in tool_calls_raw
+        ]
+    return msg