mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-05-09 15:05:26 +03:00
Compare commits
No commits in common. "d113be5e190157cf3469b87cde2b6199a838871d" and "556d5d8c9d4ff7c520d8377c6e7c94ee5c297c80" have entirely different histories.
d113be5e19
...
556d5d8c9d
@ -3,11 +3,9 @@
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import operator
|
||||
import time
|
||||
from datetime import datetime
|
||||
from functools import reduce
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, Generator, List, Optional
|
||||
|
||||
import cv2
|
||||
from fastapi import APIRouter, Body, Depends, Request
|
||||
@ -19,14 +17,6 @@ from frigate.api.auth import (
|
||||
get_allowed_cameras_for_filter,
|
||||
require_camera_access,
|
||||
)
|
||||
from frigate.api.chat_util import (
|
||||
chunk_content,
|
||||
distance_to_score,
|
||||
format_events_with_local_time,
|
||||
fuse_scores,
|
||||
hydrate_event,
|
||||
parse_iso_to_timestamp,
|
||||
)
|
||||
from frigate.api.defs.query.events_query_parameters import EventsQueryParams
|
||||
from frigate.api.defs.request.chat_body import ChatCompletionRequest
|
||||
from frigate.api.defs.response.chat_response import (
|
||||
@ -42,13 +32,55 @@ from frigate.jobs.vlm_watch import (
|
||||
start_vlm_watch_job,
|
||||
stop_vlm_watch_job,
|
||||
)
|
||||
from frigate.models import Event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(tags=[Tags.chat])
|
||||
|
||||
|
||||
def _chunk_content(content: str, chunk_size: int = 80) -> Generator[str, None, None]:
|
||||
"""Yield content in word-aware chunks for streaming."""
|
||||
if not content:
|
||||
return
|
||||
words = content.split(" ")
|
||||
current: List[str] = []
|
||||
current_len = 0
|
||||
for w in words:
|
||||
current.append(w)
|
||||
current_len += len(w) + 1
|
||||
if current_len >= chunk_size:
|
||||
yield " ".join(current) + " "
|
||||
current = []
|
||||
current_len = 0
|
||||
if current:
|
||||
yield " ".join(current)
|
||||
|
||||
|
||||
def _format_events_with_local_time(
|
||||
events_list: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Add human-readable local start/end times to each event for the LLM."""
|
||||
result = []
|
||||
for evt in events_list:
|
||||
if not isinstance(evt, dict):
|
||||
result.append(evt)
|
||||
continue
|
||||
copy_evt = dict(evt)
|
||||
try:
|
||||
start_ts = evt.get("start_time")
|
||||
end_ts = evt.get("end_time")
|
||||
if start_ts is not None:
|
||||
dt_start = datetime.fromtimestamp(start_ts)
|
||||
copy_evt["start_time_local"] = dt_start.strftime("%Y-%m-%d %I:%M:%S %p")
|
||||
if end_ts is not None:
|
||||
dt_end = datetime.fromtimestamp(end_ts)
|
||||
copy_evt["end_time_local"] = dt_end.strftime("%Y-%m-%d %I:%M:%S %p")
|
||||
except (TypeError, ValueError, OSError):
|
||||
pass
|
||||
result.append(copy_evt)
|
||||
return result
|
||||
|
||||
|
||||
class ToolExecuteRequest(BaseModel):
|
||||
"""Request model for tool execution."""
|
||||
|
||||
@ -126,76 +158,6 @@ def get_tool_definitions() -> List[Dict[str, Any]]:
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find_similar_objects",
|
||||
"description": (
|
||||
"Find tracked objects that are visually and semantically similar "
|
||||
"to a specific past event. Use this when the user references a "
|
||||
"particular object they have seen and wants to find other "
|
||||
"sightings of the same or similar one ('that green car', 'the "
|
||||
"person in the red jacket', 'the package that was delivered'). "
|
||||
"Prefer this over search_objects whenever the user's intent is "
|
||||
"'find more like this specific one.' Use search_objects first "
|
||||
"only if you need to locate the anchor event. Requires semantic "
|
||||
"search to be enabled."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"event_id": {
|
||||
"type": "string",
|
||||
"description": "The id of the anchor event to find similar objects to.",
|
||||
},
|
||||
"after": {
|
||||
"type": "string",
|
||||
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
||||
},
|
||||
"cameras": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of cameras to restrict to. Defaults to all.",
|
||||
},
|
||||
"labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of labels to restrict to. Defaults to the anchor event's label.",
|
||||
},
|
||||
"sub_labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of sub_labels (names) to restrict to.",
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of zones. An event matches if any of its zones overlap.",
|
||||
},
|
||||
"similarity_mode": {
|
||||
"type": "string",
|
||||
"enum": ["visual", "semantic", "fused"],
|
||||
"description": "Which similarity signal(s) to use. 'fused' (default) combines visual and semantic.",
|
||||
"default": "fused",
|
||||
},
|
||||
"min_score": {
|
||||
"type": "number",
|
||||
"description": "Drop matches with a similarity score below this threshold (0.0-1.0).",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of matches to return (default: 10).",
|
||||
"default": 10,
|
||||
},
|
||||
},
|
||||
"required": ["event_id"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
@ -472,166 +434,6 @@ async def _execute_search_objects(
|
||||
)
|
||||
|
||||
|
||||
async def _execute_find_similar_objects(
|
||||
request: Request,
|
||||
arguments: Dict[str, Any],
|
||||
allowed_cameras: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute the find_similar_objects tool.
|
||||
|
||||
Returns a plain dict (not JSONResponse) so the chat loop can embed it
|
||||
directly in tool-result messages.
|
||||
"""
|
||||
# 1. Semantic search enabled?
|
||||
config = request.app.frigate_config
|
||||
if not getattr(config.semantic_search, "enabled", False):
|
||||
return {
|
||||
"error": "semantic_search_disabled",
|
||||
"message": (
|
||||
"Semantic search must be enabled to find similar objects. "
|
||||
"Enable it in the Frigate config under semantic_search."
|
||||
),
|
||||
}
|
||||
|
||||
context = request.app.embeddings
|
||||
if context is None:
|
||||
return {
|
||||
"error": "semantic_search_disabled",
|
||||
"message": "Embeddings context is not available.",
|
||||
}
|
||||
|
||||
# 2. Anchor lookup.
|
||||
event_id = arguments.get("event_id")
|
||||
if not event_id:
|
||||
return {"error": "missing_event_id", "message": "event_id is required."}
|
||||
|
||||
try:
|
||||
anchor = Event.get(Event.id == event_id)
|
||||
except Event.DoesNotExist:
|
||||
return {
|
||||
"error": "anchor_not_found",
|
||||
"message": f"Could not find event {event_id}.",
|
||||
}
|
||||
|
||||
# 3. Parse params.
|
||||
after = parse_iso_to_timestamp(arguments.get("after"))
|
||||
before = parse_iso_to_timestamp(arguments.get("before"))
|
||||
|
||||
cameras = arguments.get("cameras")
|
||||
if cameras:
|
||||
# Respect RBAC: intersect with the user's allowed cameras.
|
||||
cameras = [c for c in cameras if c in allowed_cameras]
|
||||
else:
|
||||
cameras = list(allowed_cameras) if allowed_cameras else None
|
||||
|
||||
labels = arguments.get("labels") or [anchor.label]
|
||||
sub_labels = arguments.get("sub_labels")
|
||||
zones = arguments.get("zones")
|
||||
|
||||
similarity_mode = arguments.get("similarity_mode", "fused")
|
||||
if similarity_mode not in ("visual", "semantic", "fused"):
|
||||
similarity_mode = "fused"
|
||||
|
||||
min_score = arguments.get("min_score")
|
||||
limit = int(arguments.get("limit", 10))
|
||||
limit = max(1, min(limit, 50))
|
||||
|
||||
# 4. Run similarity searches. We deliberately do NOT pass event_ids into
|
||||
# the vec queries — the IN filter on sqlite-vec is broken in the installed
|
||||
# version (see frigate/embeddings/__init__.py). Mirror the pattern used by
|
||||
# frigate/api/event.py events_search: fetch top-k globally, then intersect
|
||||
# with the structured filters via Peewee.
|
||||
visual_distances: Dict[str, float] = {}
|
||||
description_distances: Dict[str, float] = {}
|
||||
|
||||
try:
|
||||
if similarity_mode in ("visual", "fused"):
|
||||
rows = context.search_thumbnail(anchor)
|
||||
visual_distances = {row[0]: row[1] for row in rows}
|
||||
|
||||
if similarity_mode in ("semantic", "fused"):
|
||||
query_text = (
|
||||
(anchor.data or {}).get("description")
|
||||
or anchor.sub_label
|
||||
or anchor.label
|
||||
)
|
||||
rows = context.search_description(query_text)
|
||||
description_distances = {row[0]: row[1] for row in rows}
|
||||
except Exception:
|
||||
logger.exception("Similarity search failed")
|
||||
return {
|
||||
"error": "similarity_search_failed",
|
||||
"message": "Failed to run similarity search.",
|
||||
}
|
||||
|
||||
vec_ids = set(visual_distances) | set(description_distances)
|
||||
vec_ids.discard(anchor.id)
|
||||
# vec layer returns up to k=100 per modality; flag when we hit that ceiling
|
||||
# so the LLM can mention there may be more matches beyond what we saw.
|
||||
candidate_truncated = (
|
||||
len(visual_distances) >= 100 or len(description_distances) >= 100
|
||||
)
|
||||
|
||||
if not vec_ids:
|
||||
return {
|
||||
"anchor": hydrate_event(anchor),
|
||||
"results": [],
|
||||
"similarity_mode": similarity_mode,
|
||||
"candidate_truncated": candidate_truncated,
|
||||
}
|
||||
|
||||
# 5. Apply structured filters, intersected with vec hits.
|
||||
clauses = [Event.id.in_(list(vec_ids))]
|
||||
if after is not None:
|
||||
clauses.append(Event.start_time >= after)
|
||||
if before is not None:
|
||||
clauses.append(Event.start_time <= before)
|
||||
if cameras:
|
||||
clauses.append(Event.camera.in_(cameras))
|
||||
if labels:
|
||||
clauses.append(Event.label.in_(labels))
|
||||
if sub_labels:
|
||||
clauses.append(Event.sub_label.in_(sub_labels))
|
||||
if zones:
|
||||
# Mirror the pattern used by frigate/api/event.py for JSON-array zone match.
|
||||
zone_clauses = [Event.zones.cast("text") % f'*"{zone}"*' for zone in zones]
|
||||
clauses.append(reduce(operator.or_, zone_clauses))
|
||||
|
||||
eligible = {e.id: e for e in Event.select().where(reduce(operator.and_, clauses))}
|
||||
|
||||
# 6. Fuse and rank.
|
||||
scored: List[tuple[str, float]] = []
|
||||
for eid in eligible:
|
||||
v_score = (
|
||||
distance_to_score(visual_distances[eid], context.thumb_stats)
|
||||
if eid in visual_distances
|
||||
else None
|
||||
)
|
||||
d_score = (
|
||||
distance_to_score(description_distances[eid], context.desc_stats)
|
||||
if eid in description_distances
|
||||
else None
|
||||
)
|
||||
fused = fuse_scores(v_score, d_score)
|
||||
if fused is None:
|
||||
continue
|
||||
if min_score is not None and fused < min_score:
|
||||
continue
|
||||
scored.append((eid, fused))
|
||||
|
||||
scored.sort(key=lambda pair: pair[1], reverse=True)
|
||||
scored = scored[:limit]
|
||||
|
||||
results = [hydrate_event(eligible[eid], score=score) for eid, score in scored]
|
||||
|
||||
return {
|
||||
"anchor": hydrate_event(anchor),
|
||||
"results": results,
|
||||
"similarity_mode": similarity_mode,
|
||||
"candidate_truncated": candidate_truncated,
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/chat/execute",
|
||||
dependencies=[Depends(allow_any_authenticated())],
|
||||
@ -657,13 +459,6 @@ async def execute_tool(
|
||||
if tool_name == "search_objects":
|
||||
return await _execute_search_objects(arguments, allowed_cameras)
|
||||
|
||||
if tool_name == "find_similar_objects":
|
||||
result = await _execute_find_similar_objects(
|
||||
request, arguments, allowed_cameras
|
||||
)
|
||||
status_code = 200 if "error" not in result else 400
|
||||
return JSONResponse(content=result, status_code=status_code)
|
||||
|
||||
if tool_name == "set_camera_state":
|
||||
result = await _execute_set_camera_state(request, arguments)
|
||||
return JSONResponse(
|
||||
@ -847,8 +642,6 @@ async def _execute_tool_internal(
|
||||
except (json.JSONDecodeError, AttributeError) as e:
|
||||
logger.warning(f"Failed to extract tool result: {e}")
|
||||
return {"error": "Failed to parse tool result"}
|
||||
elif tool_name == "find_similar_objects":
|
||||
return await _execute_find_similar_objects(request, arguments, allowed_cameras)
|
||||
elif tool_name == "set_camera_state":
|
||||
return await _execute_set_camera_state(request, arguments)
|
||||
elif tool_name == "get_live_context":
|
||||
@ -871,9 +664,8 @@ async def _execute_tool_internal(
|
||||
return _execute_get_recap(arguments, allowed_cameras)
|
||||
else:
|
||||
logger.error(
|
||||
"Tool call failed: unknown tool %r. Expected one of: search_objects, find_similar_objects, "
|
||||
"get_live_context, start_camera_watch, stop_camera_watch, get_profile_status, get_recap. "
|
||||
"Arguments received: %s",
|
||||
"Tool call failed: unknown tool %r. Expected one of: search_objects, get_live_context, "
|
||||
"start_camera_watch, stop_camera_watch, get_profile_status, get_recap. Arguments received: %s",
|
||||
tool_name,
|
||||
json.dumps(arguments),
|
||||
)
|
||||
@ -1135,7 +927,7 @@ async def _execute_pending_tools(
|
||||
json.dumps(tool_args),
|
||||
)
|
||||
if tool_name == "search_objects" and isinstance(tool_result, list):
|
||||
tool_result = format_events_with_local_time(tool_result)
|
||||
tool_result = _format_events_with_local_time(tool_result)
|
||||
_keys = {
|
||||
"id",
|
||||
"camera",
|
||||
@ -1288,9 +1080,7 @@ Do not start your response with phrases like "I will check...", "Let me see...",
|
||||
Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields.
|
||||
When users ask about "today", "yesterday", "this week", etc., use the current date above as reference.
|
||||
When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today).
|
||||
Always be accurate with time calculations based on the current date provided.
|
||||
|
||||
When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:<id>], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{cameras_section}"""
|
||||
Always be accurate with time calculations based on the current date provided.{cameras_section}"""
|
||||
|
||||
conversation.append(
|
||||
{
|
||||
@ -1328,9 +1118,6 @@ When a user refers to a specific object they have seen or describe with identify
|
||||
async def stream_body_llm():
|
||||
nonlocal conversation, stream_tool_calls, stream_iterations
|
||||
while stream_iterations < max_iterations:
|
||||
if await request.is_disconnected():
|
||||
logger.debug("Client disconnected, stopping chat stream")
|
||||
return
|
||||
logger.debug(
|
||||
f"Streaming LLM (iteration {stream_iterations + 1}/{max_iterations}) "
|
||||
f"with {len(conversation)} message(s)"
|
||||
@ -1340,9 +1127,6 @@ When a user refers to a specific object they have seen or describe with identify
|
||||
tools=tools if tools else None,
|
||||
tool_choice="auto",
|
||||
):
|
||||
if await request.is_disconnected():
|
||||
logger.debug("Client disconnected, stopping chat stream")
|
||||
return
|
||||
kind, value = event
|
||||
if kind == "content_delta":
|
||||
yield (
|
||||
@ -1372,11 +1156,6 @@ When a user refers to a specific object they have seen or describe with identify
|
||||
msg.get("content"), pending
|
||||
)
|
||||
)
|
||||
if await request.is_disconnected():
|
||||
logger.debug(
|
||||
"Client disconnected before tool execution"
|
||||
)
|
||||
return
|
||||
(
|
||||
executed_calls,
|
||||
tool_results,
|
||||
@ -1461,7 +1240,7 @@ When a user refers to a specific object they have seen or describe with identify
|
||||
+ b"\n"
|
||||
)
|
||||
# Stream content in word-sized chunks for smooth UX
|
||||
for part in chunk_content(final_content):
|
||||
for part in _chunk_content(final_content):
|
||||
yield (
|
||||
json.dumps({"type": "content", "delta": part}).encode(
|
||||
"utf-8"
|
||||
|
||||
@ -1,135 +0,0 @@
|
||||
"""Pure, stateless helpers used by the chat tool dispatchers.
|
||||
|
||||
These were extracted from frigate/api/chat.py to keep that module focused on
|
||||
route handlers, tool dispatchers, and streaming loop internals. Nothing in
|
||||
this file touches the FastAPI request, the embeddings context, or the chat
|
||||
loop state — all inputs and outputs are plain data.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Generator, List, Optional
|
||||
|
||||
from frigate.embeddings.util import ZScoreNormalization
|
||||
from frigate.models import Event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Similarity fusion weights for find_similar_objects.
|
||||
# Visual dominates because the feature's primary use case is "same specific object."
|
||||
# If these change, update the test in test_chat_find_similar_objects.py.
|
||||
VISUAL_WEIGHT = 0.65
|
||||
DESCRIPTION_WEIGHT = 0.35
|
||||
|
||||
|
||||
def chunk_content(content: str, chunk_size: int = 80) -> Generator[str, None, None]:
|
||||
"""Yield content in word-aware chunks for streaming."""
|
||||
if not content:
|
||||
return
|
||||
words = content.split(" ")
|
||||
current: List[str] = []
|
||||
current_len = 0
|
||||
for w in words:
|
||||
current.append(w)
|
||||
current_len += len(w) + 1
|
||||
if current_len >= chunk_size:
|
||||
yield " ".join(current) + " "
|
||||
current = []
|
||||
current_len = 0
|
||||
if current:
|
||||
yield " ".join(current)
|
||||
|
||||
|
||||
def format_events_with_local_time(
|
||||
events_list: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Add human-readable local start/end times to each event for the LLM."""
|
||||
result = []
|
||||
for evt in events_list:
|
||||
if not isinstance(evt, dict):
|
||||
result.append(evt)
|
||||
continue
|
||||
copy_evt = dict(evt)
|
||||
try:
|
||||
start_ts = evt.get("start_time")
|
||||
end_ts = evt.get("end_time")
|
||||
if start_ts is not None:
|
||||
dt_start = datetime.fromtimestamp(start_ts)
|
||||
copy_evt["start_time_local"] = dt_start.strftime("%Y-%m-%d %I:%M:%S %p")
|
||||
if end_ts is not None:
|
||||
dt_end = datetime.fromtimestamp(end_ts)
|
||||
copy_evt["end_time_local"] = dt_end.strftime("%Y-%m-%d %I:%M:%S %p")
|
||||
except (TypeError, ValueError, OSError):
|
||||
pass
|
||||
result.append(copy_evt)
|
||||
return result
|
||||
|
||||
|
||||
def distance_to_score(distance: float, stats: ZScoreNormalization) -> float:
|
||||
"""Convert a cosine distance to a [0, 1] similarity score.
|
||||
|
||||
Uses the existing ZScoreNormalization stats maintained by EmbeddingsContext
|
||||
to normalize across deployments, then a bounded sigmoid. Lower distance ->
|
||||
higher score. If stats are uninitialized (stddev == 0), returns a neutral
|
||||
0.5 so the fallback ordering by raw distance still dominates.
|
||||
"""
|
||||
if stats.stddev == 0:
|
||||
return 0.5
|
||||
z = (distance - stats.mean) / stats.stddev
|
||||
# Sigmoid on -z so that small distance (good) -> high score.
|
||||
return 1.0 / (1.0 + math.exp(z))
|
||||
|
||||
|
||||
def fuse_scores(
|
||||
visual_score: Optional[float],
|
||||
description_score: Optional[float],
|
||||
) -> Optional[float]:
|
||||
"""Weighted fusion of visual and description similarity scores.
|
||||
|
||||
If one side is missing (e.g., no description embedding for this event),
|
||||
the other side's score is returned alone with no penalty. If both are
|
||||
missing, returns None and the caller should drop the event.
|
||||
"""
|
||||
if visual_score is None and description_score is None:
|
||||
return None
|
||||
if visual_score is None:
|
||||
return description_score
|
||||
if description_score is None:
|
||||
return visual_score
|
||||
return VISUAL_WEIGHT * visual_score + DESCRIPTION_WEIGHT * description_score
|
||||
|
||||
|
||||
def parse_iso_to_timestamp(value: Optional[str]) -> Optional[float]:
|
||||
"""Parse an ISO-8601 string as server-local time -> unix timestamp.
|
||||
|
||||
Mirrors the parsing _execute_search_objects uses so both tools accept the
|
||||
same format from the LLM.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
s = value.replace("Z", "").strip()[:19]
|
||||
dt = datetime.strptime(s, "%Y-%m-%dT%H:%M:%S")
|
||||
return time.mktime(dt.timetuple())
|
||||
except (ValueError, AttributeError, TypeError):
|
||||
logger.warning("Invalid timestamp format: %s", value)
|
||||
return None
|
||||
|
||||
|
||||
def hydrate_event(event: Event, score: Optional[float] = None) -> Dict[str, Any]:
|
||||
"""Convert an Event row into the dict shape returned by find_similar_objects."""
|
||||
data: Dict[str, Any] = {
|
||||
"id": event.id,
|
||||
"camera": event.camera,
|
||||
"label": event.label,
|
||||
"sub_label": event.sub_label,
|
||||
"start_time": event.start_time,
|
||||
"end_time": event.end_time,
|
||||
"zones": event.zones,
|
||||
}
|
||||
if score is not None:
|
||||
data["score"] = score
|
||||
return data
|
||||
@ -1,303 +0,0 @@
|
||||
"""Tests for the find_similar_objects chat tool."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from playhouse.sqlite_ext import SqliteExtDatabase
|
||||
|
||||
from frigate.api.chat import (
|
||||
_execute_find_similar_objects,
|
||||
get_tool_definitions,
|
||||
)
|
||||
from frigate.api.chat_util import (
|
||||
DESCRIPTION_WEIGHT,
|
||||
VISUAL_WEIGHT,
|
||||
distance_to_score,
|
||||
fuse_scores,
|
||||
)
|
||||
from frigate.embeddings.util import ZScoreNormalization
|
||||
from frigate.models import Event
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
class TestDistanceToScore(unittest.TestCase):
|
||||
def test_lower_distance_gives_higher_score(self):
|
||||
stats = ZScoreNormalization()
|
||||
# Seed the stats with a small distribution so stddev > 0.
|
||||
stats._update([0.1, 0.2, 0.3, 0.4, 0.5])
|
||||
|
||||
close_score = distance_to_score(0.1, stats)
|
||||
far_score = distance_to_score(0.5, stats)
|
||||
|
||||
self.assertGreater(close_score, far_score)
|
||||
self.assertGreaterEqual(close_score, 0.0)
|
||||
self.assertLessEqual(close_score, 1.0)
|
||||
self.assertGreaterEqual(far_score, 0.0)
|
||||
self.assertLessEqual(far_score, 1.0)
|
||||
|
||||
def test_uninitialized_stats_returns_neutral_score(self):
|
||||
stats = ZScoreNormalization() # n == 0, stddev == 0
|
||||
self.assertEqual(distance_to_score(0.3, stats), 0.5)
|
||||
|
||||
|
||||
class TestFuseScores(unittest.TestCase):
|
||||
def test_weights_sum_to_one(self):
|
||||
self.assertAlmostEqual(VISUAL_WEIGHT + DESCRIPTION_WEIGHT, 1.0)
|
||||
|
||||
def test_fuses_both_sides(self):
|
||||
fused = fuse_scores(visual_score=0.8, description_score=0.4)
|
||||
expected = VISUAL_WEIGHT * 0.8 + DESCRIPTION_WEIGHT * 0.4
|
||||
self.assertAlmostEqual(fused, expected)
|
||||
|
||||
def test_missing_description_uses_visual_only(self):
|
||||
fused = fuse_scores(visual_score=0.7, description_score=None)
|
||||
self.assertAlmostEqual(fused, 0.7)
|
||||
|
||||
def test_missing_visual_uses_description_only(self):
|
||||
fused = fuse_scores(visual_score=None, description_score=0.6)
|
||||
self.assertAlmostEqual(fused, 0.6)
|
||||
|
||||
def test_both_missing_returns_none(self):
|
||||
self.assertIsNone(fuse_scores(visual_score=None, description_score=None))
|
||||
|
||||
|
||||
class TestToolDefinition(unittest.TestCase):
|
||||
def test_find_similar_objects_is_registered(self):
|
||||
tools = get_tool_definitions()
|
||||
names = [t["function"]["name"] for t in tools]
|
||||
self.assertIn("find_similar_objects", names)
|
||||
|
||||
def test_find_similar_objects_schema(self):
|
||||
tools = get_tool_definitions()
|
||||
tool = next(t for t in tools if t["function"]["name"] == "find_similar_objects")
|
||||
params = tool["function"]["parameters"]["properties"]
|
||||
self.assertIn("event_id", params)
|
||||
self.assertIn("after", params)
|
||||
self.assertIn("before", params)
|
||||
self.assertIn("cameras", params)
|
||||
self.assertIn("labels", params)
|
||||
self.assertIn("sub_labels", params)
|
||||
self.assertIn("zones", params)
|
||||
self.assertIn("similarity_mode", params)
|
||||
self.assertIn("min_score", params)
|
||||
self.assertIn("limit", params)
|
||||
self.assertEqual(tool["function"]["parameters"]["required"], ["event_id"])
|
||||
self.assertEqual(
|
||||
params["similarity_mode"]["enum"], ["visual", "semantic", "fused"]
|
||||
)
|
||||
|
||||
|
||||
class TestExecuteFindSimilarObjects(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
|
||||
self.tmp.close()
|
||||
self.db = SqliteExtDatabase(self.tmp.name)
|
||||
Event.bind(self.db, bind_refs=False, bind_backrefs=False)
|
||||
self.db.connect()
|
||||
self.db.create_tables([Event])
|
||||
|
||||
# Insert an anchor plus two candidates.
|
||||
def make(event_id, label="car", camera="driveway", start=1_700_000_100):
|
||||
Event.create(
|
||||
id=event_id,
|
||||
label=label,
|
||||
sub_label=None,
|
||||
camera=camera,
|
||||
start_time=start,
|
||||
end_time=start + 10,
|
||||
top_score=0.9,
|
||||
score=0.9,
|
||||
false_positive=False,
|
||||
zones=[],
|
||||
thumbnail="",
|
||||
has_clip=True,
|
||||
has_snapshot=True,
|
||||
region=[0, 0, 1, 1],
|
||||
box=[0, 0, 1, 1],
|
||||
area=1,
|
||||
retain_indefinitely=False,
|
||||
ratio=1.0,
|
||||
plus_id="",
|
||||
model_hash="",
|
||||
detector_type="",
|
||||
model_type="",
|
||||
data={"description": "a green sedan"},
|
||||
)
|
||||
|
||||
make("anchor", start=1_700_000_200)
|
||||
make("cand_a", start=1_700_000_100)
|
||||
make("cand_b", start=1_700_000_150)
|
||||
self.make = make
|
||||
|
||||
def tearDown(self):
|
||||
self.db.close()
|
||||
os.unlink(self.tmp.name)
|
||||
|
||||
def _make_request(self, semantic_enabled=True, embeddings=None):
|
||||
app = SimpleNamespace(
|
||||
embeddings=embeddings,
|
||||
frigate_config=SimpleNamespace(
|
||||
semantic_search=SimpleNamespace(enabled=semantic_enabled),
|
||||
),
|
||||
)
|
||||
return SimpleNamespace(app=app)
|
||||
|
||||
def test_semantic_search_disabled_returns_error(self):
|
||||
req = self._make_request(semantic_enabled=False)
|
||||
result = _run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "anchor"},
|
||||
allowed_cameras=["driveway"],
|
||||
)
|
||||
)
|
||||
self.assertEqual(result["error"], "semantic_search_disabled")
|
||||
|
||||
def test_anchor_not_found_returns_error(self):
|
||||
embeddings = MagicMock()
|
||||
req = self._make_request(embeddings=embeddings)
|
||||
result = _run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "nope"},
|
||||
allowed_cameras=["driveway"],
|
||||
)
|
||||
)
|
||||
self.assertEqual(result["error"], "anchor_not_found")
|
||||
|
||||
def test_empty_candidates_returns_empty_results(self):
|
||||
embeddings = MagicMock()
|
||||
req = self._make_request(embeddings=embeddings)
|
||||
# Filter to a camera with no other events.
|
||||
result = _run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "anchor", "cameras": ["nonexistent_cam"]},
|
||||
allowed_cameras=["nonexistent_cam"],
|
||||
)
|
||||
)
|
||||
self.assertEqual(result["results"], [])
|
||||
self.assertFalse(result["candidate_truncated"])
|
||||
self.assertEqual(result["anchor"]["id"], "anchor")
|
||||
|
||||
def test_fused_calls_both_searches_and_ranks(self):
|
||||
embeddings = MagicMock()
|
||||
# cand_a visually closer, cand_b semantically closer.
|
||||
embeddings.search_thumbnail.return_value = [
|
||||
("cand_a", 0.10),
|
||||
("cand_b", 0.40),
|
||||
]
|
||||
embeddings.search_description.return_value = [
|
||||
("cand_a", 0.50),
|
||||
("cand_b", 0.20),
|
||||
]
|
||||
embeddings.thumb_stats = ZScoreNormalization()
|
||||
embeddings.thumb_stats._update([0.1, 0.2, 0.3, 0.4, 0.5])
|
||||
embeddings.desc_stats = ZScoreNormalization()
|
||||
embeddings.desc_stats._update([0.1, 0.2, 0.3, 0.4, 0.5])
|
||||
|
||||
req = self._make_request(embeddings=embeddings)
|
||||
result = _run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "anchor"},
|
||||
allowed_cameras=["driveway"],
|
||||
)
|
||||
)
|
||||
embeddings.search_thumbnail.assert_called_once()
|
||||
embeddings.search_description.assert_called_once()
|
||||
# cand_a should rank first because visual is weighted higher.
|
||||
self.assertEqual(result["results"][0]["id"], "cand_a")
|
||||
self.assertIn("score", result["results"][0])
|
||||
self.assertEqual(result["similarity_mode"], "fused")
|
||||
|
||||
def test_visual_mode_only_calls_thumbnail(self):
|
||||
embeddings = MagicMock()
|
||||
embeddings.search_thumbnail.return_value = [("cand_a", 0.1)]
|
||||
embeddings.thumb_stats = ZScoreNormalization()
|
||||
embeddings.thumb_stats._update([0.1, 0.2, 0.3])
|
||||
|
||||
req = self._make_request(embeddings=embeddings)
|
||||
_run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "anchor", "similarity_mode": "visual"},
|
||||
allowed_cameras=["driveway"],
|
||||
)
|
||||
)
|
||||
embeddings.search_thumbnail.assert_called_once()
|
||||
embeddings.search_description.assert_not_called()
|
||||
|
||||
def test_semantic_mode_only_calls_description(self):
|
||||
embeddings = MagicMock()
|
||||
embeddings.search_description.return_value = [("cand_a", 0.1)]
|
||||
embeddings.desc_stats = ZScoreNormalization()
|
||||
embeddings.desc_stats._update([0.1, 0.2, 0.3])
|
||||
|
||||
req = self._make_request(embeddings=embeddings)
|
||||
_run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "anchor", "similarity_mode": "semantic"},
|
||||
allowed_cameras=["driveway"],
|
||||
)
|
||||
)
|
||||
embeddings.search_description.assert_called_once()
|
||||
embeddings.search_thumbnail.assert_not_called()
|
||||
|
||||
def test_min_score_drops_low_scoring_results(self):
|
||||
embeddings = MagicMock()
|
||||
embeddings.search_thumbnail.return_value = [
|
||||
("cand_a", 0.10),
|
||||
("cand_b", 0.90),
|
||||
]
|
||||
embeddings.search_description.return_value = []
|
||||
embeddings.thumb_stats = ZScoreNormalization()
|
||||
embeddings.thumb_stats._update([0.1, 0.2, 0.3, 0.4, 0.5])
|
||||
embeddings.desc_stats = ZScoreNormalization()
|
||||
|
||||
req = self._make_request(embeddings=embeddings)
|
||||
result = _run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "anchor", "similarity_mode": "visual", "min_score": 0.6},
|
||||
allowed_cameras=["driveway"],
|
||||
)
|
||||
)
|
||||
ids = [r["id"] for r in result["results"]]
|
||||
self.assertIn("cand_a", ids)
|
||||
self.assertNotIn("cand_b", ids)
|
||||
|
||||
def test_labels_defaults_to_anchor_label(self):
|
||||
self.make("person_a", label="person")
|
||||
embeddings = MagicMock()
|
||||
embeddings.search_thumbnail.return_value = [
|
||||
("cand_a", 0.1),
|
||||
("cand_b", 0.2),
|
||||
]
|
||||
embeddings.search_description.return_value = []
|
||||
embeddings.thumb_stats = ZScoreNormalization()
|
||||
embeddings.thumb_stats._update([0.1, 0.2, 0.3])
|
||||
embeddings.desc_stats = ZScoreNormalization()
|
||||
|
||||
req = self._make_request(embeddings=embeddings)
|
||||
result = _run(
|
||||
_execute_find_similar_objects(
|
||||
req,
|
||||
{"event_id": "anchor", "similarity_mode": "visual"},
|
||||
allowed_cameras=["driveway"],
|
||||
)
|
||||
)
|
||||
ids = [r["id"] for r in result["results"]]
|
||||
self.assertNotIn("person_a", ids)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@ -1,116 +0,0 @@
|
||||
/**
|
||||
* Global allowlist of regex patterns that the error collector ignores.
|
||||
*
|
||||
* Each entry MUST include a comment explaining what it silences and why.
|
||||
* The allowlist is filtered at collection time, so failure messages list
|
||||
* only unfiltered errors.
|
||||
*
|
||||
* Per-spec additions go through the `expectedErrors` test fixture parameter
|
||||
* (see error-collector.ts), not by editing this file. That keeps allowlist
|
||||
* drift visible per-PR rather than buried in shared infrastructure.
|
||||
*
|
||||
* NOTE ON CONSOLE vs REQUEST ERRORS:
|
||||
* When a network request returns a 5xx response, the browser emits two
|
||||
* events that the error collector captures:
|
||||
* [request] "500 Internal Server Error <url>" — from onResponse (URL included)
|
||||
* [console] "Failed to load resource: ..." — from onConsole (URL NOT included)
|
||||
*
|
||||
* The request-level message includes the URL, so those patterns are specific.
|
||||
* The console-level message text (from ConsoleMessage.text()) does NOT include
|
||||
* the URL — the URL is stored separately in e.url. Therefore the console
|
||||
* pattern for HTTP 500s cannot be URL-discriminated, and a single pattern
|
||||
* covers all such browser echoes. This is safe because every such console
|
||||
* error is already caught (and specifically matched) by its paired [request]
|
||||
* entry below.
|
||||
*/
|
||||
|
||||
export const GLOBAL_ALLOWLIST: RegExp[] = [
|
||||
// -------------------------------------------------------------------------
|
||||
// Browser echo of HTTP 5xx responses (console mirror of [request] events).
|
||||
//
|
||||
// Whenever the browser receives a 5xx response it emits a console error:
|
||||
// "Failed to load resource: the server responded with a status of 500
|
||||
// (Internal Server Error)"
|
||||
// The URL is NOT part of ConsoleMessage.text() — it is stored separately.
|
||||
// Every console error of this form is therefore paired with a specific
|
||||
// [request] 500 entry below that names the exact endpoint. Allowlisting
|
||||
// this pattern here silences the browser echo; the request-level entries
|
||||
// enforce specificity.
|
||||
// -------------------------------------------------------------------------
|
||||
/Failed to load resource: the server responded with a status of 500/,
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Mock infrastructure gaps — API endpoints not yet covered by ApiMocker.
|
||||
//
|
||||
// These produce 500s because Vite's preview server has no handler for them.
|
||||
// Each is a TODO(real-bug): the mock should be extended so these endpoints
|
||||
// return sensible fixture data in tests.
|
||||
//
|
||||
// Only [request] patterns are listed here; the paired [console] mirror is
|
||||
// covered by the "Failed to load resource" entry above.
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// TODO(real-bug): ApiMocker registers "**/api/reviews**" (plural) but the
|
||||
// app fetches /api/review (singular) for the review list and timeline.
|
||||
// Affects: review.spec.ts, navigation.spec.ts, live.spec.ts, auth.spec.ts.
|
||||
// Fix: add route handlers for /api/review and /api/review/** in api-mocker.ts.
|
||||
/500 Internal Server Error.*\/api\/review(\?|\/|$)/,
|
||||
|
||||
// TODO(real-bug): /api/stats/history is not mocked; the system page fetches
|
||||
// it for the detector/process history charts.
|
||||
// Fix: add route handler for /api/stats/history in api-mocker.ts.
|
||||
/500 Internal Server Error.*\/api\/stats\/history/,
|
||||
|
||||
// TODO(real-bug): /api/event_ids is not mocked; the explore/search page
|
||||
// fetches it to resolve event IDs for display.
|
||||
// Fix: add route handler for /api/event_ids in api-mocker.ts.
|
||||
/500 Internal Server Error.*\/api\/event_ids/,
|
||||
|
||||
// TODO(real-bug): /api/sub_labels?split_joined=1 returns 500; the mock
|
||||
// registers "**/api/sub_labels" which may not match when a query string is
|
||||
// present, or route registration order causes the catch-all to win first.
|
||||
// Fix: change the mock route to "**/api/sub_labels**" in api-mocker.ts.
|
||||
/500 Internal Server Error.*\/api\/sub_labels/,
|
||||
|
||||
// TODO(real-bug): MediaMocker handles /api/*/latest.jpg but the app also
|
||||
// requests /api/*/latest.webp (webp format) for camera snapshots.
|
||||
// Affects: live.spec.ts, review.spec.ts, auth.spec.ts, navigation.spec.ts.
|
||||
// Fix: add route handler for /api/*/latest.webp in MediaMocker.install().
|
||||
/500 Internal Server Error.*\/api\/[^/]+\/latest\.webp/,
|
||||
/failed: net::ERR_ABORTED.*\/api\/[^/]+\/latest\.webp/,
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Mock infrastructure gap — WebSocket streams.
|
||||
//
|
||||
// Playwright's page.route() does not intercept WebSocket connections.
|
||||
// The jsmpeg live-stream WS connections to /live/jsmpeg/* always fail
|
||||
// with a 500 handshake error because the Vite preview server has no WS
|
||||
// handler. TODO(real-bug): add WsMocker support for jsmpeg WebSocket
|
||||
// connections, or suppress the connection attempt in the test environment.
|
||||
// Affects: live.spec.ts (single camera view), auth.spec.ts.
|
||||
// -------------------------------------------------------------------------
|
||||
/WebSocket connection to '.*\/live\/jsmpeg\/.*' failed/,
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Benign — lazy-loaded chunk aborts during navigation.
|
||||
//
|
||||
// When a test navigates away from a page while the browser is still
|
||||
// fetching lazily-split JS/CSS asset chunks, the in-flight fetch is
|
||||
// cancelled (net::ERR_ABORTED). This is normal browser behaviour on
|
||||
// navigation and does not indicate a real error; the assets load fine
|
||||
// on a stable connection.
|
||||
// -------------------------------------------------------------------------
|
||||
/failed: net::ERR_ABORTED.*\/assets\//,
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Real app bug — Radix UI DialogContent missing accessible title.
|
||||
//
|
||||
// TODO(real-bug): A dialog somewhere in the app renders <DialogContent>
|
||||
// without a <DialogTitle>, violating Radix UI's accessibility contract.
|
||||
// The warning originates from the bundled main-*.js. Investigate which
|
||||
// dialog component is missing the title and add a VisuallyHidden DialogTitle.
|
||||
// Likely candidate: face-library or search-detail dialog in explore page.
|
||||
// See: https://radix-ui.com/primitives/docs/components/dialog
|
||||
// -------------------------------------------------------------------------
|
||||
/`DialogContent` requires a `DialogTitle`/,
|
||||
];
|
||||
@ -1,122 +0,0 @@
|
||||
/**
|
||||
* Collects console errors, page errors, and failed network requests
|
||||
* during a Playwright test, with regex-based allowlist filtering.
|
||||
*
|
||||
* Usage:
|
||||
* const collector = installErrorCollector(page, [...GLOBAL_ALLOWLIST]);
|
||||
* // ... run test ...
|
||||
* collector.assertClean(); // throws if any non-allowlisted error
|
||||
*
|
||||
* The collector is wired into the `frigateApp` fixture so every test
|
||||
* gets it for free. Tests that intentionally trigger an error pass
|
||||
* additional regexes via the `expectedErrors` fixture parameter.
|
||||
*/
|
||||
|
||||
import type { Page, Request, Response, ConsoleMessage } from "@playwright/test";
|
||||
|
||||
export type CollectedError = {
|
||||
kind: "console" | "pageerror" | "request";
|
||||
message: string;
|
||||
url?: string;
|
||||
stack?: string;
|
||||
};
|
||||
|
||||
export type ErrorCollector = {
|
||||
errors: CollectedError[];
|
||||
assertClean(): void;
|
||||
};
|
||||
|
||||
function isAllowlisted(message: string, allowlist: RegExp[]): boolean {
|
||||
return allowlist.some((pattern) => pattern.test(message));
|
||||
}
|
||||
|
||||
function firstStackFrame(stack: string | undefined): string | undefined {
|
||||
if (!stack) return undefined;
|
||||
const lines = stack
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
// Skip the error message line (line 0); return the first "at ..." frame
|
||||
return lines.find((l) => l.startsWith("at "));
|
||||
}
|
||||
|
||||
function isSameOrigin(url: string, baseURL: string | undefined): boolean {
|
||||
if (!baseURL) return true;
|
||||
try {
|
||||
return new URL(url).origin === new URL(baseURL).origin;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function installErrorCollector(
|
||||
page: Page,
|
||||
allowlist: RegExp[],
|
||||
): ErrorCollector {
|
||||
const errors: CollectedError[] = [];
|
||||
const baseURL = (
|
||||
page.context() as unknown as { _options?: { baseURL?: string } }
|
||||
)._options?.baseURL;
|
||||
|
||||
const onConsole = (msg: ConsoleMessage) => {
|
||||
if (msg.type() !== "error") return;
|
||||
const text = msg.text();
|
||||
if (isAllowlisted(text, allowlist)) return;
|
||||
errors.push({
|
||||
kind: "console",
|
||||
message: text,
|
||||
url: msg.location().url,
|
||||
});
|
||||
};
|
||||
|
||||
const onPageError = (err: Error) => {
|
||||
const text = err.message;
|
||||
if (isAllowlisted(text, allowlist)) return;
|
||||
errors.push({
|
||||
kind: "pageerror",
|
||||
message: text,
|
||||
stack: firstStackFrame(err.stack),
|
||||
});
|
||||
};
|
||||
|
||||
const onResponse = (response: Response) => {
|
||||
const status = response.status();
|
||||
if (status < 500) return;
|
||||
const url = response.url();
|
||||
if (!isSameOrigin(url, baseURL)) return;
|
||||
const text = `${status} ${response.statusText()} ${url}`;
|
||||
if (isAllowlisted(text, allowlist)) return;
|
||||
errors.push({ kind: "request", message: text, url });
|
||||
};
|
||||
|
||||
const onRequestFailed = (request: Request) => {
|
||||
const url = request.url();
|
||||
if (!isSameOrigin(url, baseURL)) return;
|
||||
const failure = request.failure();
|
||||
const text = `failed: ${failure?.errorText ?? "unknown"} ${url}`;
|
||||
if (isAllowlisted(text, allowlist)) return;
|
||||
errors.push({ kind: "request", message: text, url });
|
||||
};
|
||||
|
||||
page.on("console", onConsole);
|
||||
page.on("pageerror", onPageError);
|
||||
page.on("response", onResponse);
|
||||
page.on("requestfailed", onRequestFailed);
|
||||
|
||||
return {
|
||||
errors,
|
||||
assertClean() {
|
||||
if (errors.length === 0) return;
|
||||
const formatted = errors
|
||||
.map((e, i) => {
|
||||
const stack = e.stack ? `\n ${e.stack}` : "";
|
||||
const url = e.url && e.url !== e.message ? ` (${e.url})` : "";
|
||||
return ` ${i + 1}. [${e.kind}] ${e.message}${url}${stack}`;
|
||||
})
|
||||
.join("\n");
|
||||
throw new Error(
|
||||
`Page emitted ${errors.length} unexpected error${errors.length === 1 ? "" : "s"}:\n${formatted}`,
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -6,11 +6,6 @@
|
||||
* @playwright/test directly. The `frigateApp` fixture provides a
|
||||
* fully mocked Frigate frontend ready for interaction.
|
||||
*
|
||||
* The fixture also installs the error collector (see error-collector.ts).
|
||||
* Any console error, page error, or same-origin failed request that is
|
||||
* not on the global allowlist or the test's `expectedErrors` list will
|
||||
* fail the test in the fixture's teardown.
|
||||
*
|
||||
* CRITICAL: All route/WS handlers are registered before page.goto()
|
||||
* to prevent AuthProvider from redirecting to login.html.
|
||||
*/
|
||||
@ -22,8 +17,6 @@ import {
|
||||
type ApiMockOverrides,
|
||||
} from "../helpers/api-mocker";
|
||||
import { WsMocker } from "../helpers/ws-mocker";
|
||||
import { installErrorCollector, type ErrorCollector } from "./error-collector";
|
||||
import { GLOBAL_ALLOWLIST } from "./error-allowlist";
|
||||
|
||||
export class FrigateApp {
|
||||
public api: ApiMocker;
|
||||
@ -74,43 +67,10 @@ export class FrigateApp {
|
||||
|
||||
type FrigateFixtures = {
|
||||
frigateApp: FrigateApp;
|
||||
/**
|
||||
* Per-test additional allowlist regex patterns. Tests that intentionally
|
||||
* trigger errors (e.g. error-state tests that hit a mocked 500) declare
|
||||
* their expected errors here so the collector ignores them.
|
||||
*
|
||||
* Default is `[]` — most tests should not need this.
|
||||
*/
|
||||
expectedErrors: RegExp[];
|
||||
errorCollector: ErrorCollector;
|
||||
};
|
||||
|
||||
export const test = base.extend<FrigateFixtures>({
|
||||
expectedErrors: [[], { option: true }],
|
||||
|
||||
errorCollector: async ({ page, expectedErrors }, use, testInfo) => {
|
||||
const collector = installErrorCollector(page, [
|
||||
...GLOBAL_ALLOWLIST,
|
||||
...expectedErrors,
|
||||
]);
|
||||
await use(collector);
|
||||
if (process.env.E2E_STRICT_ERRORS === "1") {
|
||||
collector.assertClean();
|
||||
} else if (collector.errors.length > 0) {
|
||||
// Soft mode: attach errors to the test report so they're visible
|
||||
// without failing the run.
|
||||
await testInfo.attach("collected-errors.txt", {
|
||||
body: collector.errors
|
||||
.map((e) => `[${e.kind}] ${e.message}${e.url ? ` (${e.url})` : ""}`)
|
||||
.join("\n"),
|
||||
contentType: "text/plain",
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
frigateApp: async ({ page, errorCollector }, use, testInfo) => {
|
||||
// Reference the collector so its `use()` runs and teardown fires
|
||||
void errorCollector;
|
||||
frigateApp: async ({ page }, use, testInfo) => {
|
||||
const app = new FrigateApp(page, testInfo.project.name);
|
||||
await app.installDefaults();
|
||||
await use(app);
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
/**
|
||||
* Per-test mock overrides for driving empty / loading / error states.
|
||||
*
|
||||
* Playwright route handlers are LIFO: the most recently registered handler
|
||||
* matching a URL takes precedence. The frigateApp fixture installs default
|
||||
* mocks before the test body runs, so these helpers — called inside the
|
||||
* test body — register AFTER the defaults and therefore win.
|
||||
*
|
||||
* Always call these BEFORE the navigation that triggers the request.
|
||||
*
|
||||
* Example:
|
||||
* await mockEmpty(page, "**\/api\/exports**");
|
||||
* await frigateApp.goto("/export");
|
||||
* // Page now renders the empty state
|
||||
*/
|
||||
|
||||
import type { Page } from "@playwright/test";
|
||||
|
||||
/** Return an empty array for the matched endpoint. */
|
||||
export async function mockEmpty(
|
||||
page: Page,
|
||||
urlPattern: string | RegExp,
|
||||
): Promise<void> {
|
||||
await page.route(urlPattern, (route) => route.fulfill({ json: [] }));
|
||||
}
|
||||
|
||||
/** Return an HTTP error for the matched endpoint. Default status 500. */
|
||||
export async function mockError(
|
||||
page: Page,
|
||||
urlPattern: string | RegExp,
|
||||
status = 500,
|
||||
): Promise<void> {
|
||||
await page.route(urlPattern, (route) =>
|
||||
route.fulfill({
|
||||
status,
|
||||
json: { success: false, message: "Mocked error" },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delay the response by `ms` milliseconds before fulfilling with the
|
||||
* provided body. Use to assert loading-state UI is visible during the
|
||||
* delay window.
|
||||
*/
|
||||
export async function mockDelay(
|
||||
page: Page,
|
||||
urlPattern: string | RegExp,
|
||||
ms: number,
|
||||
body: unknown = [],
|
||||
): Promise<void> {
|
||||
await page.route(urlPattern, async (route) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
await route.fulfill({ json: body });
|
||||
});
|
||||
}
|
||||
@ -79,57 +79,4 @@ export class BasePage {
|
||||
async waitForPageLoad() {
|
||||
await this.page.waitForSelector("#pageRoot", { timeout: 10_000 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the mobile-only export pane / sheet that slides up from the
|
||||
* bottom on the export page. No-op on desktop. Returns the pane locator
|
||||
* so the caller can assert against its contents.
|
||||
*/
|
||||
async openMobilePane(): Promise<Locator> {
|
||||
if (this.isDesktop) {
|
||||
// Return the desktop equivalent (the main content area itself)
|
||||
return this.pageRoot;
|
||||
}
|
||||
// Look for any element that opens a sheet/dialog on tap.
|
||||
// Specific views override this with their own selector.
|
||||
const pane = this.page.locator('[role="dialog"]').first();
|
||||
return pane;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a side drawer (e.g. mobile filter drawer). View-specific page
|
||||
* objects should override this with their actual trigger selector.
|
||||
* The default implementation looks for a button labelled "Open menu"
|
||||
* or "Filters" and clicks it, then returns the drawer locator.
|
||||
*/
|
||||
async openDrawer(): Promise<Locator> {
|
||||
if (this.isDesktop) {
|
||||
return this.pageRoot;
|
||||
}
|
||||
const trigger = this.page
|
||||
.getByRole("button", { name: /menu|filter/i })
|
||||
.first();
|
||||
if (await trigger.count()) {
|
||||
await trigger.click();
|
||||
}
|
||||
return this.page.locator('[role="dialog"], [data-state="open"]').first();
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a bottom sheet (vaul). View-specific page objects should
|
||||
* override this with their actual trigger selector.
|
||||
*/
|
||||
async openBottomSheet(): Promise<Locator> {
|
||||
if (this.isDesktop) {
|
||||
return this.pageRoot;
|
||||
}
|
||||
return this.page.locator("[vaul-drawer]").first();
|
||||
}
|
||||
|
||||
/** Close any currently-open mobile overlay (drawer, sheet, dialog). */
|
||||
async closeMobileOverlay(): Promise<void> {
|
||||
if (this.isDesktop) return;
|
||||
// Press Escape — Radix dialogs and vaul both close on Escape
|
||||
await this.page.keyboard.press("Escape");
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,160 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Lint script for e2e specs. Bans lenient test patterns and requires
|
||||
* a @mobile-tagged test in every spec under specs/ (excluding _meta/).
|
||||
*
|
||||
* Banned patterns:
|
||||
* - page.waitForTimeout( — use expect().toPass() or waitFor instead
|
||||
* - if (await ... .isVisible()) — assertions must be unconditional
|
||||
* - if ((await ... .count()) > 0) — same as above
|
||||
* - expect(... .length).toBeGreaterThan(0) on textContent results
|
||||
*
|
||||
* Escape hatch: append `// e2e-lint-allow` on any line to silence the
|
||||
* check for that line. Use sparingly and explain why in a comment above.
|
||||
*
|
||||
* @mobile rule: every .spec.ts under specs/ (not specs/_meta/) must
|
||||
* contain at least one test title or describe with the substring "@mobile".
|
||||
*
|
||||
* Specs in PENDING_REWRITE are exempt from all rules until they are
|
||||
* rewritten with proper assertions and mobile coverage. Remove each
|
||||
* entry when its spec is updated.
|
||||
*/
|
||||
|
||||
import { readFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { join, relative, resolve, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const SPECS_DIR = resolve(__dirname, "..", "specs");
|
||||
const META_PREFIX = resolve(SPECS_DIR, "_meta");
|
||||
|
||||
// Specs exempt from lint rules until they are rewritten with proper
|
||||
// assertions and mobile coverage. Remove each entry when its spec is updated.
|
||||
const PENDING_REWRITE = new Set([
|
||||
"auth.spec.ts",
|
||||
"chat.spec.ts",
|
||||
"classification.spec.ts",
|
||||
"config-editor.spec.ts",
|
||||
"explore.spec.ts",
|
||||
"export.spec.ts",
|
||||
"face-library.spec.ts",
|
||||
"live.spec.ts",
|
||||
"logs.spec.ts",
|
||||
"navigation.spec.ts",
|
||||
"replay.spec.ts",
|
||||
"review.spec.ts",
|
||||
"system.spec.ts",
|
||||
]);
|
||||
|
||||
const BANNED_PATTERNS = [
|
||||
{
|
||||
name: "page.waitForTimeout",
|
||||
regex: /\bwaitForTimeout\s*\(/,
|
||||
advice:
|
||||
"Use expect.poll(), expect(...).toPass(), or waitFor() with a real condition.",
|
||||
},
|
||||
{
|
||||
name: "conditional isVisible() assertion",
|
||||
regex: /\bif\s*\(\s*await\s+[^)]*\.isVisible\s*\(/,
|
||||
advice:
|
||||
"Assertions must be unconditional. Use expect(...).toBeVisible() instead.",
|
||||
},
|
||||
{
|
||||
name: "conditional count() assertion",
|
||||
regex: /\bif\s*\(\s*\(?\s*await\s+[^)]*\.count\s*\(\s*\)\s*\)?\s*[><=!]/,
|
||||
advice:
|
||||
"Assertions must be unconditional. Use expect(...).toHaveCount(n).",
|
||||
},
|
||||
{
|
||||
name: "vacuous textContent length assertion",
|
||||
regex: /expect\([^)]*\.length\)\.toBeGreaterThan\(0\)/,
|
||||
advice:
|
||||
"Assert specific content, not that some text exists.",
|
||||
},
|
||||
];
|
||||
|
||||
function walk(dir) {
|
||||
const entries = readdirSync(dir);
|
||||
const out = [];
|
||||
for (const entry of entries) {
|
||||
const full = join(dir, entry);
|
||||
const st = statSync(full);
|
||||
if (st.isDirectory()) {
|
||||
out.push(...walk(full));
|
||||
} else if (entry.endsWith(".spec.ts")) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function lintFile(file) {
|
||||
const basename = file.split("/").pop();
|
||||
if (PENDING_REWRITE.has(basename)) return [];
|
||||
if (file.includes("/specs/settings/")) return [];
|
||||
|
||||
const errors = [];
|
||||
const text = readFileSync(file, "utf8");
|
||||
const lines = text.split("\n");
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line.includes("e2e-lint-allow")) continue;
|
||||
for (const pat of BANNED_PATTERNS) {
|
||||
if (pat.regex.test(line)) {
|
||||
errors.push({
|
||||
file,
|
||||
line: i + 1,
|
||||
col: 1,
|
||||
rule: pat.name,
|
||||
message: `${pat.name}: ${pat.advice}`,
|
||||
source: line.trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// @mobile rule: skip _meta
|
||||
const isMeta = file.startsWith(META_PREFIX);
|
||||
if (!isMeta) {
|
||||
if (!/@mobile\b/.test(text)) {
|
||||
errors.push({
|
||||
file,
|
||||
line: 1,
|
||||
col: 1,
|
||||
rule: "missing @mobile test",
|
||||
message:
|
||||
'Spec must contain at least one test or describe tagged with "@mobile".',
|
||||
source: "",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const files = walk(SPECS_DIR);
|
||||
const allErrors = [];
|
||||
for (const f of files) {
|
||||
allErrors.push(...lintFile(f));
|
||||
}
|
||||
|
||||
if (allErrors.length === 0) {
|
||||
console.log(`e2e:lint: ${files.length} spec files OK`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
for (const err of allErrors) {
|
||||
const rel = relative(process.cwd(), err.file);
|
||||
console.error(`${rel}:${err.line}:${err.col} ${err.rule}`);
|
||||
console.error(` ${err.message}`);
|
||||
if (err.source) console.error(` > ${err.source}`);
|
||||
}
|
||||
console.error(
|
||||
`\ne2e:lint: ${allErrors.length} error${allErrors.length === 1 ? "" : "s"} in ${files.length} files`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
main();
|
||||
@ -1,112 +0,0 @@
|
||||
/**
|
||||
* Self-tests for the error collector fixture itself.
|
||||
*
|
||||
* These guard against future regressions in the safety net. Each test
|
||||
* deliberately triggers (or avoids triggering) an error to verify the
|
||||
* collector behaves correctly. Tests that expect to fail use the
|
||||
* `expectedErrors` fixture parameter to allowlist their own errors.
|
||||
*/
|
||||
|
||||
import { test, expect } from "../../fixtures/frigate-test";
|
||||
|
||||
// test.use applies to a whole describe block in Playwright, so each test
|
||||
// that needs a custom allowlist gets its own describe.
|
||||
|
||||
test.describe("Error Collector — clean @meta", () => {
|
||||
test("clean page passes", async ({ frigateApp }) => {
|
||||
await frigateApp.goto("/");
|
||||
// No errors triggered. The fixture teardown should not throw.
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Error Collector — unallowlisted console error fails @meta", () => {
|
||||
test("console.error fails the test when not allowlisted", async ({
|
||||
page,
|
||||
frigateApp,
|
||||
}) => {
|
||||
test.skip(
|
||||
process.env.E2E_STRICT_ERRORS !== "1",
|
||||
"Requires E2E_STRICT_ERRORS=1 to assert failure",
|
||||
);
|
||||
test.fail(); // We expect the fixture teardown to throw
|
||||
await frigateApp.goto("/");
|
||||
await page.evaluate(() => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error("UNEXPECTED_DELIBERATE_TEST_ERROR_xyz123");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Error Collector — allowlisted console error passes @meta", () => {
|
||||
test.use({ expectedErrors: [/ALLOWED_DELIBERATE_TEST_ERROR_xyz123/] });
|
||||
|
||||
test("console.error is silenced when allowlisted via expectedErrors", async ({
|
||||
page,
|
||||
frigateApp,
|
||||
}) => {
|
||||
await frigateApp.goto("/");
|
||||
await page.evaluate(() => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error("ALLOWED_DELIBERATE_TEST_ERROR_xyz123");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Error Collector — uncaught pageerror fails @meta", () => {
|
||||
test("uncaught pageerror fails the test", async ({ page, frigateApp }) => {
|
||||
test.skip(
|
||||
process.env.E2E_STRICT_ERRORS !== "1",
|
||||
"Requires E2E_STRICT_ERRORS=1 to assert failure",
|
||||
);
|
||||
test.fail();
|
||||
await frigateApp.goto("/");
|
||||
await page.evaluate(() => {
|
||||
setTimeout(() => {
|
||||
throw new Error("UNCAUGHT_DELIBERATE_TEST_ERROR_xyz789");
|
||||
}, 0);
|
||||
});
|
||||
// Wait a frame to let the throw propagate before fixture teardown.
|
||||
// The marker below silences the e2e:lint banned-pattern check on this line.
|
||||
await page.waitForTimeout(100); // e2e-lint-allow: deliberate; need to await async throw
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Error Collector — 5xx fails @meta", () => {
|
||||
test("same-origin 5xx response fails the test", async ({
|
||||
page,
|
||||
frigateApp,
|
||||
}) => {
|
||||
test.skip(
|
||||
process.env.E2E_STRICT_ERRORS !== "1",
|
||||
"Requires E2E_STRICT_ERRORS=1 to assert failure",
|
||||
);
|
||||
test.fail();
|
||||
await page.route("**/api/version", (route) =>
|
||||
route.fulfill({ status: 500, body: "boom" }),
|
||||
);
|
||||
await frigateApp.goto("/");
|
||||
await page.evaluate(() => fetch("/api/version").catch(() => {}));
|
||||
// Give the response listener a microtask to fire
|
||||
await expect.poll(async () => true).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Error Collector — allowlisted 5xx passes @meta", () => {
|
||||
// Use a single alternation regex so test.use() receives a 1-element array.
|
||||
// Playwright's isFixtureTuple() treats any [value, object] pair as a fixture
|
||||
// tuple, so a 2-element array whose second item is a RegExp would be
|
||||
// misinterpreted as [defaultValue, options]. Both the request collector
|
||||
// error ("500 … /api/version") and the browser console error
|
||||
// ("Failed to load resource … 500") are matched by the alternation below.
|
||||
test.use({
|
||||
expectedErrors: [/500.*\/api\/version|Failed to load resource.*500/],
|
||||
});
|
||||
|
||||
test("allowlisted 5xx passes", async ({ page, frigateApp }) => {
|
||||
await page.route("**/api/version", (route) =>
|
||||
route.fulfill({ status: 500, body: "boom" }),
|
||||
);
|
||||
await frigateApp.goto("/");
|
||||
await page.evaluate(() => fetch("/api/version").catch(() => {}));
|
||||
});
|
||||
});
|
||||
@ -1,73 +0,0 @@
|
||||
/**
|
||||
* Self-tests for the mock override helpers. Verifies each helper
|
||||
* intercepts the matched URL and returns the expected payload/status.
|
||||
*/
|
||||
|
||||
import { test, expect } from "../../fixtures/frigate-test";
|
||||
import { mockEmpty, mockError, mockDelay } from "../../helpers/mock-overrides";
|
||||
|
||||
test.describe("Mock Overrides — empty @meta", () => {
|
||||
test("mockEmpty returns []", async ({ page, frigateApp }) => {
|
||||
await mockEmpty(page, "**/api/__meta_test__");
|
||||
await frigateApp.goto("/");
|
||||
const result = await page.evaluate(async () => {
|
||||
const r = await fetch("/api/__meta_test__");
|
||||
return { status: r.status, body: await r.json() };
|
||||
});
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.body).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Mock Overrides — error default @meta", () => {
|
||||
// Match both the collected request error and the browser's console echo.
|
||||
// Using a single alternation regex avoids Playwright's isFixtureTuple
|
||||
// collision with multi-element RegExp arrays.
|
||||
test.use({
|
||||
expectedErrors: [/500.*__meta_test__|Failed to load resource.*500/],
|
||||
});
|
||||
|
||||
test("mockError returns 500 by default", async ({ page, frigateApp }) => {
|
||||
await mockError(page, "**/api/__meta_test__");
|
||||
await frigateApp.goto("/");
|
||||
const status = await page.evaluate(async () => {
|
||||
const r = await fetch("/api/__meta_test__");
|
||||
return r.status;
|
||||
});
|
||||
expect(status).toBe(500);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Mock Overrides — error custom status @meta", () => {
|
||||
// The browser emits a "Failed to load resource" console.error for 404s,
|
||||
// which the error collector catches even though 404 is not a 5xx.
|
||||
test.use({
|
||||
expectedErrors: [/Failed to load resource.*404|404.*__meta_test_404__/],
|
||||
});
|
||||
|
||||
test("mockError accepts a custom status", async ({ page, frigateApp }) => {
|
||||
await mockError(page, "**/api/__meta_test_404__", 404);
|
||||
await frigateApp.goto("/");
|
||||
const status = await page.evaluate(async () => {
|
||||
const r = await fetch("/api/__meta_test_404__");
|
||||
return r.status;
|
||||
});
|
||||
expect(status).toBe(404);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Mock Overrides — delay @meta", () => {
|
||||
test("mockDelay delays response by the requested ms", async ({
|
||||
page,
|
||||
frigateApp,
|
||||
}) => {
|
||||
await mockDelay(page, "**/api/__meta_test_delay__", 300, ["delayed"]);
|
||||
await frigateApp.goto("/");
|
||||
const elapsed = await page.evaluate(async () => {
|
||||
const start = performance.now();
|
||||
await fetch("/api/__meta_test_delay__");
|
||||
return performance.now() - start;
|
||||
});
|
||||
expect(elapsed).toBeGreaterThanOrEqual(250);
|
||||
});
|
||||
});
|
||||
@ -7,8 +7,7 @@
|
||||
"dev": "vite --host",
|
||||
"postinstall": "patch-package",
|
||||
"build": "tsc && vite build --base=/BASE_PATH/",
|
||||
"lint": "eslint --ext .jsx,.js,.tsx,.ts --ignore-path .gitignore . && npm run e2e:lint",
|
||||
"e2e:lint": "node e2e/scripts/lint-specs.mjs",
|
||||
"lint": "eslint --ext .jsx,.js,.tsx,.ts --ignore-path .gitignore .",
|
||||
"lint:fix": "eslint --ext .jsx,.js,.tsx,.ts --ignore-path .gitignore --fix .",
|
||||
"preview": "vite preview",
|
||||
"prettier:write": "prettier -u -w --ignore-path .gitignore \"*.{ts,tsx,js,jsx,css,html}\"",
|
||||
|
||||
@ -12,23 +12,6 @@
|
||||
"result": "Result",
|
||||
"arguments": "Arguments:",
|
||||
"response": "Response:",
|
||||
"attachment_chip_label": "{{label}} on {{camera}}",
|
||||
"attachment_chip_remove": "Remove attachment",
|
||||
"open_in_explore": "Open in Explore",
|
||||
"attach_event_aria": "Attach event {{eventId}}",
|
||||
"attachment_picker_paste_label": "Or paste event ID",
|
||||
"attachment_picker_attach": "Attach",
|
||||
"attachment_picker_placeholder": "Attach an event",
|
||||
"quick_reply_find_similar": "Find similar sightings",
|
||||
"quick_reply_tell_me_more": "Tell me more about this",
|
||||
"quick_reply_when_else": "When else was it seen?",
|
||||
"quick_reply_find_similar_text": "Find similar sightings to this.",
|
||||
"quick_reply_tell_me_more_text": "Tell me more about this one.",
|
||||
"quick_reply_when_else_text": "When else was this seen?",
|
||||
"anchor": "Reference",
|
||||
"similarity_score": "Similarity",
|
||||
"no_similar_objects_found": "No similar objects found.",
|
||||
"semantic_search_required": "Semantic search must be enabled to find similar objects.",
|
||||
"send": "Send",
|
||||
"suggested_requests": "Try asking:",
|
||||
"starting_requests": {
|
||||
|
||||
@ -1,111 +0,0 @@
|
||||
import { useApiHost } from "@/api";
|
||||
import { useCameraFriendlyName } from "@/hooks/use-camera-friendly-name";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import useSWR from "swr";
|
||||
import { LuX, LuExternalLink } from "react-icons/lu";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import ActivityIndicator from "@/components/indicators/activity-indicator";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { getTranslatedLabel } from "@/utils/i18n";
|
||||
|
||||
type ChatAttachmentChipProps = {
|
||||
eventId: string;
|
||||
mode: "composer" | "bubble";
|
||||
onRemove?: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* Small horizontal chip rendering an event as an "attachment": a thumbnail,
|
||||
* a friendly label like "Person on driveway", an optional remove X (composer
|
||||
* mode), and an external-link icon that opens the event in Explore.
|
||||
*/
|
||||
export function ChatAttachmentChip({
|
||||
eventId,
|
||||
mode,
|
||||
onRemove,
|
||||
}: ChatAttachmentChipProps) {
|
||||
const apiHost = useApiHost();
|
||||
const { t } = useTranslation(["views/chat"]);
|
||||
|
||||
const { data: eventData } = useSWR<{ label: string; camera: string }[]>(
|
||||
`event_ids?ids=${eventId}`,
|
||||
);
|
||||
const evt = eventData?.[0];
|
||||
const cameraName = useCameraFriendlyName(evt?.camera);
|
||||
const displayLabel = evt
|
||||
? t("attachment_chip_label", {
|
||||
label: getTranslatedLabel(evt.label),
|
||||
camera: cameraName,
|
||||
})
|
||||
: eventId;
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"inline-flex max-w-full items-center gap-2 rounded-lg border border-border bg-background/80 p-1.5 pr-2",
|
||||
mode === "bubble" && "border-primary-foreground/30 bg-transparent",
|
||||
)}
|
||||
>
|
||||
<div className="relative size-10 shrink-0 overflow-hidden rounded-md">
|
||||
<img
|
||||
className="size-full object-cover"
|
||||
src={`${apiHost}api/events/${eventId}/thumbnail.webp`}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
onError={(e) => {
|
||||
(e.currentTarget as HTMLImageElement).style.visibility = "hidden";
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{evt ? (
|
||||
<span
|
||||
className={cn(
|
||||
"truncate text-xs",
|
||||
mode === "bubble"
|
||||
? "text-primary-foreground/90"
|
||||
: "text-foreground",
|
||||
)}
|
||||
>
|
||||
{displayLabel}
|
||||
</span>
|
||||
) : (
|
||||
<ActivityIndicator className="size-4" />
|
||||
)}
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<a
|
||||
href={`/explore?event_id=${eventId}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className={cn(
|
||||
"flex size-6 shrink-0 items-center justify-center rounded text-muted-foreground hover:text-foreground",
|
||||
mode === "bubble" &&
|
||||
"text-primary-foreground/70 hover:text-primary-foreground",
|
||||
)}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
aria-label={t("open_in_explore")}
|
||||
>
|
||||
<LuExternalLink className="size-3.5" />
|
||||
</a>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t("open_in_explore")}</TooltipContent>
|
||||
</Tooltip>
|
||||
{mode === "composer" && onRemove && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-6 shrink-0 text-muted-foreground hover:text-foreground"
|
||||
onClick={onRemove}
|
||||
aria-label={t("attachment_chip_remove")}
|
||||
>
|
||||
<LuX className="size-3.5" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -1,52 +1,31 @@
|
||||
import { useApiHost } from "@/api";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { LuExternalLink } from "react-icons/lu";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
type ChatEvent = { id: string; score?: number };
|
||||
|
||||
type ChatEventThumbnailsRowProps = {
|
||||
events: ChatEvent[];
|
||||
anchor?: { id: string } | null;
|
||||
onAttach?: (eventId: string) => void;
|
||||
events: { id: string }[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Horizontal scroll row of event thumbnail images for chat.
|
||||
* Optionally renders an anchor thumbnail with a "reference" badge above the
|
||||
* results, and per-event similarity scores when provided.
|
||||
* Clicking a thumbnail calls onAttach; a small external-link overlay opens
|
||||
* the event in Explore.
|
||||
* Renders nothing when there is nothing to show.
|
||||
* Horizontal scroll row of event thumbnail images for chat (e.g. after search_objects).
|
||||
* Renders nothing when events is empty.
|
||||
*/
|
||||
export function ChatEventThumbnailsRow({
|
||||
events,
|
||||
anchor = null,
|
||||
onAttach,
|
||||
}: ChatEventThumbnailsRowProps) {
|
||||
const apiHost = useApiHost();
|
||||
const { t } = useTranslation(["views/chat"]);
|
||||
|
||||
if (events.length === 0 && !anchor) return null;
|
||||
if (events.length === 0) return null;
|
||||
|
||||
const renderThumb = (event: ChatEvent, isAnchor = false) => (
|
||||
<div
|
||||
return (
|
||||
<div className="flex min-w-0 max-w-full flex-col gap-1 self-start">
|
||||
<div className="scrollbar-container min-w-0 overflow-x-auto">
|
||||
<div className="flex w-max gap-2">
|
||||
{events.map((event) => (
|
||||
<a
|
||||
key={event.id}
|
||||
className={cn(
|
||||
"relative aspect-square size-32 shrink-0 overflow-hidden rounded-lg",
|
||||
isAnchor && "ring-2 ring-primary",
|
||||
)}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
className="block size-full"
|
||||
onClick={() => onAttach?.(event.id)}
|
||||
aria-label={t("attach_event_aria", { eventId: event.id })}
|
||||
href={`/explore?event_id=${event.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="relative aspect-square size-32 shrink-0 overflow-hidden rounded-lg"
|
||||
>
|
||||
<img
|
||||
className="size-full object-cover"
|
||||
@ -54,44 +33,10 @@ export function ChatEventThumbnailsRow({
|
||||
alt=""
|
||||
loading="lazy"
|
||||
/>
|
||||
</button>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<a
|
||||
href={`/explore?event_id=${event.id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className="absolute right-1 top-1 flex size-6 items-center justify-center rounded bg-black/60 text-white hover:bg-black/80"
|
||||
aria-label={t("open_in_explore")}
|
||||
>
|
||||
<LuExternalLink className="size-3" />
|
||||
</a>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t("open_in_explore")}</TooltipContent>
|
||||
</Tooltip>
|
||||
{isAnchor && (
|
||||
<span className="pointer-events-none absolute left-1 top-1 rounded bg-primary px-1 text-[10px] text-primary-foreground">
|
||||
{t("anchor")}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="flex min-w-0 max-w-full flex-col gap-2 self-start">
|
||||
{anchor && (
|
||||
<div className="scrollbar-container min-w-0 overflow-x-auto">
|
||||
<div className="flex w-max gap-2">{renderThumb(anchor, true)}</div>
|
||||
</div>
|
||||
)}
|
||||
{events.length > 0 && (
|
||||
<div className="scrollbar-container min-w-0 overflow-x-auto">
|
||||
<div className="flex w-max gap-2">
|
||||
{events.map((event) => renderThumb(event))}
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@ -15,8 +15,6 @@ import {
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ChatAttachmentChip } from "@/components/chat/ChatAttachmentChip";
|
||||
import { parseAttachedEvent } from "@/utils/chatUtil";
|
||||
|
||||
type MessageBubbleProps = {
|
||||
role: "user" | "assistant";
|
||||
@ -128,10 +126,6 @@ export function MessageBubble({
|
||||
);
|
||||
}
|
||||
|
||||
const { eventId: attachedEventId, body: displayContent } = isUser
|
||||
? parseAttachedEvent(content)
|
||||
: { eventId: null, body: content };
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
@ -146,20 +140,9 @@ export function MessageBubble({
|
||||
)}
|
||||
>
|
||||
{isUser ? (
|
||||
<div className="flex flex-col gap-2">
|
||||
{attachedEventId && (
|
||||
<ChatAttachmentChip eventId={attachedEventId} mode="bubble" />
|
||||
)}
|
||||
<div className="whitespace-pre-wrap">{displayContent}</div>
|
||||
</div>
|
||||
content
|
||||
) : (
|
||||
<div
|
||||
className={cn(
|
||||
"[&>*:last-child]:inline",
|
||||
!isComplete &&
|
||||
"after:ml-0.5 after:inline-block after:h-4 after:w-2 after:animate-cursor-blink after:rounded-sm after:bg-foreground after:align-middle after:content-['']",
|
||||
)}
|
||||
>
|
||||
<>
|
||||
<ReactMarkdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
components={{
|
||||
@ -185,7 +168,10 @@ export function MessageBubble({
|
||||
>
|
||||
{content}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
{!isComplete && (
|
||||
<span className="ml-1 inline-block h-4 w-0.5 animate-pulse bg-foreground align-middle" />
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-0.5">
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
import { useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { LuPaperclip } from "react-icons/lu";
|
||||
import { useApiHost } from "@/api";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from "@/components/ui/popover";
|
||||
|
||||
const EVENT_ID_RE = /^[A-Za-z0-9._-]+$/;
|
||||
|
||||
type ChatPaperclipButtonProps = {
|
||||
recentEventIds: string[];
|
||||
onAttach: (eventId: string) => void;
|
||||
disabled?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Paperclip button with a popover for picking an event to attach.
|
||||
* Shows a grid of recent thumbnails (from the latest assistant message) and a
|
||||
* "paste event ID" fallback input.
|
||||
*/
|
||||
export function ChatPaperclipButton({
|
||||
recentEventIds,
|
||||
onAttach,
|
||||
disabled = false,
|
||||
}: ChatPaperclipButtonProps) {
|
||||
const apiHost = useApiHost();
|
||||
const { t } = useTranslation(["views/chat"]);
|
||||
const [open, setOpen] = useState(false);
|
||||
const [pasteId, setPasteId] = useState("");
|
||||
|
||||
const handlePickThumbnail = (eventId: string) => {
|
||||
onAttach(eventId);
|
||||
setOpen(false);
|
||||
setPasteId("");
|
||||
};
|
||||
|
||||
const handlePasteSubmit = () => {
|
||||
const trimmed = pasteId.trim();
|
||||
if (!trimmed || !EVENT_ID_RE.test(trimmed)) return;
|
||||
onAttach(trimmed);
|
||||
setOpen(false);
|
||||
setPasteId("");
|
||||
};
|
||||
|
||||
const handlePasteKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === "Enter") {
|
||||
e.preventDefault();
|
||||
handlePasteSubmit();
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Popover open={open} onOpenChange={setOpen}>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-10 shrink-0 rounded-full"
|
||||
disabled={disabled}
|
||||
aria-label={t("attachment_picker_placeholder")}
|
||||
>
|
||||
<LuPaperclip className="size-4" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-72" align="start">
|
||||
<div className="flex flex-col gap-3">
|
||||
{recentEventIds.length > 0 && (
|
||||
<div className="grid grid-cols-4 gap-2">
|
||||
{recentEventIds.slice(0, 8).map((id) => (
|
||||
<button
|
||||
key={id}
|
||||
type="button"
|
||||
onClick={() => handlePickThumbnail(id)}
|
||||
className="relative aspect-square overflow-hidden rounded-md ring-offset-background hover:ring-2 hover:ring-primary"
|
||||
aria-label={t("attach_event_aria", { eventId: id })}
|
||||
>
|
||||
<img
|
||||
className="size-full object-cover"
|
||||
src={`${apiHost}api/events/${id}/thumbnail.webp`}
|
||||
alt=""
|
||||
loading="lazy"
|
||||
/>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
placeholder={t("attachment_picker_paste_label")}
|
||||
value={pasteId}
|
||||
onChange={(e) => setPasteId(e.target.value)}
|
||||
onKeyDown={handlePasteKeyDown}
|
||||
className="h-8 text-xs"
|
||||
/>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="select"
|
||||
className="h-8"
|
||||
disabled={!pasteId.trim() || !EVENT_ID_RE.test(pasteId.trim())}
|
||||
onClick={handlePasteSubmit}
|
||||
>
|
||||
{t("attachment_picker_attach")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
type QuickReply = { labelKey: string; textKey: string };
|
||||
|
||||
const REPLIES: QuickReply[] = [
|
||||
{
|
||||
labelKey: "quick_reply_find_similar",
|
||||
textKey: "quick_reply_find_similar_text",
|
||||
},
|
||||
{
|
||||
labelKey: "quick_reply_tell_me_more",
|
||||
textKey: "quick_reply_tell_me_more_text",
|
||||
},
|
||||
{ labelKey: "quick_reply_when_else", textKey: "quick_reply_when_else_text" },
|
||||
];
|
||||
|
||||
type ChatQuickRepliesProps = {
|
||||
onSend: (text: string) => void;
|
||||
disabled?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Row of pill buttons shown in the composer while an attachment is pending.
|
||||
* Clicking a pill immediately calls onSend with the canned text.
|
||||
*/
|
||||
export function ChatQuickReplies({
|
||||
onSend,
|
||||
disabled = false,
|
||||
}: ChatQuickRepliesProps) {
|
||||
const { t } = useTranslation(["views/chat"]);
|
||||
|
||||
return (
|
||||
<div className="flex w-full flex-wrap gap-2">
|
||||
{REPLIES.map((reply) => (
|
||||
<Button
|
||||
key={reply.labelKey}
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-7 rounded-full px-3 text-xs"
|
||||
disabled={disabled}
|
||||
onClick={() => onSend(t(reply.textKey))}
|
||||
>
|
||||
{t(reply.labelKey)}
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -1,22 +1,17 @@
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { FaArrowUpLong, FaStop } from "react-icons/fa6";
|
||||
import { FaArrowUpLong } from "react-icons/fa6";
|
||||
import { LuCircleAlert } from "react-icons/lu";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useState, useCallback, useRef, useEffect, useMemo } from "react";
|
||||
import { useState, useCallback, useRef, useEffect } from "react";
|
||||
import axios from "axios";
|
||||
import { ChatEventThumbnailsRow } from "@/components/chat/ChatEventThumbnailsRow";
|
||||
import { MessageBubble } from "@/components/chat/ChatMessage";
|
||||
import { ToolCallsGroup } from "@/components/chat/ToolCallsGroup";
|
||||
import { ChatStartingState } from "@/components/chat/ChatStartingState";
|
||||
import { ChatAttachmentChip } from "@/components/chat/ChatAttachmentChip";
|
||||
import { ChatQuickReplies } from "@/components/chat/ChatQuickReplies";
|
||||
import { ChatPaperclipButton } from "@/components/chat/ChatPaperclipButton";
|
||||
import type { ChatMessage } from "@/types/chat";
|
||||
import {
|
||||
getEventIdsFromSearchObjectsToolCalls,
|
||||
getFindSimilarObjectsFromToolCalls,
|
||||
prependAttachment,
|
||||
streamChatCompletion,
|
||||
} from "@/utils/chatUtil";
|
||||
|
||||
@ -26,9 +21,7 @@ export default function ChatPage() {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [attachedEventId, setAttachedEventId] = useState<string | null>(null);
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
document.title = t("documentTitle");
|
||||
@ -71,59 +64,22 @@ export default function ChatPage() {
|
||||
...(axios.defaults.headers.common as Record<string, string>),
|
||||
};
|
||||
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
await streamChatCompletion(
|
||||
url,
|
||||
headers,
|
||||
apiMessages,
|
||||
{
|
||||
await streamChatCompletion(url, headers, apiMessages, {
|
||||
updateMessages: (updater) => setMessages(updater),
|
||||
onError: (message) => setError(message),
|
||||
onDone: () => {
|
||||
abortRef.current = null;
|
||||
setIsLoading(false);
|
||||
},
|
||||
onDone: () => setIsLoading(false),
|
||||
defaultErrorMessage: t("error"),
|
||||
},
|
||||
controller.signal,
|
||||
);
|
||||
});
|
||||
},
|
||||
[isLoading, t],
|
||||
);
|
||||
|
||||
const recentEventIds = useMemo(() => {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i];
|
||||
if (msg.role !== "assistant" || !msg.toolCalls) continue;
|
||||
const similar = getFindSimilarObjectsFromToolCalls(msg.toolCalls);
|
||||
if (similar) return similar.results.map((e) => e.id);
|
||||
const events = getEventIdsFromSearchObjectsToolCalls(msg.toolCalls);
|
||||
if (events.length > 0) return events.map((e) => e.id);
|
||||
}
|
||||
return [];
|
||||
}, [messages]);
|
||||
|
||||
const sendMessage = useCallback(
|
||||
(textOverride?: string) => {
|
||||
const text = (textOverride ?? input).trim();
|
||||
const sendMessage = useCallback(() => {
|
||||
const text = input.trim();
|
||||
if (!text || isLoading) return;
|
||||
const wireText = attachedEventId
|
||||
? prependAttachment(text, attachedEventId)
|
||||
: text;
|
||||
setInput("");
|
||||
setAttachedEventId(null);
|
||||
submitConversation([...messages, { role: "user", content: wireText }]);
|
||||
},
|
||||
[attachedEventId, input, isLoading, messages, submitConversation],
|
||||
);
|
||||
|
||||
const stopGeneration = useCallback(() => {
|
||||
abortRef.current?.abort();
|
||||
abortRef.current = null;
|
||||
setIsLoading(false);
|
||||
}, []);
|
||||
submitConversation([...messages, { role: "user", content: text }]);
|
||||
}, [input, isLoading, messages, submitConversation]);
|
||||
|
||||
const handleEditSubmit = useCallback(
|
||||
(messageIndex: number, newContent: string) => {
|
||||
@ -136,10 +92,6 @@ export default function ChatPage() {
|
||||
[messages, submitConversation],
|
||||
);
|
||||
|
||||
const handleClearAttachment = useCallback(() => {
|
||||
setAttachedEventId(null);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="flex size-full justify-center p-2 md:p-4">
|
||||
<div className="flex size-full flex-col xl:w-[50%] 3xl:w-[35%]">
|
||||
@ -209,27 +161,10 @@ export default function ChatPage() {
|
||||
{msg.role === "assistant" &&
|
||||
isComplete &&
|
||||
(() => {
|
||||
const similar = getFindSimilarObjectsFromToolCalls(
|
||||
msg.toolCalls,
|
||||
);
|
||||
if (similar) {
|
||||
return (
|
||||
<ChatEventThumbnailsRow
|
||||
events={similar.results}
|
||||
anchor={similar.anchor}
|
||||
onAttach={setAttachedEventId}
|
||||
/>
|
||||
);
|
||||
}
|
||||
const events = getEventIdsFromSearchObjectsToolCalls(
|
||||
msg.toolCalls,
|
||||
);
|
||||
return (
|
||||
<ChatEventThumbnailsRow
|
||||
events={events}
|
||||
onAttach={setAttachedEventId}
|
||||
/>
|
||||
);
|
||||
return <ChatEventThumbnailsRow events={events} />;
|
||||
})()}
|
||||
</div>
|
||||
);
|
||||
@ -253,11 +188,6 @@ export default function ChatPage() {
|
||||
sendMessage={sendMessage}
|
||||
isLoading={isLoading}
|
||||
placeholder={t("placeholder")}
|
||||
attachedEventId={attachedEventId}
|
||||
onClearAttachment={handleClearAttachment}
|
||||
onAttach={setAttachedEventId}
|
||||
onStop={stopGeneration}
|
||||
recentEventIds={recentEventIds}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
@ -268,14 +198,9 @@ export default function ChatPage() {
|
||||
type ChatEntryProps = {
|
||||
input: string;
|
||||
setInput: (value: string) => void;
|
||||
sendMessage: (textOverride?: string) => void;
|
||||
sendMessage: () => void;
|
||||
isLoading: boolean;
|
||||
placeholder: string;
|
||||
attachedEventId: string | null;
|
||||
onClearAttachment: () => void;
|
||||
onAttach: (eventId: string) => void;
|
||||
onStop: () => void;
|
||||
recentEventIds: string[];
|
||||
};
|
||||
|
||||
function ChatEntry({
|
||||
@ -284,11 +209,6 @@ function ChatEntry({
|
||||
sendMessage,
|
||||
isLoading,
|
||||
placeholder,
|
||||
attachedEventId,
|
||||
onClearAttachment,
|
||||
onAttach,
|
||||
onStop,
|
||||
recentEventIds,
|
||||
}: ChatEntryProps) {
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
@ -298,28 +218,8 @@ function ChatEntry({
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="mt-2 flex w-full flex-col items-stretch justify-center gap-2 rounded-xl bg-secondary p-3">
|
||||
{attachedEventId && (
|
||||
<div className="flex items-center">
|
||||
<ChatAttachmentChip
|
||||
eventId={attachedEventId}
|
||||
mode="composer"
|
||||
onRemove={onClearAttachment}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{attachedEventId && (
|
||||
<ChatQuickReplies
|
||||
onSend={(text) => sendMessage(text)}
|
||||
disabled={isLoading}
|
||||
/>
|
||||
)}
|
||||
<div className="mt-2 flex w-full flex-col items-center justify-center rounded-xl bg-secondary p-3">
|
||||
<div className="flex w-full flex-row items-center gap-2">
|
||||
<ChatPaperclipButton
|
||||
recentEventIds={recentEventIds}
|
||||
onAttach={onAttach}
|
||||
disabled={isLoading || attachedEventId != null}
|
||||
/>
|
||||
<Input
|
||||
className="w-full flex-1 border-transparent bg-transparent shadow-none focus-visible:ring-0 dark:bg-transparent"
|
||||
placeholder={placeholder}
|
||||
@ -328,24 +228,14 @@ function ChatEntry({
|
||||
onKeyDown={handleKeyDown}
|
||||
aria-busy={isLoading}
|
||||
/>
|
||||
{isLoading ? (
|
||||
<Button
|
||||
variant="destructive"
|
||||
className="size-10 shrink-0 rounded-full"
|
||||
onClick={onStop}
|
||||
>
|
||||
<FaStop className="size-3" />
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
variant="select"
|
||||
className="size-10 shrink-0 rounded-full"
|
||||
disabled={!input.trim()}
|
||||
onClick={() => sendMessage()}
|
||||
disabled={!input.trim() || isLoading}
|
||||
onClick={sendMessage}
|
||||
>
|
||||
<FaArrowUpLong className="size-4" />
|
||||
<FaArrowUpLong size="16" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@ -25,7 +25,6 @@ export async function streamChatCompletion(
|
||||
headers: Record<string, string>,
|
||||
apiMessages: { role: string; content: string }[],
|
||||
callbacks: StreamChatCallbacks,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const {
|
||||
updateMessages,
|
||||
@ -39,7 +38,6 @@ export async function streamChatCompletion(
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ messages: apiMessages, stream: true }),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
@ -154,15 +152,11 @@ export async function streamChatCompletion(
|
||||
return next;
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof DOMException && err.name === "AbortError") {
|
||||
// User stopped generation — not an error
|
||||
} else {
|
||||
} catch {
|
||||
onError(defaultErrorMessage);
|
||||
updateMessages((prev) =>
|
||||
prev.filter((m) => !(m.role === "assistant" && m.content === "")),
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
onDone();
|
||||
}
|
||||
@ -197,72 +191,3 @@ export function getEventIdsFromSearchObjectsToolCalls(
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
const ATTACHED_EVENT_MARKER = /^\[attached_event:([A-Za-z0-9._-]+)\]\s*\n?/;
|
||||
|
||||
export function parseAttachedEvent(content: string): {
|
||||
eventId: string | null;
|
||||
body: string;
|
||||
} {
|
||||
if (!content) return { eventId: null, body: content };
|
||||
const match = content.match(ATTACHED_EVENT_MARKER);
|
||||
if (!match) return { eventId: null, body: content };
|
||||
const body = content.slice(match[0].length).replace(/^\n+/, "");
|
||||
return { eventId: match[1], body };
|
||||
}
|
||||
|
||||
export function prependAttachment(body: string, eventId: string): string {
|
||||
return `[attached_event:${eventId}]\n\n${body}`;
|
||||
}
|
||||
|
||||
export type FindSimilarObjectsResult = {
|
||||
anchor: { id: string } | null;
|
||||
results: { id: string; score?: number }[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse find_similar_objects tool call response(s) into anchor + ranked results.
|
||||
* Returns null if no find_similar_objects call is present so the caller can
|
||||
* decide whether to render.
|
||||
*/
|
||||
export function getFindSimilarObjectsFromToolCalls(
|
||||
toolCalls: ToolCall[] | undefined,
|
||||
): FindSimilarObjectsResult | null {
|
||||
if (!toolCalls?.length) return null;
|
||||
for (const tc of toolCalls) {
|
||||
if (tc.name !== "find_similar_objects" || !tc.response?.trim()) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(tc.response) as {
|
||||
anchor?: { id?: unknown };
|
||||
results?: unknown;
|
||||
};
|
||||
const anchorId =
|
||||
parsed.anchor && typeof parsed.anchor.id === "string"
|
||||
? parsed.anchor.id
|
||||
: null;
|
||||
const anchor = anchorId ? { id: anchorId } : null;
|
||||
const results: { id: string; score?: number }[] = [];
|
||||
if (Array.isArray(parsed.results)) {
|
||||
for (const item of parsed.results) {
|
||||
if (
|
||||
item &&
|
||||
typeof item === "object" &&
|
||||
"id" in item &&
|
||||
typeof (item as { id: unknown }).id === "string"
|
||||
) {
|
||||
const entry: { id: string; score?: number } = {
|
||||
id: (item as { id: string }).id,
|
||||
};
|
||||
const rawScore = (item as { score?: unknown }).score;
|
||||
if (typeof rawScore === "number") entry.score = rawScore;
|
||||
results.push(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { anchor, results };
|
||||
} catch {
|
||||
// ignore parse errors
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -49,7 +49,6 @@ module.exports = {
|
||||
scale4: "scale4 3s ease-in-out infinite",
|
||||
"timeline-zoom-in": "timeline-zoom-in 0.3s ease-out",
|
||||
"timeline-zoom-out": "timeline-zoom-out 0.3s ease-out",
|
||||
"cursor-blink": "cursor-blink 1s step-end infinite",
|
||||
},
|
||||
aspectRatio: {
|
||||
wide: "32 / 9",
|
||||
@ -190,10 +189,6 @@ module.exports = {
|
||||
"50%": { transform: "translateY(0%)", opacity: "0.5" },
|
||||
"100%": { transform: "translateY(0)", opacity: "1" },
|
||||
},
|
||||
"cursor-blink": {
|
||||
"0%, 100%": { opacity: "1" },
|
||||
"50%": { opacity: "0" },
|
||||
},
|
||||
},
|
||||
screens: {
|
||||
xs: "480px",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user