mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-07-01 17:41:13 +03:00
Move chat prompts to prompts
This commit is contained in:
Nicolas Mowen
parent
f5b7395d69
commit
31f9611d34
@ -37,8 +37,11 @@ from frigate.api.defs.response.chat_response import (
|
|||||||
from frigate.api.defs.tags import Tags
|
from frigate.api.defs.tags import Tags
|
||||||
from frigate.api.event import _build_attribute_filter_clause, events
|
from frigate.api.event import _build_attribute_filter_clause, events
|
||||||
from frigate.config import FrigateConfig
|
from frigate.config import FrigateConfig
|
||||||
from frigate.config.classification import ObjectClassificationType
|
from frigate.genai.prompts import (
|
||||||
from frigate.config.ui import UnitSystemEnum
|
build_chat_system_prompt,
|
||||||
|
get_attribute_classifications,
|
||||||
|
get_tool_definitions,
|
||||||
|
)
|
||||||
from frigate.genai.utils import build_assistant_message_for_conversation
|
from frigate.genai.utils import build_assistant_message_for_conversation
|
||||||
from frigate.jobs.vlm_watch import (
|
from frigate.jobs.vlm_watch import (
|
||||||
get_vlm_watch_job,
|
get_vlm_watch_job,
|
||||||
@ -69,440 +72,6 @@ class VLMMonitorRequest(BaseModel):
|
|||||||
zones: List[str] = []
|
zones: List[str] = []
|
||||||
|
|
||||||
|
|
||||||
def get_attribute_classifications(config: FrigateConfig) -> List[Dict[str, Any]]:
|
|
||||||
"""Return enabled custom classification models of `attribute` type.
|
|
||||||
|
|
||||||
Each entry: {"name": <model name>, "objects": [<object label>, ...]}.
|
|
||||||
These models attach attribute metadata to events on the listed object
|
|
||||||
types, which can later be filtered via the search_objects `attribute`
|
|
||||||
field.
|
|
||||||
"""
|
|
||||||
result: List[Dict[str, Any]] = []
|
|
||||||
|
|
||||||
for model_key, model_config in config.classification.custom.items():
|
|
||||||
if not model_config.enabled or model_config.object_config is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if (
|
|
||||||
model_config.object_config.classification_type
|
|
||||||
!= ObjectClassificationType.attribute
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
|
|
||||||
result.append(
|
|
||||||
{
|
|
||||||
"name": model_config.name or model_key,
|
|
||||||
"objects": list(model_config.object_config.objects or []),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def get_tool_definitions(
|
|
||||||
semantic_search_enabled: bool = False,
|
|
||||||
attribute_classifications: Optional[List[Dict[str, Any]]] = None,
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""
|
|
||||||
Get OpenAI-compatible tool definitions for Frigate.
|
|
||||||
|
|
||||||
Returns a list of tool definitions that can be used with OpenAI-compatible
|
|
||||||
function calling APIs. When semantic search is enabled, the search_objects
|
|
||||||
tool exposes an additional `semantic_query` parameter for descriptive
|
|
||||||
queries (e.g. "person riding a lawn mower") and find_similar_objects is
|
|
||||||
included. When attribute classification models are configured, an
|
|
||||||
`attribute` parameter is exposed for filtering by their labels.
|
|
||||||
"""
|
|
||||||
search_objects_properties: Dict[str, Any] = {
|
|
||||||
"camera": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Camera name to filter by (optional).",
|
|
||||||
},
|
|
||||||
"label": {
|
|
||||||
"type": "string",
|
|
||||||
"description": (
|
|
||||||
"Generic object class to filter by — one of the tracked detector "
|
|
||||||
"labels such as 'person', 'package', 'car', 'dog', 'bird'. Use "
|
|
||||||
"this for broad queries like 'show me all cars today'. Combine "
|
|
||||||
"with semantic_query when the user also describes appearance or "
|
|
||||||
"behavior (e.g. label='person', semantic_query='riding a lawn "
|
|
||||||
"mower')."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"sub_label": {
|
|
||||||
"type": "string",
|
|
||||||
"description": (
|
|
||||||
"Filter by a DISCRETE NAMED entity recognized in the detection. "
|
|
||||||
"Use this for: a known person's name ('John'), a delivery "
|
|
||||||
"company ('Amazon', 'UPS'), a recognized animal species or "
|
|
||||||
"breed ('blue jay', 'cardinal', 'golden retriever'), or a "
|
|
||||||
"license plate string. When filtering by a specific name, set "
|
|
||||||
"only sub_label and leave label unset. Do NOT use sub_label "
|
|
||||||
"for descriptions of appearance, clothing, or actions — those "
|
|
||||||
"belong in semantic_query."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"after": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
|
||||||
},
|
|
||||||
"before": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
|
||||||
},
|
|
||||||
"zones": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "string"},
|
|
||||||
"description": "List of zone names to filter by.",
|
|
||||||
},
|
|
||||||
"limit": {
|
|
||||||
"type": "integer",
|
|
||||||
"description": "Maximum number of objects to return (default: 25).",
|
|
||||||
"default": 25,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if attribute_classifications:
|
|
||||||
model_outline = "; ".join(
|
|
||||||
f"{m['name']} (applies to {', '.join(m['objects']) or 'any object'})"
|
|
||||||
for m in attribute_classifications
|
|
||||||
)
|
|
||||||
search_objects_properties["attribute"] = {
|
|
||||||
"type": "string",
|
|
||||||
"description": (
|
|
||||||
"Filter by a classification attribute label produced by a "
|
|
||||||
"configured attribute classification model. Use this INSTEAD "
|
|
||||||
"of semantic_query when the user's request matches one of "
|
|
||||||
"these classifications. Configured models: "
|
|
||||||
f"{model_outline}. "
|
|
||||||
"Set the value to the attribute label that matches the user's "
|
|
||||||
"phrasing (case-sensitive)."
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
if semantic_search_enabled:
|
|
||||||
search_objects_properties["semantic_query"] = {
|
|
||||||
"type": "string",
|
|
||||||
"description": (
|
|
||||||
"Optional natural-language description of a PHYSICAL "
|
|
||||||
"CHARACTERISTIC, APPEARANCE, or ACTIVITY the user mentioned, "
|
|
||||||
"used to semantically narrow results. Only set this when the "
|
|
||||||
"user describes something beyond what label and sub_label can "
|
|
||||||
"express on their own.\n"
|
|
||||||
"USE for descriptive phrases like: 'riding a lawn mower', "
|
|
||||||
"'wearing a red jacket', 'carrying a package', 'walking a "
|
|
||||||
"dog', 'on a bicycle', 'holding an umbrella'.\n"
|
|
||||||
"DO NOT USE for:\n"
|
|
||||||
"- specific named people, pets, or delivery companies → use sub_label\n"
|
|
||||||
"- animal species or breed names like 'blue jay', 'cardinal', "
|
|
||||||
"'golden retriever' → use sub_label\n"
|
|
||||||
"- license plate strings → use sub_label\n"
|
|
||||||
"- generic object queries like 'all cars today' or 'every "
|
|
||||||
"person' → use label alone with no semantic_query\n"
|
|
||||||
"When set, combine with label/time/camera/zone filters as "
|
|
||||||
"usual (e.g. label='person', semantic_query='riding a lawn "
|
|
||||||
"mower', after='2024-05-01T00:00:00Z')."
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
search_objects_description = (
|
|
||||||
"Search the historical record of detected objects in Frigate. "
|
|
||||||
"Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', "
|
|
||||||
"'when was the last car?', 'show me detections from yesterday'. "
|
|
||||||
"Do NOT use this for monitoring or alerting requests about future events — "
|
|
||||||
"use start_camera_watch instead for those. "
|
|
||||||
"An 'object' in Frigate represents a tracked detection (e.g., a person, package, car).\n\n"
|
|
||||||
"Choose filters based on what the user is asking for:\n"
|
|
||||||
"- Generic class query ('show me all cars today'): set `label` only.\n"
|
|
||||||
"- Specific NAMED entity (known person, delivery company, animal "
|
|
||||||
"species/breed like 'blue jay' or 'golden retriever', license "
|
|
||||||
"plate): set `sub_label` only and leave `label` unset.\n"
|
|
||||||
)
|
|
||||||
if semantic_search_enabled:
|
|
||||||
search_objects_description += (
|
|
||||||
"- Physical CHARACTERISTIC, APPEARANCE, or ACTIVITY that is not a "
|
|
||||||
"discrete name ('person riding a lawn mower', 'someone in a red "
|
|
||||||
"jacket', 'person carrying a package'): set `semantic_query` with "
|
|
||||||
"the descriptive phrase, optionally alongside `label` for the "
|
|
||||||
"object class. Do NOT put descriptive phrases in sub_label."
|
|
||||||
)
|
|
||||||
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "search_objects",
|
|
||||||
"description": search_objects_description,
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": search_objects_properties,
|
|
||||||
},
|
|
||||||
"required": [],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "find_similar_objects",
|
|
||||||
"description": (
|
|
||||||
"Find tracked objects that are visually and semantically similar "
|
|
||||||
"to a specific past event. Use this when the user references a "
|
|
||||||
"particular object they have seen and wants to find other "
|
|
||||||
"sightings of the same or similar one ('that green car', 'the "
|
|
||||||
"person in the red jacket', 'the package that was delivered'). "
|
|
||||||
"Prefer this over search_objects whenever the user's intent is "
|
|
||||||
"'find more like this specific one.' Use search_objects first "
|
|
||||||
"only if you need to locate the anchor event. Requires semantic "
|
|
||||||
"search to be enabled."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"event_id": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "The id of the anchor event to find similar objects to.",
|
|
||||||
},
|
|
||||||
"after": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
|
||||||
},
|
|
||||||
"before": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
|
||||||
},
|
|
||||||
"cameras": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "string"},
|
|
||||||
"description": "Optional list of cameras to restrict to. Defaults to all.",
|
|
||||||
},
|
|
||||||
"labels": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "string"},
|
|
||||||
"description": "Optional list of labels to restrict to. Defaults to the anchor event's label.",
|
|
||||||
},
|
|
||||||
"sub_labels": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "string"},
|
|
||||||
"description": "Optional list of sub_labels (names) to restrict to.",
|
|
||||||
},
|
|
||||||
"zones": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "string"},
|
|
||||||
"description": "Optional list of zones. An event matches if any of its zones overlap.",
|
|
||||||
},
|
|
||||||
"similarity_mode": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["visual", "semantic", "fused"],
|
|
||||||
"description": "Which similarity signal(s) to use. 'fused' (default) combines visual and semantic.",
|
|
||||||
"default": "fused",
|
|
||||||
},
|
|
||||||
"min_score": {
|
|
||||||
"type": "number",
|
|
||||||
"description": "Drop matches with a similarity score below this threshold (0.0-1.0).",
|
|
||||||
},
|
|
||||||
"limit": {
|
|
||||||
"type": "integer",
|
|
||||||
"description": "Maximum number of matches to return (default: 10).",
|
|
||||||
"default": 10,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["event_id"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "set_camera_state",
|
|
||||||
"description": (
|
|
||||||
"Change a camera's feature state (e.g., turn detection on/off, enable/disable recordings). "
|
|
||||||
"Use camera='*' to apply to all cameras at once. "
|
|
||||||
"Only call this tool when the user explicitly asks to change a camera setting. "
|
|
||||||
"Requires admin privileges."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"camera": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Camera name to target, or '*' to target all cameras.",
|
|
||||||
},
|
|
||||||
"feature": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": [
|
|
||||||
"detect",
|
|
||||||
"record",
|
|
||||||
"snapshots",
|
|
||||||
"audio",
|
|
||||||
"motion",
|
|
||||||
"enabled",
|
|
||||||
"birdseye",
|
|
||||||
"birdseye_mode",
|
|
||||||
"improve_contrast",
|
|
||||||
"ptz_autotracker",
|
|
||||||
"motion_contour_area",
|
|
||||||
"motion_threshold",
|
|
||||||
"notifications",
|
|
||||||
"audio_transcription",
|
|
||||||
"review_alerts",
|
|
||||||
"review_detections",
|
|
||||||
"object_descriptions",
|
|
||||||
"review_descriptions",
|
|
||||||
"profile",
|
|
||||||
],
|
|
||||||
"description": (
|
|
||||||
"The feature to change. Most features accept ON or OFF. "
|
|
||||||
"birdseye_mode accepts CONTINUOUS, MOTION, or OBJECTS. "
|
|
||||||
"motion_contour_area and motion_threshold accept a number. "
|
|
||||||
"profile accepts a profile name or 'none' to deactivate (requires camera='*')."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"value": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "The value to set. ON or OFF for toggles, a number for thresholds, a profile name or 'none' for profile.",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["camera", "feature", "value"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "get_live_context",
|
|
||||||
"description": (
|
|
||||||
"Get the current live image and detection information for a camera: objects being tracked, "
|
|
||||||
"zones, timestamps. Use this to understand what is visible in the live view. "
|
|
||||||
"Call this when answering questions about what is happening right now on a specific camera."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"camera": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Camera name to get live context for.",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["camera"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "start_camera_watch",
|
|
||||||
"description": (
|
|
||||||
"Start a continuous VLM watch job that monitors a camera and sends a notification "
|
|
||||||
"when a specified condition is met. Use this when the user wants to be alerted about "
|
|
||||||
"a future event, e.g. 'tell me when guests arrive' or 'notify me when the package is picked up'. "
|
|
||||||
"Only one watch job can run at a time. Returns a job ID."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"camera": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Camera ID to monitor.",
|
|
||||||
},
|
|
||||||
"condition": {
|
|
||||||
"type": "string",
|
|
||||||
"description": (
|
|
||||||
"Natural-language description of the condition to watch for, "
|
|
||||||
"e.g. 'a person arrives at the front door'."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"max_duration_minutes": {
|
|
||||||
"type": "integer",
|
|
||||||
"description": "Maximum time to watch before giving up (minutes, default 60).",
|
|
||||||
"default": 60,
|
|
||||||
},
|
|
||||||
"labels": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "string"},
|
|
||||||
"description": "Object labels that should trigger a VLM check (e.g. ['person', 'car']). If omitted, any detection on the camera triggers a check.",
|
|
||||||
},
|
|
||||||
"zones": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {"type": "string"},
|
|
||||||
"description": "Zone names to filter by. If specified, only detections in these zones trigger a VLM check.",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["camera", "condition"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "stop_camera_watch",
|
|
||||||
"description": (
|
|
||||||
"Cancel the currently running VLM watch job. Use this when the user wants to "
|
|
||||||
"stop a previously started watch, e.g. 'stop watching the front door'."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {},
|
|
||||||
"required": [],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "get_profile_status",
|
|
||||||
"description": (
|
|
||||||
"Get the current profile status including the active profile and "
|
|
||||||
"timestamps of when each profile was last activated. Use this to "
|
|
||||||
"determine time periods for recap requests — e.g. when the user asks "
|
|
||||||
"'what happened while I was away?', call this first to find the relevant "
|
|
||||||
"time window based on profile activation history."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {},
|
|
||||||
"required": [],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": "get_recap",
|
|
||||||
"description": (
|
|
||||||
"Get a recap of all activity (alerts and detections) for a given time period. "
|
|
||||||
"Use this after calling get_profile_status to retrieve what happened during "
|
|
||||||
"a specific window — e.g. 'what happened while I was away?'. Returns a "
|
|
||||||
"chronological list of activity with camera, objects, zones, and GenAI-generated "
|
|
||||||
"descriptions when available. Summarize the results for the user."
|
|
||||||
),
|
|
||||||
"parameters": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"after": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Start of the time period in ISO 8601 format (e.g. '2025-03-15T08:00:00').",
|
|
||||||
},
|
|
||||||
"before": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "End of the time period in ISO 8601 format (e.g. '2025-03-15T17:00:00').",
|
|
||||||
},
|
|
||||||
"cameras": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Comma-separated camera IDs to include, or 'all' for all cameras. Default is 'all'.",
|
|
||||||
},
|
|
||||||
"severity": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["alert", "detection"],
|
|
||||||
"description": "Filter by severity level. Omit to include both alerts and detections.",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"required": ["after", "before"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
"/chat/tools",
|
"/chat/tools",
|
||||||
dependencies=[Depends(allow_any_authenticated())],
|
dependencies=[Depends(allow_any_authenticated())],
|
||||||
@ -1550,81 +1119,12 @@ async def chat_completion(
|
|||||||
)
|
)
|
||||||
conversation = []
|
conversation = []
|
||||||
|
|
||||||
current_datetime = datetime.now()
|
system_prompt = build_chat_system_prompt(
|
||||||
current_date_str = current_datetime.strftime("%Y-%m-%d")
|
config=config,
|
||||||
current_time_str = current_datetime.strftime("%I:%M:%S %p")
|
allowed_cameras=allowed_cameras,
|
||||||
|
semantic_search_enabled=semantic_search_enabled,
|
||||||
cameras_info = []
|
attribute_classifications=attribute_classifications,
|
||||||
has_speed_zone = False
|
)
|
||||||
for camera_id in allowed_cameras:
|
|
||||||
if camera_id not in config.cameras:
|
|
||||||
continue
|
|
||||||
camera_config = config.cameras[camera_id]
|
|
||||||
friendly_name = (
|
|
||||||
camera_config.friendly_name
|
|
||||||
if camera_config.friendly_name
|
|
||||||
else camera_id.replace("_", " ").title()
|
|
||||||
)
|
|
||||||
zone_names = list(camera_config.zones.keys())
|
|
||||||
if not has_speed_zone:
|
|
||||||
has_speed_zone = any(
|
|
||||||
zone.distances for zone in camera_config.zones.values()
|
|
||||||
)
|
|
||||||
if zone_names:
|
|
||||||
cameras_info.append(
|
|
||||||
f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
cameras_info.append(f" - {friendly_name} (ID: {camera_id})")
|
|
||||||
|
|
||||||
cameras_section = ""
|
|
||||||
if cameras_info:
|
|
||||||
cameras_section = (
|
|
||||||
"\n\nAvailable cameras:\n"
|
|
||||||
+ "\n".join(cameras_info)
|
|
||||||
+ "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls."
|
|
||||||
)
|
|
||||||
|
|
||||||
speed_units_section = ""
|
|
||||||
if has_speed_zone:
|
|
||||||
speed_unit = (
|
|
||||||
"mph" if config.ui.unit_system == UnitSystemEnum.imperial else "km/h"
|
|
||||||
)
|
|
||||||
speed_units_section = f"\n\nReport object speeds to the user in {speed_unit}."
|
|
||||||
|
|
||||||
semantic_search_section = ""
|
|
||||||
if semantic_search_enabled:
|
|
||||||
semantic_search_section = (
|
|
||||||
"\n\nWhen routing a search_objects call, pick filters by the shape of the user's request:\n"
|
|
||||||
"- Generic class ('show me all cars today'): set `label` only.\n"
|
|
||||||
"- Specific named entity — a known person ('John'), delivery company ('Amazon'), animal species/breed ('blue jay', 'cardinal', 'golden retriever'), or license plate: set `sub_label` only and leave `label` unset.\n"
|
|
||||||
"- Physical characteristic, appearance, or activity that is NOT a discrete name ('find me people riding a lawn mower', 'someone in a red jacket', 'a person carrying a package'): set `semantic_query` with the descriptive phrase, optionally combined with `label` for the object class. Never put descriptive phrases in `sub_label`."
|
|
||||||
)
|
|
||||||
|
|
||||||
attribute_classification_section = ""
|
|
||||||
if attribute_classifications:
|
|
||||||
model_lines = "\n".join(
|
|
||||||
f"- {m['name']}: applies to {', '.join(m['objects']) or 'any object'}"
|
|
||||||
for m in attribute_classifications
|
|
||||||
)
|
|
||||||
attribute_classification_section = (
|
|
||||||
"\n\nAttribute classification models are configured for the following object types:\n"
|
|
||||||
f"{model_lines}\n"
|
|
||||||
"When the user's request matches one of these classifications, set the search_objects `attribute` field to the matching label rather than using `semantic_query`. Reserve `semantic_query` for descriptive phrases that fall outside the configured attribute labels."
|
|
||||||
)
|
|
||||||
|
|
||||||
system_prompt = f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events.
|
|
||||||
|
|
||||||
Current server local date and time: {current_date_str} at {current_time_str}
|
|
||||||
|
|
||||||
Do not start your response with phrases like "I will check...", "Let me see...", or "Let me look...". Answer directly.
|
|
||||||
|
|
||||||
Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields.
|
|
||||||
When users ask about "today", "yesterday", "this week", etc., use the current date above as reference.
|
|
||||||
When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today).
|
|
||||||
Always be accurate with time calculations based on the current date provided.
|
|
||||||
|
|
||||||
When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:<id>], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{semantic_search_section}{attribute_classification_section}{cameras_section}{speed_units_section}"""
|
|
||||||
|
|
||||||
conversation.append(
|
conversation.append(
|
||||||
{
|
{
|
||||||
|
|||||||
@ -6,11 +6,13 @@ transport.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
from typing import Any
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
from playhouse.shortcuts import model_to_dict
|
from playhouse.shortcuts import model_to_dict
|
||||||
|
|
||||||
from frigate.config import CameraConfig
|
from frigate.config import CameraConfig, FrigateConfig
|
||||||
|
from frigate.config.classification import ObjectClassificationType
|
||||||
|
from frigate.config.ui import UnitSystemEnum
|
||||||
from frigate.data_processing.post.types import ReviewMetadata
|
from frigate.data_processing.post.types import ReviewMetadata
|
||||||
from frigate.models import Event
|
from frigate.models import Event
|
||||||
|
|
||||||
@ -212,3 +214,526 @@ def build_object_description_prompt(
|
|||||||
camera_config.objects.genai.prompt,
|
camera_config.objects.genai.prompt,
|
||||||
)
|
)
|
||||||
return template.format(**model_to_dict(event))
|
return template.format(**model_to_dict(event))
|
||||||
|
|
||||||
|
|
||||||
|
def get_attribute_classifications(config: FrigateConfig) -> List[Dict[str, Any]]:
|
||||||
|
"""Return enabled custom classification models of `attribute` type.
|
||||||
|
|
||||||
|
Each entry: {"name": <model name>, "objects": [<object label>, ...]}.
|
||||||
|
These models attach attribute metadata to events on the listed object
|
||||||
|
types, which can later be filtered via the search_objects `attribute`
|
||||||
|
field.
|
||||||
|
"""
|
||||||
|
result: List[Dict[str, Any]] = []
|
||||||
|
|
||||||
|
for model_key, model_config in config.classification.custom.items():
|
||||||
|
if not model_config.enabled or model_config.object_config is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if (
|
||||||
|
model_config.object_config.classification_type
|
||||||
|
!= ObjectClassificationType.attribute
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
result.append(
|
||||||
|
{
|
||||||
|
"name": model_config.name or model_key,
|
||||||
|
"objects": list(model_config.object_config.objects or []),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def get_tool_definitions(
|
||||||
|
semantic_search_enabled: bool = False,
|
||||||
|
attribute_classifications: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Get OpenAI-compatible tool definitions for Frigate.
|
||||||
|
|
||||||
|
Returns a list of tool definitions that can be used with OpenAI-compatible
|
||||||
|
function calling APIs. When semantic search is enabled, the search_objects
|
||||||
|
tool exposes an additional `semantic_query` parameter for descriptive
|
||||||
|
queries (e.g. "person riding a lawn mower") and find_similar_objects is
|
||||||
|
included. When attribute classification models are configured, an
|
||||||
|
`attribute` parameter is exposed for filtering by their labels.
|
||||||
|
"""
|
||||||
|
search_objects_properties: Dict[str, Any] = {
|
||||||
|
"camera": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Camera name to filter by (optional).",
|
||||||
|
},
|
||||||
|
"label": {
|
||||||
|
"type": "string",
|
||||||
|
"description": (
|
||||||
|
"Generic object class to filter by — one of the tracked detector "
|
||||||
|
"labels such as 'person', 'package', 'car', 'dog', 'bird'. Use "
|
||||||
|
"this for broad queries like 'show me all cars today'. Combine "
|
||||||
|
"with semantic_query when the user also describes appearance or "
|
||||||
|
"behavior (e.g. label='person', semantic_query='riding a lawn "
|
||||||
|
"mower')."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"sub_label": {
|
||||||
|
"type": "string",
|
||||||
|
"description": (
|
||||||
|
"Filter by a DISCRETE NAMED entity recognized in the detection. "
|
||||||
|
"Use this for: a known person's name ('John'), a delivery "
|
||||||
|
"company ('Amazon', 'UPS'), a recognized animal species or "
|
||||||
|
"breed ('blue jay', 'cardinal', 'golden retriever'), or a "
|
||||||
|
"license plate string. When filtering by a specific name, set "
|
||||||
|
"only sub_label and leave label unset. Do NOT use sub_label "
|
||||||
|
"for descriptions of appearance, clothing, or actions — those "
|
||||||
|
"belong in semantic_query."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"after": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
||||||
|
},
|
||||||
|
"before": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
||||||
|
},
|
||||||
|
"zones": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "List of zone names to filter by.",
|
||||||
|
},
|
||||||
|
"limit": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Maximum number of objects to return (default: 25).",
|
||||||
|
"default": 25,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if attribute_classifications:
|
||||||
|
model_outline = "; ".join(
|
||||||
|
f"{m['name']} (applies to {', '.join(m['objects']) or 'any object'})"
|
||||||
|
for m in attribute_classifications
|
||||||
|
)
|
||||||
|
search_objects_properties["attribute"] = {
|
||||||
|
"type": "string",
|
||||||
|
"description": (
|
||||||
|
"Filter by a classification attribute label produced by a "
|
||||||
|
"configured attribute classification model. Use this INSTEAD "
|
||||||
|
"of semantic_query when the user's request matches one of "
|
||||||
|
"these classifications. Configured models: "
|
||||||
|
f"{model_outline}. "
|
||||||
|
"Set the value to the attribute label that matches the user's "
|
||||||
|
"phrasing (case-sensitive)."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
if semantic_search_enabled:
|
||||||
|
search_objects_properties["semantic_query"] = {
|
||||||
|
"type": "string",
|
||||||
|
"description": (
|
||||||
|
"Optional natural-language description of a PHYSICAL "
|
||||||
|
"CHARACTERISTIC, APPEARANCE, or ACTIVITY the user mentioned, "
|
||||||
|
"used to semantically narrow results. Only set this when the "
|
||||||
|
"user describes something beyond what label and sub_label can "
|
||||||
|
"express on their own.\n"
|
||||||
|
"USE for descriptive phrases like: 'riding a lawn mower', "
|
||||||
|
"'wearing a red jacket', 'carrying a package', 'walking a "
|
||||||
|
"dog', 'on a bicycle', 'holding an umbrella'.\n"
|
||||||
|
"DO NOT USE for:\n"
|
||||||
|
"- specific named people, pets, or delivery companies → use sub_label\n"
|
||||||
|
"- animal species or breed names like 'blue jay', 'cardinal', "
|
||||||
|
"'golden retriever' → use sub_label\n"
|
||||||
|
"- license plate strings → use sub_label\n"
|
||||||
|
"- generic object queries like 'all cars today' or 'every "
|
||||||
|
"person' → use label alone with no semantic_query\n"
|
||||||
|
"When set, combine with label/time/camera/zone filters as "
|
||||||
|
"usual (e.g. label='person', semantic_query='riding a lawn "
|
||||||
|
"mower', after='2024-05-01T00:00:00Z')."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
search_objects_description = (
|
||||||
|
"Search the historical record of detected objects in Frigate. "
|
||||||
|
"Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', "
|
||||||
|
"'when was the last car?', 'show me detections from yesterday'. "
|
||||||
|
"Do NOT use this for monitoring or alerting requests about future events — "
|
||||||
|
"use start_camera_watch instead for those. "
|
||||||
|
"An 'object' in Frigate represents a tracked detection (e.g., a person, package, car).\n\n"
|
||||||
|
"Choose filters based on what the user is asking for:\n"
|
||||||
|
"- Generic class query ('show me all cars today'): set `label` only.\n"
|
||||||
|
"- Specific NAMED entity (known person, delivery company, animal "
|
||||||
|
"species/breed like 'blue jay' or 'golden retriever', license "
|
||||||
|
"plate): set `sub_label` only and leave `label` unset.\n"
|
||||||
|
)
|
||||||
|
if semantic_search_enabled:
|
||||||
|
search_objects_description += (
|
||||||
|
"- Physical CHARACTERISTIC, APPEARANCE, or ACTIVITY that is not a "
|
||||||
|
"discrete name ('person riding a lawn mower', 'someone in a red "
|
||||||
|
"jacket', 'person carrying a package'): set `semantic_query` with "
|
||||||
|
"the descriptive phrase, optionally alongside `label` for the "
|
||||||
|
"object class. Do NOT put descriptive phrases in sub_label."
|
||||||
|
)
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "search_objects",
|
||||||
|
"description": search_objects_description,
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": search_objects_properties,
|
||||||
|
},
|
||||||
|
"required": [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "find_similar_objects",
|
||||||
|
"description": (
|
||||||
|
"Find tracked objects that are visually and semantically similar "
|
||||||
|
"to a specific past event. Use this when the user references a "
|
||||||
|
"particular object they have seen and wants to find other "
|
||||||
|
"sightings of the same or similar one ('that green car', 'the "
|
||||||
|
"person in the red jacket', 'the package that was delivered'). "
|
||||||
|
"Prefer this over search_objects whenever the user's intent is "
|
||||||
|
"'find more like this specific one.' Use search_objects first "
|
||||||
|
"only if you need to locate the anchor event. Requires semantic "
|
||||||
|
"search to be enabled."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"event_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The id of the anchor event to find similar objects to.",
|
||||||
|
},
|
||||||
|
"after": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
||||||
|
},
|
||||||
|
"before": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
||||||
|
},
|
||||||
|
"cameras": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "Optional list of cameras to restrict to. Defaults to all.",
|
||||||
|
},
|
||||||
|
"labels": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "Optional list of labels to restrict to. Defaults to the anchor event's label.",
|
||||||
|
},
|
||||||
|
"sub_labels": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "Optional list of sub_labels (names) to restrict to.",
|
||||||
|
},
|
||||||
|
"zones": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "Optional list of zones. An event matches if any of its zones overlap.",
|
||||||
|
},
|
||||||
|
"similarity_mode": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["visual", "semantic", "fused"],
|
||||||
|
"description": "Which similarity signal(s) to use. 'fused' (default) combines visual and semantic.",
|
||||||
|
"default": "fused",
|
||||||
|
},
|
||||||
|
"min_score": {
|
||||||
|
"type": "number",
|
||||||
|
"description": "Drop matches with a similarity score below this threshold (0.0-1.0).",
|
||||||
|
},
|
||||||
|
"limit": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Maximum number of matches to return (default: 10).",
|
||||||
|
"default": 10,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": ["event_id"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "set_camera_state",
|
||||||
|
"description": (
|
||||||
|
"Change a camera's feature state (e.g., turn detection on/off, enable/disable recordings). "
|
||||||
|
"Use camera='*' to apply to all cameras at once. "
|
||||||
|
"Only call this tool when the user explicitly asks to change a camera setting. "
|
||||||
|
"Requires admin privileges."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"camera": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Camera name to target, or '*' to target all cameras.",
|
||||||
|
},
|
||||||
|
"feature": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"detect",
|
||||||
|
"record",
|
||||||
|
"snapshots",
|
||||||
|
"audio",
|
||||||
|
"motion",
|
||||||
|
"enabled",
|
||||||
|
"birdseye",
|
||||||
|
"birdseye_mode",
|
||||||
|
"improve_contrast",
|
||||||
|
"ptz_autotracker",
|
||||||
|
"motion_contour_area",
|
||||||
|
"motion_threshold",
|
||||||
|
"notifications",
|
||||||
|
"audio_transcription",
|
||||||
|
"review_alerts",
|
||||||
|
"review_detections",
|
||||||
|
"object_descriptions",
|
||||||
|
"review_descriptions",
|
||||||
|
"profile",
|
||||||
|
],
|
||||||
|
"description": (
|
||||||
|
"The feature to change. Most features accept ON or OFF. "
|
||||||
|
"birdseye_mode accepts CONTINUOUS, MOTION, or OBJECTS. "
|
||||||
|
"motion_contour_area and motion_threshold accept a number. "
|
||||||
|
"profile accepts a profile name or 'none' to deactivate (requires camera='*')."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"value": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The value to set. ON or OFF for toggles, a number for thresholds, a profile name or 'none' for profile.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": ["camera", "feature", "value"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "get_live_context",
|
||||||
|
"description": (
|
||||||
|
"Get the current live image and detection information for a camera: objects being tracked, "
|
||||||
|
"zones, timestamps. Use this to understand what is visible in the live view. "
|
||||||
|
"Call this when answering questions about what is happening right now on a specific camera."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"camera": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Camera name to get live context for.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": ["camera"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "start_camera_watch",
|
||||||
|
"description": (
|
||||||
|
"Start a continuous VLM watch job that monitors a camera and sends a notification "
|
||||||
|
"when a specified condition is met. Use this when the user wants to be alerted about "
|
||||||
|
"a future event, e.g. 'tell me when guests arrive' or 'notify me when the package is picked up'. "
|
||||||
|
"Only one watch job can run at a time. Returns a job ID."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"camera": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Camera ID to monitor.",
|
||||||
|
},
|
||||||
|
"condition": {
|
||||||
|
"type": "string",
|
||||||
|
"description": (
|
||||||
|
"Natural-language description of the condition to watch for, "
|
||||||
|
"e.g. 'a person arrives at the front door'."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"max_duration_minutes": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Maximum time to watch before giving up (minutes, default 60).",
|
||||||
|
"default": 60,
|
||||||
|
},
|
||||||
|
"labels": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "Object labels that should trigger a VLM check (e.g. ['person', 'car']). If omitted, any detection on the camera triggers a check.",
|
||||||
|
},
|
||||||
|
"zones": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {"type": "string"},
|
||||||
|
"description": "Zone names to filter by. If specified, only detections in these zones trigger a VLM check.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": ["camera", "condition"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "stop_camera_watch",
|
||||||
|
"description": (
|
||||||
|
"Cancel the currently running VLM watch job. Use this when the user wants to "
|
||||||
|
"stop a previously started watch, e.g. 'stop watching the front door'."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {},
|
||||||
|
"required": [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "get_profile_status",
|
||||||
|
"description": (
|
||||||
|
"Get the current profile status including the active profile and "
|
||||||
|
"timestamps of when each profile was last activated. Use this to "
|
||||||
|
"determine time periods for recap requests — e.g. when the user asks "
|
||||||
|
"'what happened while I was away?', call this first to find the relevant "
|
||||||
|
"time window based on profile activation history."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {},
|
||||||
|
"required": [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "get_recap",
|
||||||
|
"description": (
|
||||||
|
"Get a recap of all activity (alerts and detections) for a given time period. "
|
||||||
|
"Use this after calling get_profile_status to retrieve what happened during "
|
||||||
|
"a specific window — e.g. 'what happened while I was away?'. Returns a "
|
||||||
|
"chronological list of activity with camera, objects, zones, and GenAI-generated "
|
||||||
|
"descriptions when available. Summarize the results for the user."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"after": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Start of the time period in ISO 8601 format (e.g. '2025-03-15T08:00:00').",
|
||||||
|
},
|
||||||
|
"before": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "End of the time period in ISO 8601 format (e.g. '2025-03-15T17:00:00').",
|
||||||
|
},
|
||||||
|
"cameras": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Comma-separated camera IDs to include, or 'all' for all cameras. Default is 'all'.",
|
||||||
|
},
|
||||||
|
"severity": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["alert", "detection"],
|
||||||
|
"description": "Filter by severity level. Omit to include both alerts and detections.",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"required": ["after", "before"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def build_chat_system_prompt(
|
||||||
|
config: FrigateConfig,
|
||||||
|
allowed_cameras: List[str],
|
||||||
|
semantic_search_enabled: bool,
|
||||||
|
attribute_classifications: List[Dict[str, Any]],
|
||||||
|
) -> str:
|
||||||
|
"""Build the system prompt for the chat completion endpoint.
|
||||||
|
|
||||||
|
Composes the static framing with conditional sections describing the
|
||||||
|
available cameras, speed units, semantic-search routing guidance, and
|
||||||
|
configured attribute classifications.
|
||||||
|
"""
|
||||||
|
current_datetime = datetime.datetime.now()
|
||||||
|
current_date_str = current_datetime.strftime("%Y-%m-%d")
|
||||||
|
current_time_str = current_datetime.strftime("%I:%M:%S %p")
|
||||||
|
|
||||||
|
cameras_info: List[str] = []
|
||||||
|
has_speed_zone = False
|
||||||
|
for camera_id in allowed_cameras:
|
||||||
|
if camera_id not in config.cameras:
|
||||||
|
continue
|
||||||
|
camera_config = config.cameras[camera_id]
|
||||||
|
friendly_name = (
|
||||||
|
camera_config.friendly_name
|
||||||
|
if camera_config.friendly_name
|
||||||
|
else camera_id.replace("_", " ").title()
|
||||||
|
)
|
||||||
|
zone_names = list(camera_config.zones.keys())
|
||||||
|
if not has_speed_zone:
|
||||||
|
has_speed_zone = any(
|
||||||
|
zone.distances for zone in camera_config.zones.values()
|
||||||
|
)
|
||||||
|
if zone_names:
|
||||||
|
cameras_info.append(
|
||||||
|
f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
cameras_info.append(f" - {friendly_name} (ID: {camera_id})")
|
||||||
|
|
||||||
|
cameras_section = ""
|
||||||
|
if cameras_info:
|
||||||
|
cameras_section = (
|
||||||
|
"\n\nAvailable cameras:\n"
|
||||||
|
+ "\n".join(cameras_info)
|
||||||
|
+ "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls."
|
||||||
|
)
|
||||||
|
|
||||||
|
speed_units_section = ""
|
||||||
|
if has_speed_zone:
|
||||||
|
speed_unit = (
|
||||||
|
"mph" if config.ui.unit_system == UnitSystemEnum.imperial else "km/h"
|
||||||
|
)
|
||||||
|
speed_units_section = f"\n\nReport object speeds to the user in {speed_unit}."
|
||||||
|
|
||||||
|
semantic_search_section = ""
|
||||||
|
if semantic_search_enabled:
|
||||||
|
semantic_search_section = (
|
||||||
|
"\n\nWhen routing a search_objects call, pick filters by the shape of the user's request:\n"
|
||||||
|
"- Generic class ('show me all cars today'): set `label` only.\n"
|
||||||
|
"- Specific named entity — a known person ('John'), delivery company ('Amazon'), animal species/breed ('blue jay', 'cardinal', 'golden retriever'), or license plate: set `sub_label` only and leave `label` unset.\n"
|
||||||
|
"- Physical characteristic, appearance, or activity that is NOT a discrete name ('find me people riding a lawn mower', 'someone in a red jacket', 'a person carrying a package'): set `semantic_query` with the descriptive phrase, optionally combined with `label` for the object class. Never put descriptive phrases in `sub_label`."
|
||||||
|
)
|
||||||
|
|
||||||
|
attribute_classification_section = ""
|
||||||
|
if attribute_classifications:
|
||||||
|
model_lines = "\n".join(
|
||||||
|
f"- {m['name']}: applies to {', '.join(m['objects']) or 'any object'}"
|
||||||
|
for m in attribute_classifications
|
||||||
|
)
|
||||||
|
attribute_classification_section = (
|
||||||
|
"\n\nAttribute classification models are configured for the following object types:\n"
|
||||||
|
f"{model_lines}\n"
|
||||||
|
"When the user's request matches one of these classifications, set the search_objects `attribute` field to the matching label rather than using `semantic_query`. Reserve `semantic_query` for descriptive phrases that fall outside the configured attribute labels."
|
||||||
|
)
|
||||||
|
|
||||||
|
return f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events.
|
||||||
|
|
||||||
|
Current server local date and time: {current_date_str} at {current_time_str}
|
||||||
|
|
||||||
|
Do not start your response with phrases like "I will check...", "Let me see...", or "Let me look...". Answer directly.
|
||||||
|
|
||||||
|
Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields.
|
||||||
|
When users ask about "today", "yesterday", "this week", etc., use the current date above as reference.
|
||||||
|
When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today).
|
||||||
|
Always be accurate with time calculations based on the current date provided.
|
||||||
|
|
||||||
|
When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:<id>], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{semantic_search_section}{attribute_classification_section}{cameras_section}{speed_units_section}"""
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user