mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-06-21 03:41:55 +03:00
Move chat prompts to prompts
This commit is contained in:
Nicolas Mowen
parent
f5b7395d69
commit
31f9611d34
@ -37,8 +37,11 @@ from frigate.api.defs.response.chat_response import (
|
||||
from frigate.api.defs.tags import Tags
|
||||
from frigate.api.event import _build_attribute_filter_clause, events
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.classification import ObjectClassificationType
|
||||
from frigate.config.ui import UnitSystemEnum
|
||||
from frigate.genai.prompts import (
|
||||
build_chat_system_prompt,
|
||||
get_attribute_classifications,
|
||||
get_tool_definitions,
|
||||
)
|
||||
from frigate.genai.utils import build_assistant_message_for_conversation
|
||||
from frigate.jobs.vlm_watch import (
|
||||
get_vlm_watch_job,
|
||||
@ -69,440 +72,6 @@ class VLMMonitorRequest(BaseModel):
|
||||
zones: List[str] = []
|
||||
|
||||
|
||||
def get_attribute_classifications(config: FrigateConfig) -> List[Dict[str, Any]]:
|
||||
"""Return enabled custom classification models of `attribute` type.
|
||||
|
||||
Each entry: {"name": <model name>, "objects": [<object label>, ...]}.
|
||||
These models attach attribute metadata to events on the listed object
|
||||
types, which can later be filtered via the search_objects `attribute`
|
||||
field.
|
||||
"""
|
||||
result: List[Dict[str, Any]] = []
|
||||
|
||||
for model_key, model_config in config.classification.custom.items():
|
||||
if not model_config.enabled or model_config.object_config is None:
|
||||
continue
|
||||
|
||||
if (
|
||||
model_config.object_config.classification_type
|
||||
!= ObjectClassificationType.attribute
|
||||
):
|
||||
continue
|
||||
|
||||
result.append(
|
||||
{
|
||||
"name": model_config.name or model_key,
|
||||
"objects": list(model_config.object_config.objects or []),
|
||||
}
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_tool_definitions(
|
||||
semantic_search_enabled: bool = False,
|
||||
attribute_classifications: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get OpenAI-compatible tool definitions for Frigate.
|
||||
|
||||
Returns a list of tool definitions that can be used with OpenAI-compatible
|
||||
function calling APIs. When semantic search is enabled, the search_objects
|
||||
tool exposes an additional `semantic_query` parameter for descriptive
|
||||
queries (e.g. "person riding a lawn mower") and find_similar_objects is
|
||||
included. When attribute classification models are configured, an
|
||||
`attribute` parameter is exposed for filtering by their labels.
|
||||
"""
|
||||
search_objects_properties: Dict[str, Any] = {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera name to filter by (optional).",
|
||||
},
|
||||
"label": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Generic object class to filter by — one of the tracked detector "
|
||||
"labels such as 'person', 'package', 'car', 'dog', 'bird'. Use "
|
||||
"this for broad queries like 'show me all cars today'. Combine "
|
||||
"with semantic_query when the user also describes appearance or "
|
||||
"behavior (e.g. label='person', semantic_query='riding a lawn "
|
||||
"mower')."
|
||||
),
|
||||
},
|
||||
"sub_label": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Filter by a DISCRETE NAMED entity recognized in the detection. "
|
||||
"Use this for: a known person's name ('John'), a delivery "
|
||||
"company ('Amazon', 'UPS'), a recognized animal species or "
|
||||
"breed ('blue jay', 'cardinal', 'golden retriever'), or a "
|
||||
"license plate string. When filtering by a specific name, set "
|
||||
"only sub_label and leave label unset. Do NOT use sub_label "
|
||||
"for descriptions of appearance, clothing, or actions — those "
|
||||
"belong in semantic_query."
|
||||
),
|
||||
},
|
||||
"after": {
|
||||
"type": "string",
|
||||
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of zone names to filter by.",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of objects to return (default: 25).",
|
||||
"default": 25,
|
||||
},
|
||||
}
|
||||
|
||||
if attribute_classifications:
|
||||
model_outline = "; ".join(
|
||||
f"{m['name']} (applies to {', '.join(m['objects']) or 'any object'})"
|
||||
for m in attribute_classifications
|
||||
)
|
||||
search_objects_properties["attribute"] = {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Filter by a classification attribute label produced by a "
|
||||
"configured attribute classification model. Use this INSTEAD "
|
||||
"of semantic_query when the user's request matches one of "
|
||||
"these classifications. Configured models: "
|
||||
f"{model_outline}. "
|
||||
"Set the value to the attribute label that matches the user's "
|
||||
"phrasing (case-sensitive)."
|
||||
),
|
||||
}
|
||||
|
||||
if semantic_search_enabled:
|
||||
search_objects_properties["semantic_query"] = {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Optional natural-language description of a PHYSICAL "
|
||||
"CHARACTERISTIC, APPEARANCE, or ACTIVITY the user mentioned, "
|
||||
"used to semantically narrow results. Only set this when the "
|
||||
"user describes something beyond what label and sub_label can "
|
||||
"express on their own.\n"
|
||||
"USE for descriptive phrases like: 'riding a lawn mower', "
|
||||
"'wearing a red jacket', 'carrying a package', 'walking a "
|
||||
"dog', 'on a bicycle', 'holding an umbrella'.\n"
|
||||
"DO NOT USE for:\n"
|
||||
"- specific named people, pets, or delivery companies → use sub_label\n"
|
||||
"- animal species or breed names like 'blue jay', 'cardinal', "
|
||||
"'golden retriever' → use sub_label\n"
|
||||
"- license plate strings → use sub_label\n"
|
||||
"- generic object queries like 'all cars today' or 'every "
|
||||
"person' → use label alone with no semantic_query\n"
|
||||
"When set, combine with label/time/camera/zone filters as "
|
||||
"usual (e.g. label='person', semantic_query='riding a lawn "
|
||||
"mower', after='2024-05-01T00:00:00Z')."
|
||||
),
|
||||
}
|
||||
|
||||
search_objects_description = (
|
||||
"Search the historical record of detected objects in Frigate. "
|
||||
"Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', "
|
||||
"'when was the last car?', 'show me detections from yesterday'. "
|
||||
"Do NOT use this for monitoring or alerting requests about future events — "
|
||||
"use start_camera_watch instead for those. "
|
||||
"An 'object' in Frigate represents a tracked detection (e.g., a person, package, car).\n\n"
|
||||
"Choose filters based on what the user is asking for:\n"
|
||||
"- Generic class query ('show me all cars today'): set `label` only.\n"
|
||||
"- Specific NAMED entity (known person, delivery company, animal "
|
||||
"species/breed like 'blue jay' or 'golden retriever', license "
|
||||
"plate): set `sub_label` only and leave `label` unset.\n"
|
||||
)
|
||||
if semantic_search_enabled:
|
||||
search_objects_description += (
|
||||
"- Physical CHARACTERISTIC, APPEARANCE, or ACTIVITY that is not a "
|
||||
"discrete name ('person riding a lawn mower', 'someone in a red "
|
||||
"jacket', 'person carrying a package'): set `semantic_query` with "
|
||||
"the descriptive phrase, optionally alongside `label` for the "
|
||||
"object class. Do NOT put descriptive phrases in sub_label."
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_objects",
|
||||
"description": search_objects_description,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": search_objects_properties,
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find_similar_objects",
|
||||
"description": (
|
||||
"Find tracked objects that are visually and semantically similar "
|
||||
"to a specific past event. Use this when the user references a "
|
||||
"particular object they have seen and wants to find other "
|
||||
"sightings of the same or similar one ('that green car', 'the "
|
||||
"person in the red jacket', 'the package that was delivered'). "
|
||||
"Prefer this over search_objects whenever the user's intent is "
|
||||
"'find more like this specific one.' Use search_objects first "
|
||||
"only if you need to locate the anchor event. Requires semantic "
|
||||
"search to be enabled."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"event_id": {
|
||||
"type": "string",
|
||||
"description": "The id of the anchor event to find similar objects to.",
|
||||
},
|
||||
"after": {
|
||||
"type": "string",
|
||||
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
||||
},
|
||||
"cameras": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of cameras to restrict to. Defaults to all.",
|
||||
},
|
||||
"labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of labels to restrict to. Defaults to the anchor event's label.",
|
||||
},
|
||||
"sub_labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of sub_labels (names) to restrict to.",
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of zones. An event matches if any of its zones overlap.",
|
||||
},
|
||||
"similarity_mode": {
|
||||
"type": "string",
|
||||
"enum": ["visual", "semantic", "fused"],
|
||||
"description": "Which similarity signal(s) to use. 'fused' (default) combines visual and semantic.",
|
||||
"default": "fused",
|
||||
},
|
||||
"min_score": {
|
||||
"type": "number",
|
||||
"description": "Drop matches with a similarity score below this threshold (0.0-1.0).",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of matches to return (default: 10).",
|
||||
"default": 10,
|
||||
},
|
||||
},
|
||||
"required": ["event_id"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "set_camera_state",
|
||||
"description": (
|
||||
"Change a camera's feature state (e.g., turn detection on/off, enable/disable recordings). "
|
||||
"Use camera='*' to apply to all cameras at once. "
|
||||
"Only call this tool when the user explicitly asks to change a camera setting. "
|
||||
"Requires admin privileges."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera name to target, or '*' to target all cameras.",
|
||||
},
|
||||
"feature": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"detect",
|
||||
"record",
|
||||
"snapshots",
|
||||
"audio",
|
||||
"motion",
|
||||
"enabled",
|
||||
"birdseye",
|
||||
"birdseye_mode",
|
||||
"improve_contrast",
|
||||
"ptz_autotracker",
|
||||
"motion_contour_area",
|
||||
"motion_threshold",
|
||||
"notifications",
|
||||
"audio_transcription",
|
||||
"review_alerts",
|
||||
"review_detections",
|
||||
"object_descriptions",
|
||||
"review_descriptions",
|
||||
"profile",
|
||||
],
|
||||
"description": (
|
||||
"The feature to change. Most features accept ON or OFF. "
|
||||
"birdseye_mode accepts CONTINUOUS, MOTION, or OBJECTS. "
|
||||
"motion_contour_area and motion_threshold accept a number. "
|
||||
"profile accepts a profile name or 'none' to deactivate (requires camera='*')."
|
||||
),
|
||||
},
|
||||
"value": {
|
||||
"type": "string",
|
||||
"description": "The value to set. ON or OFF for toggles, a number for thresholds, a profile name or 'none' for profile.",
|
||||
},
|
||||
},
|
||||
"required": ["camera", "feature", "value"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_live_context",
|
||||
"description": (
|
||||
"Get the current live image and detection information for a camera: objects being tracked, "
|
||||
"zones, timestamps. Use this to understand what is visible in the live view. "
|
||||
"Call this when answering questions about what is happening right now on a specific camera."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera name to get live context for.",
|
||||
},
|
||||
},
|
||||
"required": ["camera"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "start_camera_watch",
|
||||
"description": (
|
||||
"Start a continuous VLM watch job that monitors a camera and sends a notification "
|
||||
"when a specified condition is met. Use this when the user wants to be alerted about "
|
||||
"a future event, e.g. 'tell me when guests arrive' or 'notify me when the package is picked up'. "
|
||||
"Only one watch job can run at a time. Returns a job ID."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera ID to monitor.",
|
||||
},
|
||||
"condition": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Natural-language description of the condition to watch for, "
|
||||
"e.g. 'a person arrives at the front door'."
|
||||
),
|
||||
},
|
||||
"max_duration_minutes": {
|
||||
"type": "integer",
|
||||
"description": "Maximum time to watch before giving up (minutes, default 60).",
|
||||
"default": 60,
|
||||
},
|
||||
"labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Object labels that should trigger a VLM check (e.g. ['person', 'car']). If omitted, any detection on the camera triggers a check.",
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Zone names to filter by. If specified, only detections in these zones trigger a VLM check.",
|
||||
},
|
||||
},
|
||||
"required": ["camera", "condition"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "stop_camera_watch",
|
||||
"description": (
|
||||
"Cancel the currently running VLM watch job. Use this when the user wants to "
|
||||
"stop a previously started watch, e.g. 'stop watching the front door'."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_profile_status",
|
||||
"description": (
|
||||
"Get the current profile status including the active profile and "
|
||||
"timestamps of when each profile was last activated. Use this to "
|
||||
"determine time periods for recap requests — e.g. when the user asks "
|
||||
"'what happened while I was away?', call this first to find the relevant "
|
||||
"time window based on profile activation history."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_recap",
|
||||
"description": (
|
||||
"Get a recap of all activity (alerts and detections) for a given time period. "
|
||||
"Use this after calling get_profile_status to retrieve what happened during "
|
||||
"a specific window — e.g. 'what happened while I was away?'. Returns a "
|
||||
"chronological list of activity with camera, objects, zones, and GenAI-generated "
|
||||
"descriptions when available. Summarize the results for the user."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"after": {
|
||||
"type": "string",
|
||||
"description": "Start of the time period in ISO 8601 format (e.g. '2025-03-15T08:00:00').",
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": "End of the time period in ISO 8601 format (e.g. '2025-03-15T17:00:00').",
|
||||
},
|
||||
"cameras": {
|
||||
"type": "string",
|
||||
"description": "Comma-separated camera IDs to include, or 'all' for all cameras. Default is 'all'.",
|
||||
},
|
||||
"severity": {
|
||||
"type": "string",
|
||||
"enum": ["alert", "detection"],
|
||||
"description": "Filter by severity level. Omit to include both alerts and detections.",
|
||||
},
|
||||
},
|
||||
"required": ["after", "before"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@router.get(
|
||||
"/chat/tools",
|
||||
dependencies=[Depends(allow_any_authenticated())],
|
||||
@ -1550,81 +1119,12 @@ async def chat_completion(
|
||||
)
|
||||
conversation = []
|
||||
|
||||
current_datetime = datetime.now()
|
||||
current_date_str = current_datetime.strftime("%Y-%m-%d")
|
||||
current_time_str = current_datetime.strftime("%I:%M:%S %p")
|
||||
|
||||
cameras_info = []
|
||||
has_speed_zone = False
|
||||
for camera_id in allowed_cameras:
|
||||
if camera_id not in config.cameras:
|
||||
continue
|
||||
camera_config = config.cameras[camera_id]
|
||||
friendly_name = (
|
||||
camera_config.friendly_name
|
||||
if camera_config.friendly_name
|
||||
else camera_id.replace("_", " ").title()
|
||||
)
|
||||
zone_names = list(camera_config.zones.keys())
|
||||
if not has_speed_zone:
|
||||
has_speed_zone = any(
|
||||
zone.distances for zone in camera_config.zones.values()
|
||||
)
|
||||
if zone_names:
|
||||
cameras_info.append(
|
||||
f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})"
|
||||
)
|
||||
else:
|
||||
cameras_info.append(f" - {friendly_name} (ID: {camera_id})")
|
||||
|
||||
cameras_section = ""
|
||||
if cameras_info:
|
||||
cameras_section = (
|
||||
"\n\nAvailable cameras:\n"
|
||||
+ "\n".join(cameras_info)
|
||||
+ "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls."
|
||||
)
|
||||
|
||||
speed_units_section = ""
|
||||
if has_speed_zone:
|
||||
speed_unit = (
|
||||
"mph" if config.ui.unit_system == UnitSystemEnum.imperial else "km/h"
|
||||
)
|
||||
speed_units_section = f"\n\nReport object speeds to the user in {speed_unit}."
|
||||
|
||||
semantic_search_section = ""
|
||||
if semantic_search_enabled:
|
||||
semantic_search_section = (
|
||||
"\n\nWhen routing a search_objects call, pick filters by the shape of the user's request:\n"
|
||||
"- Generic class ('show me all cars today'): set `label` only.\n"
|
||||
"- Specific named entity — a known person ('John'), delivery company ('Amazon'), animal species/breed ('blue jay', 'cardinal', 'golden retriever'), or license plate: set `sub_label` only and leave `label` unset.\n"
|
||||
"- Physical characteristic, appearance, or activity that is NOT a discrete name ('find me people riding a lawn mower', 'someone in a red jacket', 'a person carrying a package'): set `semantic_query` with the descriptive phrase, optionally combined with `label` for the object class. Never put descriptive phrases in `sub_label`."
|
||||
)
|
||||
|
||||
attribute_classification_section = ""
|
||||
if attribute_classifications:
|
||||
model_lines = "\n".join(
|
||||
f"- {m['name']}: applies to {', '.join(m['objects']) or 'any object'}"
|
||||
for m in attribute_classifications
|
||||
)
|
||||
attribute_classification_section = (
|
||||
"\n\nAttribute classification models are configured for the following object types:\n"
|
||||
f"{model_lines}\n"
|
||||
"When the user's request matches one of these classifications, set the search_objects `attribute` field to the matching label rather than using `semantic_query`. Reserve `semantic_query` for descriptive phrases that fall outside the configured attribute labels."
|
||||
)
|
||||
|
||||
system_prompt = f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events.
|
||||
|
||||
Current server local date and time: {current_date_str} at {current_time_str}
|
||||
|
||||
Do not start your response with phrases like "I will check...", "Let me see...", or "Let me look...". Answer directly.
|
||||
|
||||
Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields.
|
||||
When users ask about "today", "yesterday", "this week", etc., use the current date above as reference.
|
||||
When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today).
|
||||
Always be accurate with time calculations based on the current date provided.
|
||||
|
||||
When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:<id>], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{semantic_search_section}{attribute_classification_section}{cameras_section}{speed_units_section}"""
|
||||
system_prompt = build_chat_system_prompt(
|
||||
config=config,
|
||||
allowed_cameras=allowed_cameras,
|
||||
semantic_search_enabled=semantic_search_enabled,
|
||||
attribute_classifications=attribute_classifications,
|
||||
)
|
||||
|
||||
conversation.append(
|
||||
{
|
||||
|
||||
@ -6,11 +6,13 @@ transport.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
from typing import Any
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from playhouse.shortcuts import model_to_dict
|
||||
|
||||
from frigate.config import CameraConfig
|
||||
from frigate.config import CameraConfig, FrigateConfig
|
||||
from frigate.config.classification import ObjectClassificationType
|
||||
from frigate.config.ui import UnitSystemEnum
|
||||
from frigate.data_processing.post.types import ReviewMetadata
|
||||
from frigate.models import Event
|
||||
|
||||
@ -212,3 +214,526 @@ def build_object_description_prompt(
|
||||
camera_config.objects.genai.prompt,
|
||||
)
|
||||
return template.format(**model_to_dict(event))
|
||||
|
||||
|
||||
def get_attribute_classifications(config: FrigateConfig) -> List[Dict[str, Any]]:
|
||||
"""Return enabled custom classification models of `attribute` type.
|
||||
|
||||
Each entry: {"name": <model name>, "objects": [<object label>, ...]}.
|
||||
These models attach attribute metadata to events on the listed object
|
||||
types, which can later be filtered via the search_objects `attribute`
|
||||
field.
|
||||
"""
|
||||
result: List[Dict[str, Any]] = []
|
||||
|
||||
for model_key, model_config in config.classification.custom.items():
|
||||
if not model_config.enabled or model_config.object_config is None:
|
||||
continue
|
||||
|
||||
if (
|
||||
model_config.object_config.classification_type
|
||||
!= ObjectClassificationType.attribute
|
||||
):
|
||||
continue
|
||||
|
||||
result.append(
|
||||
{
|
||||
"name": model_config.name or model_key,
|
||||
"objects": list(model_config.object_config.objects or []),
|
||||
}
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_tool_definitions(
|
||||
semantic_search_enabled: bool = False,
|
||||
attribute_classifications: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get OpenAI-compatible tool definitions for Frigate.
|
||||
|
||||
Returns a list of tool definitions that can be used with OpenAI-compatible
|
||||
function calling APIs. When semantic search is enabled, the search_objects
|
||||
tool exposes an additional `semantic_query` parameter for descriptive
|
||||
queries (e.g. "person riding a lawn mower") and find_similar_objects is
|
||||
included. When attribute classification models are configured, an
|
||||
`attribute` parameter is exposed for filtering by their labels.
|
||||
"""
|
||||
search_objects_properties: Dict[str, Any] = {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera name to filter by (optional).",
|
||||
},
|
||||
"label": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Generic object class to filter by — one of the tracked detector "
|
||||
"labels such as 'person', 'package', 'car', 'dog', 'bird'. Use "
|
||||
"this for broad queries like 'show me all cars today'. Combine "
|
||||
"with semantic_query when the user also describes appearance or "
|
||||
"behavior (e.g. label='person', semantic_query='riding a lawn "
|
||||
"mower')."
|
||||
),
|
||||
},
|
||||
"sub_label": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Filter by a DISCRETE NAMED entity recognized in the detection. "
|
||||
"Use this for: a known person's name ('John'), a delivery "
|
||||
"company ('Amazon', 'UPS'), a recognized animal species or "
|
||||
"breed ('blue jay', 'cardinal', 'golden retriever'), or a "
|
||||
"license plate string. When filtering by a specific name, set "
|
||||
"only sub_label and leave label unset. Do NOT use sub_label "
|
||||
"for descriptions of appearance, clothing, or actions — those "
|
||||
"belong in semantic_query."
|
||||
),
|
||||
},
|
||||
"after": {
|
||||
"type": "string",
|
||||
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of zone names to filter by.",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of objects to return (default: 25).",
|
||||
"default": 25,
|
||||
},
|
||||
}
|
||||
|
||||
if attribute_classifications:
|
||||
model_outline = "; ".join(
|
||||
f"{m['name']} (applies to {', '.join(m['objects']) or 'any object'})"
|
||||
for m in attribute_classifications
|
||||
)
|
||||
search_objects_properties["attribute"] = {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Filter by a classification attribute label produced by a "
|
||||
"configured attribute classification model. Use this INSTEAD "
|
||||
"of semantic_query when the user's request matches one of "
|
||||
"these classifications. Configured models: "
|
||||
f"{model_outline}. "
|
||||
"Set the value to the attribute label that matches the user's "
|
||||
"phrasing (case-sensitive)."
|
||||
),
|
||||
}
|
||||
|
||||
if semantic_search_enabled:
|
||||
search_objects_properties["semantic_query"] = {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Optional natural-language description of a PHYSICAL "
|
||||
"CHARACTERISTIC, APPEARANCE, or ACTIVITY the user mentioned, "
|
||||
"used to semantically narrow results. Only set this when the "
|
||||
"user describes something beyond what label and sub_label can "
|
||||
"express on their own.\n"
|
||||
"USE for descriptive phrases like: 'riding a lawn mower', "
|
||||
"'wearing a red jacket', 'carrying a package', 'walking a "
|
||||
"dog', 'on a bicycle', 'holding an umbrella'.\n"
|
||||
"DO NOT USE for:\n"
|
||||
"- specific named people, pets, or delivery companies → use sub_label\n"
|
||||
"- animal species or breed names like 'blue jay', 'cardinal', "
|
||||
"'golden retriever' → use sub_label\n"
|
||||
"- license plate strings → use sub_label\n"
|
||||
"- generic object queries like 'all cars today' or 'every "
|
||||
"person' → use label alone with no semantic_query\n"
|
||||
"When set, combine with label/time/camera/zone filters as "
|
||||
"usual (e.g. label='person', semantic_query='riding a lawn "
|
||||
"mower', after='2024-05-01T00:00:00Z')."
|
||||
),
|
||||
}
|
||||
|
||||
search_objects_description = (
|
||||
"Search the historical record of detected objects in Frigate. "
|
||||
"Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', "
|
||||
"'when was the last car?', 'show me detections from yesterday'. "
|
||||
"Do NOT use this for monitoring or alerting requests about future events — "
|
||||
"use start_camera_watch instead for those. "
|
||||
"An 'object' in Frigate represents a tracked detection (e.g., a person, package, car).\n\n"
|
||||
"Choose filters based on what the user is asking for:\n"
|
||||
"- Generic class query ('show me all cars today'): set `label` only.\n"
|
||||
"- Specific NAMED entity (known person, delivery company, animal "
|
||||
"species/breed like 'blue jay' or 'golden retriever', license "
|
||||
"plate): set `sub_label` only and leave `label` unset.\n"
|
||||
)
|
||||
if semantic_search_enabled:
|
||||
search_objects_description += (
|
||||
"- Physical CHARACTERISTIC, APPEARANCE, or ACTIVITY that is not a "
|
||||
"discrete name ('person riding a lawn mower', 'someone in a red "
|
||||
"jacket', 'person carrying a package'): set `semantic_query` with "
|
||||
"the descriptive phrase, optionally alongside `label` for the "
|
||||
"object class. Do NOT put descriptive phrases in sub_label."
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_objects",
|
||||
"description": search_objects_description,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": search_objects_properties,
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "find_similar_objects",
|
||||
"description": (
|
||||
"Find tracked objects that are visually and semantically similar "
|
||||
"to a specific past event. Use this when the user references a "
|
||||
"particular object they have seen and wants to find other "
|
||||
"sightings of the same or similar one ('that green car', 'the "
|
||||
"person in the red jacket', 'the package that was delivered'). "
|
||||
"Prefer this over search_objects whenever the user's intent is "
|
||||
"'find more like this specific one.' Use search_objects first "
|
||||
"only if you need to locate the anchor event. Requires semantic "
|
||||
"search to be enabled."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"event_id": {
|
||||
"type": "string",
|
||||
"description": "The id of the anchor event to find similar objects to.",
|
||||
},
|
||||
"after": {
|
||||
"type": "string",
|
||||
"description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
|
||||
},
|
||||
"cameras": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of cameras to restrict to. Defaults to all.",
|
||||
},
|
||||
"labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of labels to restrict to. Defaults to the anchor event's label.",
|
||||
},
|
||||
"sub_labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of sub_labels (names) to restrict to.",
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional list of zones. An event matches if any of its zones overlap.",
|
||||
},
|
||||
"similarity_mode": {
|
||||
"type": "string",
|
||||
"enum": ["visual", "semantic", "fused"],
|
||||
"description": "Which similarity signal(s) to use. 'fused' (default) combines visual and semantic.",
|
||||
"default": "fused",
|
||||
},
|
||||
"min_score": {
|
||||
"type": "number",
|
||||
"description": "Drop matches with a similarity score below this threshold (0.0-1.0).",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of matches to return (default: 10).",
|
||||
"default": 10,
|
||||
},
|
||||
},
|
||||
"required": ["event_id"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "set_camera_state",
|
||||
"description": (
|
||||
"Change a camera's feature state (e.g., turn detection on/off, enable/disable recordings). "
|
||||
"Use camera='*' to apply to all cameras at once. "
|
||||
"Only call this tool when the user explicitly asks to change a camera setting. "
|
||||
"Requires admin privileges."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera name to target, or '*' to target all cameras.",
|
||||
},
|
||||
"feature": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"detect",
|
||||
"record",
|
||||
"snapshots",
|
||||
"audio",
|
||||
"motion",
|
||||
"enabled",
|
||||
"birdseye",
|
||||
"birdseye_mode",
|
||||
"improve_contrast",
|
||||
"ptz_autotracker",
|
||||
"motion_contour_area",
|
||||
"motion_threshold",
|
||||
"notifications",
|
||||
"audio_transcription",
|
||||
"review_alerts",
|
||||
"review_detections",
|
||||
"object_descriptions",
|
||||
"review_descriptions",
|
||||
"profile",
|
||||
],
|
||||
"description": (
|
||||
"The feature to change. Most features accept ON or OFF. "
|
||||
"birdseye_mode accepts CONTINUOUS, MOTION, or OBJECTS. "
|
||||
"motion_contour_area and motion_threshold accept a number. "
|
||||
"profile accepts a profile name or 'none' to deactivate (requires camera='*')."
|
||||
),
|
||||
},
|
||||
"value": {
|
||||
"type": "string",
|
||||
"description": "The value to set. ON or OFF for toggles, a number for thresholds, a profile name or 'none' for profile.",
|
||||
},
|
||||
},
|
||||
"required": ["camera", "feature", "value"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_live_context",
|
||||
"description": (
|
||||
"Get the current live image and detection information for a camera: objects being tracked, "
|
||||
"zones, timestamps. Use this to understand what is visible in the live view. "
|
||||
"Call this when answering questions about what is happening right now on a specific camera."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera name to get live context for.",
|
||||
},
|
||||
},
|
||||
"required": ["camera"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "start_camera_watch",
|
||||
"description": (
|
||||
"Start a continuous VLM watch job that monitors a camera and sends a notification "
|
||||
"when a specified condition is met. Use this when the user wants to be alerted about "
|
||||
"a future event, e.g. 'tell me when guests arrive' or 'notify me when the package is picked up'. "
|
||||
"Only one watch job can run at a time. Returns a job ID."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"camera": {
|
||||
"type": "string",
|
||||
"description": "Camera ID to monitor.",
|
||||
},
|
||||
"condition": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Natural-language description of the condition to watch for, "
|
||||
"e.g. 'a person arrives at the front door'."
|
||||
),
|
||||
},
|
||||
"max_duration_minutes": {
|
||||
"type": "integer",
|
||||
"description": "Maximum time to watch before giving up (minutes, default 60).",
|
||||
"default": 60,
|
||||
},
|
||||
"labels": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Object labels that should trigger a VLM check (e.g. ['person', 'car']). If omitted, any detection on the camera triggers a check.",
|
||||
},
|
||||
"zones": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Zone names to filter by. If specified, only detections in these zones trigger a VLM check.",
|
||||
},
|
||||
},
|
||||
"required": ["camera", "condition"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "stop_camera_watch",
|
||||
"description": (
|
||||
"Cancel the currently running VLM watch job. Use this when the user wants to "
|
||||
"stop a previously started watch, e.g. 'stop watching the front door'."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_profile_status",
|
||||
"description": (
|
||||
"Get the current profile status including the active profile and "
|
||||
"timestamps of when each profile was last activated. Use this to "
|
||||
"determine time periods for recap requests — e.g. when the user asks "
|
||||
"'what happened while I was away?', call this first to find the relevant "
|
||||
"time window based on profile activation history."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_recap",
|
||||
"description": (
|
||||
"Get a recap of all activity (alerts and detections) for a given time period. "
|
||||
"Use this after calling get_profile_status to retrieve what happened during "
|
||||
"a specific window — e.g. 'what happened while I was away?'. Returns a "
|
||||
"chronological list of activity with camera, objects, zones, and GenAI-generated "
|
||||
"descriptions when available. Summarize the results for the user."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"after": {
|
||||
"type": "string",
|
||||
"description": "Start of the time period in ISO 8601 format (e.g. '2025-03-15T08:00:00').",
|
||||
},
|
||||
"before": {
|
||||
"type": "string",
|
||||
"description": "End of the time period in ISO 8601 format (e.g. '2025-03-15T17:00:00').",
|
||||
},
|
||||
"cameras": {
|
||||
"type": "string",
|
||||
"description": "Comma-separated camera IDs to include, or 'all' for all cameras. Default is 'all'.",
|
||||
},
|
||||
"severity": {
|
||||
"type": "string",
|
||||
"enum": ["alert", "detection"],
|
||||
"description": "Filter by severity level. Omit to include both alerts and detections.",
|
||||
},
|
||||
},
|
||||
"required": ["after", "before"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def build_chat_system_prompt(
|
||||
config: FrigateConfig,
|
||||
allowed_cameras: List[str],
|
||||
semantic_search_enabled: bool,
|
||||
attribute_classifications: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
"""Build the system prompt for the chat completion endpoint.
|
||||
|
||||
Composes the static framing with conditional sections describing the
|
||||
available cameras, speed units, semantic-search routing guidance, and
|
||||
configured attribute classifications.
|
||||
"""
|
||||
current_datetime = datetime.datetime.now()
|
||||
current_date_str = current_datetime.strftime("%Y-%m-%d")
|
||||
current_time_str = current_datetime.strftime("%I:%M:%S %p")
|
||||
|
||||
cameras_info: List[str] = []
|
||||
has_speed_zone = False
|
||||
for camera_id in allowed_cameras:
|
||||
if camera_id not in config.cameras:
|
||||
continue
|
||||
camera_config = config.cameras[camera_id]
|
||||
friendly_name = (
|
||||
camera_config.friendly_name
|
||||
if camera_config.friendly_name
|
||||
else camera_id.replace("_", " ").title()
|
||||
)
|
||||
zone_names = list(camera_config.zones.keys())
|
||||
if not has_speed_zone:
|
||||
has_speed_zone = any(
|
||||
zone.distances for zone in camera_config.zones.values()
|
||||
)
|
||||
if zone_names:
|
||||
cameras_info.append(
|
||||
f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})"
|
||||
)
|
||||
else:
|
||||
cameras_info.append(f" - {friendly_name} (ID: {camera_id})")
|
||||
|
||||
cameras_section = ""
|
||||
if cameras_info:
|
||||
cameras_section = (
|
||||
"\n\nAvailable cameras:\n"
|
||||
+ "\n".join(cameras_info)
|
||||
+ "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls."
|
||||
)
|
||||
|
||||
speed_units_section = ""
|
||||
if has_speed_zone:
|
||||
speed_unit = (
|
||||
"mph" if config.ui.unit_system == UnitSystemEnum.imperial else "km/h"
|
||||
)
|
||||
speed_units_section = f"\n\nReport object speeds to the user in {speed_unit}."
|
||||
|
||||
semantic_search_section = ""
|
||||
if semantic_search_enabled:
|
||||
semantic_search_section = (
|
||||
"\n\nWhen routing a search_objects call, pick filters by the shape of the user's request:\n"
|
||||
"- Generic class ('show me all cars today'): set `label` only.\n"
|
||||
"- Specific named entity — a known person ('John'), delivery company ('Amazon'), animal species/breed ('blue jay', 'cardinal', 'golden retriever'), or license plate: set `sub_label` only and leave `label` unset.\n"
|
||||
"- Physical characteristic, appearance, or activity that is NOT a discrete name ('find me people riding a lawn mower', 'someone in a red jacket', 'a person carrying a package'): set `semantic_query` with the descriptive phrase, optionally combined with `label` for the object class. Never put descriptive phrases in `sub_label`."
|
||||
)
|
||||
|
||||
attribute_classification_section = ""
|
||||
if attribute_classifications:
|
||||
model_lines = "\n".join(
|
||||
f"- {m['name']}: applies to {', '.join(m['objects']) or 'any object'}"
|
||||
for m in attribute_classifications
|
||||
)
|
||||
attribute_classification_section = (
|
||||
"\n\nAttribute classification models are configured for the following object types:\n"
|
||||
f"{model_lines}\n"
|
||||
"When the user's request matches one of these classifications, set the search_objects `attribute` field to the matching label rather than using `semantic_query`. Reserve `semantic_query` for descriptive phrases that fall outside the configured attribute labels."
|
||||
)
|
||||
|
||||
return f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events.
|
||||
|
||||
Current server local date and time: {current_date_str} at {current_time_str}
|
||||
|
||||
Do not start your response with phrases like "I will check...", "Let me see...", or "Let me look...". Answer directly.
|
||||
|
||||
Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields.
|
||||
When users ask about "today", "yesterday", "this week", etc., use the current date above as reference.
|
||||
When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today).
|
||||
Always be accurate with time calculations based on the current date provided.
|
||||
|
||||
When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:<id>], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{semantic_search_section}{attribute_classification_section}{cameras_section}{speed_units_section}"""
|
||||
|
||||
Loading…
Reference in New Issue
Block a user