diff --git a/frigate/api/chat.py b/frigate/api/chat.py index 72b805a0d5..a701a04bd5 100644 --- a/frigate/api/chat.py +++ b/frigate/api/chat.py @@ -37,8 +37,11 @@ from frigate.api.defs.response.chat_response import ( from frigate.api.defs.tags import Tags from frigate.api.event import _build_attribute_filter_clause, events from frigate.config import FrigateConfig -from frigate.config.classification import ObjectClassificationType -from frigate.config.ui import UnitSystemEnum +from frigate.genai.prompts import ( + build_chat_system_prompt, + get_attribute_classifications, + get_tool_definitions, +) from frigate.genai.utils import build_assistant_message_for_conversation from frigate.jobs.vlm_watch import ( get_vlm_watch_job, @@ -69,440 +72,6 @@ class VLMMonitorRequest(BaseModel): zones: List[str] = [] -def get_attribute_classifications(config: FrigateConfig) -> List[Dict[str, Any]]: - """Return enabled custom classification models of `attribute` type. - - Each entry: {"name": , "objects": [, ...]}. - These models attach attribute metadata to events on the listed object - types, which can later be filtered via the search_objects `attribute` - field. - """ - result: List[Dict[str, Any]] = [] - - for model_key, model_config in config.classification.custom.items(): - if not model_config.enabled or model_config.object_config is None: - continue - - if ( - model_config.object_config.classification_type - != ObjectClassificationType.attribute - ): - continue - - result.append( - { - "name": model_config.name or model_key, - "objects": list(model_config.object_config.objects or []), - } - ) - - return result - - -def get_tool_definitions( - semantic_search_enabled: bool = False, - attribute_classifications: Optional[List[Dict[str, Any]]] = None, -) -> List[Dict[str, Any]]: - """ - Get OpenAI-compatible tool definitions for Frigate. - - Returns a list of tool definitions that can be used with OpenAI-compatible - function calling APIs. When semantic search is enabled, the search_objects - tool exposes an additional `semantic_query` parameter for descriptive - queries (e.g. "person riding a lawn mower") and find_similar_objects is - included. When attribute classification models are configured, an - `attribute` parameter is exposed for filtering by their labels. - """ - search_objects_properties: Dict[str, Any] = { - "camera": { - "type": "string", - "description": "Camera name to filter by (optional).", - }, - "label": { - "type": "string", - "description": ( - "Generic object class to filter by — one of the tracked detector " - "labels such as 'person', 'package', 'car', 'dog', 'bird'. Use " - "this for broad queries like 'show me all cars today'. Combine " - "with semantic_query when the user also describes appearance or " - "behavior (e.g. label='person', semantic_query='riding a lawn " - "mower')." - ), - }, - "sub_label": { - "type": "string", - "description": ( - "Filter by a DISCRETE NAMED entity recognized in the detection. " - "Use this for: a known person's name ('John'), a delivery " - "company ('Amazon', 'UPS'), a recognized animal species or " - "breed ('blue jay', 'cardinal', 'golden retriever'), or a " - "license plate string. When filtering by a specific name, set " - "only sub_label and leave label unset. Do NOT use sub_label " - "for descriptions of appearance, clothing, or actions — those " - "belong in semantic_query." - ), - }, - "after": { - "type": "string", - "description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').", - }, - "before": { - "type": "string", - "description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').", - }, - "zones": { - "type": "array", - "items": {"type": "string"}, - "description": "List of zone names to filter by.", - }, - "limit": { - "type": "integer", - "description": "Maximum number of objects to return (default: 25).", - "default": 25, - }, - } - - if attribute_classifications: - model_outline = "; ".join( - f"{m['name']} (applies to {', '.join(m['objects']) or 'any object'})" - for m in attribute_classifications - ) - search_objects_properties["attribute"] = { - "type": "string", - "description": ( - "Filter by a classification attribute label produced by a " - "configured attribute classification model. Use this INSTEAD " - "of semantic_query when the user's request matches one of " - "these classifications. Configured models: " - f"{model_outline}. " - "Set the value to the attribute label that matches the user's " - "phrasing (case-sensitive)." - ), - } - - if semantic_search_enabled: - search_objects_properties["semantic_query"] = { - "type": "string", - "description": ( - "Optional natural-language description of a PHYSICAL " - "CHARACTERISTIC, APPEARANCE, or ACTIVITY the user mentioned, " - "used to semantically narrow results. Only set this when the " - "user describes something beyond what label and sub_label can " - "express on their own.\n" - "USE for descriptive phrases like: 'riding a lawn mower', " - "'wearing a red jacket', 'carrying a package', 'walking a " - "dog', 'on a bicycle', 'holding an umbrella'.\n" - "DO NOT USE for:\n" - "- specific named people, pets, or delivery companies → use sub_label\n" - "- animal species or breed names like 'blue jay', 'cardinal', " - "'golden retriever' → use sub_label\n" - "- license plate strings → use sub_label\n" - "- generic object queries like 'all cars today' or 'every " - "person' → use label alone with no semantic_query\n" - "When set, combine with label/time/camera/zone filters as " - "usual (e.g. label='person', semantic_query='riding a lawn " - "mower', after='2024-05-01T00:00:00Z')." - ), - } - - search_objects_description = ( - "Search the historical record of detected objects in Frigate. " - "Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', " - "'when was the last car?', 'show me detections from yesterday'. " - "Do NOT use this for monitoring or alerting requests about future events — " - "use start_camera_watch instead for those. " - "An 'object' in Frigate represents a tracked detection (e.g., a person, package, car).\n\n" - "Choose filters based on what the user is asking for:\n" - "- Generic class query ('show me all cars today'): set `label` only.\n" - "- Specific NAMED entity (known person, delivery company, animal " - "species/breed like 'blue jay' or 'golden retriever', license " - "plate): set `sub_label` only and leave `label` unset.\n" - ) - if semantic_search_enabled: - search_objects_description += ( - "- Physical CHARACTERISTIC, APPEARANCE, or ACTIVITY that is not a " - "discrete name ('person riding a lawn mower', 'someone in a red " - "jacket', 'person carrying a package'): set `semantic_query` with " - "the descriptive phrase, optionally alongside `label` for the " - "object class. Do NOT put descriptive phrases in sub_label." - ) - - return [ - { - "type": "function", - "function": { - "name": "search_objects", - "description": search_objects_description, - "parameters": { - "type": "object", - "properties": search_objects_properties, - }, - "required": [], - }, - }, - { - "type": "function", - "function": { - "name": "find_similar_objects", - "description": ( - "Find tracked objects that are visually and semantically similar " - "to a specific past event. Use this when the user references a " - "particular object they have seen and wants to find other " - "sightings of the same or similar one ('that green car', 'the " - "person in the red jacket', 'the package that was delivered'). " - "Prefer this over search_objects whenever the user's intent is " - "'find more like this specific one.' Use search_objects first " - "only if you need to locate the anchor event. Requires semantic " - "search to be enabled." - ), - "parameters": { - "type": "object", - "properties": { - "event_id": { - "type": "string", - "description": "The id of the anchor event to find similar objects to.", - }, - "after": { - "type": "string", - "description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').", - }, - "before": { - "type": "string", - "description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').", - }, - "cameras": { - "type": "array", - "items": {"type": "string"}, - "description": "Optional list of cameras to restrict to. Defaults to all.", - }, - "labels": { - "type": "array", - "items": {"type": "string"}, - "description": "Optional list of labels to restrict to. Defaults to the anchor event's label.", - }, - "sub_labels": { - "type": "array", - "items": {"type": "string"}, - "description": "Optional list of sub_labels (names) to restrict to.", - }, - "zones": { - "type": "array", - "items": {"type": "string"}, - "description": "Optional list of zones. An event matches if any of its zones overlap.", - }, - "similarity_mode": { - "type": "string", - "enum": ["visual", "semantic", "fused"], - "description": "Which similarity signal(s) to use. 'fused' (default) combines visual and semantic.", - "default": "fused", - }, - "min_score": { - "type": "number", - "description": "Drop matches with a similarity score below this threshold (0.0-1.0).", - }, - "limit": { - "type": "integer", - "description": "Maximum number of matches to return (default: 10).", - "default": 10, - }, - }, - "required": ["event_id"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "set_camera_state", - "description": ( - "Change a camera's feature state (e.g., turn detection on/off, enable/disable recordings). " - "Use camera='*' to apply to all cameras at once. " - "Only call this tool when the user explicitly asks to change a camera setting. " - "Requires admin privileges." - ), - "parameters": { - "type": "object", - "properties": { - "camera": { - "type": "string", - "description": "Camera name to target, or '*' to target all cameras.", - }, - "feature": { - "type": "string", - "enum": [ - "detect", - "record", - "snapshots", - "audio", - "motion", - "enabled", - "birdseye", - "birdseye_mode", - "improve_contrast", - "ptz_autotracker", - "motion_contour_area", - "motion_threshold", - "notifications", - "audio_transcription", - "review_alerts", - "review_detections", - "object_descriptions", - "review_descriptions", - "profile", - ], - "description": ( - "The feature to change. Most features accept ON or OFF. " - "birdseye_mode accepts CONTINUOUS, MOTION, or OBJECTS. " - "motion_contour_area and motion_threshold accept a number. " - "profile accepts a profile name or 'none' to deactivate (requires camera='*')." - ), - }, - "value": { - "type": "string", - "description": "The value to set. ON or OFF for toggles, a number for thresholds, a profile name or 'none' for profile.", - }, - }, - "required": ["camera", "feature", "value"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_live_context", - "description": ( - "Get the current live image and detection information for a camera: objects being tracked, " - "zones, timestamps. Use this to understand what is visible in the live view. " - "Call this when answering questions about what is happening right now on a specific camera." - ), - "parameters": { - "type": "object", - "properties": { - "camera": { - "type": "string", - "description": "Camera name to get live context for.", - }, - }, - "required": ["camera"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "start_camera_watch", - "description": ( - "Start a continuous VLM watch job that monitors a camera and sends a notification " - "when a specified condition is met. Use this when the user wants to be alerted about " - "a future event, e.g. 'tell me when guests arrive' or 'notify me when the package is picked up'. " - "Only one watch job can run at a time. Returns a job ID." - ), - "parameters": { - "type": "object", - "properties": { - "camera": { - "type": "string", - "description": "Camera ID to monitor.", - }, - "condition": { - "type": "string", - "description": ( - "Natural-language description of the condition to watch for, " - "e.g. 'a person arrives at the front door'." - ), - }, - "max_duration_minutes": { - "type": "integer", - "description": "Maximum time to watch before giving up (minutes, default 60).", - "default": 60, - }, - "labels": { - "type": "array", - "items": {"type": "string"}, - "description": "Object labels that should trigger a VLM check (e.g. ['person', 'car']). If omitted, any detection on the camera triggers a check.", - }, - "zones": { - "type": "array", - "items": {"type": "string"}, - "description": "Zone names to filter by. If specified, only detections in these zones trigger a VLM check.", - }, - }, - "required": ["camera", "condition"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "stop_camera_watch", - "description": ( - "Cancel the currently running VLM watch job. Use this when the user wants to " - "stop a previously started watch, e.g. 'stop watching the front door'." - ), - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_profile_status", - "description": ( - "Get the current profile status including the active profile and " - "timestamps of when each profile was last activated. Use this to " - "determine time periods for recap requests — e.g. when the user asks " - "'what happened while I was away?', call this first to find the relevant " - "time window based on profile activation history." - ), - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_recap", - "description": ( - "Get a recap of all activity (alerts and detections) for a given time period. " - "Use this after calling get_profile_status to retrieve what happened during " - "a specific window — e.g. 'what happened while I was away?'. Returns a " - "chronological list of activity with camera, objects, zones, and GenAI-generated " - "descriptions when available. Summarize the results for the user." - ), - "parameters": { - "type": "object", - "properties": { - "after": { - "type": "string", - "description": "Start of the time period in ISO 8601 format (e.g. '2025-03-15T08:00:00').", - }, - "before": { - "type": "string", - "description": "End of the time period in ISO 8601 format (e.g. '2025-03-15T17:00:00').", - }, - "cameras": { - "type": "string", - "description": "Comma-separated camera IDs to include, or 'all' for all cameras. Default is 'all'.", - }, - "severity": { - "type": "string", - "enum": ["alert", "detection"], - "description": "Filter by severity level. Omit to include both alerts and detections.", - }, - }, - "required": ["after", "before"], - }, - }, - }, - ] - - @router.get( "/chat/tools", dependencies=[Depends(allow_any_authenticated())], @@ -1550,81 +1119,12 @@ async def chat_completion( ) conversation = [] - current_datetime = datetime.now() - current_date_str = current_datetime.strftime("%Y-%m-%d") - current_time_str = current_datetime.strftime("%I:%M:%S %p") - - cameras_info = [] - has_speed_zone = False - for camera_id in allowed_cameras: - if camera_id not in config.cameras: - continue - camera_config = config.cameras[camera_id] - friendly_name = ( - camera_config.friendly_name - if camera_config.friendly_name - else camera_id.replace("_", " ").title() - ) - zone_names = list(camera_config.zones.keys()) - if not has_speed_zone: - has_speed_zone = any( - zone.distances for zone in camera_config.zones.values() - ) - if zone_names: - cameras_info.append( - f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})" - ) - else: - cameras_info.append(f" - {friendly_name} (ID: {camera_id})") - - cameras_section = "" - if cameras_info: - cameras_section = ( - "\n\nAvailable cameras:\n" - + "\n".join(cameras_info) - + "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls." - ) - - speed_units_section = "" - if has_speed_zone: - speed_unit = ( - "mph" if config.ui.unit_system == UnitSystemEnum.imperial else "km/h" - ) - speed_units_section = f"\n\nReport object speeds to the user in {speed_unit}." - - semantic_search_section = "" - if semantic_search_enabled: - semantic_search_section = ( - "\n\nWhen routing a search_objects call, pick filters by the shape of the user's request:\n" - "- Generic class ('show me all cars today'): set `label` only.\n" - "- Specific named entity — a known person ('John'), delivery company ('Amazon'), animal species/breed ('blue jay', 'cardinal', 'golden retriever'), or license plate: set `sub_label` only and leave `label` unset.\n" - "- Physical characteristic, appearance, or activity that is NOT a discrete name ('find me people riding a lawn mower', 'someone in a red jacket', 'a person carrying a package'): set `semantic_query` with the descriptive phrase, optionally combined with `label` for the object class. Never put descriptive phrases in `sub_label`." - ) - - attribute_classification_section = "" - if attribute_classifications: - model_lines = "\n".join( - f"- {m['name']}: applies to {', '.join(m['objects']) or 'any object'}" - for m in attribute_classifications - ) - attribute_classification_section = ( - "\n\nAttribute classification models are configured for the following object types:\n" - f"{model_lines}\n" - "When the user's request matches one of these classifications, set the search_objects `attribute` field to the matching label rather than using `semantic_query`. Reserve `semantic_query` for descriptive phrases that fall outside the configured attribute labels." - ) - - system_prompt = f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events. - -Current server local date and time: {current_date_str} at {current_time_str} - -Do not start your response with phrases like "I will check...", "Let me see...", or "Let me look...". Answer directly. - -Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields. -When users ask about "today", "yesterday", "this week", etc., use the current date above as reference. -When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today). -Always be accurate with time calculations based on the current date provided. - -When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{semantic_search_section}{attribute_classification_section}{cameras_section}{speed_units_section}""" + system_prompt = build_chat_system_prompt( + config=config, + allowed_cameras=allowed_cameras, + semantic_search_enabled=semantic_search_enabled, + attribute_classifications=attribute_classifications, + ) conversation.append( { diff --git a/frigate/genai/prompts.py b/frigate/genai/prompts.py index 58c668d4d5..9ff81cdebd 100644 --- a/frigate/genai/prompts.py +++ b/frigate/genai/prompts.py @@ -6,11 +6,13 @@ transport. """ import datetime -from typing import Any +from typing import Any, Dict, List, Optional from playhouse.shortcuts import model_to_dict -from frigate.config import CameraConfig +from frigate.config import CameraConfig, FrigateConfig +from frigate.config.classification import ObjectClassificationType +from frigate.config.ui import UnitSystemEnum from frigate.data_processing.post.types import ReviewMetadata from frigate.models import Event @@ -212,3 +214,526 @@ def build_object_description_prompt( camera_config.objects.genai.prompt, ) return template.format(**model_to_dict(event)) + + +def get_attribute_classifications(config: FrigateConfig) -> List[Dict[str, Any]]: + """Return enabled custom classification models of `attribute` type. + + Each entry: {"name": , "objects": [, ...]}. + These models attach attribute metadata to events on the listed object + types, which can later be filtered via the search_objects `attribute` + field. + """ + result: List[Dict[str, Any]] = [] + + for model_key, model_config in config.classification.custom.items(): + if not model_config.enabled or model_config.object_config is None: + continue + + if ( + model_config.object_config.classification_type + != ObjectClassificationType.attribute + ): + continue + + result.append( + { + "name": model_config.name or model_key, + "objects": list(model_config.object_config.objects or []), + } + ) + + return result + + +def get_tool_definitions( + semantic_search_enabled: bool = False, + attribute_classifications: Optional[List[Dict[str, Any]]] = None, +) -> List[Dict[str, Any]]: + """ + Get OpenAI-compatible tool definitions for Frigate. + + Returns a list of tool definitions that can be used with OpenAI-compatible + function calling APIs. When semantic search is enabled, the search_objects + tool exposes an additional `semantic_query` parameter for descriptive + queries (e.g. "person riding a lawn mower") and find_similar_objects is + included. When attribute classification models are configured, an + `attribute` parameter is exposed for filtering by their labels. + """ + search_objects_properties: Dict[str, Any] = { + "camera": { + "type": "string", + "description": "Camera name to filter by (optional).", + }, + "label": { + "type": "string", + "description": ( + "Generic object class to filter by — one of the tracked detector " + "labels such as 'person', 'package', 'car', 'dog', 'bird'. Use " + "this for broad queries like 'show me all cars today'. Combine " + "with semantic_query when the user also describes appearance or " + "behavior (e.g. label='person', semantic_query='riding a lawn " + "mower')." + ), + }, + "sub_label": { + "type": "string", + "description": ( + "Filter by a DISCRETE NAMED entity recognized in the detection. " + "Use this for: a known person's name ('John'), a delivery " + "company ('Amazon', 'UPS'), a recognized animal species or " + "breed ('blue jay', 'cardinal', 'golden retriever'), or a " + "license plate string. When filtering by a specific name, set " + "only sub_label and leave label unset. Do NOT use sub_label " + "for descriptions of appearance, clothing, or actions — those " + "belong in semantic_query." + ), + }, + "after": { + "type": "string", + "description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').", + }, + "before": { + "type": "string", + "description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').", + }, + "zones": { + "type": "array", + "items": {"type": "string"}, + "description": "List of zone names to filter by.", + }, + "limit": { + "type": "integer", + "description": "Maximum number of objects to return (default: 25).", + "default": 25, + }, + } + + if attribute_classifications: + model_outline = "; ".join( + f"{m['name']} (applies to {', '.join(m['objects']) or 'any object'})" + for m in attribute_classifications + ) + search_objects_properties["attribute"] = { + "type": "string", + "description": ( + "Filter by a classification attribute label produced by a " + "configured attribute classification model. Use this INSTEAD " + "of semantic_query when the user's request matches one of " + "these classifications. Configured models: " + f"{model_outline}. " + "Set the value to the attribute label that matches the user's " + "phrasing (case-sensitive)." + ), + } + + if semantic_search_enabled: + search_objects_properties["semantic_query"] = { + "type": "string", + "description": ( + "Optional natural-language description of a PHYSICAL " + "CHARACTERISTIC, APPEARANCE, or ACTIVITY the user mentioned, " + "used to semantically narrow results. Only set this when the " + "user describes something beyond what label and sub_label can " + "express on their own.\n" + "USE for descriptive phrases like: 'riding a lawn mower', " + "'wearing a red jacket', 'carrying a package', 'walking a " + "dog', 'on a bicycle', 'holding an umbrella'.\n" + "DO NOT USE for:\n" + "- specific named people, pets, or delivery companies → use sub_label\n" + "- animal species or breed names like 'blue jay', 'cardinal', " + "'golden retriever' → use sub_label\n" + "- license plate strings → use sub_label\n" + "- generic object queries like 'all cars today' or 'every " + "person' → use label alone with no semantic_query\n" + "When set, combine with label/time/camera/zone filters as " + "usual (e.g. label='person', semantic_query='riding a lawn " + "mower', after='2024-05-01T00:00:00Z')." + ), + } + + search_objects_description = ( + "Search the historical record of detected objects in Frigate. " + "Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', " + "'when was the last car?', 'show me detections from yesterday'. " + "Do NOT use this for monitoring or alerting requests about future events — " + "use start_camera_watch instead for those. " + "An 'object' in Frigate represents a tracked detection (e.g., a person, package, car).\n\n" + "Choose filters based on what the user is asking for:\n" + "- Generic class query ('show me all cars today'): set `label` only.\n" + "- Specific NAMED entity (known person, delivery company, animal " + "species/breed like 'blue jay' or 'golden retriever', license " + "plate): set `sub_label` only and leave `label` unset.\n" + ) + if semantic_search_enabled: + search_objects_description += ( + "- Physical CHARACTERISTIC, APPEARANCE, or ACTIVITY that is not a " + "discrete name ('person riding a lawn mower', 'someone in a red " + "jacket', 'person carrying a package'): set `semantic_query` with " + "the descriptive phrase, optionally alongside `label` for the " + "object class. Do NOT put descriptive phrases in sub_label." + ) + + return [ + { + "type": "function", + "function": { + "name": "search_objects", + "description": search_objects_description, + "parameters": { + "type": "object", + "properties": search_objects_properties, + }, + "required": [], + }, + }, + { + "type": "function", + "function": { + "name": "find_similar_objects", + "description": ( + "Find tracked objects that are visually and semantically similar " + "to a specific past event. Use this when the user references a " + "particular object they have seen and wants to find other " + "sightings of the same or similar one ('that green car', 'the " + "person in the red jacket', 'the package that was delivered'). " + "Prefer this over search_objects whenever the user's intent is " + "'find more like this specific one.' Use search_objects first " + "only if you need to locate the anchor event. Requires semantic " + "search to be enabled." + ), + "parameters": { + "type": "object", + "properties": { + "event_id": { + "type": "string", + "description": "The id of the anchor event to find similar objects to.", + }, + "after": { + "type": "string", + "description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').", + }, + "before": { + "type": "string", + "description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').", + }, + "cameras": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional list of cameras to restrict to. Defaults to all.", + }, + "labels": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional list of labels to restrict to. Defaults to the anchor event's label.", + }, + "sub_labels": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional list of sub_labels (names) to restrict to.", + }, + "zones": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional list of zones. An event matches if any of its zones overlap.", + }, + "similarity_mode": { + "type": "string", + "enum": ["visual", "semantic", "fused"], + "description": "Which similarity signal(s) to use. 'fused' (default) combines visual and semantic.", + "default": "fused", + }, + "min_score": { + "type": "number", + "description": "Drop matches with a similarity score below this threshold (0.0-1.0).", + }, + "limit": { + "type": "integer", + "description": "Maximum number of matches to return (default: 10).", + "default": 10, + }, + }, + "required": ["event_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "set_camera_state", + "description": ( + "Change a camera's feature state (e.g., turn detection on/off, enable/disable recordings). " + "Use camera='*' to apply to all cameras at once. " + "Only call this tool when the user explicitly asks to change a camera setting. " + "Requires admin privileges." + ), + "parameters": { + "type": "object", + "properties": { + "camera": { + "type": "string", + "description": "Camera name to target, or '*' to target all cameras.", + }, + "feature": { + "type": "string", + "enum": [ + "detect", + "record", + "snapshots", + "audio", + "motion", + "enabled", + "birdseye", + "birdseye_mode", + "improve_contrast", + "ptz_autotracker", + "motion_contour_area", + "motion_threshold", + "notifications", + "audio_transcription", + "review_alerts", + "review_detections", + "object_descriptions", + "review_descriptions", + "profile", + ], + "description": ( + "The feature to change. Most features accept ON or OFF. " + "birdseye_mode accepts CONTINUOUS, MOTION, or OBJECTS. " + "motion_contour_area and motion_threshold accept a number. " + "profile accepts a profile name or 'none' to deactivate (requires camera='*')." + ), + }, + "value": { + "type": "string", + "description": "The value to set. ON or OFF for toggles, a number for thresholds, a profile name or 'none' for profile.", + }, + }, + "required": ["camera", "feature", "value"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_live_context", + "description": ( + "Get the current live image and detection information for a camera: objects being tracked, " + "zones, timestamps. Use this to understand what is visible in the live view. " + "Call this when answering questions about what is happening right now on a specific camera." + ), + "parameters": { + "type": "object", + "properties": { + "camera": { + "type": "string", + "description": "Camera name to get live context for.", + }, + }, + "required": ["camera"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "start_camera_watch", + "description": ( + "Start a continuous VLM watch job that monitors a camera and sends a notification " + "when a specified condition is met. Use this when the user wants to be alerted about " + "a future event, e.g. 'tell me when guests arrive' or 'notify me when the package is picked up'. " + "Only one watch job can run at a time. Returns a job ID." + ), + "parameters": { + "type": "object", + "properties": { + "camera": { + "type": "string", + "description": "Camera ID to monitor.", + }, + "condition": { + "type": "string", + "description": ( + "Natural-language description of the condition to watch for, " + "e.g. 'a person arrives at the front door'." + ), + }, + "max_duration_minutes": { + "type": "integer", + "description": "Maximum time to watch before giving up (minutes, default 60).", + "default": 60, + }, + "labels": { + "type": "array", + "items": {"type": "string"}, + "description": "Object labels that should trigger a VLM check (e.g. ['person', 'car']). If omitted, any detection on the camera triggers a check.", + }, + "zones": { + "type": "array", + "items": {"type": "string"}, + "description": "Zone names to filter by. If specified, only detections in these zones trigger a VLM check.", + }, + }, + "required": ["camera", "condition"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "stop_camera_watch", + "description": ( + "Cancel the currently running VLM watch job. Use this when the user wants to " + "stop a previously started watch, e.g. 'stop watching the front door'." + ), + "parameters": { + "type": "object", + "properties": {}, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_profile_status", + "description": ( + "Get the current profile status including the active profile and " + "timestamps of when each profile was last activated. Use this to " + "determine time periods for recap requests — e.g. when the user asks " + "'what happened while I was away?', call this first to find the relevant " + "time window based on profile activation history." + ), + "parameters": { + "type": "object", + "properties": {}, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_recap", + "description": ( + "Get a recap of all activity (alerts and detections) for a given time period. " + "Use this after calling get_profile_status to retrieve what happened during " + "a specific window — e.g. 'what happened while I was away?'. Returns a " + "chronological list of activity with camera, objects, zones, and GenAI-generated " + "descriptions when available. Summarize the results for the user." + ), + "parameters": { + "type": "object", + "properties": { + "after": { + "type": "string", + "description": "Start of the time period in ISO 8601 format (e.g. '2025-03-15T08:00:00').", + }, + "before": { + "type": "string", + "description": "End of the time period in ISO 8601 format (e.g. '2025-03-15T17:00:00').", + }, + "cameras": { + "type": "string", + "description": "Comma-separated camera IDs to include, or 'all' for all cameras. Default is 'all'.", + }, + "severity": { + "type": "string", + "enum": ["alert", "detection"], + "description": "Filter by severity level. Omit to include both alerts and detections.", + }, + }, + "required": ["after", "before"], + }, + }, + }, + ] + + +def build_chat_system_prompt( + config: FrigateConfig, + allowed_cameras: List[str], + semantic_search_enabled: bool, + attribute_classifications: List[Dict[str, Any]], +) -> str: + """Build the system prompt for the chat completion endpoint. + + Composes the static framing with conditional sections describing the + available cameras, speed units, semantic-search routing guidance, and + configured attribute classifications. + """ + current_datetime = datetime.datetime.now() + current_date_str = current_datetime.strftime("%Y-%m-%d") + current_time_str = current_datetime.strftime("%I:%M:%S %p") + + cameras_info: List[str] = [] + has_speed_zone = False + for camera_id in allowed_cameras: + if camera_id not in config.cameras: + continue + camera_config = config.cameras[camera_id] + friendly_name = ( + camera_config.friendly_name + if camera_config.friendly_name + else camera_id.replace("_", " ").title() + ) + zone_names = list(camera_config.zones.keys()) + if not has_speed_zone: + has_speed_zone = any( + zone.distances for zone in camera_config.zones.values() + ) + if zone_names: + cameras_info.append( + f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})" + ) + else: + cameras_info.append(f" - {friendly_name} (ID: {camera_id})") + + cameras_section = "" + if cameras_info: + cameras_section = ( + "\n\nAvailable cameras:\n" + + "\n".join(cameras_info) + + "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls." + ) + + speed_units_section = "" + if has_speed_zone: + speed_unit = ( + "mph" if config.ui.unit_system == UnitSystemEnum.imperial else "km/h" + ) + speed_units_section = f"\n\nReport object speeds to the user in {speed_unit}." + + semantic_search_section = "" + if semantic_search_enabled: + semantic_search_section = ( + "\n\nWhen routing a search_objects call, pick filters by the shape of the user's request:\n" + "- Generic class ('show me all cars today'): set `label` only.\n" + "- Specific named entity — a known person ('John'), delivery company ('Amazon'), animal species/breed ('blue jay', 'cardinal', 'golden retriever'), or license plate: set `sub_label` only and leave `label` unset.\n" + "- Physical characteristic, appearance, or activity that is NOT a discrete name ('find me people riding a lawn mower', 'someone in a red jacket', 'a person carrying a package'): set `semantic_query` with the descriptive phrase, optionally combined with `label` for the object class. Never put descriptive phrases in `sub_label`." + ) + + attribute_classification_section = "" + if attribute_classifications: + model_lines = "\n".join( + f"- {m['name']}: applies to {', '.join(m['objects']) or 'any object'}" + for m in attribute_classifications + ) + attribute_classification_section = ( + "\n\nAttribute classification models are configured for the following object types:\n" + f"{model_lines}\n" + "When the user's request matches one of these classifications, set the search_objects `attribute` field to the matching label rather than using `semantic_query`. Reserve `semantic_query` for descriptive phrases that fall outside the configured attribute labels." + ) + + return f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events. + +Current server local date and time: {current_date_str} at {current_time_str} + +Do not start your response with phrases like "I will check...", "Let me see...", or "Let me look...". Answer directly. + +Always present times to the user in the server's local timezone. When tool results include start_time_local and end_time_local, use those exact strings when listing or describing detection times—do not convert or invent timestamps. Do not use UTC or ISO format with Z for the user-facing answer unless the tool result only provides Unix timestamps without local time fields. +When users ask about "today", "yesterday", "this week", etc., use the current date above as reference. +When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today). +Always be accurate with time calculations based on the current date provided. + +When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{semantic_search_section}{attribute_classification_section}{cameras_section}{speed_units_section}"""