diff --git a/frigate/api/chat.py b/frigate/api/chat.py index 0543d5f8a6..939512dd52 100644 --- a/frigate/api/chat.py +++ b/frigate/api/chat.py @@ -36,6 +36,7 @@ from frigate.api.defs.response.chat_response import ( ) from frigate.api.defs.tags import Tags from frigate.api.event import events +from frigate.config import FrigateConfig from frigate.genai.utils import build_assistant_message_for_conversation from frigate.jobs.vlm_watch import ( get_vlm_watch_job, @@ -401,9 +402,38 @@ def get_tools() -> JSONResponse: return JSONResponse(content={"tools": tools}) +def _resolve_zones( + zones: List[str], + config: FrigateConfig, + target_cameras: List[str], +) -> List[str]: + """Map zone names to their canonical config keys, case-insensitively. + + LLMs frequently echo a user's casing ("Front Yard") instead of the + configured key ("front_yard"). The downstream zone filter is a SQLite GLOB + over the JSON-encoded zones column, which is case-sensitive — so an + unnormalized name silently returns zero matches. Build a lookup over the + relevant cameras' configured zones and substitute when we find a match; + unknown names pass through so behavior matches what the model asked for. + """ + if not zones: + return zones + + lookup: Dict[str, str] = {} + for camera_id in target_cameras: + camera_config = config.cameras.get(camera_id) + if camera_config is None: + continue + for zone_name in camera_config.zones.keys(): + lookup.setdefault(zone_name.lower(), zone_name) + + return [lookup.get(z.lower(), z) for z in zones] + + async def _execute_search_objects( arguments: Dict[str, Any], allowed_cameras: List[str], + config: FrigateConfig, ) -> JSONResponse: """ Execute the search_objects tool. @@ -437,6 +467,11 @@ async def _execute_search_objects( # Convert zones array to comma-separated string if provided zones = arguments.get("zones") if isinstance(zones, list): + camera_arg = arguments.get("camera") + target_cameras = ( + [camera_arg] if camera_arg and camera_arg != "all" else allowed_cameras + ) + zones = _resolve_zones(zones, config, target_cameras) zones = ",".join(zones) elif zones is None: zones = "all" @@ -528,6 +563,11 @@ async def _execute_find_similar_objects( sub_labels = arguments.get("sub_labels") zones = arguments.get("zones") + if zones: + zones = _resolve_zones( + zones, request.app.frigate_config, cameras or list(allowed_cameras) + ) + similarity_mode = arguments.get("similarity_mode", "fused") if similarity_mode not in ("visual", "semantic", "fused"): similarity_mode = "fused" @@ -655,7 +695,9 @@ async def execute_tool( logger.debug(f"Executing tool: {tool_name} with arguments: {arguments}") if tool_name == "search_objects": - return await _execute_search_objects(arguments, allowed_cameras) + return await _execute_search_objects( + arguments, allowed_cameras, request.app.frigate_config + ) if tool_name == "find_similar_objects": result = await _execute_find_similar_objects( @@ -835,7 +877,9 @@ async def _execute_tool_internal( This is used by the chat completion endpoint to execute tools. """ if tool_name == "search_objects": - response = await _execute_search_objects(arguments, allowed_cameras) + response = await _execute_search_objects( + arguments, allowed_cameras, request.app.frigate_config + ) try: if hasattr(response, "body"): body_str = response.body.decode("utf-8") @@ -899,6 +943,9 @@ async def _execute_start_camera_watch( await require_camera_access(camera, request=request) + if zones: + zones = _resolve_zones(zones, config, [camera]) + genai_manager = request.app.genai_manager chat_client = genai_manager.chat_client if chat_client is None or not chat_client.supports_vision: