Miscellaneous fixes (0.18 beta) (#23716)

* resolve zone friendly names against the correct camera

* Improve handling of zone names in chat prompt

* show a numeric keyboard for numeric config form fields on mobile

* Specify english only for semantic search tool when model is JinaV1

* resolve export hwaccel args global value against the correct config path

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

frigate/api/chat.py

@@ -7,7 +7,7 @@ import operator
 import time
 from datetime import datetime
 from functools import reduce
-from typing import Any
+from typing import Any, Literal
 
 import cv2
 from fastapi import APIRouter, Body, Depends, HTTPException, Request
@@ -37,6 +37,7 @@ from frigate.api.defs.response.chat_response import (
 from frigate.api.defs.tags import Tags
 from frigate.api.event import _build_attribute_filter_clause, events
 from frigate.config import FrigateConfig
+from frigate.config.classification import SemanticSearchModelEnum
 from frigate.genai.prompts import (
     build_chat_system_prompt,
     get_attribute_classifications,
@@ -86,10 +87,23 @@ def get_tools(request: Request) -> JSONResponse:
     tools = get_tool_definitions(
         semantic_search_enabled=semantic_search_enabled,
         attribute_classifications=attribute_classifications,
+        embeddings_language=_embeddings_language(config),
     )
     return JSONResponse(content={"tools": tools})
 
 
+def _embeddings_language(config: FrigateConfig) -> Literal["english", "multi"]:
+    """Return the language capability of the configured embeddings model.
+
+    JinaV1 is English-only; every other option (JinaV2 or a GenAI embeddings
+    provider) handles multiple languages.
+    """
+    if config.semantic_search.model == SemanticSearchModelEnum.jinav1:
+        return "english"
+
+    return "multi"
+
+
 def _resolve_zones(
     zones: list[str],
     config: FrigateConfig,
@@ -98,11 +112,14 @@ def _resolve_zones(
     """Map zone names to their canonical config keys, case-insensitively.
 
     LLMs frequently echo a user's casing ("Front Yard") instead of the
-    configured key ("front_yard"). The downstream zone filter is a SQLite GLOB
-    over the JSON-encoded zones column, which is case-sensitive — so an
-    unnormalized name silently returns zero matches. Build a lookup over the
-    relevant cameras' configured zones and substitute when we find a match;
-    unknown names pass through so behavior matches what the model asked for.
+    configured key ("front_yard"), or fall back to a zone's friendly name
+    ("Front Walkway") instead of its ID ("front_walk"). The downstream zone
+    filter is a SQLite GLOB over the JSON-encoded zones column, which stores
+    config keys and is case-sensitive — so an unnormalized name silently
+    returns zero matches. Build a lookup over the relevant cameras' configured
+    zones, keyed by both the config key and the friendly name, and substitute
+    when we find a match; unknown names pass through so behavior matches what
+    the model asked for.
     """
     if not zones:
         return zones
@@ -112,8 +129,11 @@ def _resolve_zones(
         camera_config = config.cameras.get(camera_id)
         if camera_config is None:
             continue
-        for zone_name in camera_config.zones.keys():
+        for zone_name, zone_config in camera_config.zones.items():
             lookup.setdefault(zone_name.lower(), zone_name)
+            lookup.setdefault(
+                zone_config.get_formatted_name(zone_name).lower(), zone_name
+            )
 
     return [lookup.get(z.lower(), z) for z in zones]
 
@@ -1134,6 +1154,7 @@ async def chat_completion(
     tools = get_tool_definitions(
         semantic_search_enabled=semantic_search_enabled,
         attribute_classifications=attribute_classifications,
+        embeddings_language=_embeddings_language(config),
     )
     conversation = []
 

frigate/genai/prompts.py

@@ -6,7 +6,7 @@ transport.
 """
 
 import datetime
-from typing import Any
+from typing import Any, Literal
 
 from playhouse.shortcuts import model_to_dict
 
@@ -249,6 +249,7 @@ def get_attribute_classifications(config: FrigateConfig) -> list[dict[str, Any]]
 def get_tool_definitions(
     semantic_search_enabled: bool = False,
     attribute_classifications: list[dict[str, Any]] | None = None,
+    embeddings_language: Literal["english", "multi"] = "multi",
 ) -> list[dict[str, Any]]:
     """
     Get OpenAI-compatible tool definitions for Frigate.
@@ -258,7 +259,9 @@ def get_tool_definitions(
     tool exposes an additional `semantic_query` parameter for descriptive
     queries (e.g. "person riding a lawn mower") and find_similar_objects is
     included. When attribute classification models are configured, an
-    `attribute` parameter is exposed for filtering by their labels.
+    `attribute` parameter is exposed for filtering by their labels. When the
+    embeddings model only understands English (JinaV1), the `semantic_query`
+    description instructs the model to write the query in English.
     """
     search_objects_properties: dict[str, Any] = {
         "camera": {
@@ -349,6 +352,14 @@ def get_tool_definitions(
                 "When set, combine with label/time/camera/zone filters as "
                 "usual (e.g. label='person', semantic_query='riding a lawn "
                 "mower', after='2024-05-01T00:00:00Z')."
+                + (
+                    " The configured embeddings model only understands "
+                    "English, so always write semantic_query in English, "
+                    "translating the user's description if they phrased it "
+                    "in another language."
+                    if embeddings_language == "english"
+                    else ""
+                )
             ),
         }
 
@@ -682,14 +693,17 @@ def build_chat_system_prompt(
             if camera_config.friendly_name
             else camera_id.replace("_", " ").title()
         )
-        zone_names = list(camera_config.zones.keys())
+        zone_descriptors = [
+            f"{zone_config.get_formatted_name(zone_name)} (ID: {zone_name})"
+            for zone_name, zone_config in camera_config.zones.items()
+        ]
         if not has_speed_zone:
             has_speed_zone = any(
                 zone.distances for zone in camera_config.zones.values()
             )
-        if zone_names:
+        if zone_descriptors:
             cameras_info.append(
-                f"  - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})"
+                f"  - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_descriptors)})"
             )
         else:
             cameras_info.append(f"  - {friendly_name} (ID: {camera_id})")
@@ -699,7 +713,7 @@ def build_chat_system_prompt(
         cameras_section = (
             "\n\nAvailable cameras:\n"
             + "\n".join(cameras_info)
-            + "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls."
+            + "\n\nWhen users refer to cameras or zones by their friendly name (e.g., 'Back Deck Camera', 'Front Walkway'), use the corresponding ID (e.g., 'back_deck_cam', 'front_walk') in tool calls. Tool results also identify zones by their ID, so when presenting cameras or zones back to the user, translate the ID to its friendly name."
         )
 
     speed_units_section = ""