Miscellaneous fixes (0.18 beta) (#23716)

* resolve zone friendly names against the correct camera

* Improve handling of zone names in chat prompt

* show a numeric keyboard for numeric config form fields on mobile

* Specify english only for semantic search tool when model is JinaV1

* resolve export hwaccel args global value against the correct config path

---------

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

frigate/api/chat.py

@@ -7,7 +7,7 @@ import operator
 import time
 from datetime import datetime
 from functools import reduce
-from typing import Any
+from typing import Any, Literal
 
 import cv2
 from fastapi import APIRouter, Body, Depends, HTTPException, Request
@@ -37,6 +37,7 @@ from frigate.api.defs.response.chat_response import (
 from frigate.api.defs.tags import Tags
 from frigate.api.event import _build_attribute_filter_clause, events
 from frigate.config import FrigateConfig
+from frigate.config.classification import SemanticSearchModelEnum
 from frigate.genai.prompts import (
     build_chat_system_prompt,
     get_attribute_classifications,
@@ -86,10 +87,23 @@ def get_tools(request: Request) -> JSONResponse:
     tools = get_tool_definitions(
         semantic_search_enabled=semantic_search_enabled,
         attribute_classifications=attribute_classifications,
+        embeddings_language=_embeddings_language(config),
     )
     return JSONResponse(content={"tools": tools})
 
 
+def _embeddings_language(config: FrigateConfig) -> Literal["english", "multi"]:
+    """Return the language capability of the configured embeddings model.
+
+    JinaV1 is English-only; every other option (JinaV2 or a GenAI embeddings
+    provider) handles multiple languages.
+    """
+    if config.semantic_search.model == SemanticSearchModelEnum.jinav1:
+        return "english"
+
+    return "multi"
+
+
 def _resolve_zones(
     zones: list[str],
     config: FrigateConfig,
@@ -98,11 +112,14 @@ def _resolve_zones(
     """Map zone names to their canonical config keys, case-insensitively.
 
     LLMs frequently echo a user's casing ("Front Yard") instead of the
-    configured key ("front_yard"). The downstream zone filter is a SQLite GLOB
+    configured key ("front_yard"), or fall back to a zone's friendly name
-    over the JSON-encoded zones column, which is case-sensitive — so an
+    ("Front Walkway") instead of its ID ("front_walk"). The downstream zone
-    unnormalized name silently returns zero matches. Build a lookup over the
+    filter is a SQLite GLOB over the JSON-encoded zones column, which stores
-    relevant cameras' configured zones and substitute when we find a match;
+    config keys and is case-sensitive — so an unnormalized name silently
-    unknown names pass through so behavior matches what the model asked for.
+    returns zero matches. Build a lookup over the relevant cameras' configured
+    zones, keyed by both the config key and the friendly name, and substitute
+    when we find a match; unknown names pass through so behavior matches what
+    the model asked for.
     """
     if not zones:
         return zones
@@ -112,8 +129,11 @@ def _resolve_zones(
         camera_config = config.cameras.get(camera_id)
         if camera_config is None:
             continue
-        for zone_name in camera_config.zones.keys():
+        for zone_name, zone_config in camera_config.zones.items():
             lookup.setdefault(zone_name.lower(), zone_name)
+            lookup.setdefault(
+                zone_config.get_formatted_name(zone_name).lower(), zone_name
+            )
 
     return [lookup.get(z.lower(), z) for z in zones]
 
@@ -1134,6 +1154,7 @@ async def chat_completion(
     tools = get_tool_definitions(
         semantic_search_enabled=semantic_search_enabled,
         attribute_classifications=attribute_classifications,
+        embeddings_language=_embeddings_language(config),
     )
     conversation = []
 

frigate/genai/prompts.py

@@ -6,7 +6,7 @@ transport.
 """
 
 import datetime
-from typing import Any
+from typing import Any, Literal
 
 from playhouse.shortcuts import model_to_dict
 
@@ -249,6 +249,7 @@ def get_attribute_classifications(config: FrigateConfig) -> list[dict[str, Any]]
 def get_tool_definitions(
     semantic_search_enabled: bool = False,
     attribute_classifications: list[dict[str, Any]] | None = None,
+    embeddings_language: Literal["english", "multi"] = "multi",
 ) -> list[dict[str, Any]]:
     """
     Get OpenAI-compatible tool definitions for Frigate.
@@ -258,7 +259,9 @@ def get_tool_definitions(
     tool exposes an additional `semantic_query` parameter for descriptive
     queries (e.g. "person riding a lawn mower") and find_similar_objects is
     included. When attribute classification models are configured, an
-    `attribute` parameter is exposed for filtering by their labels.
+    `attribute` parameter is exposed for filtering by their labels. When the
+    embeddings model only understands English (JinaV1), the `semantic_query`
+    description instructs the model to write the query in English.
     """
     search_objects_properties: dict[str, Any] = {
         "camera": {
@@ -349,6 +352,14 @@ def get_tool_definitions(
                 "When set, combine with label/time/camera/zone filters as "
                 "usual (e.g. label='person', semantic_query='riding a lawn "
                 "mower', after='2024-05-01T00:00:00Z')."
+                + (
+                    " The configured embeddings model only understands "
+                    "English, so always write semantic_query in English, "
+                    "translating the user's description if they phrased it "
+                    "in another language."
+                    if embeddings_language == "english"
+                    else ""
+                )
             ),
         }
 
@@ -682,14 +693,17 @@ def build_chat_system_prompt(
             if camera_config.friendly_name
             else camera_id.replace("_", " ").title()
         )
-        zone_names = list(camera_config.zones.keys())
+        zone_descriptors = [
+            f"{zone_config.get_formatted_name(zone_name)} (ID: {zone_name})"
+            for zone_name, zone_config in camera_config.zones.items()
+        ]
         if not has_speed_zone:
             has_speed_zone = any(
                 zone.distances for zone in camera_config.zones.values()
             )
-        if zone_names:
+        if zone_descriptors:
             cameras_info.append(
-                f"  - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})"
+                f"  - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_descriptors)})"
             )
         else:
             cameras_info.append(f"  - {friendly_name} (ID: {camera_id})")
@@ -699,7 +713,7 @@ def build_chat_system_prompt(
         cameras_section = (
             "\n\nAvailable cameras:\n"
             + "\n".join(cameras_info)
-            + "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls."
+            + "\n\nWhen users refer to cameras or zones by their friendly name (e.g., 'Back Deck Camera', 'Front Walkway'), use the corresponding ID (e.g., 'back_deck_cam', 'front_walk') in tool calls. Tool results also identify zones by their ID, so when presenting cameras or zones back to the user, translate the ID to its friendly name."
         )
 
     speed_units_section = ""

web/src/components/config-form/section-configs/detect.ts

@@ -169,6 +169,13 @@ const detect: SectionConfigOverrides = {
       resolution: ["width", "height", "fps"],
       tracking: ["min_initialized", "max_disappeared"],
     },
+    uiSchema: {
+      annotation_offset: {
+        "ui:options": {
+          signed: true,
+        },
+      },
+    },
     hiddenFields: ["enabled_in_config"],
     advancedFields: [
       "min_initialized",

web/src/components/config-form/section-configs/record.ts

@@ -57,6 +57,7 @@ const record: SectionConfigOverrides = {
           "ui:options": {
             suppressMultiSchema: true,
             ffmpegPresetField: "hwaccel_args",
+            ffmpegGlobalFieldPath: "export.hwaccel_args",
           },
         },
       },

web/src/components/config-form/theme/utils/index.ts

@@ -17,3 +17,4 @@ export {
   isSubtreeModified,
 } from "./overrides";
 export { getSizedFieldClassName } from "./fieldSizing";
+export { getNumericInputMode } from "./inputMode";

web/src/components/config-form/theme/utils/inputMode.ts

@@ -0,0 +1,51 @@
+import type { RJSFSchema } from "@rjsf/utils";
+
+type NumericInputOptions = {
+  signed?: boolean;
+};
+
+/**
+ * Derive the on-screen keyboard hint for a schema field.
+ *
+ * Numeric config fields render as text inputs because RJSF's NumberField
+ * relies on the widget echoing raw strings back, so that trailing "." and "0"
+ * characters survive while a value is being typed. That means the numeric
+ * keypad has to be requested explicitly. Desktop browsers ignore inputMode, so
+ * this only affects virtual keyboards.
+ *
+ * Fields accepting negative values opt out, since the iOS numeric and decimal
+ * keypads have no minus key. Most numeric fields declare no minimum even
+ * though they are non-negative, so signed fields are marked explicitly with
+ * ui:options.signed.
+ *
+ * Args:
+ *     schema: The JSON schema for the field being rendered
+ *     options: The resolved ui:options for the field
+ *
+ * Returns:
+ *     The inputMode to apply, or undefined to leave the keyboard alone
+ */
+export function getNumericInputMode(
+  schema: RJSFSchema,
+  options: unknown,
+): "numeric" | "decimal" | undefined {
+  const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+  const isInteger = types.includes("integer");
+
+  if (!isInteger && !types.includes("number")) {
+    return undefined;
+  }
+
+  const numericOptions =
+    typeof options === "object" && options !== null
+      ? (options as NumericInputOptions)
+      : undefined;
+
+  const minimum = schema.minimum ?? schema.exclusiveMinimum;
+
+  if (numericOptions?.signed || (minimum ?? 0) < 0) {
+    return undefined;
+  }
+
+  return isInteger ? "numeric" : "decimal";
+}

web/src/components/config-form/theme/widgets/FfmpegArgsWidget.tsx

@@ -120,6 +120,12 @@ export function FfmpegArgsWidget(props: WidgetProps) {
     id,
   } = props;
   const presetField = options?.ffmpegPresetField as PresetField | undefined;
+  // Path to this field within its config section. This is usually the same as
+  // the preset field, but the two diverge when the field sits below the
+  // section root: record.export.hwaccel_args uses the hwaccel_args preset list
+  // while living at export.hwaccel_args inside the record section.
+  const globalFieldPath =
+    (options?.ffmpegGlobalFieldPath as string | undefined) ?? presetField;
   const allowInherit = options?.allowInherit === true;
   const hideDescription = options?.hideDescription === true;
   const useSplitLayout = options?.splitLayout !== false;
@@ -131,11 +137,18 @@ export function FfmpegArgsWidget(props: WidgetProps) {
 
   // Extract the global value for this specific field to detect inheritance
   const globalFieldValue = useMemo(() => {
-    if (!showUseGlobalSetting || !formContext?.globalValue || !presetField) {
+    if (
+      !showUseGlobalSetting ||
+      !formContext?.globalValue ||
+      !globalFieldPath
+    ) {
       return undefined;
     }
-    return get(formContext.globalValue as Record<string, unknown>, presetField);
+    return get(
-  }, [showUseGlobalSetting, formContext?.globalValue, presetField]);
+      formContext.globalValue as Record<string, unknown>,
+      globalFieldPath,
+    );
+  }, [showUseGlobalSetting, formContext?.globalValue, globalFieldPath]);
 
   const { data } = useSWR<FfmpegPresetResponse>("ffmpeg/presets");
 

web/src/components/config-form/theme/widgets/TextWidget.tsx

@@ -2,7 +2,7 @@
 import type { WidgetProps } from "@rjsf/utils";
 import { Input } from "@/components/ui/input";
 import { cn } from "@/lib/utils";
-import { getSizedFieldClassName } from "../utils";
+import { getNumericInputMode, getSizedFieldClassName } from "../utils";
 
 export function TextWidget(props: WidgetProps) {
   const {
@@ -28,6 +28,7 @@ export function TextWidget(props: WidgetProps) {
       id={id}
       className={cn(fieldClassName)}
       type="text"
+      inputMode={getNumericInputMode(schema, options)}
       value={value ?? ""}
       disabled={disabled || readonly}
       placeholder={placeholder || (options.placeholder as string) || ""}

web/src/components/overlay/ObjectTrackOverlay.tsx

@@ -127,8 +127,12 @@ export default function ObjectTrackOverlay({
     },
   );
 
-  const getZonesFriendlyNames = (zones: string[], config: FrigateConfig) => {
+  const getZonesFriendlyNames = (
-    return zones?.map((zone) => resolveZoneName(config, zone)) ?? [];
+    zones: string[],
+    config: FrigateConfig,
+    cameraId?: string,
+  ) => {
+    return zones?.map((zone) => resolveZoneName(config, zone, cameraId)) ?? [];
   };
 
   const timelineResults = useMemo(() => {
@@ -151,7 +155,7 @@ export default function ObjectTrackOverlay({
         data: {
           ...event.data,
           zones_friendly_names: config
-            ? getZonesFriendlyNames(event.data?.zones, config)
+            ? getZonesFriendlyNames(event.data?.zones, config, event.camera)
             : [],
         },
       }));

web/src/components/overlay/detail/ObjectPath.tsx

@@ -61,7 +61,11 @@ export function ObjectPath({
                 ...pos.lifecycle_item?.data,
                 zones_friendly_names: pos.lifecycle_item?.data.zones.map(
                   (zone) => {
-                    return resolveZoneName(config, zone);
+                    return resolveZoneName(
+                      config,
+                      zone,
+                      pos.lifecycle_item?.camera,
+                    );
                   },
                 ),
               },

web/src/components/overlay/detail/TrackingDetails.tsx

@@ -301,11 +301,19 @@ export function TrackingDetails({
     [recordings, actualVideoStart],
   );
 
-  eventSequence?.map((event) => {
+  const sequence = useMemo(
-    event.data.zones_friendly_names = event.data?.zones?.map((zone) => {
+    () =>
-      return resolveZoneName(config, zone);
+      eventSequence?.map((item) => ({
-    });
+        ...item,
-  });
+        data: {
+          ...item.data,
+          zones_friendly_names: item.data?.zones?.map((zone) =>
+            resolveZoneName(config, zone, item.camera),
+          ),
+        },
+      })),
+    [eventSequence, config],
+  );
 
   // Use manualOverride (set when seeking in image mode) if present so
   // lifecycle rows and overlays follow image-mode seeks. Otherwise fall
@@ -849,9 +857,9 @@ export function TrackingDetails({
             </div>
 
             <div className="mt-2">
-              {!eventSequence ? (
+              {!sequence ? (
                 <ActivityIndicator className="size-2" size={2} />
-              ) : eventSequence.length === 0 ? (
+              ) : sequence.length === 0 ? (
                 <div className="py-2 text-muted-foreground">
                   {t("detail.noObjectDetailData", { ns: "views/events" })}
                 </div>
@@ -871,7 +879,7 @@ export function TrackingDetails({
                     />
                   )}
                   <div className="space-y-2">
-                    {eventSequence.map((item, idx) => {
+                    {sequence.map((item, idx) => {
                       return (
                         <div
                           key={`${item.timestamp}-${item.source_id ?? ""}-${idx}`}

web/src/components/timeline/DetailStream.tsx

@@ -1032,7 +1032,7 @@ function ObjectTimeline({
         data: {
           ...event.data,
           zones_friendly_names: event.data?.zones?.map((zone) =>
-            resolveZoneName(config, zone),
+            resolveZoneName(config, zone, event.camera),
           ),
         },
       }));