From 81b53b78356577ef6f5b075123aebfdf2e1f8ffb Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:42:26 -0500 Subject: [PATCH] Miscellaneous fixes (0.18 beta) (#23716) * resolve zone friendly names against the correct camera * Improve handling of zone names in chat prompt * show a numeric keyboard for numeric config form fields on mobile * Specify english only for semantic search tool when model is JinaV1 * resolve export hwaccel args global value against the correct config path --------- Co-authored-by: Nicolas Mowen --- frigate/api/chat.py | 35 ++++++++++--- frigate/genai/prompts.py | 26 +++++++--- .../config-form/section-configs/detect.ts | 7 +++ .../config-form/section-configs/record.ts | 1 + .../config-form/theme/utils/index.ts | 1 + .../config-form/theme/utils/inputMode.ts | 51 +++++++++++++++++++ .../theme/widgets/FfmpegArgsWidget.tsx | 19 +++++-- .../config-form/theme/widgets/TextWidget.tsx | 3 +- .../components/overlay/ObjectTrackOverlay.tsx | 10 ++-- .../components/overlay/detail/ObjectPath.tsx | 6 ++- .../overlay/detail/TrackingDetails.tsx | 24 ++++++--- web/src/components/timeline/DetailStream.tsx | 2 +- 12 files changed, 155 insertions(+), 30 deletions(-) create mode 100644 web/src/components/config-form/theme/utils/inputMode.ts diff --git a/frigate/api/chat.py b/frigate/api/chat.py index ae6ff7a948..fa4510cf14 100644 --- a/frigate/api/chat.py +++ b/frigate/api/chat.py @@ -7,7 +7,7 @@ import operator import time from datetime import datetime from functools import reduce -from typing import Any +from typing import Any, Literal import cv2 from fastapi import APIRouter, Body, Depends, HTTPException, Request @@ -37,6 +37,7 @@ from frigate.api.defs.response.chat_response import ( from frigate.api.defs.tags import Tags from frigate.api.event import _build_attribute_filter_clause, events from frigate.config import FrigateConfig +from frigate.config.classification import SemanticSearchModelEnum from frigate.genai.prompts import ( build_chat_system_prompt, get_attribute_classifications, @@ -86,10 +87,23 @@ def get_tools(request: Request) -> JSONResponse: tools = get_tool_definitions( semantic_search_enabled=semantic_search_enabled, attribute_classifications=attribute_classifications, + embeddings_language=_embeddings_language(config), ) return JSONResponse(content={"tools": tools}) +def _embeddings_language(config: FrigateConfig) -> Literal["english", "multi"]: + """Return the language capability of the configured embeddings model. + + JinaV1 is English-only; every other option (JinaV2 or a GenAI embeddings + provider) handles multiple languages. + """ + if config.semantic_search.model == SemanticSearchModelEnum.jinav1: + return "english" + + return "multi" + + def _resolve_zones( zones: list[str], config: FrigateConfig, @@ -98,11 +112,14 @@ def _resolve_zones( """Map zone names to their canonical config keys, case-insensitively. LLMs frequently echo a user's casing ("Front Yard") instead of the - configured key ("front_yard"). The downstream zone filter is a SQLite GLOB - over the JSON-encoded zones column, which is case-sensitive — so an - unnormalized name silently returns zero matches. Build a lookup over the - relevant cameras' configured zones and substitute when we find a match; - unknown names pass through so behavior matches what the model asked for. + configured key ("front_yard"), or fall back to a zone's friendly name + ("Front Walkway") instead of its ID ("front_walk"). The downstream zone + filter is a SQLite GLOB over the JSON-encoded zones column, which stores + config keys and is case-sensitive — so an unnormalized name silently + returns zero matches. Build a lookup over the relevant cameras' configured + zones, keyed by both the config key and the friendly name, and substitute + when we find a match; unknown names pass through so behavior matches what + the model asked for. """ if not zones: return zones @@ -112,8 +129,11 @@ def _resolve_zones( camera_config = config.cameras.get(camera_id) if camera_config is None: continue - for zone_name in camera_config.zones.keys(): + for zone_name, zone_config in camera_config.zones.items(): lookup.setdefault(zone_name.lower(), zone_name) + lookup.setdefault( + zone_config.get_formatted_name(zone_name).lower(), zone_name + ) return [lookup.get(z.lower(), z) for z in zones] @@ -1134,6 +1154,7 @@ async def chat_completion( tools = get_tool_definitions( semantic_search_enabled=semantic_search_enabled, attribute_classifications=attribute_classifications, + embeddings_language=_embeddings_language(config), ) conversation = [] diff --git a/frigate/genai/prompts.py b/frigate/genai/prompts.py index 5f1d328d7a..33045606eb 100644 --- a/frigate/genai/prompts.py +++ b/frigate/genai/prompts.py @@ -6,7 +6,7 @@ transport. """ import datetime -from typing import Any +from typing import Any, Literal from playhouse.shortcuts import model_to_dict @@ -249,6 +249,7 @@ def get_attribute_classifications(config: FrigateConfig) -> list[dict[str, Any]] def get_tool_definitions( semantic_search_enabled: bool = False, attribute_classifications: list[dict[str, Any]] | None = None, + embeddings_language: Literal["english", "multi"] = "multi", ) -> list[dict[str, Any]]: """ Get OpenAI-compatible tool definitions for Frigate. @@ -258,7 +259,9 @@ def get_tool_definitions( tool exposes an additional `semantic_query` parameter for descriptive queries (e.g. "person riding a lawn mower") and find_similar_objects is included. When attribute classification models are configured, an - `attribute` parameter is exposed for filtering by their labels. + `attribute` parameter is exposed for filtering by their labels. When the + embeddings model only understands English (JinaV1), the `semantic_query` + description instructs the model to write the query in English. """ search_objects_properties: dict[str, Any] = { "camera": { @@ -349,6 +352,14 @@ def get_tool_definitions( "When set, combine with label/time/camera/zone filters as " "usual (e.g. label='person', semantic_query='riding a lawn " "mower', after='2024-05-01T00:00:00Z')." + + ( + " The configured embeddings model only understands " + "English, so always write semantic_query in English, " + "translating the user's description if they phrased it " + "in another language." + if embeddings_language == "english" + else "" + ) ), } @@ -682,14 +693,17 @@ def build_chat_system_prompt( if camera_config.friendly_name else camera_id.replace("_", " ").title() ) - zone_names = list(camera_config.zones.keys()) + zone_descriptors = [ + f"{zone_config.get_formatted_name(zone_name)} (ID: {zone_name})" + for zone_name, zone_config in camera_config.zones.items() + ] if not has_speed_zone: has_speed_zone = any( zone.distances for zone in camera_config.zones.values() ) - if zone_names: + if zone_descriptors: cameras_info.append( - f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_names)})" + f" - {friendly_name} (ID: {camera_id}, zones: {', '.join(zone_descriptors)})" ) else: cameras_info.append(f" - {friendly_name} (ID: {camera_id})") @@ -699,7 +713,7 @@ def build_chat_system_prompt( cameras_section = ( "\n\nAvailable cameras:\n" + "\n".join(cameras_info) - + "\n\nWhen users refer to cameras by their friendly name (e.g., 'Back Deck Camera'), use the corresponding camera ID (e.g., 'back_deck_cam') in tool calls." + + "\n\nWhen users refer to cameras or zones by their friendly name (e.g., 'Back Deck Camera', 'Front Walkway'), use the corresponding ID (e.g., 'back_deck_cam', 'front_walk') in tool calls. Tool results also identify zones by their ID, so when presenting cameras or zones back to the user, translate the ID to its friendly name." ) speed_units_section = "" diff --git a/web/src/components/config-form/section-configs/detect.ts b/web/src/components/config-form/section-configs/detect.ts index 4213a68448..776fbb6567 100644 --- a/web/src/components/config-form/section-configs/detect.ts +++ b/web/src/components/config-form/section-configs/detect.ts @@ -169,6 +169,13 @@ const detect: SectionConfigOverrides = { resolution: ["width", "height", "fps"], tracking: ["min_initialized", "max_disappeared"], }, + uiSchema: { + annotation_offset: { + "ui:options": { + signed: true, + }, + }, + }, hiddenFields: ["enabled_in_config"], advancedFields: [ "min_initialized", diff --git a/web/src/components/config-form/section-configs/record.ts b/web/src/components/config-form/section-configs/record.ts index d4dd481b86..9271e025fb 100644 --- a/web/src/components/config-form/section-configs/record.ts +++ b/web/src/components/config-form/section-configs/record.ts @@ -57,6 +57,7 @@ const record: SectionConfigOverrides = { "ui:options": { suppressMultiSchema: true, ffmpegPresetField: "hwaccel_args", + ffmpegGlobalFieldPath: "export.hwaccel_args", }, }, }, diff --git a/web/src/components/config-form/theme/utils/index.ts b/web/src/components/config-form/theme/utils/index.ts index bb27f297b7..2c483ea231 100644 --- a/web/src/components/config-form/theme/utils/index.ts +++ b/web/src/components/config-form/theme/utils/index.ts @@ -17,3 +17,4 @@ export { isSubtreeModified, } from "./overrides"; export { getSizedFieldClassName } from "./fieldSizing"; +export { getNumericInputMode } from "./inputMode"; diff --git a/web/src/components/config-form/theme/utils/inputMode.ts b/web/src/components/config-form/theme/utils/inputMode.ts new file mode 100644 index 0000000000..02189e42d1 --- /dev/null +++ b/web/src/components/config-form/theme/utils/inputMode.ts @@ -0,0 +1,51 @@ +import type { RJSFSchema } from "@rjsf/utils"; + +type NumericInputOptions = { + signed?: boolean; +}; + +/** + * Derive the on-screen keyboard hint for a schema field. + * + * Numeric config fields render as text inputs because RJSF's NumberField + * relies on the widget echoing raw strings back, so that trailing "." and "0" + * characters survive while a value is being typed. That means the numeric + * keypad has to be requested explicitly. Desktop browsers ignore inputMode, so + * this only affects virtual keyboards. + * + * Fields accepting negative values opt out, since the iOS numeric and decimal + * keypads have no minus key. Most numeric fields declare no minimum even + * though they are non-negative, so signed fields are marked explicitly with + * ui:options.signed. + * + * Args: + * schema: The JSON schema for the field being rendered + * options: The resolved ui:options for the field + * + * Returns: + * The inputMode to apply, or undefined to leave the keyboard alone + */ +export function getNumericInputMode( + schema: RJSFSchema, + options: unknown, +): "numeric" | "decimal" | undefined { + const types = Array.isArray(schema.type) ? schema.type : [schema.type]; + const isInteger = types.includes("integer"); + + if (!isInteger && !types.includes("number")) { + return undefined; + } + + const numericOptions = + typeof options === "object" && options !== null + ? (options as NumericInputOptions) + : undefined; + + const minimum = schema.minimum ?? schema.exclusiveMinimum; + + if (numericOptions?.signed || (minimum ?? 0) < 0) { + return undefined; + } + + return isInteger ? "numeric" : "decimal"; +} diff --git a/web/src/components/config-form/theme/widgets/FfmpegArgsWidget.tsx b/web/src/components/config-form/theme/widgets/FfmpegArgsWidget.tsx index e523bfd466..527789c814 100644 --- a/web/src/components/config-form/theme/widgets/FfmpegArgsWidget.tsx +++ b/web/src/components/config-form/theme/widgets/FfmpegArgsWidget.tsx @@ -120,6 +120,12 @@ export function FfmpegArgsWidget(props: WidgetProps) { id, } = props; const presetField = options?.ffmpegPresetField as PresetField | undefined; + // Path to this field within its config section. This is usually the same as + // the preset field, but the two diverge when the field sits below the + // section root: record.export.hwaccel_args uses the hwaccel_args preset list + // while living at export.hwaccel_args inside the record section. + const globalFieldPath = + (options?.ffmpegGlobalFieldPath as string | undefined) ?? presetField; const allowInherit = options?.allowInherit === true; const hideDescription = options?.hideDescription === true; const useSplitLayout = options?.splitLayout !== false; @@ -131,11 +137,18 @@ export function FfmpegArgsWidget(props: WidgetProps) { // Extract the global value for this specific field to detect inheritance const globalFieldValue = useMemo(() => { - if (!showUseGlobalSetting || !formContext?.globalValue || !presetField) { + if ( + !showUseGlobalSetting || + !formContext?.globalValue || + !globalFieldPath + ) { return undefined; } - return get(formContext.globalValue as Record, presetField); - }, [showUseGlobalSetting, formContext?.globalValue, presetField]); + return get( + formContext.globalValue as Record, + globalFieldPath, + ); + }, [showUseGlobalSetting, formContext?.globalValue, globalFieldPath]); const { data } = useSWR("ffmpeg/presets"); diff --git a/web/src/components/config-form/theme/widgets/TextWidget.tsx b/web/src/components/config-form/theme/widgets/TextWidget.tsx index 9c408797d1..b833a70618 100644 --- a/web/src/components/config-form/theme/widgets/TextWidget.tsx +++ b/web/src/components/config-form/theme/widgets/TextWidget.tsx @@ -2,7 +2,7 @@ import type { WidgetProps } from "@rjsf/utils"; import { Input } from "@/components/ui/input"; import { cn } from "@/lib/utils"; -import { getSizedFieldClassName } from "../utils"; +import { getNumericInputMode, getSizedFieldClassName } from "../utils"; export function TextWidget(props: WidgetProps) { const { @@ -28,6 +28,7 @@ export function TextWidget(props: WidgetProps) { id={id} className={cn(fieldClassName)} type="text" + inputMode={getNumericInputMode(schema, options)} value={value ?? ""} disabled={disabled || readonly} placeholder={placeholder || (options.placeholder as string) || ""} diff --git a/web/src/components/overlay/ObjectTrackOverlay.tsx b/web/src/components/overlay/ObjectTrackOverlay.tsx index 8f78adcd74..4ed243a188 100644 --- a/web/src/components/overlay/ObjectTrackOverlay.tsx +++ b/web/src/components/overlay/ObjectTrackOverlay.tsx @@ -127,8 +127,12 @@ export default function ObjectTrackOverlay({ }, ); - const getZonesFriendlyNames = (zones: string[], config: FrigateConfig) => { - return zones?.map((zone) => resolveZoneName(config, zone)) ?? []; + const getZonesFriendlyNames = ( + zones: string[], + config: FrigateConfig, + cameraId?: string, + ) => { + return zones?.map((zone) => resolveZoneName(config, zone, cameraId)) ?? []; }; const timelineResults = useMemo(() => { @@ -151,7 +155,7 @@ export default function ObjectTrackOverlay({ data: { ...event.data, zones_friendly_names: config - ? getZonesFriendlyNames(event.data?.zones, config) + ? getZonesFriendlyNames(event.data?.zones, config, event.camera) : [], }, })); diff --git a/web/src/components/overlay/detail/ObjectPath.tsx b/web/src/components/overlay/detail/ObjectPath.tsx index 4af68a0d4f..8be0c18d9c 100644 --- a/web/src/components/overlay/detail/ObjectPath.tsx +++ b/web/src/components/overlay/detail/ObjectPath.tsx @@ -61,7 +61,11 @@ export function ObjectPath({ ...pos.lifecycle_item?.data, zones_friendly_names: pos.lifecycle_item?.data.zones.map( (zone) => { - return resolveZoneName(config, zone); + return resolveZoneName( + config, + zone, + pos.lifecycle_item?.camera, + ); }, ), }, diff --git a/web/src/components/overlay/detail/TrackingDetails.tsx b/web/src/components/overlay/detail/TrackingDetails.tsx index 88e48f8012..1e6d3e6306 100644 --- a/web/src/components/overlay/detail/TrackingDetails.tsx +++ b/web/src/components/overlay/detail/TrackingDetails.tsx @@ -301,11 +301,19 @@ export function TrackingDetails({ [recordings, actualVideoStart], ); - eventSequence?.map((event) => { - event.data.zones_friendly_names = event.data?.zones?.map((zone) => { - return resolveZoneName(config, zone); - }); - }); + const sequence = useMemo( + () => + eventSequence?.map((item) => ({ + ...item, + data: { + ...item.data, + zones_friendly_names: item.data?.zones?.map((zone) => + resolveZoneName(config, zone, item.camera), + ), + }, + })), + [eventSequence, config], + ); // Use manualOverride (set when seeking in image mode) if present so // lifecycle rows and overlays follow image-mode seeks. Otherwise fall @@ -849,9 +857,9 @@ export function TrackingDetails({
- {!eventSequence ? ( + {!sequence ? ( - ) : eventSequence.length === 0 ? ( + ) : sequence.length === 0 ? (
{t("detail.noObjectDetailData", { ns: "views/events" })}
@@ -871,7 +879,7 @@ export function TrackingDetails({ /> )}
- {eventSequence.map((item, idx) => { + {sequence.map((item, idx) => { return (
- resolveZoneName(config, zone), + resolveZoneName(config, zone, event.camera), ), }, }));