Support using GenAI for audio transcription (#24396)

* Add support for running transcription with GenAI

* Improve audio joining

* Fix GenAI model capability reporting

* Support language correctly

* Migrate existing users to keep english selected

* Fix models

* Fix tests

* Fix accepted null model

* Handle slwo providers

web/src/components/config-form/section-configs/audio_transcription.ts

@@ -34,9 +34,32 @@ const audioTranscription: SectionConfigOverrides = {
     },
   },
   global: {
-    fieldOrder: ["enabled", "language", "device", "model_size"],
+    fieldOrder: ["enabled", "model", "language", "device", "model_size"],
     advancedFields: ["language", "device", "model_size"],
-    restartRequired: ["enabled", "language", "device", "model_size"],
+    restartRequired: ["enabled", "model", "language", "device", "model_size"],
+    fieldMessages: [
+      {
+        key: "genai-provider-ignores-local-settings",
+        health: (ctx) => ctx.fullConfig.audio_transcription?.enabled === true,
+        field: "device",
+        messageKey: "configMessages.audioTranscription.genaiProviderSelected",
+        severity: "info",
+        position: "after",
+        condition: (ctx) =>
+          typeof ctx.formData?.model === "string" &&
+          ctx.formData.model !== "" &&
+          ctx.formData.model !== "whisper",
+      },
+    ],
+    uiSchema: {
+      model: {
+        "ui:widget": "audioTranscriptionModel",
+      },
+      model_size: {
+        "ui:widget": "audioTranscriptionModelSize",
+        "ui:options": { size: "xs", enumI18nPrefix: "modelSize" },
+      },
+    },
   },
 };
 

web/src/components/config-form/theme/frigateTheme.ts

@@ -33,6 +33,8 @@ import { CameraPathWidget } from "./widgets/CameraPathWidget";
 import { OptionalFieldWidget } from "./widgets/OptionalFieldWidget";
 import { SemanticSearchModelWidget } from "./widgets/SemanticSearchModelWidget";
 import { SemanticSearchModelSizeWidget } from "./widgets/SemanticSearchModelSizeWidget";
+import { AudioTranscriptionModelWidget } from "./widgets/AudioTranscriptionModelWidget";
+import { AudioTranscriptionModelSizeWidget } from "./widgets/AudioTranscriptionModelSizeWidget";
 import { OnvifProfileWidget } from "./widgets/OnvifProfileWidget";
 import { PTZPresetsWidget } from "./widgets/PTZPresetsWidget";
 import { DefaultRoleWidget } from "./widgets/DefaultRoleWidget";
@@ -93,6 +95,8 @@ export const frigateTheme: FrigateTheme = {
     optionalField: OptionalFieldWidget,
     semanticSearchModel: SemanticSearchModelWidget,
     semanticSearchModelSize: SemanticSearchModelSizeWidget,
+    audioTranscriptionModel: AudioTranscriptionModelWidget,
+    audioTranscriptionModelSize: AudioTranscriptionModelSizeWidget,
     onvifProfile: OnvifProfileWidget,
     ptzPresets: PTZPresetsWidget,
     defaultRole: DefaultRoleWidget,

web/src/components/config-form/theme/widgets/AudioTranscriptionModelSizeWidget.tsx

@@ -0,0 +1,17 @@
+// audio_transcription.model_size. See GenAIBackedModelSizeWidget for the shared
+// implementation, including the clear-vs-default handling.
+import type { WidgetProps } from "@rjsf/utils";
+import { GenAIBackedModelSizeWidget } from "./GenAIBackedModelSizeWidget";
+
+export function AudioTranscriptionModelSizeWidget(props: WidgetProps) {
+  return (
+    <GenAIBackedModelSizeWidget
+      {...props}
+      options={{
+        ...props.options,
+        builtInModels: ["whisper"],
+        i18nPrefix: "audioTranscriptionModelSize",
+      }}
+    />
+  );
+}

web/src/components/config-form/theme/widgets/AudioTranscriptionModelWidget.tsx

@@ -0,0 +1,18 @@
+// audio_transcription.model: the built-in whisper backend plus GenAI providers
+// with the transcribe role. See GenAIBackedModelWidget for the shared
+// implementation.
+import type { WidgetProps } from "@rjsf/utils";
+import { GenAIBackedModelWidget } from "./GenAIBackedModelWidget";
+
+export function AudioTranscriptionModelWidget(props: WidgetProps) {
+  return (
+    <GenAIBackedModelWidget
+      {...props}
+      options={{
+        ...props.options,
+        role: "transcribe",
+        i18nPrefix: "audioTranscriptionModel",
+      }}
+    />
+  );
+}

web/src/components/config-form/theme/widgets/GenAIBackedModelSizeWidget.tsx

@@ -0,0 +1,65 @@
+// Disables model_size and shows "N/A" when a GenAI provider is selected in the
+// companion model field. Reads model via LiveFormDataContext so it re-runs even
+// when RJSF's SchemaField memoization would skip this widget. The built-in model
+// names and the i18n key prefix come from ui:options.
+import type { WidgetProps } from "@rjsf/utils";
+import { useContext, useEffect } from "react";
+import { useTranslation } from "react-i18next";
+import {
+  Select,
+  SelectContent,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { LiveFormDataContext } from "../../LiveFormDataContext";
+import { getSizedFieldClassName } from "../utils";
+import { SelectWidget } from "./SelectWidget";
+
+export function GenAIBackedModelSizeWidget(props: WidgetProps) {
+  const { t } = useTranslation(["views/settings"]);
+  const liveFormData = useContext(LiveFormDataContext);
+  const model = liveFormData?.model;
+
+  const builtInModels = (props.options?.builtInModels as string[]) ?? [];
+  const i18nPrefix =
+    (props.options?.i18nPrefix as string | undefined) ??
+    "semanticSearchModelSize";
+
+  const isProvider =
+    typeof model === "string" && model !== "" && !builtInModels.includes(model);
+
+  // model_size is unused on a GenAI provider. Only clear it (which the backend
+  // treats as "remove") for a non-default value, which can only come from the
+  // config file. A defaulted value is indistinguishable from unset in the
+  // resolved config, so clearing it would falsely dirty the field and delete a
+  // YAML key that isn't there. Restore the default when returning to a built-in model.
+  const { value, onChange, schema } = props;
+  const schemaDefault = schema?.default as string | undefined;
+  useEffect(() => {
+    if (isProvider) {
+      if (value !== undefined && value !== schemaDefault) {
+        onChange(undefined);
+      }
+    } else if (value === undefined && schemaDefault) {
+      onChange(schemaDefault);
+    }
+  }, [isProvider, value, onChange, schemaDefault]);
+
+  if (isProvider) {
+    const fieldClassName = getSizedFieldClassName(props.options ?? {}, "sm");
+    return (
+      <Select value="" disabled>
+        <SelectTrigger className={fieldClassName}>
+          <SelectValue
+            placeholder={t(`configForm.${i18nPrefix}.notApplicable`, {
+              defaultValue: "Not applicable for GenAI providers",
+            })}
+          />
+        </SelectTrigger>
+        <SelectContent />
+      </Select>
+    );
+  }
+
+  return <SelectWidget {...props} />;
+}

web/src/components/config-form/theme/widgets/GenAIBackedModelWidget.tsx

@@ -0,0 +1,164 @@
+// Combobox for a "local model or GenAI provider" field (semantic_search.model,
+// audio_transcription.model). Shows the built-in model enum values alongside the
+// GenAI providers holding the relevant role. The role and the i18n key prefix
+// come from ui:options so each field can reuse this with its own wording.
+import { useState, useMemo } from "react";
+import type { WidgetProps } from "@rjsf/utils";
+import { useTranslation } from "react-i18next";
+import { Check, ChevronsUpDown } from "lucide-react";
+import { cn } from "@/lib/utils";
+import { Button } from "@/components/ui/button";
+import {
+  Command,
+  CommandGroup,
+  CommandItem,
+  CommandList,
+} from "@/components/ui/command";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/components/ui/popover";
+import type { ConfigFormContext } from "@/types/configForm";
+import { getSizedFieldClassName } from "../utils";
+
+interface ProviderOption {
+  value: string;
+  label: string;
+}
+
+export function GenAIBackedModelWidget(props: WidgetProps) {
+  const { id, value, disabled, readonly, onChange, schema, registry, options } =
+    props;
+  const { t } = useTranslation(["views/settings"]);
+  const [open, setOpen] = useState(false);
+
+  const formContext = registry?.formContext as ConfigFormContext | undefined;
+  const fieldClassName = getSizedFieldClassName(options, "sm");
+  const role = (options?.role as string | undefined) ?? "embeddings";
+  const i18nPrefix =
+    (options?.i18nPrefix as string | undefined) ?? "semanticSearchModel";
+
+  // Built-in model options from schema.examples (populated by transformer
+  // collapsing the anyOf enum+string union)
+  const builtInModels: ProviderOption[] = useMemo(() => {
+    const examples = (schema as Record<string, unknown>).examples;
+    if (!Array.isArray(examples)) return [];
+    return examples
+      .filter((v): v is string => typeof v === "string")
+      .map((v) => ({ value: v, label: v }));
+  }, [schema]);
+
+  // GenAI providers that have the role this field is backed by
+  const roleProviders: ProviderOption[] = useMemo(() => {
+    const genai = (
+      formContext?.fullConfig as Record<string, unknown> | undefined
+    )?.genai;
+    if (!genai || typeof genai !== "object" || Array.isArray(genai)) return [];
+
+    const providers: ProviderOption[] = [];
+    for (const [key, config] of Object.entries(
+      genai as Record<string, unknown>,
+    )) {
+      if (!config || typeof config !== "object" || Array.isArray(config))
+        continue;
+      const roles = (config as Record<string, unknown>).roles;
+      if (Array.isArray(roles) && roles.includes(role)) {
+        providers.push({ value: key, label: key });
+      }
+    }
+    return providers;
+  }, [formContext?.fullConfig, role]);
+
+  const currentLabel =
+    builtInModels.find((m) => m.value === value)?.label ??
+    roleProviders.find((p) => p.value === value)?.label ??
+    (typeof value === "string" && value ? value : undefined);
+
+  return (
+    <Popover open={open} onOpenChange={setOpen}>
+      <PopoverTrigger asChild>
+        <Button
+          id={id}
+          type="button"
+          variant="outline"
+          role="combobox"
+          aria-expanded={open}
+          disabled={disabled || readonly}
+          className={cn(
+            "justify-between font-normal",
+            !currentLabel && "text-muted-foreground",
+            fieldClassName,
+          )}
+        >
+          {currentLabel ??
+            t(`configForm.${i18nPrefix}.placeholder`, {
+              ns: "views/settings",
+              defaultValue: "Select model…",
+            })}
+          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-[--radix-popover-trigger-width] p-0">
+        <Command>
+          <CommandList>
+            {builtInModels.length > 0 && (
+              <CommandGroup
+                heading={t(`configForm.${i18nPrefix}.builtIn`, {
+                  ns: "views/settings",
+                  defaultValue: "Built-in Models",
+                })}
+              >
+                {builtInModels.map((model) => (
+                  <CommandItem
+                    key={model.value}
+                    value={model.value}
+                    onSelect={() => {
+                      onChange(model.value);
+                      setOpen(false);
+                    }}
+                  >
+                    <Check
+                      className={cn(
+                        "mr-2 h-4 w-4",
+                        value === model.value ? "opacity-100" : "opacity-0",
+                      )}
+                    />
+                    {model.label}
+                  </CommandItem>
+                ))}
+              </CommandGroup>
+            )}
+            {roleProviders.length > 0 && (
+              <CommandGroup
+                heading={t(`configForm.${i18nPrefix}.genaiProviders`, {
+                  ns: "views/settings",
+                  defaultValue: "GenAI Providers",
+                })}
+              >
+                {roleProviders.map((provider) => (
+                  <CommandItem
+                    key={provider.value}
+                    value={provider.value}
+                    onSelect={() => {
+                      onChange(provider.value);
+                      setOpen(false);
+                    }}
+                  >
+                    <Check
+                      className={cn(
+                        "mr-2 h-4 w-4",
+                        value === provider.value ? "opacity-100" : "opacity-0",
+                      )}
+                    />
+                    {provider.label}
+                  </CommandItem>
+                ))}
+              </CommandGroup>
+            )}
+          </CommandList>
+        </Command>
+      </PopoverContent>
+    </Popover>
+  );
+}

web/src/components/config-form/theme/widgets/GenAIRolesWidget.tsx

@@ -4,9 +4,14 @@ import { useTranslation } from "react-i18next";
 import useSWR from "swr";
 import { Switch } from "@/components/ui/switch";
 import type { ConfigFormContext } from "@/types/configForm";
-import type { GenAIModelsResponse } from "@/types/chat";
+import type { GenAIModelCapabilities, GenAIModelsResponse } from "@/types/chat";
 
-const GENAI_ROLES = ["embeddings", "descriptions", "chat"] as const;
+const GENAI_ROLES = [
+  "embeddings",
+  "descriptions",
+  "chat",
+  "transcribe",
+] as const;
 
 function normalizeValue(value: unknown): string[] {
   if (Array.isArray(value)) {
@@ -43,18 +48,56 @@ export function GenAIRolesWidget(props: WidgetProps) {
     revalidateOnFocus: false,
   });
 
-  const embeddingsSupported = useMemo(() => {
+  // The model currently chosen in the form, which is what the roles have to
+  // reflect. Reading the saved config instead would keep reporting the previous
+  // model's capabilities until a save and a refetch.
+  const selectedModel = useMemo(() => {
+    if (!providerKey) return undefined;
+    const formData = formContext?.formData as
+      Record<string, unknown> | undefined;
+    const entry = formData?.[providerKey];
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+      return undefined;
+    }
+    const model = (entry as Record<string, unknown>).model;
+    return typeof model === "string" && model ? model : undefined;
+  }, [formContext?.formData, providerKey]);
+
+  // Capabilities the provider reported for that specific model. Absent when the
+  // provider cannot describe a model it has not loaded, in which case the
+  // entry-level flags (which describe the saved model) are the best available.
+  const modelCapabilities: GenAIModelCapabilities | undefined = useMemo(() => {
+    if (!providerKey || !selectedModel) return undefined;
+    return genaiInfo?.[providerKey]?.model_capabilities?.[selectedModel];
+  }, [genaiInfo, providerKey, selectedModel]);
+
+  const capabilityOf = (
+    key: "supports_embeddings" | "supports_transcription",
+  ): boolean => {
+    const perModel = modelCapabilities?.[key];
+    if (perModel !== undefined) return perModel;
     if (!providerKey) return true;
     const info = genaiInfo?.[providerKey];
-    return info ? info.supports_embeddings : true;
-  }, [genaiInfo, providerKey]);
+    // assume supported when nothing is known, so a role is never hidden on
+    // missing information alone
+    return info ? info[key] : true;
+  };
+
+  const embeddingsSupported = capabilityOf("supports_embeddings");
+  const transcriptionSupported = capabilityOf("supports_transcription");
+
+  const unsupportedRoles = useMemo(() => {
+    const unsupported = new Set<string>();
+
+    if (!embeddingsSupported) unsupported.add("embeddings");
+    if (!transcriptionSupported) unsupported.add("transcribe");
+
+    return unsupported;
+  }, [embeddingsSupported, transcriptionSupported]);
 
   const availableRoles = useMemo(
-    () =>
-      embeddingsSupported
-        ? GENAI_ROLES
-        : GENAI_ROLES.filter((role) => role !== "embeddings"),
-    [embeddingsSupported],
+    () => GENAI_ROLES.filter((role) => !unsupportedRoles.has(role)),
+    [unsupportedRoles],
   );
 
   const occupiedRoles = useMemo(() => {
@@ -80,11 +123,13 @@ export function GenAIRolesWidget(props: WidgetProps) {
     return occupied;
   }, [formContext?.formData, providerKey]);
 
+  // strip every unsupported role in a single onChange; two effects each
+  // rewriting the same value would race and lose one of the edits
   useEffect(() => {
-    if (!embeddingsSupported && selectedRoles.includes("embeddings")) {
-      onChange(selectedRoles.filter((role) => role !== "embeddings"));
-    }
-  }, [embeddingsSupported, selectedRoles, onChange]);
+    if (!selectedRoles.some((role) => unsupportedRoles.has(role))) return;
+
+    onChange(selectedRoles.filter((role) => !unsupportedRoles.has(role)));
+  }, [unsupportedRoles, selectedRoles, onChange]);
 
   const toggleRole = (role: string, enabled: boolean) => {
     if (enabled) {

web/src/components/config-form/theme/widgets/SemanticSearchModelSizeWidget.tsx

@@ -1,61 +1,17 @@
-// Disables model_size and shows "N/A" when a GenAI provider is selected.
-// Reads model via LiveFormDataContext so it re-runs even when RJSF's
-// SchemaField memoization would skip this widget.
+// semantic_search.model_size. See GenAIBackedModelSizeWidget for the shared
+// implementation, including the clear-vs-default handling.
 import type { WidgetProps } from "@rjsf/utils";
-import { useContext, useEffect } from "react";
-import { useTranslation } from "react-i18next";
-import {
-  Select,
-  SelectContent,
-  SelectTrigger,
-  SelectValue,
-} from "@/components/ui/select";
-import { LiveFormDataContext } from "../../LiveFormDataContext";
-import { getSizedFieldClassName } from "../utils";
-import { SelectWidget } from "./SelectWidget";
+import { GenAIBackedModelSizeWidget } from "./GenAIBackedModelSizeWidget";
 
 export function SemanticSearchModelSizeWidget(props: WidgetProps) {
-  const { t } = useTranslation(["views/settings"]);
-  const liveFormData = useContext(LiveFormDataContext);
-  const model = liveFormData?.model;
-  const isProvider =
-    typeof model === "string" &&
-    model !== "" &&
-    model !== "jinav1" &&
-    model !== "jinav2";
-
-  // model_size is unused on a GenAI provider. Only clear it (which the backend
-  // treats as "remove") for a non-default value, which can only come from the
-  // config file. A defaulted value is indistinguishable from unset in the
-  // resolved config, so clearing it would falsely dirty the field and delete a
-  // YAML key that isn't there. Restore the default when returning to a Jina model.
-  const { value, onChange, schema } = props;
-  const schemaDefault = schema?.default as string | undefined;
-  useEffect(() => {
-    if (isProvider) {
-      if (value !== undefined && value !== schemaDefault) {
-        onChange(undefined);
-      }
-    } else if (value === undefined && schemaDefault) {
-      onChange(schemaDefault);
-    }
-  }, [isProvider, value, onChange, schemaDefault]);
-
-  if (isProvider) {
-    const fieldClassName = getSizedFieldClassName(props.options ?? {}, "sm");
-    return (
-      <Select value="" disabled>
-        <SelectTrigger className={fieldClassName}>
-          <SelectValue
-            placeholder={t("configForm.semanticSearchModelSize.notApplicable", {
-              defaultValue: "Not applicable for GenAI providers",
-            })}
-          />
-        </SelectTrigger>
-        <SelectContent />
-      </Select>
-    );
-  }
-
-  return <SelectWidget {...props} />;
+  return (
+    <GenAIBackedModelSizeWidget
+      {...props}
+      options={{
+        ...props.options,
+        builtInModels: ["jinav1", "jinav2"],
+        i18nPrefix: "semanticSearchModelSize",
+      }}
+    />
+  );
 }

web/src/components/config-form/theme/widgets/SemanticSearchModelWidget.tsx

@@ -1,159 +1,17 @@
-// Combobox widget for semantic_search.model field.
-// Shows built-in model enum values and GenAI providers with the embeddings role.
-import { useState, useMemo } from "react";
+// semantic_search.model: built-in Jina models plus GenAI providers with the
+// embeddings role. See GenAIBackedModelWidget for the shared implementation.
 import type { WidgetProps } from "@rjsf/utils";
-import { useTranslation } from "react-i18next";
-import { Check, ChevronsUpDown } from "lucide-react";
-import { cn } from "@/lib/utils";
-import { Button } from "@/components/ui/button";
-import {
-  Command,
-  CommandGroup,
-  CommandItem,
-  CommandList,
-} from "@/components/ui/command";
-import {
-  Popover,
-  PopoverContent,
-  PopoverTrigger,
-} from "@/components/ui/popover";
-import type { ConfigFormContext } from "@/types/configForm";
-import { getSizedFieldClassName } from "../utils";
-
-interface ProviderOption {
-  value: string;
-  label: string;
-}
+import { GenAIBackedModelWidget } from "./GenAIBackedModelWidget";
 
 export function SemanticSearchModelWidget(props: WidgetProps) {
-  const { id, value, disabled, readonly, onChange, schema, registry, options } =
-    props;
-  const { t } = useTranslation(["views/settings"]);
-  const [open, setOpen] = useState(false);
-
-  const formContext = registry?.formContext as ConfigFormContext | undefined;
-  const fieldClassName = getSizedFieldClassName(options, "sm");
-
-  // Built-in model options from schema.examples (populated by transformer
-  // collapsing the anyOf enum+string union)
-  const builtInModels: ProviderOption[] = useMemo(() => {
-    const examples = (schema as Record<string, unknown>).examples;
-    if (!Array.isArray(examples)) return [];
-    return examples
-      .filter((v): v is string => typeof v === "string")
-      .map((v) => ({ value: v, label: v }));
-  }, [schema]);
-
-  // GenAI providers that have the "embeddings" role
-  const embeddingsProviders: ProviderOption[] = useMemo(() => {
-    const genai = (
-      formContext?.fullConfig as Record<string, unknown> | undefined
-    )?.genai;
-    if (!genai || typeof genai !== "object" || Array.isArray(genai)) return [];
-
-    const providers: ProviderOption[] = [];
-    for (const [key, config] of Object.entries(
-      genai as Record<string, unknown>,
-    )) {
-      if (!config || typeof config !== "object" || Array.isArray(config))
-        continue;
-      const roles = (config as Record<string, unknown>).roles;
-      if (Array.isArray(roles) && roles.includes("embeddings")) {
-        providers.push({ value: key, label: key });
-      }
-    }
-    return providers;
-  }, [formContext?.fullConfig]);
-
-  const currentLabel =
-    builtInModels.find((m) => m.value === value)?.label ??
-    embeddingsProviders.find((p) => p.value === value)?.label ??
-    (typeof value === "string" && value ? value : undefined);
-
   return (
-    <Popover open={open} onOpenChange={setOpen}>
-      <PopoverTrigger asChild>
-        <Button
-          id={id}
-          type="button"
-          variant="outline"
-          role="combobox"
-          aria-expanded={open}
-          disabled={disabled || readonly}
-          className={cn(
-            "justify-between font-normal",
-            !currentLabel && "text-muted-foreground",
-            fieldClassName,
-          )}
-        >
-          {currentLabel ??
-            t("configForm.semanticSearchModel.placeholder", {
-              ns: "views/settings",
-              defaultValue: "Select model…",
-            })}
-          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
-        </Button>
-      </PopoverTrigger>
-      <PopoverContent className="w-[--radix-popover-trigger-width] p-0">
-        <Command>
-          <CommandList>
-            {builtInModels.length > 0 && (
-              <CommandGroup
-                heading={t("configForm.semanticSearchModel.builtIn", {
-                  ns: "views/settings",
-                  defaultValue: "Built-in Models",
-                })}
-              >
-                {builtInModels.map((model) => (
-                  <CommandItem
-                    key={model.value}
-                    value={model.value}
-                    onSelect={() => {
-                      onChange(model.value);
-                      setOpen(false);
-                    }}
-                  >
-                    <Check
-                      className={cn(
-                        "mr-2 h-4 w-4",
-                        value === model.value ? "opacity-100" : "opacity-0",
-                      )}
-                    />
-                    {model.label}
-                  </CommandItem>
-                ))}
-              </CommandGroup>
-            )}
-            {embeddingsProviders.length > 0 && (
-              <CommandGroup
-                heading={t("configForm.semanticSearchModel.genaiProviders", {
-                  ns: "views/settings",
-                  defaultValue: "GenAI Providers",
-                })}
-              >
-                {embeddingsProviders.map((provider) => (
-                  <CommandItem
-                    key={provider.value}
-                    value={provider.value}
-                    onSelect={() => {
-                      onChange(provider.value);
-                      setOpen(false);
-                    }}
-                  >
-                    <Check
-                      className={cn(
-                        "mr-2 h-4 w-4",
-                        value === provider.value ? "opacity-100" : "opacity-0",
-                      )}
-                    />
-                    {provider.label}
-                  </CommandItem>
-                ))}
-              </CommandGroup>
-            )}
-          </CommandList>
-        </Command>
-      </PopoverContent>
-    </Popover>
+    <GenAIBackedModelWidget
+      {...props}
+      options={{
+        ...props.options,
+        role: "embeddings",
+        i18nPrefix: "semanticSearchModel",
+      }}
+    />
   );
 }

web/src/types/chat.ts

@@ -50,11 +50,23 @@ export type ChatStats = {
 
 export type ShowStatsMode = "while_generating" | "always";
 
+// Capability flags a provider can report for a model it has not loaded.
+// Keyed by model name (and alias) in GenAIProviderInfo.model_capabilities.
+export type GenAIModelCapabilities = {
+  supports_vision?: boolean;
+  supports_embeddings?: boolean;
+  supports_transcription?: boolean;
+};
+
 export type GenAIProviderInfo = {
   models: string[];
   roles: string[];
   supports_toggleable_thinking: boolean;
   supports_embeddings: boolean;
+  supports_transcription: boolean;
+  // Per-model capabilities, when the provider can report them without loading
+  // the model. The top-level flags above describe the configured model only.
+  model_capabilities?: Record<string, GenAIModelCapabilities>;
 };
 
 export type GenAIModelsResponse = Record<string, GenAIProviderInfo>;

web/src/types/frigateConfig.ts

@@ -394,7 +394,7 @@ export type AllGroupsStreamingSettings = {
   [groupName: string]: GroupStreamingSettings;
 };
 
-export type GenAIRole = "chat" | "descriptions" | "embeddings";
+export type GenAIRole = "chat" | "descriptions" | "embeddings" | "transcribe";
 
 export type GenAIAgentConfig = {
   api_key?: string;