Refactor detector and model management (#23995)

* Refactor detector and model management

* Fix model resolution field

web/src/components/card/SearchThumbnail.tsx

@@ -13,6 +13,7 @@ import { cn } from "@/lib/utils";
 import { TooltipPortal } from "@radix-ui/react-tooltip";
 import useContextMenu from "@/hooks/use-contextmenu";
 import { getTranslatedLabel } from "@/utils/i18n";
+import { isAttributeOfLabel } from "@/utils/modelUtil";
 
 type SearchThumbnailProps = {
   searchResult: SearchResult;
@@ -58,9 +59,7 @@ export default function SearchThumbnail({
     }
 
     if (
-      config.model.attributes_map[searchResult.label]?.includes(
-        searchResult.sub_label,
-      )
+      isAttributeOfLabel(config, searchResult.label, searchResult.sub_label)
     ) {
       return searchResult.sub_label;
     }
@@ -82,9 +81,7 @@ export default function SearchThumbnail({
     }
 
     if (
-      config.model.attributes_map[searchResult.label]?.includes(
-        searchResult.sub_label,
-      )
+      isAttributeOfLabel(config, searchResult.label, searchResult.sub_label)
     ) {
       return "";
     }

web/src/components/classification/ClassificationModelEditDialog.tsx

@@ -32,6 +32,7 @@ import {
 } from "@/types/frigateConfig";
 import { ClassificationDatasetResponse } from "@/types/classification";
 import { getTranslatedLabel } from "@/utils/i18n";
+import { isAttributeLabel } from "@/utils/modelUtil";
 import { zodResolver } from "@hookform/resolvers/zod";
 import axios from "axios";
 import { useCallback, useEffect, useMemo, useState } from "react";
@@ -99,7 +100,7 @@ export default function ClassificationModelEditDialog({
       }
 
       cameraConfig.objects.track.forEach((label) => {
-        if (!config.model.all_attributes.includes(label)) {
+        if (!isAttributeLabel(config, label)) {
           labels.add(label);
         }
       });

web/src/components/classification/wizard/Step1NameAndDefine.tsx

@@ -27,6 +27,7 @@ import useSWR from "swr";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { getTranslatedLabel } from "@/utils/i18n";
 import { useDocDomain } from "@/hooks/use-doc-domain";
+import { isAttributeLabel } from "@/utils/modelUtil";
 import {
   Popover,
   PopoverContent,
@@ -72,7 +73,7 @@ export default function Step1NameAndDefine({
       }
 
       cameraConfig.objects.track.forEach((label) => {
-        if (!config.model.all_attributes.includes(label)) {
+        if (!isAttributeLabel(config, label)) {
           labels.add(label);
         }
       });

web/src/components/config-form/theme/widgets/ObjectLabelSwitchesWidget.tsx

@@ -19,23 +19,16 @@ function collectLabelmapLabels(labelmap: unknown, labels: Set<string>) {
   });
 }
 
-// Read labelmap labels from the global model and detector models.
+// Read labelmap labels from every configured detection model.
 function getLabelmapLabels(context: FormContext): string[] {
   const labels = new Set<string>();
   const fullConfig = context.fullConfig as FrigateConfig | undefined;
 
-  if (fullConfig?.model) {
-    collectLabelmapLabels(fullConfig.model.labelmap, labels);
-  }
-
-  if (fullConfig?.detectors) {
-    // detectors is a map of detector configs; each may include a model labelmap.
-    Object.values(fullConfig.detectors).forEach((detector) => {
-      if (detector?.model?.labelmap) {
-        collectLabelmapLabels(detector.model.labelmap, labels);
-      }
-    });
-  }
+  fullConfig?.models?.forEach((model) => {
+    if (model?.labelmap) {
+      collectLabelmapLabels(model.labelmap, labels);
+    }
+  });
 
   return [...labels];
 }

web/src/components/filter/SearchFilterGroup.tsx

@@ -26,6 +26,7 @@ import { CalendarRangeFilterButton } from "./CalendarFilterButton";
 import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
 import { useTranslation } from "react-i18next";
 import { getTranslatedLabel } from "@/utils/i18n";
+import { isAttributeLabel } from "@/utils/modelUtil";
 import { useAllowedCameras } from "@/hooks/use-allowed-cameras";
 
 type SearchFilterGroupProps = {
@@ -73,7 +74,7 @@ export default function SearchFilterGroup({
       }
 
       cameraConfig.objects.track.forEach((label) => {
-        if (!config.model.all_attributes.includes(label)) {
+        if (!isAttributeLabel(config, label)) {
           labels.add(label);
         }
       });

web/src/components/overlay/ObjectTrackOverlay.tsx

@@ -13,6 +13,7 @@ import { cn } from "@/lib/utils";
 import { useTranslation } from "react-i18next";
 import { Event } from "@/types/event";
 import { resolveZoneName } from "@/hooks/use-zone-friendly-name";
+import { getPrimaryModel } from "@/utils/modelUtil";
 
 // Use a small tolerance (10ms) for browsers with seek precision by-design issues
 const TOLERANCE = 0.01;
@@ -178,7 +179,7 @@ export default function ObjectTrackOverlay({
 
   const getObjectColor = useCallback(
     (label: string, objectId: string) => {
-      const objectColor = config?.model?.colormap[label];
+      const objectColor = getPrimaryModel(config)?.colormap?.[label];
       if (objectColor) {
         const reversed = [...objectColor].reverse();
         return `rgb(${reversed.join(",")})`;

web/src/pages/Replay.tsx

@@ -49,6 +49,7 @@ import Logo from "@/components/Logo";
 import { Separator } from "@/components/ui/separator";
 import { useDocDomain } from "@/hooks/use-doc-domain";
 import DebugDrawingLayer from "@/components/overlay/DebugDrawingLayer";
+import { getPrimaryModel } from "@/utils/modelUtil";
 import { IoMdArrowRoundBack } from "react-icons/io";
 
 type DebugReplayStatus = {
@@ -642,7 +643,7 @@ function ObjectList({ cameraConfig, objects, config }: ObjectListProps) {
     if (!config) {
       return;
     }
-    return config.model?.colormap;
+    return getPrimaryModel(config)?.colormap;
   }, [config]);
 
   const getColorForObjectName = useCallback(

web/src/pages/Settings.tsx

@@ -115,6 +115,7 @@ import SaveAllPreviewPopover, {
   type SaveAllPreviewItem,
 } from "@/components/overlay/detail/SaveAllPreviewPopover";
 import { useRestart } from "@/api/ws";
+import { getPrimaryModel } from "@/utils/modelUtil";
 import {
   Tooltip,
   TooltipContent,
@@ -949,14 +950,16 @@ export default function Settings() {
           const pendingKeySet = Object.keys(
             sanitizedDetectors as JsonObject,
           ).sort();
-          const savedKeySet = Object.keys(config.detectors ?? {}).sort();
+          const savedKeySet = [
+            ...(getPrimaryModel(config)?.devices ?? []),
+          ].sort();
           detectorKeysChanged =
             JSON.stringify(pendingKeySet) !== JSON.stringify(savedKeySet);
         }
         let modelTabChanged = false;
         if (sanitizedModel && typeof sanitizedModel === "object") {
           const newPath = (sanitizedModel as { path?: string }).path;
-          const oldPath = config.model?.path;
+          const oldPath = getPrimaryModel(config)?.path;
           const newIsPlus =
             typeof newPath === "string" && newPath.startsWith("plus://");
           const oldIsPlus =

web/src/types/frigateConfig.ts

@@ -66,6 +66,7 @@ export interface CameraConfig {
     height: number;
     max_disappeared: number;
     min_initialized: number;
+    scene: string | null;
     stationary: {
       interval: number;
       max_frames: {
@@ -405,6 +406,32 @@ export type GenAIAgentConfig = {
   runtime_options?: Record<string, unknown>;
 };
 
+export type DetectionModelConfig = {
+  scene: string;
+  devices: string[];
+  height: number;
+  input_pixel_format: string;
+  input_tensor: string;
+  labelmap: Record<string, unknown>;
+  labelmap_path: string | null;
+  model_type: string;
+  path: string | null;
+  width: number;
+  colormap: { [key: string]: [number, number, number] };
+  attributes_map: { [key: string]: string[] };
+  all_attributes: string[];
+  plus?: {
+    name: string;
+    id: string;
+    trainDate: string;
+    baseModel: string;
+    isBaseModel: boolean;
+    supportedDetectors: string[];
+    width: number;
+    height: number;
+  } | null;
+};
+
 export interface FrigateConfig {
   version: string;
   safe_mode: boolean;
@@ -468,23 +495,6 @@ export interface FrigateConfig {
     width: number | null;
   };
 
-  detectors: {
-    coral: {
-      device: string;
-      model: {
-        height: number;
-        input_pixel_format: string;
-        input_tensor: string;
-        labelmap: Record<string, string>;
-        labelmap_path: string | null;
-        model_type: string;
-        path: string;
-        width: number;
-      };
-      type: string;
-    };
-  };
-
   environment_vars: Record<string, unknown>;
 
   face_recognition: FaceRecognitionConfig;
@@ -524,29 +534,7 @@ export interface FrigateConfig {
     logs: Record<string, string>;
   };
 
-  model: {
-    height: number;
-    input_pixel_format: string;
-    input_tensor: string;
-    labelmap: Record<string, unknown>;
-    labelmap_path: string | null;
-    model_type: string;
-    path: string | null;
-    width: number;
-    colormap: { [key: string]: [number, number, number] };
-    attributes_map: { [key: string]: string[] };
-    all_attributes: string[];
-    plus?: {
-      name: string;
-      id: string;
-      trainDate: string;
-      baseModel: string;
-      isBaseModel: boolean;
-      supportedDetectors: string[];
-      width: number;
-      height: number;
-    } | null;
-  };
+  models: DetectionModelConfig[];
 
   motion: Record<string, unknown> | null;
 

web/src/utils/configUtil.ts

@@ -493,6 +493,7 @@ export interface SectionSavePayload {
 // ---------------------------------------------------------------------------
 
 import { resolveAndCleanSchema } from "@/lib/config-schema";
+import { getAllAttributes } from "@/utils/modelUtil";
 
 type SchemaWithDefinitions = RJSFSchema & {
   $defs?: Record<string, RJSFSchema>;
@@ -796,7 +797,7 @@ export function getEffectiveAttributeLabels(
   fullCameraConfig: CameraConfig | undefined,
   level: "global" | "camera" | "replay" | undefined,
 ): string[] {
-  const all = fullConfig?.model?.all_attributes ?? [];
+  const all = getAllAttributes(fullConfig);
   if (level !== "global" && fullCameraConfig?.type === "lpr") {
     return all.filter((attr) => attr !== "license_plate");
   }

web/src/utils/iconUtil.tsx

@@ -56,8 +56,10 @@ export function getAttributeLabels(config?: FrigateConfig) {
 
   const labels = new Set();
 
-  Object.values(config.model.attributes_map).forEach((values) =>
-    values.forEach((label) => labels.add(label)),
+  config.models?.forEach((model) =>
+    Object.values(model.attributes_map ?? {}).forEach((values) =>
+      values.forEach((label) => labels.add(label)),
+    ),
   );
   return [...labels];
 }

web/src/utils/modelUtil.ts

@@ -0,0 +1,69 @@
+import { DetectionModelConfig, FrigateConfig } from "@/types/frigateConfig";
+
+/**
+ * The model a camera runs on, matched by the camera's detect scene.
+ *
+ * Falls back to the model for every scene, then to the only configured model,
+ * which is what the backend does when a camera does not name a scene.
+ */
+export function getModelForCamera(
+  config?: FrigateConfig,
+  camera?: string,
+): DetectionModelConfig | undefined {
+  const models = config?.models;
+
+  if (!models?.length) {
+    return undefined;
+  }
+
+  const scene = camera ? config?.cameras?.[camera]?.detect?.scene : undefined;
+
+  if (scene) {
+    const match = models.find((model) => model.scene == scene);
+
+    if (match) {
+      return match;
+    }
+  }
+
+  return models.find((model) => model.scene == "all") ?? models[0];
+}
+
+/** The model used when the question is not about a specific camera. */
+export function getPrimaryModel(
+  config?: FrigateConfig,
+): DetectionModelConfig | undefined {
+  return getModelForCamera(config);
+}
+
+/** Every object attribute across all configured models. */
+export function getAllAttributes(config?: FrigateConfig): string[] {
+  const attributes = new Set<string>();
+
+  config?.models?.forEach((model) =>
+    model.all_attributes?.forEach((attribute) => attributes.add(attribute)),
+  );
+
+  return [...attributes];
+}
+
+/** Whether a label is an attribute of any configured model. */
+export function isAttributeLabel(
+  config: FrigateConfig | undefined,
+  label: string,
+): boolean {
+  return !!config?.models?.some((model) =>
+    model.all_attributes?.includes(label),
+  );
+}
+
+/** Whether an attribute belongs to a parent label in any configured model. */
+export function isAttributeOfLabel(
+  config: FrigateConfig | undefined,
+  label: string,
+  attribute: string,
+): boolean {
+  return !!config?.models?.some((model) =>
+    model.attributes_map?.[label]?.includes(attribute),
+  );
+}

web/src/views/settings/DetectorsAndModelSettingsView.tsx

@@ -49,6 +49,7 @@ import {
 import { ConfigSectionTemplate } from "@/components/config-form/sections";
 import { ConfigMessageBanner } from "@/components/config-form/ConfigMessageBanner";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import { getPrimaryModel } from "@/utils/modelUtil";
 import {
   buildHiddenFieldContext,
   getSectionConfig,
@@ -115,8 +116,9 @@ const STATUS_BAR_KEY = "detectors_and_model";
 const EMPTY_PENDING: Record<string, ConfigSectionData> = {};
 
 const deriveInitialState = (config: FrigateConfig): PageState => {
-  const plusModelId = config.model?.plus?.id;
-  const modelPath = config.model?.path;
+  const primaryModel = getPrimaryModel(config);
+  const plusModelId = primaryModel?.plus?.id;
+  const modelPath = primaryModel?.path;
   const plusEnabled = Boolean(config.plus?.enabled);
 
   // The reliable signal that a Plus model is currently active is the
@@ -136,10 +138,12 @@ const deriveInitialState = (config: FrigateConfig): PageState => {
     modelTab = "custom";
   }
 
-  const { plus: _plus, ...modelWithoutPlus } = (config.model ?? {}) as Record<
-    string,
-    unknown
-  >;
+  const {
+    plus: _plus,
+    scene: _scene,
+    devices: _devices,
+    ...modelWithoutPlus
+  } = (primaryModel ?? {}) as Record<string, unknown>;
   // If a Plus model is active, the resolved `model.path` is auto-derived from
   // `plus.id` — drop it so the Custom tab starts clean and doesn't silently
   // re-save the same Plus model when the user thinks they switched modes.
@@ -148,7 +152,7 @@ const deriveInitialState = (config: FrigateConfig): PageState => {
   }
 
   return {
-    detectors: (config.detectors ?? {}) as ConfigSectionData,
+    detectors: { devices: primaryModel?.devices ?? [] } as ConfigSectionData,
     modelTab,
     plusModelId: plusModelId ?? undefined,
     customModel: modelWithoutPlus as ConfigSectionData,

web/src/views/settings/FrigatePlusSettingsView.tsx

@@ -17,6 +17,7 @@ import { CameraNameLabel } from "@/components/camera/FriendlyNameLabel";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { isReplayCamera } from "@/utils/cameraUtil";
 import type { SettingsPageProps } from "@/views/settings/SingleSectionPage";
+import { getPrimaryModel } from "@/utils/modelUtil";
 
 export default function FrigatePlusSettingsView(_props: SettingsPageProps) {
   const { t } = useTranslation("views/settings");
@@ -51,7 +52,7 @@ export default function FrigatePlusSettingsView(_props: SettingsPageProps) {
               description={
                 <>
                   <p>{t("frigatePlus.apiKey.desc")}</p>
-                  {!config?.model.plus && (
+                  {!getPrimaryModel(config)?.plus && (
                     <div className="mt-2 flex items-center text-primary-variant">
                       <Link
                         to="https://frigate.video/plus"
@@ -85,7 +86,7 @@ export default function FrigatePlusSettingsView(_props: SettingsPageProps) {
 
           {config?.plus?.enabled && (
             <FrigatePlusCurrentModelSummary
-              plusModel={config.model.plus}
+              plusModel={getPrimaryModel(config)?.plus}
               action={
                 <Button
                   size="sm"

web/src/views/settings/ObjectSettingsView.tsx

@@ -34,6 +34,7 @@ import { useCameraFriendlyName } from "@/hooks/use-camera-friendly-name";
 import { AudioLevelGraph } from "@/components/audio/AudioLevelGraph";
 import { useWs } from "@/api/ws";
 import { cn } from "@/lib/utils";
+import { getPrimaryModel } from "@/utils/modelUtil";
 
 type ObjectSettingsViewProps = {
   selectedCamera?: string;
@@ -172,11 +173,10 @@ export default function ObjectSettingsView({
         <div className="mb-5 space-y-3 text-sm text-muted-foreground">
           <p>
             {t("debug.detectorDesc", {
-              detectors: config
-                ? Object.keys(config?.detectors)
-                    .map((detector) => capitalizeFirstLetter(detector))
-                    .join(",")
-                : "",
+              detectors: (config?.models ?? [])
+                .flatMap((model) => model.devices ?? [])
+                .map((device) => capitalizeFirstLetter(device))
+                .join(","),
             })}
           </p>
           <p>{t("debug.desc")}</p>
@@ -380,7 +380,7 @@ function ObjectList({ cameraConfig, objects }: ObjectListProps) {
       return;
     }
 
-    return config.model?.colormap;
+    return getPrimaryModel(config)?.colormap;
   }, [config]);
 
   const getColorForObjectName = useCallback(

web/src/views/settings/components/FrigatePlusCurrentModelSummary.tsx

@@ -3,11 +3,11 @@ import {
   SettingsGroupCard,
   SplitCardRow,
 } from "@/components/card/SettingsGroupCard";
-import type { FrigateConfig } from "@/types/frigateConfig";
+import type { DetectionModelConfig } from "@/types/frigateConfig";
 import { useTranslation } from "react-i18next";
 
 type FrigatePlusCurrentModelSummaryProps = {
-  plusModel: FrigateConfig["model"]["plus"];
+  plusModel: DetectionModelConfig["plus"];
   action?: ReactNode;
 };