administrator/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-07-30 23:59:02 +03:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Implement multi-model object detection

Browse Source
This commit is contained in:
Nicolas Mowen
2026-07-23 17:51:21 -06:00
parent e3fa701893
commit ede06d794d
49 changed files with 1070 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/e2e/fixtures/mock-data/cases.json
+1 -1
View File
@@ -1 +1 @@
[{"id": "case-001", "name": "Package Theft Investigation", "description": "Review of suspicious activity near the front porch", "created_at": 1780597809.365581, "updated_at": 1780673409.365581}]
[{"id": "case-001", "name": "Package Theft Investigation", "description": "Review of suspicious activity near the front porch", "created_at": 1784761296.1184616, "updated_at": 1784836896.1184616}]
web/e2e/fixtures/mock-data/config-schema.json
+1 -1
View File
File diff suppressed because one or more lines are too long
web/e2e/fixtures/mock-data/config-snapshot.json
+1 -1
View File
File diff suppressed because one or more lines are too long
web/e2e/fixtures/mock-data/events.json
+1 -1
View File
@@ -1 +1 @@
[{"id": "event-person-001", "label": "person", "sub_label": null, "camera": "front_door", "start_time": 1780677009.365581, "end_time": 1780677039.365581, "false_positive": false, "zones": ["front_yard"], "thumbnail": null, "has_clip": true, "has_snapshot": true, "retain_indefinitely": false, "plus_id": null, "model_hash": "abc123", "detector_type": "cpu", "model_type": "ssd", "data": {"top_score": 0.92, "score": 0.92, "region": [0.1, 0.1, 0.5, 0.8], "box": [0.2, 0.15, 0.45, 0.75], "area": 0.18, "ratio": 0.6, "type": "object", "description": "A person walking toward the front door", "average_estimated_speed": 1.2, "velocity_angle": 45.0, "path_data": [[[0.2, 0.5], 0.0], [[0.3, 0.5], 1.0]]}}, {"id": "event-car-001", "label": "car", "sub_label": null, "camera": "backyard", "start_time": 1780673409.365581, "end_time": 1780673454.365581, "false_positive": false, "zones": ["driveway"], "thumbnail": null, "has_clip": true, "has_snapshot": true, "retain_indefinitely": false, "plus_id": null, "model_hash": "def456", "detector_type": "cpu", "model_type": "ssd", "data": {"top_score": 0.87, "score": 0.87, "region": [0.3, 0.2, 0.9, 0.7], "box": [0.35, 0.25, 0.85, 0.65], "area": 0.2, "ratio": 1.25, "type": "object", "description": "A car parked in the driveway", "average_estimated_speed": 0.0, "velocity_angle": 0.0, "path_data": []}}, {"id": "event-person-002", "label": "person", "sub_label": null, "camera": "garage", "start_time": 1780669809.365581, "end_time": 1780669829.365581, "false_positive": false, "zones": [], "thumbnail": null, "has_clip": false, "has_snapshot": true, "retain_indefinitely": false, "plus_id": null, "model_hash": "ghi789", "detector_type": "cpu", "model_type": "ssd", "data": {"top_score": 0.78, "score": 0.78, "region": [0.0, 0.0, 0.6, 0.9], "box": [0.1, 0.05, 0.5, 0.85], "area": 0.32, "ratio": 0.5, "type": "object", "description": null, "average_estimated_speed": 0.5, "velocity_angle": 90.0, "path_data": [[[0.1, 0.4], 0.0]]}}]
[{"id": "event-person-001", "label": "person", "sub_label": null, "camera": "front_door", "start_time": 1784840496.1184616, "end_time": 1784840526.1184616, "false_positive": false, "zones": ["front_yard"], "thumbnail": null, "has_clip": true, "has_snapshot": true, "retain_indefinitely": false, "plus_id": null, "model_hash": "abc123", "detector_type": "cpu", "model_type": "ssd", "data": {"top_score": 0.92, "score": 0.92, "region": [0.1, 0.1, 0.5, 0.8], "box": [0.2, 0.15, 0.45, 0.75], "area": 0.18, "ratio": 0.6, "type": "object", "description": "A person walking toward the front door", "average_estimated_speed": 1.2, "velocity_angle": 45.0, "path_data": [[[0.2, 0.5], 0.0], [[0.3, 0.5], 1.0]]}}, {"id": "event-car-001", "label": "car", "sub_label": null, "camera": "backyard", "start_time": 1784836896.1184616, "end_time": 1784836941.1184616, "false_positive": false, "zones": ["driveway"], "thumbnail": null, "has_clip": true, "has_snapshot": true, "retain_indefinitely": false, "plus_id": null, "model_hash": "def456", "detector_type": "cpu", "model_type": "ssd", "data": {"top_score": 0.87, "score": 0.87, "region": [0.3, 0.2, 0.9, 0.7], "box": [0.35, 0.25, 0.85, 0.65], "area": 0.2, "ratio": 1.25, "type": "object", "description": "A car parked in the driveway", "average_estimated_speed": 0.0, "velocity_angle": 0.0, "path_data": []}}, {"id": "event-person-002", "label": "person", "sub_label": null, "camera": "garage", "start_time": 1784833296.1184616, "end_time": 1784833316.1184616, "false_positive": false, "zones": [], "thumbnail": null, "has_clip": false, "has_snapshot": true, "retain_indefinitely": false, "plus_id": null, "model_hash": "ghi789", "detector_type": "cpu", "model_type": "ssd", "data": {"top_score": 0.78, "score": 0.78, "region": [0.0, 0.0, 0.6, 0.9], "box": [0.1, 0.05, 0.5, 0.85], "area": 0.32, "ratio": 0.5, "type": "object", "description": null, "average_estimated_speed": 0.5, "velocity_angle": 90.0, "path_data": [[[0.1, 0.4], 0.0]]}}]
web/e2e/fixtures/mock-data/exports.json
+1 -1
View File
@@ -1 +1 @@
[{"id": "export-001", "camera": "front_door", "name": "Front Door - Person Alert", "date": 1780680609.365581, "video_path": "/exports/export-001.mp4", "thumb_path": "/exports/export-001-thumb.jpg", "in_progress": false, "export_case_id": null}, {"id": "export-002", "camera": "backyard", "name": "Backyard - Car Detection", "date": 1780673409.365581, "video_path": "/exports/export-002.mp4", "thumb_path": "/exports/export-002-thumb.jpg", "in_progress": false, "export_case_id": "case-001"}, {"id": "export-003", "camera": "garage", "name": "Garage - In Progress", "date": 1780682409.365581, "video_path": "/exports/export-003.mp4", "thumb_path": "/exports/export-003-thumb.jpg", "in_progress": true, "export_case_id": null}]
[{"id": "export-001", "camera": "front_door", "name": "Front Door - Person Alert", "date": 1784844096.1184616, "video_path": "/exports/export-001.mp4", "thumb_path": "/exports/export-001-thumb.jpg", "in_progress": false, "export_case_id": null}, {"id": "export-002", "camera": "backyard", "name": "Backyard - Car Detection", "date": 1784836896.1184616, "video_path": "/exports/export-002.mp4", "thumb_path": "/exports/export-002-thumb.jpg", "in_progress": false, "export_case_id": "case-001"}, {"id": "export-003", "camera": "garage", "name": "Garage - In Progress", "date": 1784845896.1184616, "video_path": "/exports/export-003.mp4", "thumb_path": "/exports/export-003-thumb.jpg", "in_progress": true, "export_case_id": null}]
web/e2e/fixtures/mock-data/generate-mock-data.py
+11 -5
View File
@@ -102,11 +102,17 @@ def generate_config():
snapshot = config.model_dump()
# Runtime-computed fields not in the Pydantic dump
all_attrs = set()
for attrs in snapshot.get("model", {}).get("attributes_map", {}).values():
all_attrs.update(attrs)
snapshot["model"]["all_attributes"] = sorted(all_attrs)
snapshot["model"]["colormap"] = {}
for model_dict in snapshot.get("models", {}).values():
all_attrs = set()
for attrs in model_dict.get("attributes_map", {}).values():
all_attrs.update(attrs)
model_dict["all_attributes"] = sorted(all_attrs)
model_dict["colormap"] = {}
# legacy single-model block mirrors the default model, matching /api/config
models = snapshot.get("models", {})
default_key = "default" if "default" in models else next(iter(models))
snapshot["model"] = models[default_key]
return snapshot
web/e2e/fixtures/mock-data/review-summary.json
+1 -1
View File
@@ -1 +1 @@
{"2026-06-05": {"day": "2026-06-05", "reviewed_alert": 1, "reviewed_detection": 0, "total_alert": 2, "total_detection": 2}, "2026-06-04": {"day": "2026-06-04", "reviewed_alert": 3, "reviewed_detection": 2, "total_alert": 3, "total_detection": 4}}
{"2026-07-23": {"day": "2026-07-23", "reviewed_alert": 1, "reviewed_detection": 0, "total_alert": 2, "total_detection": 2}, "2026-07-22": {"day": "2026-07-22", "reviewed_alert": 3, "reviewed_detection": 2, "total_alert": 3, "total_detection": 4}}
web/e2e/fixtures/mock-data/reviews.json
+1 -1
View File
@@ -1 +1 @@
[{"id": "review-alert-001", "camera": "front_door", "start_time": "2026-06-05T11:30:09.365581", "end_time": "2026-06-05T11:30:39.365581", "has_been_reviewed": false, "severity": "alert", "thumb_path": "/clips/front_door/review-alert-001-thumb.jpg", "data": {"audio": [], "detections": ["person-abc123"], "objects": ["person"], "sub_labels": [], "significant_motion_areas": [], "zones": ["front_yard"]}}, {"id": "review-alert-002", "camera": "backyard", "start_time": "2026-06-05T10:30:09.365581", "end_time": "2026-06-05T10:30:54.365581", "has_been_reviewed": true, "severity": "alert", "thumb_path": "/clips/backyard/review-alert-002-thumb.jpg", "data": {"audio": [], "detections": ["car-def456"], "objects": ["car"], "sub_labels": [], "significant_motion_areas": [], "zones": ["driveway"]}}, {"id": "review-detect-001", "camera": "garage", "start_time": "2026-06-05T09:30:09.365581", "end_time": "2026-06-05T09:30:29.365581", "has_been_reviewed": false, "severity": "detection", "thumb_path": "/clips/garage/review-detect-001-thumb.jpg", "data": {"audio": [], "detections": ["person-ghi789"], "objects": ["person"], "sub_labels": [], "significant_motion_areas": [], "zones": []}}, {"id": "review-detect-002", "camera": "front_door", "start_time": "2026-06-05T08:30:09.365581", "end_time": "2026-06-05T08:30:24.365581", "has_been_reviewed": false, "severity": "detection", "thumb_path": "/clips/front_door/review-detect-002-thumb.jpg", "data": {"audio": [], "detections": ["car-jkl012"], "objects": ["car"], "sub_labels": [], "significant_motion_areas": [], "zones": ["front_yard"]}}]
[{"id": "review-alert-001", "camera": "front_door", "start_time": "2026-07-23T21:01:36.118462", "end_time": "2026-07-23T21:02:06.118462", "has_been_reviewed": false, "severity": "alert", "thumb_path": "/clips/front_door/review-alert-001-thumb.jpg", "data": {"audio": [], "detections": ["person-abc123"], "objects": ["person"], "sub_labels": [], "significant_motion_areas": [], "zones": ["front_yard"]}}, {"id": "review-alert-002", "camera": "backyard", "start_time": "2026-07-23T20:01:36.118462", "end_time": "2026-07-23T20:02:21.118462", "has_been_reviewed": true, "severity": "alert", "thumb_path": "/clips/backyard/review-alert-002-thumb.jpg", "data": {"audio": [], "detections": ["car-def456"], "objects": ["car"], "sub_labels": [], "significant_motion_areas": [], "zones": ["driveway"]}}, {"id": "review-detect-001", "camera": "garage", "start_time": "2026-07-23T19:01:36.118462", "end_time": "2026-07-23T19:01:56.118462", "has_been_reviewed": false, "severity": "detection", "thumb_path": "/clips/garage/review-detect-001-thumb.jpg", "data": {"audio": [], "detections": ["person-ghi789"], "objects": ["person"], "sub_labels": [], "significant_motion_areas": [], "zones": []}}, {"id": "review-detect-002", "camera": "front_door", "start_time": "2026-07-23T18:01:36.118462", "end_time": "2026-07-23T18:01:51.118462", "has_been_reviewed": false, "severity": "detection", "thumb_path": "/clips/front_door/review-detect-002-thumb.jpg", "data": {"audio": [], "detections": ["car-jkl012"], "objects": ["car"], "sub_labels": [], "significant_motion_areas": [], "zones": ["front_yard"]}}]
web/public/locales/en/config/cameras.json
+4
View File
@@ -86,6 +86,10 @@
"label": "Enable object detection",
"description": "Enable or disable object detection for this camera."
},
"model": {
"label": "Detection model name",
"description": "Name of the model (key under `models`) used by this camera. Defaults to the only defined model, or the model named 'default'."
},
"height": {
"label": "Detect height",
"description": "Height (pixels) of frames used for the detect stream; leave empty to use the native stream resolution."
web/public/locales/en/config/global.json
+11 -3
View File
@@ -329,6 +329,10 @@
"label": "Detector specific model path",
"description": "File path to the detector model binary if required by the chosen detector."
},
"model_key": {
"label": "Assigned model name",
"description": "Name of the model (key under `models`) this detector instance serves. Set automatically at runtime, users should not set this."
},
"axengine": {
"label": "AXEngine NPU",
"description": "AXERA AX650N/AX8850N NPU detector running compiled .axmodel files via the AXEngine runtime."
@@ -454,9 +458,9 @@
}
}
},
"model": {
"label": "Detection model",
"description": "Settings to configure a custom object detection model and its input shape.",
"models": {
"label": "Detection models",
"description": "Named object detection models. Cameras select a model with detect.model; detectors are assigned to models automatically.",
"path": {
"label": "Custom object detector model path",
"description": "Path to a custom detection model file (or plus://<model_id> for Frigate+ models)."
@@ -625,6 +629,10 @@
"label": "Enable object detection",
"description": "Enable or disable object detection for all cameras; can be overridden per-camera."
},
"model": {
"label": "Detection model name",
"description": "Name of the model (key under `models`) used by this camera. Defaults to the only defined model, or the model named 'default'."
},
"height": {
"label": "Detect height",
"description": "Height (pixels) of frames used for the detect stream; leave empty to use the native stream resolution."
web/src/components/config-form/sections/CameraOverridesBadge.tsx
+1
View File
@@ -74,6 +74,7 @@ const SECTIONS_WITHOUT_OVERRIDE_BADGE = new Set([
"birdseye",
"detectors",
"model",
"models",
]);
type CameraEntryProps = {
web/src/hooks/use-config-schema.ts
+11 -1
View File
@@ -70,7 +70,17 @@ export function extractSectionSchema(
// For global level, get from root properties
if (schemaObj.properties) {
const props = schemaObj.properties;
const sectionProp = props[sectionPath];
let sectionProp = props[sectionPath];
// the model editor edits a single entry of the `models` map, so
// resolve the map value schema since there is no root `model` property
if (!sectionProp && sectionPath === "model") {
const modelsProp = props["models"] as SchemaWithDefinitions | undefined;
const additional = modelsProp?.additionalProperties;
if (additional && typeof additional === "object") {
sectionProp = additional as RJSFSchema;
}
}
if (sectionProp && typeof sectionProp === "object") {
if ("$ref" in sectionProp && typeof sectionProp.$ref === "string") {
web/src/types/frigateConfig.ts
+30 -23
View File
@@ -59,6 +59,7 @@ export interface CameraConfig {
height: number;
max_disappeared: number;
min_initialized: number;
model: string;
stationary: {
interval: number;
max_frames: {
@@ -394,6 +395,30 @@ export type GenAIAgentConfig = {
runtime_options?: Record<string, unknown>;
};
export interface ModelConfig {
height: number;
input_pixel_format: string;
input_tensor: string;
labelmap: Record<string, unknown>;
labelmap_path: string | null;
model_type: string;
path: string | null;
width: number;
colormap: { [key: string]: [number, number, number] };
attributes_map: { [key: string]: string[] };
all_attributes: string[];
plus?: {
name: string;
id: string;
trainDate: string;
baseModel: string;
isBaseModel: boolean;
supportedDetectors: string[];
width: number;
height: number;
} | null;
}
export interface FrigateConfig {
version: string;
safe_mode: boolean;
@@ -446,6 +471,7 @@ export interface FrigateConfig {
height: number | null;
max_disappeared: number | null;
min_initialized: number | null;
model: string | null;
stationary: {
interval: number | null;
max_frames: {
@@ -512,29 +538,10 @@ export interface FrigateConfig {
logs: Record<string, string>;
};
model: {
height: number;
input_pixel_format: string;
input_tensor: string;
labelmap: Record<string, unknown>;
labelmap_path: string | null;
model_type: string;
path: string | null;
width: number;
colormap: { [key: string]: [number, number, number] };
attributes_map: { [key: string]: string[] };
all_attributes: string[];
plus?: {
name: string;
id: string;
trainDate: string;
baseModel: string;
isBaseModel: boolean;
supportedDetectors: string[];
width: number;
height: number;
} | null;
};
// legacy single-model block, mirrors the default entry of `models`
model: ModelConfig;
models: { [modelKey: string]: ModelConfig };
motion: Record<string, unknown> | null;
web/src/views/settings/DetectorsAndModelSettingsView.tsx
+9 -9
View File
@@ -115,8 +115,10 @@ const STATUS_BAR_KEY = "detectors_and_model";
const EMPTY_PENDING: Record<string, ConfigSectionData> = {};
const deriveInitialState = (config: FrigateConfig): PageState => {
const plusModelId = config.model?.plus?.id;
const modelPath = config.model?.path;
// this view edits the default model; other named models are untouched
const defaultModel = config.models?.default ?? config.model;
const plusModelId = defaultModel?.plus?.id;
const modelPath = defaultModel?.path;
const plusEnabled = Boolean(config.plus?.enabled);
// The reliable signal that a Plus model is currently active is the
@@ -136,10 +138,8 @@ const deriveInitialState = (config: FrigateConfig): PageState => {
modelTab = "custom";
}
const { plus: _plus, ...modelWithoutPlus } = (config.model ?? {}) as Record<
string,
unknown
>;
const { plus: _plus, ...modelWithoutPlus } = (defaultModel ??
{}) as unknown as Record<string, unknown>;
// If a Plus model is active, the resolved `model.path` is auto-derived from
// `plus.id` — drop it so the Custom tab starts clean and doesn't silently
// re-save the same Plus model when the user thinks they switched modes.
@@ -476,7 +476,7 @@ export default function DetectorsAndModelSettingsView({
try {
await axios.put("config/set", {
requires_restart: 0,
config_data: { detectors: null, model: null },
config_data: { detectors: null, models: { default: null } },
});
preCleared = true;
} catch {
@@ -488,7 +488,7 @@ export default function DetectorsAndModelSettingsView({
requires_restart: 0,
config_data: {
detectors: sanitizedDetectors,
model: modelPayload,
models: { default: modelPayload },
},
});
@@ -541,7 +541,7 @@ export default function DetectorsAndModelSettingsView({
snapshot.detectors,
detectorHiddenFields,
),
model: restoreModel,
models: { default: restoreModel },
},
});
} catch {