Review labels widget (#22664)

* add review labels widget

* register widget and add to review section

* i18n

* add border to switches widget

* padding tweaks

* don't show audio labels if audio is not enabled

* add docs links

* ability to add custom labels to review

* add hint for empty selection in review labels and SwitchesWidget

* language consistency

frigate/comms/webpush.py

@@ -542,9 +542,9 @@ class WebPushClient(Communicator):
 
         self.check_registrations()
 
-        reasoning: str = payload.get("reasoning", "")
+        text: str = payload.get("message") or payload.get("reasoning", "")
         title = f"{camera_name}: Monitoring Alert"
-        message = (reasoning[:197] + "...") if len(reasoning) > 200 else reasoning
+        message = (text[:197] + "...") if len(text) > 200 else text
 
         logger.debug(f"Sending camera monitoring push notification for {camera_name}")
 

frigate/config/camera/review.py

@@ -188,7 +188,7 @@ class ReviewConfig(FrigateBaseModel):
     detections: DetectionsConfig = Field(
         default_factory=DetectionsConfig,
         title="Detections config",
-        description="Settings for creating detection events (non-alert) and how long to keep them.",
+        description="Settings for which tracked objects generate detections (non-alert) and how detections are retained.",
     )
     genai: GenAIReviewConfig = Field(
         default_factory=GenAIReviewConfig,

frigate/genai/gemini.py

@@ -50,9 +50,9 @@ class GeminiClient(GenAIClient):
         response_format: Optional[dict] = None,
     ) -> Optional[str]:
         """Submit a request to Gemini."""
-        contents = [
+        contents = [prompt] + [
             types.Part.from_bytes(data=img, mime_type="image/jpeg") for img in images
-        ] + [prompt]
+        ]
         try:
             # Merge runtime_options into generation_config if provided
             generation_config_dict: dict[str, Any] = {"candidate_count": 1}

frigate/genai/openai.py

@@ -44,7 +44,12 @@ class OpenAIClient(GenAIClient):
     ) -> Optional[str]:
         """Submit a request to OpenAI."""
         encoded_images = [base64.b64encode(image).decode("utf-8") for image in images]
-        messages_content = []
+        messages_content: list[dict] = [
+            {
+                "type": "text",
+                "text": prompt,
+            }
+        ]
         for image in encoded_images:
             messages_content.append(
                 {
@@ -55,12 +60,6 @@ class OpenAIClient(GenAIClient):
                     },
                 }
             )
-        messages_content.append(
-            {
-                "type": "text",
-                "text": prompt,
-            }
-        )
         try:
             request_params = {
                 "model": self.genai_config.model,

frigate/jobs/vlm_watch.py

@@ -25,6 +25,9 @@ logger = logging.getLogger(__name__)
 _MIN_INTERVAL = 1
 _MAX_INTERVAL = 300
 
+# Minimum seconds between VLM iterations when woken by detections (no zone filter)
+_DETECTION_COOLDOWN_WITHOUT_ZONE = 10
+
 # Max user/assistant turn pairs to keep in conversation history
 _MAX_HISTORY = 10
 
@@ -40,6 +43,7 @@ class VLMWatchJob(Job):
     labels: list = field(default_factory=list)
     zones: list = field(default_factory=list)
     last_reasoning: str = ""
+    notification_message: str = ""
     iteration_count: int = 0
 
     def to_dict(self) -> dict[str, Any]:
@@ -196,6 +200,7 @@ class VLMWatchRunner(threading.Thread):
                 min(_MAX_INTERVAL, int(parsed.get("next_run_in", 30))),
             )
             reasoning = str(parsed.get("reasoning", ""))
+            notification_message = str(parsed.get("notification_message", ""))
         except (json.JSONDecodeError, ValueError, TypeError) as e:
             logger.warning(
                 "VLM watch job %s: failed to parse VLM response: %s", self.job.id, e
@@ -203,6 +208,7 @@ class VLMWatchRunner(threading.Thread):
             return 30
 
         self.job.last_reasoning = reasoning
+        self.job.notification_message = notification_message
         self.job.iteration_count += 1
         self._broadcast_status()
 
@@ -213,19 +219,35 @@ class VLMWatchRunner(threading.Thread):
                 self.job.camera,
                 reasoning,
             )
-            self._send_notification(reasoning)
+            self._send_notification(notification_message or reasoning)
             self.job.status = JobStatusTypesEnum.success
             return 0
 
         return next_run_in
 
     def _wait_for_trigger(self, max_wait: float) -> None:
-        """Wait up to max_wait seconds, returning early if a relevant detection fires on the target camera."""
-        deadline = time.time() + max_wait
+        """Wait up to max_wait seconds, returning early if a relevant detection fires on the target camera.
+
+        With zones configured, a matching detection wakes immediately (events
+        are already filtered).  Without zones, detections are frequent so a
+        cooldown is enforced: messages are continuously drained to prevent
+        queue backup, but the loop only exits once a match has been seen
+        *and* the cooldown period has elapsed.
+        """
+        now = time.time()
+        deadline = now + max_wait
+        use_cooldown = not self.job.zones
+        earliest_wake = now + _DETECTION_COOLDOWN_WITHOUT_ZONE if use_cooldown else 0
+        triggered = False
+
         while not self.cancel_event.is_set():
             remaining = deadline - time.time()
             if remaining <= 0:
                 break
+
+            if triggered and time.time() >= earliest_wake:
+                break
+
             result = self.detection_subscriber.check_for_update(
                 timeout=min(1.0, remaining)
             )
@@ -250,6 +272,7 @@ class VLMWatchRunner(threading.Thread):
             if cam != self.job.camera or not tracked_objects:
                 continue
             if self._detection_matches_filters(tracked_objects):
+                if not use_cooldown:
                     logger.debug(
                         "VLM watch job %s: woken early by detection event on %s",
                         self.job.id,
@@ -257,6 +280,15 @@ class VLMWatchRunner(threading.Thread):
                     )
                     break
 
+                if not triggered:
+                    logger.debug(
+                        "VLM watch job %s: detection match on %s, draining for %.0fs",
+                        self.job.id,
+                        self.job.camera,
+                        max(0, earliest_wake - time.time()),
+                    )
+                    triggered = True
+
     def _detection_matches_filters(self, tracked_objects: list) -> bool:
         """Return True if any tracked object passes the label and zone filters."""
         labels = self.job.labels
@@ -284,7 +316,11 @@ class VLMWatchRunner(threading.Thread):
             f"You will receive a sequence of frames over time. Use the conversation history to understand "
             f"what is stationary vs. actively changing.\n\n"
             f"For each frame respond with JSON only:\n"
-            f'{{"condition_met": <true/false>, "next_run_in": <integer seconds 1-300>, "reasoning": "<brief explanation>"}}\n\n'
+            f'{{"condition_met": <true/false>, "next_run_in": <integer seconds 1-300>, "reasoning": "<brief explanation>", "notification_message": "<natural language notification>"}}\n\n'
+            f"Guidelines for notification_message:\n"
+            f"- Only required when condition_met is true.\n"
+            f"- Write a short, natural notification a user would want to receive on their phone.\n"
+            f'- Example: "Your package has been delivered to the front porch."\n\n'
             f"Guidelines for next_run_in:\n"
             f"- Scene is empty / nothing of interest visible: 60-300.\n"
             f"- Relevant object(s) visible anywhere in frame (even outside the target zone): 3-10. "
@@ -294,12 +330,13 @@ class VLMWatchRunner(threading.Thread):
             f"- Keep reasoning to 1-2 sentences."
         )
 
-    def _send_notification(self, reasoning: str) -> None:
+    def _send_notification(self, message: str) -> None:
         """Publish a camera_monitoring event so downstream handlers (web push, MQTT) can notify users."""
         payload = {
             "camera": self.job.camera,
             "condition": self.job.condition,
-            "reasoning": reasoning,
+            "message": message,
+            "reasoning": self.job.last_reasoning,
             "job_id": self.job.id,
         }
 

frigate/stats/util.py

@@ -19,6 +19,7 @@ from frigate.types import StatsTrackingTypes
 from frigate.util.services import (
     calculate_shm_requirements,
     get_amd_gpu_stats,
+    get_axcl_npu_stats,
     get_bandwidth_stats,
     get_cpu_stats,
     get_fs_type,
@@ -324,6 +325,10 @@ async def set_npu_usages(config: FrigateConfig, all_stats: dict[str, Any]) -> No
             # OpenVINO NPU usage
             ov_usage = get_openvino_npu_stats()
             stats["openvino"] = ov_usage
+        elif detector.type == "axengine":
+            # AXERA NPU usage
+            axcl_usage = get_axcl_npu_stats()
+            stats["axengine"] = axcl_usage
 
     if stats:
         all_stats["npu_usages"] = stats

frigate/util/services.py

@@ -488,6 +488,43 @@ def get_rockchip_npu_stats() -> Optional[dict[str, float | str]]:
     return stats
 
 
+def get_axcl_npu_stats() -> Optional[dict[str, str | float]]:
+    """Get NPU stats using axcl."""
+    # Check if axcl-smi exists
+    axcl_smi_path = "/usr/bin/axcl/axcl-smi"
+    if not os.path.exists(axcl_smi_path):
+        return None
+
+    try:
+        # Run axcl-smi command to get NPU stats
+        axcl_command = [axcl_smi_path, "sh", "cat", "/proc/ax_proc/npu/top"]
+        p = sp.run(
+            axcl_command,
+            capture_output=True,
+            text=True,
+        )
+
+        if p.returncode != 0:
+            pass
+        else:
+            utilization = None
+
+            for line in p.stdout.strip().splitlines():
+                line = line.strip()
+                if line.startswith("utilization:"):
+                    match = re.search(r"utilization:(\d+)%", line)
+                    if match:
+                        utilization = float(match.group(1))
+
+            if utilization is not None:
+                stats: dict[str, str | float] = {"npu": utilization, "mem": "-%"}
+                return stats
+    except Exception:
+        pass
+
+    return None
+
+
 def try_get_info(f, h, default="N/A", sensor=None):
     try:
         if h:

web/public/locales/en/config/cameras.json

@@ -529,7 +529,7 @@
     },
     "detections": {
       "label": "Detections config",
-      "description": "Settings for creating detection events (non-alert) and how long to keep them.",
+      "description": "Settings for which tracked objects generate detections (non-alert) and how detections are retained.",
       "enabled": {
         "label": "Enable detections",
         "description": "Enable or disable detection events for this camera."

web/public/locales/en/config/global.json

@@ -1044,7 +1044,7 @@
     },
     "detections": {
       "label": "Detections config",
-      "description": "Settings for creating detection events (non-alert) and how long to keep them.",
+      "description": "Settings for which tracked objects generate detections (non-alert) and how detections are retained.",
       "enabled": {
         "label": "Enable detections",
         "description": "Enable or disable detection events for all cameras; can be overridden per-camera."

web/public/locales/en/views/settings.json

@@ -1431,6 +1431,11 @@
       "summary": "{{count}} object types selected",
       "empty": "No object labels available"
     },
+    "reviewLabels": {
+      "summary": "{{count}} labels selected",
+      "empty": "No labels available",
+      "allNonAlertDetections": "All non-alert activity will be included as detections."
+    },
     "filters": {
       "objectFieldLabel": "{{field}} for {{label}}"
     },
@@ -1474,7 +1479,8 @@
     "timestamp_style": {
       "title": "Timestamp Settings"
     },
-    "searchPlaceholder": "Search..."
+    "searchPlaceholder": "Search...",
+    "addCustomLabel": "Add custom label..."
   },
   "globalConfig": {
     "title": "Global Configuration",

web/src/components/config-form/section-configs/review.ts

@@ -3,14 +3,16 @@ import type { SectionConfigOverrides } from "./types";
 const review: SectionConfigOverrides = {
   base: {
     sectionDocs: "/configuration/review",
+    fieldDocs: {
+      "alerts.labels": "/configuration/review/#alerts-and-detections",
+      "detections.labels": "/configuration/review/#alerts-and-detections",
+    },
     restartRequired: [],
     fieldOrder: ["alerts", "detections", "genai"],
     fieldGroups: {},
     hiddenFields: [
       "enabled_in_config",
-      "alerts.labels",
       "alerts.enabled_in_config",
-      "detections.labels",
       "detections.enabled_in_config",
       "genai.enabled_in_config",
     ],
@@ -18,11 +20,25 @@ const review: SectionConfigOverrides = {
     uiSchema: {
       alerts: {
         "ui:before": { render: "CameraReviewStatusToggles" },
+        labels: {
+          "ui:widget": "reviewLabels",
+          "ui:options": {
+            suppressMultiSchema: true,
+          },
+        },
         required_zones: {
           "ui:widget": "hidden",
         },
       },
       detections: {
+        labels: {
+          "ui:widget": "reviewLabels",
+          "ui:options": {
+            suppressMultiSchema: true,
+            emptySelectionHintKey:
+              "configForm.reviewLabels.allNonAlertDetections",
+          },
+        },
         required_zones: {
           "ui:widget": "hidden",
         },

web/src/components/config-form/theme/frigateTheme.ts

@@ -20,6 +20,7 @@ import { TextareaWidget } from "./widgets/TextareaWidget";
 import { SwitchesWidget } from "./widgets/SwitchesWidget";
 import { ObjectLabelSwitchesWidget } from "./widgets/ObjectLabelSwitchesWidget";
 import { AudioLabelSwitchesWidget } from "./widgets/AudioLabelSwitchesWidget";
+import { ReviewLabelSwitchesWidget } from "./widgets/ReviewLabelSwitchesWidget";
 import { ZoneSwitchesWidget } from "./widgets/ZoneSwitchesWidget";
 import { ArrayAsTextWidget } from "./widgets/ArrayAsTextWidget";
 import { FfmpegArgsWidget } from "./widgets/FfmpegArgsWidget";
@@ -76,6 +77,7 @@ export const frigateTheme: FrigateTheme = {
     switches: SwitchesWidget,
     objectLabels: ObjectLabelSwitchesWidget,
     audioLabels: AudioLabelSwitchesWidget,
+    reviewLabels: ReviewLabelSwitchesWidget,
     zoneNames: ZoneSwitchesWidget,
     timezoneSelect: TimezoneSelectWidget,
     optionalField: OptionalFieldWidget,

web/src/components/config-form/theme/widgets/ReviewLabelSwitchesWidget.tsx

@@ -0,0 +1,84 @@
+// Review Label Switches Widget - For selecting review alert/detection labels via switches.
+// Combines object labels (from objects.track) and audio labels (from audio.listen)
+// since review labels can include both types.
+import type { WidgetProps } from "@rjsf/utils";
+import { SwitchesWidget } from "./SwitchesWidget";
+import type { FormContext } from "./SwitchesWidget";
+import { getTranslatedLabel } from "@/utils/i18n";
+import type { FrigateConfig } from "@/types/frigateConfig";
+import type { JsonObject } from "@/types/configForm";
+
+function getReviewLabels(context: FormContext): string[] {
+  const labels = new Set<string>();
+  const fullConfig = context.fullConfig as FrigateConfig | undefined;
+  const fullCameraConfig = context.fullCameraConfig;
+
+  // Object labels from tracked objects (camera-level, falling back to global)
+  const trackLabels =
+    fullCameraConfig?.objects?.track ?? fullConfig?.objects?.track;
+  if (Array.isArray(trackLabels)) {
+    trackLabels.forEach((label: string) => labels.add(label));
+  }
+
+  // Audio labels from listen config, only if audio detection is enabled
+  const audioEnabled =
+    fullCameraConfig?.audio?.enabled_in_config ??
+    fullConfig?.audio?.enabled_in_config;
+  if (audioEnabled) {
+    const audioLabels =
+      fullCameraConfig?.audio?.listen ?? fullConfig?.audio?.listen;
+    if (Array.isArray(audioLabels)) {
+      audioLabels.forEach((label: string) => labels.add(label));
+    }
+  }
+
+  // Include any labels already in the review form data (alerts + detections)
+  // so that previously saved labels remain visible even if tracking config changed
+  if (context.formData && typeof context.formData === "object") {
+    const formData = context.formData as JsonObject;
+    for (const section of ["alerts", "detections"] as const) {
+      const sectionData = formData[section];
+      if (sectionData && typeof sectionData === "object") {
+        const sectionLabels = (sectionData as JsonObject).labels;
+        if (Array.isArray(sectionLabels)) {
+          sectionLabels.forEach((label) => {
+            if (typeof label === "string") {
+              labels.add(label);
+            }
+          });
+        }
+      }
+    }
+  }
+
+  return [...labels].sort();
+}
+
+function getReviewLabelDisplayName(
+  label: string,
+  context?: FormContext,
+): string {
+  const fullCameraConfig = context?.fullCameraConfig;
+  const fullConfig = context?.fullConfig as FrigateConfig | undefined;
+  const audioLabels =
+    fullCameraConfig?.audio?.listen ?? fullConfig?.audio?.listen;
+  const isAudio = Array.isArray(audioLabels) && audioLabels.includes(label);
+  return getTranslatedLabel(label, isAudio ? "audio" : "object");
+}
+
+export function ReviewLabelSwitchesWidget(props: WidgetProps) {
+  return (
+    <SwitchesWidget
+      {...props}
+      options={{
+        ...props.options,
+        getEntities: getReviewLabels,
+        getDisplayLabel: getReviewLabelDisplayName,
+        i18nKey: "reviewLabels",
+        allowCustomEntries: true,
+        listClassName:
+          "relative max-h-none overflow-visible md:max-h-64 md:overflow-y-auto md:overscroll-contain md:scrollbar-container",
+      }}
+    />
+  );
+}

web/src/components/config-form/theme/widgets/SwitchesWidget.tsx

@@ -1,6 +1,6 @@
 // Generic Switches Widget - Reusable component for selecting from any list of entities
 import { WidgetProps } from "@rjsf/utils";
-import { useMemo, useState } from "react";
+import { useCallback, useMemo, useState } from "react";
 import { Switch } from "@/components/ui/switch";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
@@ -43,6 +43,10 @@ export type SwitchesWidgetOptions = {
   listClassName?: string;
   /** Enable search input to filter the list */
   enableSearch?: boolean;
+  /** Allow users to add custom entries not in the predefined list */
+  allowCustomEntries?: boolean;
+  /** i18n key for a hint shown when no entities are selected */
+  emptySelectionHintKey?: string;
 };
 
 function normalizeValue(value: unknown): string[] {
@@ -122,20 +126,51 @@ export function SwitchesWidget(props: WidgetProps) {
     [props.options],
   );
 
+  const allowCustomEntries = useMemo(
+    () => props.options?.allowCustomEntries as boolean | undefined,
+    [props.options],
+  );
+
+  const emptySelectionHintKey = useMemo(
+    () => props.options?.emptySelectionHintKey as string | undefined,
+    [props.options],
+  );
+
   const selectedEntities = useMemo(() => normalizeValue(value), [value]);
   const [isOpen, setIsOpen] = useState(selectedEntities.length > 0);
   const [searchTerm, setSearchTerm] = useState("");
+  const [customEntries, setCustomEntries] = useState<string[]>([]);
+  const [customInput, setCustomInput] = useState("");
+
+  const allEntities = useMemo(() => {
+    if (customEntries.length === 0) {
+      return availableEntities;
+    }
+    const merged = new Set([...availableEntities, ...customEntries]);
+    return [...merged].sort();
+  }, [availableEntities, customEntries]);
 
   const filteredEntities = useMemo(() => {
     if (!enableSearch || !searchTerm.trim()) {
-      return availableEntities;
+      return allEntities;
     }
     const term = searchTerm.toLowerCase();
-    return availableEntities.filter((entity) => {
+    return allEntities.filter((entity) => {
       const displayLabel = getDisplayLabel(entity, context);
       return displayLabel.toLowerCase().includes(term);
     });
-  }, [availableEntities, searchTerm, enableSearch, getDisplayLabel, context]);
+  }, [allEntities, searchTerm, enableSearch, getDisplayLabel, context]);
+
+  const addCustomEntry = useCallback(() => {
+    const trimmed = customInput.trim().toLowerCase();
+    if (!trimmed || allEntities.includes(trimmed)) {
+      setCustomInput("");
+      return;
+    }
+    setCustomEntries((prev) => [...prev, trimmed]);
+    onChange([...selectedEntities, trimmed]);
+    setCustomInput("");
+  }, [customInput, allEntities, selectedEntities, onChange]);
 
   const toggleEntity = (entity: string, enabled: boolean) => {
     if (enabled) {
@@ -163,7 +198,7 @@ export function SwitchesWidget(props: WidgetProps) {
 
   return (
     <Collapsible open={isOpen} onOpenChange={setIsOpen}>
-      <div className="space-y-3">
+      <div className="space-y-2">
         <CollapsibleTrigger asChild>
           <Button
             type="button"
@@ -180,8 +215,14 @@ export function SwitchesWidget(props: WidgetProps) {
           </Button>
         </CollapsibleTrigger>
 
-        <CollapsibleContent className="rounded-lg bg-secondary p-2 pr-0 md:max-w-md">
-          {availableEntities.length === 0 ? (
+        {emptySelectionHintKey && selectedEntities.length === 0 && t && (
+          <div className="mt-0 pb-2 text-sm text-success">
+            {t(emptySelectionHintKey, { ns: namespace })}
+          </div>
+        )}
+
+        <CollapsibleContent className="rounded-lg border border-input bg-secondary pb-1 pr-0 pt-2 md:max-w-md">
+          {allEntities.length === 0 && !allowCustomEntries ? (
             <div className="text-sm text-muted-foreground">{emptyMessage}</div>
           ) : (
             <>
@@ -223,6 +264,26 @@ export function SwitchesWidget(props: WidgetProps) {
                   );
                 })}
               </div>
+              {allowCustomEntries && !disabled && !readonly && (
+                <div className="mx-2 mt-2 pb-1">
+                  <Input
+                    type="text"
+                    placeholder={t?.("configForm.addCustomLabel", {
+                      ns: "views/settings",
+                      defaultValue: "Add custom label...",
+                    })}
+                    value={customInput}
+                    onChange={(e) => setCustomInput(e.target.value)}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter") {
+                        e.preventDefault();
+                        addCustomEntry();
+                      }
+                    }}
+                    onBlur={addCustomEntry}
+                  />
+                </div>
+              )}
             </>
           )}
         </CollapsibleContent>