From 7a8f93e9f56bb24d5c92287fb5a78017efec24d7 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 7 Oct 2025 18:11:04 -0600 Subject: [PATCH] Review summary popup (#20383) * Add title to prompt * Add popup for genai review summary * Add animation --- frigate/data_processing/post/types.py | 1 + frigate/genai/__init__.py | 3 +- .../overlay/chip/GenAISummaryChip.tsx | 117 ++++++++++++++++++ web/src/types/review.ts | 1 + web/src/views/recording/RecordingView.tsx | 29 +++++ 5 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 web/src/components/overlay/chip/GenAISummaryChip.tsx diff --git a/frigate/data_processing/post/types.py b/frigate/data_processing/post/types.py index 1da7c5eb8..70fec9b34 100644 --- a/frigate/data_processing/post/types.py +++ b/frigate/data_processing/post/types.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, ConfigDict, Field class ReviewMetadata(BaseModel): model_config = ConfigDict(extra="ignore", protected_namespaces=()) + title: str = Field(description="A concise title for the activity.") scene: str = Field( description="A comprehensive description of the setting and entities, including relevant context and plausible inferences if supported by visual evidence." ) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index f3d77e9ee..8778b0eaa 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -87,6 +87,7 @@ When forming your description: - **Weigh all evidence holistically**: Start by checking if the activity matches the normal patterns above. If it does, assign Level 0. Only consider Level 1 if the activity clearly deviates from normal patterns or shows genuine security concerns that warrant attention. Your response MUST be a flat JSON object with: +- `title` (string): A concise, one-sentence title that captures the main activity. Include any verified recognized objects (from the "Verified recognized objects" list below) and key detected objects. Examples: "Joe walking dog in backyard", "Unknown person testing car doors at night". - `scene` (string): A narrative description of what happens across the sequence from start to finish. **Only describe actions you can actually observe happening in the frames provided.** Do not infer or assume actions that aren't visible (e.g., if you see someone walking but never see them sit, don't say they sat down). Include setting, detected objects, and their observable actions. Avoid speculation or filling in assumed behaviors. Your description should align with and support the threat level you assign. - `confidence` (float): 0-1 confidence in your analysis. Higher confidence when objects/actions are clearly visible and context is unambiguous. Lower confidence when the sequence is unclear, objects are partially obscured, or context is ambiguous. - `potential_threat_level` (integer): 0, 1, or 2 as defined below. Your threat level must be consistent with your scene description and the guidance above. @@ -167,7 +168,7 @@ Sequence details: timeline_summary_prompt = f""" You are a security officer. Time range: {time_range}. -Input: JSON list with "scene", "confidence", "potential_threat_level" (1-2), "other_concerns". +Input: JSON list with "title", "scene", "confidence", "potential_threat_level" (1-2), "other_concerns". Task: Write a concise, human-presentable security report in markdown format. diff --git a/web/src/components/overlay/chip/GenAISummaryChip.tsx b/web/src/components/overlay/chip/GenAISummaryChip.tsx new file mode 100644 index 000000000..581c6733c --- /dev/null +++ b/web/src/components/overlay/chip/GenAISummaryChip.tsx @@ -0,0 +1,117 @@ +import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog"; +import { Drawer, DrawerContent, DrawerTrigger } from "@/components/ui/drawer"; +import { cn } from "@/lib/utils"; +import { ReviewSegment, ThreatLevel } from "@/types/review"; +import { useEffect, useMemo, useState } from "react"; +import { isDesktop } from "react-device-detect"; +import { useTranslation } from "react-i18next"; +import { MdAutoAwesome } from "react-icons/md"; + +type GenAISummaryChipProps = { + review?: ReviewSegment; + onClick: () => void; +}; +export function GenAISummaryChip({ review, onClick }: GenAISummaryChipProps) { + const [isVisible, setIsVisible] = useState(false); + + useEffect(() => { + setIsVisible(review?.data?.metadata != undefined); + }, [review]); + + return ( +
+ + {review?.data.metadata?.title} +
+ ); +} + +type GenAISummaryDialogProps = { + review?: ReviewSegment; + onOpen?: (open: boolean) => void; +}; +export function GenAISummaryDialog({ + review, + onOpen, +}: GenAISummaryDialogProps) { + const { t } = useTranslation(["views/explore"]); + + // data + + const aiAnalysis = useMemo(() => review?.data?.metadata, [review]); + const aiThreatLevel = useMemo(() => { + if ( + !aiAnalysis || + (!aiAnalysis.potential_threat_level && !aiAnalysis.other_concerns) + ) { + return "None"; + } + + let concerns = ""; + switch (aiAnalysis.potential_threat_level) { + case ThreatLevel.SUSPICIOUS: + concerns = `• ${t("suspiciousActivity", { ns: "views/events" })}\n`; + break; + case ThreatLevel.DANGER: + concerns = `• ${t("threateningActivity", { ns: "views/events" })}\n`; + break; + } + + (aiAnalysis.other_concerns ?? []).forEach((c) => { + concerns += `• ${c}\n`; + }); + + return concerns || "None"; + }, [aiAnalysis, t]); + + // layout + + const [open, setOpen] = useState(false); + const Overlay = isDesktop ? Dialog : Drawer; + const Trigger = isDesktop ? DialogTrigger : DrawerTrigger; + const Content = isDesktop ? DialogContent : DrawerContent; + + useEffect(() => { + if (onOpen) { + onOpen(open); + } + }, [open, onOpen]); + + if (!aiAnalysis) { + return null; + } + + return ( + + + setOpen(true)} /> + + + {t("aiAnalysis.title")} +
+ {t("details.description.label")} +
+
{aiAnalysis.scene}
+
+ {t("details.score.label")} +
+
{aiAnalysis.confidence * 100}%
+
{t("concerns.label")}
+
{aiThreatLevel}
+ + + ); +} diff --git a/web/src/types/review.ts b/web/src/types/review.ts index e0385ae83..cd1aefff5 100644 --- a/web/src/types/review.ts +++ b/web/src/types/review.ts @@ -19,6 +19,7 @@ export type ReviewData = { significant_motion_areas: number[]; zones: string[]; metadata?: { + title: string; scene: string; confidence: number; potential_threat_level?: number; diff --git a/web/src/views/recording/RecordingView.tsx b/web/src/views/recording/RecordingView.tsx index 9af2284ce..6ae284cd7 100644 --- a/web/src/views/recording/RecordingView.tsx +++ b/web/src/views/recording/RecordingView.tsx @@ -66,6 +66,7 @@ import { } from "@/components/ui/tooltip"; import { CameraNameLabel } from "@/components/camera/CameraNameLabel"; import { useAllowedCameras } from "@/hooks/use-allowed-cameras"; +import { GenAISummaryDialog } from "@/components/overlay/chip/GenAISummaryChip"; type RecordingViewProps = { startCamera: string; @@ -458,6 +459,29 @@ export function RecordingView({ [visiblePreviewObserver.current], ); + const activeReviewItem = useMemo(() => { + if (!config?.cameras?.[mainCamera].review.genai?.enabled_in_config) { + return undefined; + } + + return mainCameraReviewItems.find( + (rev) => + rev.start_time < currentTime && + rev.end_time && + currentTime < rev.end_time, + ); + }, [config, currentTime, mainCameraReviewItems, mainCamera]); + const onAnalysisOpen = useCallback( + (open: boolean) => { + if (open) { + mainControllerRef.current?.pause(); + } else { + mainControllerRef.current?.play(); + } + }, + [mainControllerRef], + ); + return (
@@ -654,6 +678,11 @@ export function RecordingView({ : Math.max(1, getCameraAspect(mainCamera) ?? 0), }} > + +