From e2532ad6e52849b83d897c9da9d16667f64e9413 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 11 Aug 2025 06:58:27 -0600 Subject: [PATCH] Generate review item summaries with requests --- frigate/api/review.py | 36 +++++++++++++++- frigate/comms/embeddings_updater.py | 2 + frigate/config/camera/genai.py | 4 +- frigate/data_processing/post/api.py | 2 +- .../post/review_descriptions.py | 30 ++++++++++++- frigate/embeddings/__init__.py | 6 +++ frigate/embeddings/maintainer.py | 4 +- frigate/genai/__init__.py | 42 +++++++++++++++++++ 8 files changed, 119 insertions(+), 7 deletions(-) diff --git a/frigate/api/review.py b/frigate/api/review.py index e6d010db7..2ff97eeea 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -6,7 +6,7 @@ from functools import reduce from pathlib import Path import pandas as pd -from fastapi import APIRouter +from fastapi import APIRouter, Request from fastapi.params import Depends from fastapi.responses import JSONResponse from peewee import Case, DoesNotExist, IntegrityError, fn, operator @@ -26,6 +26,8 @@ from frigate.api.defs.response.review_response import ( ReviewSummaryResponse, ) from frigate.api.defs.tags import Tags +from frigate.config import FrigateConfig +from frigate.embeddings import EmbeddingsContext from frigate.models import Recordings, ReviewSegment, UserReviewStatus from frigate.review.types import SeverityEnum from frigate.util.builtin import get_tz_modifiers @@ -606,3 +608,35 @@ async def set_not_reviewed( content=({"success": True, "message": f"Set Review {review_id} as not viewed"}), status_code=200, ) + + +@router.post( + "/review/summarize/start/{start_ts}/end/{end_ts}", + description="Use GenAI to summarize review items over a period of time.", +) +def generate_review_summary(request: Request, start_ts: float, end_ts: float): + config: FrigateConfig = request.app.frigate_config + + if not config.genai.provider: + return JSONResponse( + content=( + { + "success": False, + "message": "GenAI must be configured to use this feature.", + } + ), + status_code=400, + ) + + context: EmbeddingsContext = request.app.embeddings + summary = context.generate_review_summary(start_ts, end_ts) + + if summary: + return JSONResponse( + content=({"success": True, "summary": summary}), status_code=200 + ) + else: + return JSONResponse( + content=({"success": False, "message": "Failed to create summary."}), + status_code=500, + ) diff --git a/frigate/comms/embeddings_updater.py b/frigate/comms/embeddings_updater.py index 58829733b..f7fd9c2bf 100644 --- a/frigate/comms/embeddings_updater.py +++ b/frigate/comms/embeddings_updater.py @@ -29,6 +29,8 @@ class EmbeddingsRequestEnum(Enum): reindex = "reindex" # LPR reprocess_plate = "reprocess_plate" + # Review Descriptions + summarize_review = "summarize_review" class EmbeddingsResponder: diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index b47839dd5..059aeefdd 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -22,4 +22,6 @@ class GenAIConfig(FrigateBaseModel): api_key: Optional[EnvString] = Field(default=None, title="Provider API key.") base_url: Optional[str] = Field(default=None, title="Provider base url.") model: str = Field(default="gpt-4o", title="GenAI model.") - provider: GenAIProviderEnum | None = Field(default=None, title="GenAI provider.") + provider: GenAIProviderEnum | None = Field( + default=None, title="GenAI provider." + ) diff --git a/frigate/data_processing/post/api.py b/frigate/data_processing/post/api.py index cd6dda128..425e61ebb 100644 --- a/frigate/data_processing/post/api.py +++ b/frigate/data_processing/post/api.py @@ -39,7 +39,7 @@ class PostProcessorApi(ABC): pass @abstractmethod - def handle_request(self, request_data: dict[str, Any]) -> dict[str, Any] | None: + def handle_request(self, topic: str, request_data: dict[str, Any]) -> dict[str, Any] | None: """Handle metadata requests. Args: request_data (dict): containing data about requested change to process. diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index cc116d291..b7d5b4a77 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -10,11 +10,13 @@ from pathlib import Path import cv2 +from frigate.comms.embeddings_updater import EmbeddingsRequestEnum from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig from frigate.const import CACHE_DIR, CLIPS_DIR, UPDATE_REVIEW_DESCRIPTION from frigate.data_processing.types import PostProcessDataEnum from frigate.genai import GenAIClient +from frigate.models import ReviewSegment from frigate.util.builtin import EventsPerSecond, InferenceSpeed from ..post.api import PostProcessorApi @@ -116,8 +118,32 @@ class ReviewDescriptionProcessor(PostProcessorApi): ), ).start() - def handle_request(self, request_data): - pass + def handle_request(self, topic, request_data): + if topic == EmbeddingsRequestEnum.summarize_review.value: + start_ts = request_data["start_ts"] + end_ts = request_data["end_ts"] + items = [ + r["data"]["metadata"] + for r in ( + ReviewSegment.select(ReviewSegment.data) + .where( + (ReviewSegment.data["metadata"].is_null(False)) + & (ReviewSegment.start_time < end_ts) + & (ReviewSegment.end_time > start_ts) + ) + .order_by(ReviewSegment.start_time.asc()) + .dicts() + .iterator() + ) + ] + + if len(items) == 0: + logger.debug("No review items with metadata found during time period") + return None + + return self.genai_client.generate_review_summary(start_ts, end_ts, items) + else: + return None def get_cache_frames( self, camera: str, start_time: float, end_time: float diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index 03316ca58..0a854fcfa 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -313,3 +313,9 @@ class EmbeddingsContext: EmbeddingsRequestEnum.embed_thumbnail.value, {"id": str(event_id), "thumbnail": str(thumbnail), "upsert": False}, ) + + def generate_review_summary(self, start_ts: float, end_ts: float) -> str | None: + return self.requestor.send_data( + EmbeddingsRequestEnum.summarize_review.value, + {"start_ts": start_ts, "end_ts": end_ts}, + ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index ae613b623..a129b9677 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -66,7 +66,7 @@ from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataE from frigate.db.sqlitevecq import SqliteVecQueueDatabase from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum from frigate.genai import get_genai_client -from frigate.models import Event, Recordings, Trigger +from frigate.models import Event, Recordings, ReviewSegment, Trigger from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import serialize from frigate.util.image import ( @@ -121,7 +121,7 @@ class EmbeddingMaintainer(threading.Thread): ), load_vec_extension=True, ) - models = [Event, Recordings, Trigger] + models = [Event, Recordings, ReviewSegment, Trigger] db.bind(models) if config.semantic_search.enabled: diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 6e7071fce..7ad634299 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -1,5 +1,6 @@ """Generative AI module for Frigate.""" +import datetime import importlib import logging import os @@ -112,6 +113,47 @@ Your response **MUST** be a flat JSON object with: else: return None + def generate_review_summary( + self, start_ts: float, end_ts: float, segments: list[dict[str, Any]] + ) -> str | None: + """Generate a summary of review item descriptions over a period of time.""" + time_range = f"{datetime.datetime.fromtimestamp(start_ts).strftime('%I:%M %p')} to {datetime.datetime.fromtimestamp(end_ts).strftime('%I:%M %p')}" + timeline_summary_prompt = f""" +Analyze security camera metadata for {time_range} and write a professional security report. + +INPUT FORMAT: JSON objects with "scene", "confidence", and "potential_threat_level" (0-3). + +OUTPUT FORMAT: +Security Summary - {time_range} +[One sentence overview of general activity] + +[Chronological timeline with timestamps when available] + +[Final threat assessment statement] + +REPORT REQUIREMENTS: +- Write chronologically using timestamps +- Highlight any potential_threat_level ≥ 2 incidents with times +- Note unusual events even if not threats +- State "only normal activity observed" if no threats detected +- Use factual, professional security language + +STRICT RULES: +- Output ONLY the security report +- NO introductory phrases like "Here's a breakdown" +- NO recommendations, suggestions, or system commentary +- NO follow-up questions +- Write as a human security officer would + """ + + for item in segments: + timeline_summary_prompt += f"\n {item}" + + with open("/config/prompt.txt", "w") as f: + f.write(timeline_summary_prompt) + + return self._send(timeline_summary_prompt, []) + def generate_object_description( self, camera_config: CameraConfig,