diff --git a/docs/static/frigate-api.yaml b/docs/static/frigate-api.yaml index 250155aec1..14060039cb 100644 --- a/docs/static/frigate-api.yaml +++ b/docs/static/frigate-api.yaml @@ -2283,6 +2283,42 @@ paths: - frigateUserAuth: [] x-required-role: camera description: '**Access:** Authenticated user with access to the referenced camera.' + /review/{review_id}/regenerate_description: + put: + tags: + - Review + summary: Generate a review item description + description: |- + **Access:** Admin role required. + + Re-runs a review item through the GenAI descriptions process. + Frames are always taken from recordings, and both alerts and detections are + accepted regardless of the camera's GenAI alerts/detections toggles. + operationId: + regenerate_review_description_review__review_id__regenerate_description_put + parameters: + - name: review_id + in: path + required: true + schema: + type: string + title: Review Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/GenericResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + security: + - frigateAdminAuth: [] + x-required-role: admin /review/{review_id}/viewed: delete: tags: @@ -2482,6 +2518,25 @@ paths: security: - frigateAdminAuth: [] x-required-role: admin + /genai/roles: + get: + tags: + - App + summary: Get the model assigned to each GenAI role + description: |- + **Access:** Admin role required. + + Returns the selected model and its context size for each configured GenAI role. Reads only what the client saved when it initialized, so the provider is not queried for its model list. + operationId: genai_roles_genai_roles_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: {} + security: + - frigateAdminAuth: [] + x-required-role: admin /genai/probe: post: tags: diff --git a/frigate/api/app.py b/frigate/api/app.py index 7427f4d106..73a2fe4b56 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -190,6 +190,20 @@ def genai_models(request: Request): return JSONResponse(content=request.app.genai_manager.list_models()) +@router.get( + "/genai/roles", + dependencies=[Depends(allow_any_authenticated())], + summary="Get the model assigned to each GenAI role", + description=( + "Returns the selected model and its context size for each configured " + "GenAI role. Reads only what the client saved when it initialized, so " + "the provider is not queried for its model list." + ), +) +def genai_roles(request: Request): + return JSONResponse(content=request.app.genai_manager.role_info()) + + @router.post( "/genai/probe", dependencies=[Depends(require_role(["admin"]))], diff --git a/frigate/api/review.py b/frigate/api/review.py index 6b12e0f5af..65776e570f 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -734,6 +734,73 @@ async def get_review(request: Request, review_id: str): ) +@router.put( + "/review/{review_id}/regenerate_description", + response_model=GenericResponse, + dependencies=[Depends(require_role(["admin"]))], + summary="Generate a review item description", + description="""Re-runs a review item through the GenAI descriptions process. + Frames are always taken from recordings, and both alerts and detections are + accepted regardless of the camera's GenAI alerts/detections toggles. + """, +) +async def regenerate_review_description(request: Request, review_id: str): + try: + review: ReviewSegment = ReviewSegment.get(ReviewSegment.id == review_id) + except DoesNotExist: + return JSONResponse( + content={ + "success": False, + "message": "Review " + review_id + " not found", + }, + status_code=404, + ) + + await require_camera_access(review.camera, request=request) + + if review.end_time is None: + return JSONResponse( + content={ + "success": False, + "message": "Review " + review_id + " has not ended yet", + }, + status_code=400, + ) + + camera_config = request.app.frigate_config.cameras.get(review.camera) + + if camera_config is None or not camera_config.review.genai.enabled: + return JSONResponse( + content={ + "success": False, + "message": "GenAI descriptions must be enabled for this camera", + }, + status_code=400, + ) + + if request.app.genai_manager.description_client is None: + return JSONResponse( + content={ + "success": False, + "message": "A GenAI provider with the descriptions role must be configured", + }, + status_code=400, + ) + + context: EmbeddingsContext = request.app.embeddings + context.regenerate_review_description(review_id) + + return JSONResponse( + content={ + "success": True, + "message": "Review " + + review_id + + " description generation has been requested", + }, + status_code=202, + ) + + @router.delete( "/review/{review_id}/viewed", response_model=GenericResponse, diff --git a/frigate/comms/embeddings_updater.py b/frigate/comms/embeddings_updater.py index cd83709f0b..1b612504e1 100644 --- a/frigate/comms/embeddings_updater.py +++ b/frigate/comms/embeddings_updater.py @@ -32,6 +32,7 @@ class EmbeddingsRequestEnum(Enum): reprocess_plate = "reprocess_plate" # Review Descriptions summarize_review = "summarize_review" + regenerate_review_description = "regenerate_review_description" class EmbeddingsResponder: diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 9c7b3793b0..300b862cab 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -12,6 +12,7 @@ from typing import Any import cv2 from peewee import DoesNotExist +from playhouse.shortcuts import model_to_dict from titlecase import titlecase from frigate.comms.embeddings_updater import EmbeddingsRequestEnum @@ -167,17 +168,9 @@ class ReviewDescriptionProcessor(PostProcessorApi): image_source = camera_config.review.genai.image_source if image_source == ImageSourceEnum.recordings: - duration = final_data["end_time"] - final_data["start_time"] - buffer_extension = min(5, duration * RECORDING_BUFFER_EXTENSION_PERCENT) - - # Ensure minimum total duration for short review items - # This provides better context for brief events - total_duration = duration + (2 * buffer_extension) - if total_duration < MIN_RECORDING_DURATION: - # Expand buffer to reach minimum duration, still respecting max of 5s per side - additional_buffer_per_side = (MIN_RECORDING_DURATION - duration) / 2 - buffer_extension = min(5, additional_buffer_per_side) - + buffer_extension = get_recording_buffer_extension( + final_data["end_time"] - final_data["start_time"] + ) final_data["start_time"] -= buffer_extension final_data["end_time"] += buffer_extension @@ -202,16 +195,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): camera_config.review.genai.debug_save_thumbnails, ) elif camera_config.review.genai.debug_save_thumbnails: - # Save debug thumbnails for recordings - Path(os.path.join(CLIPS_DIR, "genai-requests", id)).mkdir( - parents=True, exist_ok=True - ) - for idx, frame_bytes in enumerate(thumbs): - with open( - os.path.join(CLIPS_DIR, f"genai-requests/{id}/{idx}.jpg"), - "wb", - ) as f: - f.write(frame_bytes) + self.save_debug_recording_frames(id, thumbs) else: # Use preview frames thumbs = self.get_preview_frames_as_bytes( @@ -223,25 +207,23 @@ class ReviewDescriptionProcessor(PostProcessorApi): camera_config.review.genai.debug_save_thumbnails, ) - # kickoff analysis - self.review_desc_dps.update() - threading.Thread( - target=run_analysis, - args=( - self.requestor, - self.genai_manager.description_client, - self.review_desc_speed, - camera_config, - final_data, - thumbs, - camera_config.review.genai, - sorted(self.config.all_labels), - self.config.all_attributes, - ), - ).start() + self.start_analysis(camera_config, final_data, thumbs) def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None: - if topic == EmbeddingsRequestEnum.summarize_review.value: + if topic == EmbeddingsRequestEnum.regenerate_review_description.value: + review_id = request_data["review_id"] + logger.debug("Found GenAI Review description request for %s", review_id) + + # frame extraction shells out to ffmpeg once per frame, so run the + # whole thing off the maintainer loop and answer the caller now + threading.Thread( + target=self.regenerate_description, + name=f"regenerate_review_description_{review_id}", + daemon=True, + args=(review_id,), + ).start() + return "started" + elif topic == EmbeddingsRequestEnum.summarize_review.value: start_ts = request_data["start_ts"] end_ts = request_data["end_ts"] logger.debug( @@ -360,6 +342,104 @@ class ReviewDescriptionProcessor(PostProcessorApi): else: return None + def regenerate_description(self, review_id: str) -> None: + """Re-run a finished review item through the description process. + + Frames always come from recordings: preview frames only live in the + cache briefly and are much harder to sample from once they have been + compressed into a preview clip. Alerts and detections are both accepted + regardless of the per-camera alerts/detections toggles, since the run + was asked for explicitly. + """ + client = self.genai_manager.description_client + + if client is None: + logger.error("No GenAI provider is assigned the descriptions role") + return + + try: + review: ReviewSegment = ReviewSegment.get(ReviewSegment.id == review_id) + except DoesNotExist: + logger.error( + "Review item %s not found for description generation", review_id + ) + return + + camera_config = self.config.cameras.get(str(review.camera)) + + if camera_config is None: + logger.error("Camera %s no longer exists", review.camera) + return + + if not camera_config.review.genai.enabled: + logger.error( + "GenAI review descriptions are not enabled for %s", review.camera + ) + return + + final_data = model_to_dict(review) + + if final_data["end_time"] is None: + logger.error("Review item %s has not ended yet", review_id) + return + + buffer_extension = get_recording_buffer_extension( + final_data["end_time"] - final_data["start_time"] + ) + thumbs = self.get_recording_frames( + str(review.camera), + final_data["start_time"] - buffer_extension, + final_data["end_time"] + buffer_extension, + height=480, + ) + + if not thumbs: + logger.error( + "No recording frames are available for review item %s", review_id + ) + return + + if camera_config.review.genai.debug_save_thumbnails: + self.save_debug_recording_frames(review_id, thumbs) + + self.start_analysis(camera_config, final_data, thumbs) + + def start_analysis( + self, + camera_config: CameraConfig, + final_data: dict[str, Any], + thumbs: list[bytes], + ) -> None: + """Kick off description generation for a review item in the background.""" + self.review_desc_dps.update() + threading.Thread( + target=run_analysis, + args=( + self.requestor, + self.genai_manager.description_client, + self.review_desc_speed, + camera_config, + final_data, + thumbs, + camera_config.review.genai, + sorted(self.config.all_labels), + self.config.all_attributes, + ), + ).start() + + def save_debug_recording_frames(self, review_id: str, thumbs: list[bytes]) -> None: + """Write the recording frames sent to the provider out for debugging.""" + Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir( + parents=True, exist_ok=True + ) + + for idx, frame_bytes in enumerate(thumbs): + with open( + os.path.join(CLIPS_DIR, f"genai-requests/{review_id}/{idx}.jpg"), + "wb", + ) as f: + f.write(frame_bytes) + def get_cache_frames( self, camera: str, @@ -539,6 +619,20 @@ class ReviewDescriptionProcessor(PostProcessorApi): return thumbs +def get_recording_buffer_extension(duration: float) -> float: + """Seconds of padding to add to each side of a review item when pulling + recording frames, so brief items still carry enough context.""" + buffer_extension = min(5, duration * RECORDING_BUFFER_EXTENSION_PERCENT) + + # Ensure minimum total duration for short review items + # This provides better context for brief events + if duration + (2 * buffer_extension) < MIN_RECORDING_DURATION: + # Expand buffer to reach minimum duration, still respecting max of 5s per side + buffer_extension = min(5, (MIN_RECORDING_DURATION - duration) / 2) + + return buffer_extension + + def run_analysis( requestor: InterProcessRequestor, genai_client: GenAIClient, diff --git a/frigate/embeddings/__init__.py b/frigate/embeddings/__init__.py index d9b45e7c65..61e9f45fb5 100644 --- a/frigate/embeddings/__init__.py +++ b/frigate/embeddings/__init__.py @@ -334,3 +334,9 @@ class EmbeddingsContext: EmbeddingsRequestEnum.summarize_review.value, {"start_ts": start_ts, "end_ts": end_ts}, ) + + def regenerate_review_description(self, review_id: str) -> None: + self.requestor.send_data( + EmbeddingsRequestEnum.regenerate_review_description.value, + {"review_id": review_id}, + ) diff --git a/frigate/genai/manager.py b/frigate/genai/manager.py index 1301f1b7a1..a8e38400ef 100644 --- a/frigate/genai/manager.py +++ b/frigate/genai/manager.py @@ -110,6 +110,28 @@ class GenAIClientManager: name = self._role_map.get(GenAIRoleEnum.embeddings) return self._get_client(name) if name else None + def role_info(self) -> dict[str, dict[str, Any]]: + """Return the model selected for each configured role and its context size. + + Only reads state the client saved when it initialized, so unlike + list_models() this does not ask the provider for its catalog. + """ + result: dict[str, dict[str, Any]] = {} + + for role, name in self._role_map.items(): + client = self._get_client(name) + + if not client: + continue + + result[role.value] = { + "name": name, + "model": self._configs[name].model, + "context_size": client.get_context_size(), + } + + return result + def list_models(self) -> dict[str, dict[str, Any]]: """Return per-entry model lists and capabilities, keyed by config entry name.""" result: dict[str, dict[str, Any]] = {} diff --git a/web/public/locales/en/components/dialog.json b/web/public/locales/en/components/dialog.json index 21576cdf25..1a875b3a1f 100644 --- a/web/public/locales/en/components/dialog.json +++ b/web/public/locales/en/components/dialog.json @@ -191,7 +191,14 @@ "export": "Export", "markAsReviewed": "Mark as reviewed", "markAsUnreviewed": "Mark as unreviewed", - "deleteNow": "Delete Now" + "deleteNow": "Delete Now", + "generateDescription": "Generate description" + }, + "genaiDescription": { + "toast": { + "success": "A GenAI description has been requested for this review item.", + "error": "Failed to request description: {{error}}" + } } }, "imagePicker": { diff --git a/web/src/components/filter/ReviewActionGroup.tsx b/web/src/components/filter/ReviewActionGroup.tsx index 389d12104a..3434c2c4eb 100644 --- a/web/src/components/filter/ReviewActionGroup.tsx +++ b/web/src/components/filter/ReviewActionGroup.tsx @@ -21,7 +21,9 @@ import useKeyboardListener from "@/hooks/use-keyboard-listener"; import { Trans, useTranslation } from "react-i18next"; import { toast } from "sonner"; import { useIsAdmin } from "@/hooks/use-is-admin"; +import { useReviewDescriptions } from "@/hooks/use-review-descriptions"; import MultiExportDialog from "../overlay/MultiExportDialog"; +import { MdAutoAwesome } from "react-icons/md"; type ReviewActionGroupProps = { selectedReviews: ReviewSegment[]; @@ -45,6 +47,13 @@ export default function ReviewActionGroup({ (review) => review.has_been_reviewed, ); + const { canGenerateDescription, generateDescription } = + useReviewDescriptions(); + + // only a single item can be sent through the descriptions process at a time + const showGenerateDescription = + selectedReviews.length == 1 && canGenerateDescription(selectedReviews[0]); + const onToggleReviewed = useCallback(async () => { const ids = selectedReviews.map((review) => review.id); await axios.post(`reviews/viewed`, { @@ -166,6 +175,24 @@ export default function ReviewActionGroup({ )} )} + {showGenerateDescription && ( + + )} {selectedReviews.length >= 2 && selectedReviews.length <= MAX_BATCH_EXPORT_ITEMS && ( ("config"); + const isAdmin = useIsAdmin(); + + const { data: genaiRoles } = useSWR( + isAdmin ? "genai/roles" : null, + { revalidateOnFocus: false }, + ); + + const hasSufficientContext = useMemo( + () => + (genaiRoles?.descriptions?.context_size ?? 0) >= + MIN_REVIEW_DESCRIPTION_CONTEXT, + [genaiRoles], + ); + + const canGenerateDescription = useCallback( + (review: ReviewSegment) => + isAdmin && + hasSufficientContext && + !!review.end_time && + !!config?.cameras[review.camera]?.review?.genai?.enabled, + [config, hasSufficientContext, isAdmin], + ); + + const generateDescription = useCallback( + (review: ReviewSegment) => { + axios + .put(`review/${review.id}/regenerate_description`) + .then(() => { + toast.success(t("recording.genaiDescription.toast.success"), { + position: "top-center", + }); + }) + .catch((error) => { + const errorMessage = + error.response?.data?.message || error.message || "Unknown error"; + toast.error( + t("recording.genaiDescription.toast.error", { + error: errorMessage, + }), + { position: "top-center" }, + ); + }); + }, + [t], + ); + + return { canGenerateDescription, generateDescription }; +} diff --git a/web/src/types/chat.ts b/web/src/types/chat.ts index fe3fdd4d18..1b40ea8193 100644 --- a/web/src/types/chat.ts +++ b/web/src/types/chat.ts @@ -1,3 +1,4 @@ +import { GenAIRole } from "@/types/frigateConfig"; export type ToolCallFunction = { name: string; arguments: string; @@ -57,3 +58,11 @@ export type GenAIProviderInfo = { }; export type GenAIModelsResponse = Record; + +export type GenAIRoleInfo = { + name: string; + model: string; + context_size: number; +}; + +export type GenAIRolesResponse = Partial>;