Add ability to manually run Review Descriptions from the UI / API (#24222)

* Add ability to manually run Review Descriptions from the UI / API

* Fix not handling None type for call

docs/static/frigate-api.yaml

@@ -2283,6 +2283,42 @@ paths:
         - frigateUserAuth: []
       x-required-role: camera
       description: '**Access:** Authenticated user with access to the referenced camera.'
+  /review/{review_id}/regenerate_description:
+    put:
+      tags:
+        - Review
+      summary: Generate a review item description
+      description: |-
+        **Access:** Admin role required.
+
+        Re-runs a review item through the GenAI descriptions process.
+            Frames are always taken from recordings, and both alerts and detections are
+            accepted regardless of the camera's GenAI alerts/detections toggles.
+      operationId: 
+        regenerate_review_description_review__review_id__regenerate_description_put
+      parameters:
+        - name: review_id
+          in: path
+          required: true
+          schema:
+            type: string
+            title: Review Id
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GenericResponse'
+        '422':
+          description: Validation Error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/HTTPValidationError'
+      security:
+        - frigateAdminAuth: []
+      x-required-role: admin
   /review/{review_id}/viewed:
     delete:
       tags:
@@ -2482,6 +2518,25 @@ paths:
       security:
         - frigateAdminAuth: []
       x-required-role: admin
+  /genai/roles:
+    get:
+      tags:
+        - App
+      summary: Get the model assigned to each GenAI role
+      description: |-
+        **Access:** Admin role required.
+
+        Returns the selected model and its context size for each configured GenAI role. Reads only what the client saved when it initialized, so the provider is not queried for its model list.
+      operationId: genai_roles_genai_roles_get
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema: {}
+      security:
+        - frigateAdminAuth: []
+      x-required-role: admin
   /genai/probe:
     post:
       tags:

frigate/api/app.py

@@ -190,6 +190,20 @@ def genai_models(request: Request):
     return JSONResponse(content=request.app.genai_manager.list_models())
 
 
+@router.get(
+    "/genai/roles",
+    dependencies=[Depends(allow_any_authenticated())],
+    summary="Get the model assigned to each GenAI role",
+    description=(
+        "Returns the selected model and its context size for each configured "
+        "GenAI role. Reads only what the client saved when it initialized, so "
+        "the provider is not queried for its model list."
+    ),
+)
+def genai_roles(request: Request):
+    return JSONResponse(content=request.app.genai_manager.role_info())
+
+
 @router.post(
     "/genai/probe",
     dependencies=[Depends(require_role(["admin"]))],

frigate/api/review.py

@@ -734,6 +734,73 @@ async def get_review(request: Request, review_id: str):
         )
 
 
+@router.put(
+    "/review/{review_id}/regenerate_description",
+    response_model=GenericResponse,
+    dependencies=[Depends(require_role(["admin"]))],
+    summary="Generate a review item description",
+    description="""Re-runs a review item through the GenAI descriptions process.
+    Frames are always taken from recordings, and both alerts and detections are
+    accepted regardless of the camera's GenAI alerts/detections toggles.
+    """,
+)
+async def regenerate_review_description(request: Request, review_id: str):
+    try:
+        review: ReviewSegment = ReviewSegment.get(ReviewSegment.id == review_id)
+    except DoesNotExist:
+        return JSONResponse(
+            content={
+                "success": False,
+                "message": "Review " + review_id + " not found",
+            },
+            status_code=404,
+        )
+
+    await require_camera_access(review.camera, request=request)
+
+    if review.end_time is None:
+        return JSONResponse(
+            content={
+                "success": False,
+                "message": "Review " + review_id + " has not ended yet",
+            },
+            status_code=400,
+        )
+
+    camera_config = request.app.frigate_config.cameras.get(review.camera)
+
+    if camera_config is None or not camera_config.review.genai.enabled:
+        return JSONResponse(
+            content={
+                "success": False,
+                "message": "GenAI descriptions must be enabled for this camera",
+            },
+            status_code=400,
+        )
+
+    if request.app.genai_manager.description_client is None:
+        return JSONResponse(
+            content={
+                "success": False,
+                "message": "A GenAI provider with the descriptions role must be configured",
+            },
+            status_code=400,
+        )
+
+    context: EmbeddingsContext = request.app.embeddings
+    context.regenerate_review_description(review_id)
+
+    return JSONResponse(
+        content={
+            "success": True,
+            "message": "Review "
+            + review_id
+            + " description generation has been requested",
+        },
+        status_code=202,
+    )
+
+
 @router.delete(
     "/review/{review_id}/viewed",
     response_model=GenericResponse,

frigate/comms/embeddings_updater.py

@@ -32,6 +32,7 @@ class EmbeddingsRequestEnum(Enum):
     reprocess_plate = "reprocess_plate"
     # Review Descriptions
     summarize_review = "summarize_review"
+    regenerate_review_description = "regenerate_review_description"
 
 
 class EmbeddingsResponder:

frigate/data_processing/post/review_descriptions.py

@@ -12,6 +12,7 @@ from typing import Any
 
 import cv2
 from peewee import DoesNotExist
+from playhouse.shortcuts import model_to_dict
 from titlecase import titlecase
 
 from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
@@ -167,17 +168,9 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             image_source = camera_config.review.genai.image_source
 
             if image_source == ImageSourceEnum.recordings:
-                duration = final_data["end_time"] - final_data["start_time"]
-                buffer_extension = min(5, duration * RECORDING_BUFFER_EXTENSION_PERCENT)
-
-                # Ensure minimum total duration for short review items
-                # This provides better context for brief events
-                total_duration = duration + (2 * buffer_extension)
-                if total_duration < MIN_RECORDING_DURATION:
-                    # Expand buffer to reach minimum duration, still respecting max of 5s per side
-                    additional_buffer_per_side = (MIN_RECORDING_DURATION - duration) / 2
-                    buffer_extension = min(5, additional_buffer_per_side)
-
+                buffer_extension = get_recording_buffer_extension(
+                    final_data["end_time"] - final_data["start_time"]
+                )
                 final_data["start_time"] -= buffer_extension
                 final_data["end_time"] += buffer_extension
 
@@ -202,16 +195,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                         camera_config.review.genai.debug_save_thumbnails,
                     )
                 elif camera_config.review.genai.debug_save_thumbnails:
-                    # Save debug thumbnails for recordings
-                    Path(os.path.join(CLIPS_DIR, "genai-requests", id)).mkdir(
-                        parents=True, exist_ok=True
-                    )
-                    for idx, frame_bytes in enumerate(thumbs):
-                        with open(
-                            os.path.join(CLIPS_DIR, f"genai-requests/{id}/{idx}.jpg"),
-                            "wb",
-                        ) as f:
-                            f.write(frame_bytes)
+                    self.save_debug_recording_frames(id, thumbs)
             else:
                 # Use preview frames
                 thumbs = self.get_preview_frames_as_bytes(
@@ -223,25 +207,23 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                     camera_config.review.genai.debug_save_thumbnails,
                 )
 
-            # kickoff analysis
-            self.review_desc_dps.update()
-            threading.Thread(
-                target=run_analysis,
-                args=(
-                    self.requestor,
-                    self.genai_manager.description_client,
-                    self.review_desc_speed,
-                    camera_config,
-                    final_data,
-                    thumbs,
-                    camera_config.review.genai,
-                    sorted(self.config.all_labels),
-                    self.config.all_attributes,
-                ),
-            ).start()
+            self.start_analysis(camera_config, final_data, thumbs)
 
     def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
-        if topic == EmbeddingsRequestEnum.summarize_review.value:
+        if topic == EmbeddingsRequestEnum.regenerate_review_description.value:
+            review_id = request_data["review_id"]
+            logger.debug("Found GenAI Review description request for %s", review_id)
+
+            # frame extraction shells out to ffmpeg once per frame, so run the
+            # whole thing off the maintainer loop and answer the caller now
+            threading.Thread(
+                target=self.regenerate_description,
+                name=f"regenerate_review_description_{review_id}",
+                daemon=True,
+                args=(review_id,),
+            ).start()
+            return "started"
+        elif topic == EmbeddingsRequestEnum.summarize_review.value:
             start_ts = request_data["start_ts"]
             end_ts = request_data["end_ts"]
             logger.debug(
@@ -360,6 +342,104 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         else:
             return None
 
+    def regenerate_description(self, review_id: str) -> None:
+        """Re-run a finished review item through the description process.
+
+        Frames always come from recordings: preview frames only live in the
+        cache briefly and are much harder to sample from once they have been
+        compressed into a preview clip. Alerts and detections are both accepted
+        regardless of the per-camera alerts/detections toggles, since the run
+        was asked for explicitly.
+        """
+        client = self.genai_manager.description_client
+
+        if client is None:
+            logger.error("No GenAI provider is assigned the descriptions role")
+            return
+
+        try:
+            review: ReviewSegment = ReviewSegment.get(ReviewSegment.id == review_id)
+        except DoesNotExist:
+            logger.error(
+                "Review item %s not found for description generation", review_id
+            )
+            return
+
+        camera_config = self.config.cameras.get(str(review.camera))
+
+        if camera_config is None:
+            logger.error("Camera %s no longer exists", review.camera)
+            return
+
+        if not camera_config.review.genai.enabled:
+            logger.error(
+                "GenAI review descriptions are not enabled for %s", review.camera
+            )
+            return
+
+        final_data = model_to_dict(review)
+
+        if final_data["end_time"] is None:
+            logger.error("Review item %s has not ended yet", review_id)
+            return
+
+        buffer_extension = get_recording_buffer_extension(
+            final_data["end_time"] - final_data["start_time"]
+        )
+        thumbs = self.get_recording_frames(
+            str(review.camera),
+            final_data["start_time"] - buffer_extension,
+            final_data["end_time"] + buffer_extension,
+            height=480,
+        )
+
+        if not thumbs:
+            logger.error(
+                "No recording frames are available for review item %s", review_id
+            )
+            return
+
+        if camera_config.review.genai.debug_save_thumbnails:
+            self.save_debug_recording_frames(review_id, thumbs)
+
+        self.start_analysis(camera_config, final_data, thumbs)
+
+    def start_analysis(
+        self,
+        camera_config: CameraConfig,
+        final_data: dict[str, Any],
+        thumbs: list[bytes],
+    ) -> None:
+        """Kick off description generation for a review item in the background."""
+        self.review_desc_dps.update()
+        threading.Thread(
+            target=run_analysis,
+            args=(
+                self.requestor,
+                self.genai_manager.description_client,
+                self.review_desc_speed,
+                camera_config,
+                final_data,
+                thumbs,
+                camera_config.review.genai,
+                sorted(self.config.all_labels),
+                self.config.all_attributes,
+            ),
+        ).start()
+
+    def save_debug_recording_frames(self, review_id: str, thumbs: list[bytes]) -> None:
+        """Write the recording frames sent to the provider out for debugging."""
+        Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
+            parents=True, exist_ok=True
+        )
+
+        for idx, frame_bytes in enumerate(thumbs):
+            with open(
+                os.path.join(CLIPS_DIR, f"genai-requests/{review_id}/{idx}.jpg"),
+                "wb",
+            ) as f:
+                f.write(frame_bytes)
+
     def get_cache_frames(
         self,
         camera: str,
@@ -539,6 +619,20 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         return thumbs
 
 
+def get_recording_buffer_extension(duration: float) -> float:
+    """Seconds of padding to add to each side of a review item when pulling
+    recording frames, so brief items still carry enough context."""
+    buffer_extension = min(5, duration * RECORDING_BUFFER_EXTENSION_PERCENT)
+
+    # Ensure minimum total duration for short review items
+    # This provides better context for brief events
+    if duration + (2 * buffer_extension) < MIN_RECORDING_DURATION:
+        # Expand buffer to reach minimum duration, still respecting max of 5s per side
+        buffer_extension = min(5, (MIN_RECORDING_DURATION - duration) / 2)
+
+    return buffer_extension
+
+
 def run_analysis(
     requestor: InterProcessRequestor,
     genai_client: GenAIClient,

frigate/embeddings/__init__.py

@@ -334,3 +334,9 @@ class EmbeddingsContext:
             EmbeddingsRequestEnum.summarize_review.value,
             {"start_ts": start_ts, "end_ts": end_ts},
         )
+
+    def regenerate_review_description(self, review_id: str) -> None:
+        self.requestor.send_data(
+            EmbeddingsRequestEnum.regenerate_review_description.value,
+            {"review_id": review_id},
+        )

frigate/genai/manager.py

@@ -110,6 +110,28 @@ class GenAIClientManager:
         name = self._role_map.get(GenAIRoleEnum.embeddings)
         return self._get_client(name) if name else None
 
+    def role_info(self) -> dict[str, dict[str, Any]]:
+        """Return the model selected for each configured role and its context size.
+
+        Only reads state the client saved when it initialized, so unlike
+        list_models() this does not ask the provider for its catalog.
+        """
+        result: dict[str, dict[str, Any]] = {}
+
+        for role, name in self._role_map.items():
+            client = self._get_client(name)
+
+            if not client:
+                continue
+
+            result[role.value] = {
+                "name": name,
+                "model": self._configs[name].model,
+                "context_size": client.get_context_size(),
+            }
+
+        return result
+
     def list_models(self) -> dict[str, dict[str, Any]]:
         """Return per-entry model lists and capabilities, keyed by config entry name."""
         result: dict[str, dict[str, Any]] = {}

web/public/locales/en/components/dialog.json

@@ -191,7 +191,14 @@
       "export": "Export",
       "markAsReviewed": "Mark as reviewed",
       "markAsUnreviewed": "Mark as unreviewed",
-      "deleteNow": "Delete Now"
+      "deleteNow": "Delete Now",
+      "generateDescription": "Generate description"
+    },
+    "genaiDescription": {
+      "toast": {
+        "success": "A GenAI description has been requested for this review item.",
+        "error": "Failed to request description: {{error}}"
+      }
     }
   },
   "imagePicker": {

web/src/components/filter/ReviewActionGroup.tsx

@@ -21,7 +21,9 @@ import useKeyboardListener from "@/hooks/use-keyboard-listener";
 import { Trans, useTranslation } from "react-i18next";
 import { toast } from "sonner";
 import { useIsAdmin } from "@/hooks/use-is-admin";
+import { useReviewDescriptions } from "@/hooks/use-review-descriptions";
 import MultiExportDialog from "../overlay/MultiExportDialog";
+import { MdAutoAwesome } from "react-icons/md";
 
 type ReviewActionGroupProps = {
   selectedReviews: ReviewSegment[];
@@ -45,6 +47,13 @@ export default function ReviewActionGroup({
     (review) => review.has_been_reviewed,
   );
 
+  const { canGenerateDescription, generateDescription } =
+    useReviewDescriptions();
+
+  // only a single item can be sent through the descriptions process at a time
+  const showGenerateDescription =
+    selectedReviews.length == 1 && canGenerateDescription(selectedReviews[0]);
+
   const onToggleReviewed = useCallback(async () => {
     const ids = selectedReviews.map((review) => review.id);
     await axios.post(`reviews/viewed`, {
@@ -166,6 +175,24 @@ export default function ReviewActionGroup({
               )}
             </Button>
           )}
+          {showGenerateDescription && (
+            <Button
+              className="flex items-center gap-2 p-2"
+              aria-label={t("recording.button.generateDescription")}
+              size="sm"
+              onClick={() => {
+                generateDescription(selectedReviews[0]);
+                onClearSelected();
+              }}
+            >
+              <MdAutoAwesome className="text-secondary-foreground" />
+              {isDesktop && (
+                <div className="text-primary">
+                  {t("recording.button.generateDescription")}
+                </div>
+              )}
+            </Button>
+          )}
           {selectedReviews.length >= 2 &&
             selectedReviews.length <= MAX_BATCH_EXPORT_ITEMS && (
               <MultiExportDialog

web/src/hooks/use-review-descriptions.ts

@@ -0,0 +1,68 @@
+import axios from "axios";
+import { useCallback, useMemo } from "react";
+import { useTranslation } from "react-i18next";
+import { toast } from "sonner";
+import useSWR from "swr";
+import { useIsAdmin } from "./use-is-admin";
+import { GenAIRolesResponse } from "@/types/chat";
+import { FrigateConfig } from "@/types/frigateConfig";
+import { ReviewSegment } from "@/types/review";
+
+// recordings are the only source used for on demand runs, and their frames
+// cost far more tokens than preview frames do
+export const MIN_REVIEW_DESCRIPTION_CONTEXT = 32000;
+
+/**
+ * Gating and dispatch for re-running a review item through GenAI descriptions.
+ */
+export function useReviewDescriptions() {
+  const { t } = useTranslation(["components/dialog"]);
+  const { data: config } = useSWR<FrigateConfig>("config");
+  const isAdmin = useIsAdmin();
+
+  const { data: genaiRoles } = useSWR<GenAIRolesResponse>(
+    isAdmin ? "genai/roles" : null,
+    { revalidateOnFocus: false },
+  );
+
+  const hasSufficientContext = useMemo(
+    () =>
+      (genaiRoles?.descriptions?.context_size ?? 0) >=
+      MIN_REVIEW_DESCRIPTION_CONTEXT,
+    [genaiRoles],
+  );
+
+  const canGenerateDescription = useCallback(
+    (review: ReviewSegment) =>
+      isAdmin &&
+      hasSufficientContext &&
+      !!review.end_time &&
+      !!config?.cameras[review.camera]?.review?.genai?.enabled,
+    [config, hasSufficientContext, isAdmin],
+  );
+
+  const generateDescription = useCallback(
+    (review: ReviewSegment) => {
+      axios
+        .put(`review/${review.id}/regenerate_description`)
+        .then(() => {
+          toast.success(t("recording.genaiDescription.toast.success"), {
+            position: "top-center",
+          });
+        })
+        .catch((error) => {
+          const errorMessage =
+            error.response?.data?.message || error.message || "Unknown error";
+          toast.error(
+            t("recording.genaiDescription.toast.error", {
+              error: errorMessage,
+            }),
+            { position: "top-center" },
+          );
+        });
+    },
+    [t],
+  );
+
+  return { canGenerateDescription, generateDescription };
+}

web/src/types/chat.ts

@@ -1,3 +1,4 @@
+import { GenAIRole } from "@/types/frigateConfig";
 export type ToolCallFunction = {
   name: string;
   arguments: string;
@@ -57,3 +58,11 @@ export type GenAIProviderInfo = {
 };
 
 export type GenAIModelsResponse = Record<string, GenAIProviderInfo>;
+
+export type GenAIRoleInfo = {
+  name: string;
+  model: string;
+  context_size: number;
+};
+
+export type GenAIRolesResponse = Partial<Record<GenAIRole, GenAIRoleInfo>>;