Review genai updates (#19448)

* Include extra level for normal activity

* Add dynamic toggling

* Update docs

* Add different threshold for genai

* Adjust webUI for object and review description feature

* Adjust config

* Send on startup

* Cleanup config setting

* Set config

* Fix config name

frigate/comms/dispatcher.py

@@ -75,7 +75,8 @@ class Dispatcher:
             "birdseye_mode": self._on_birdseye_mode_command,
             "review_alerts": self._on_alerts_command,
             "review_detections": self._on_detections_command,
-            "genai": self._on_genai_command,
+            "object_descriptions": self._on_object_description_command,
+            "review_descriptions": self._on_review_description_command,
         }
         self._global_settings_handlers: dict[str, Callable] = {
             "notifications": self._on_global_notification_command,
@@ -218,7 +219,12 @@ class Dispatcher:
                     ].onvif.autotracking.enabled,
                     "alerts": self.config.cameras[camera].review.alerts.enabled,
                     "detections": self.config.cameras[camera].review.detections.enabled,
-                    "genai": self.config.cameras[camera].objects.genai.enabled,
+                    "object_descriptions": self.config.cameras[
+                        camera
+                    ].objects.genai.enabled,
+                    "review_descriptions": self.config.cameras[
+                        camera
+                    ].review.genai.enabled,
                 }
 
             self.publish("camera_activity", json.dumps(camera_status))
@@ -752,8 +758,8 @@ class Dispatcher:
         )
         self.publish(f"{camera_name}/review_detections/state", payload, retain=True)
 
-    def _on_genai_command(self, camera_name: str, payload: str) -> None:
-        """Callback for GenAI topic."""
+    def _on_object_description_command(self, camera_name: str, payload: str) -> None:
+        """Callback for object description topic."""
         genai_settings = self.config.cameras[camera_name].objects.genai
 
         if payload == "ON":
@@ -764,15 +770,40 @@ class Dispatcher:
                 return
 
             if not genai_settings.enabled:
-                logger.info(f"Turning on GenAI for {camera_name}")
+                logger.info(f"Turning on object descriptions for {camera_name}")
                 genai_settings.enabled = True
         elif payload == "OFF":
             if genai_settings.enabled:
-                logger.info(f"Turning off GenAI for {camera_name}")
+                logger.info(f"Turning off object descriptions for {camera_name}")
                 genai_settings.enabled = False
 
         self.config_updater.publish_update(
-            CameraConfigUpdateTopic(CameraConfigUpdateEnum.genai, camera_name),
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.object_genai, camera_name),
             genai_settings,
         )
-        self.publish(f"{camera_name}/genai/state", payload, retain=True)
+        self.publish(f"{camera_name}/object_descriptions/state", payload, retain=True)
+
+    def _on_review_description_command(self, camera_name: str, payload: str) -> None:
+        """Callback for review description topic."""
+        genai_settings = self.config.cameras[camera_name].review.genai
+
+        if payload == "ON":
+            if not self.config.cameras[camera_name].review.genai.enabled_in_config:
+                logger.error(
+                    "GenAI Alerts or Detections must be enabled in the config to be turned on via MQTT."
+                )
+                return
+
+            if not genai_settings.enabled:
+                logger.info(f"Turning on review descriptions for {camera_name}")
+                genai_settings.enabled = True
+        elif payload == "OFF":
+            if genai_settings.enabled:
+                logger.info(f"Turning off review descriptions for {camera_name}")
+                genai_settings.enabled = False
+
+        self.config_updater.publish_update(
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.review_genai, camera_name),
+            genai_settings,
+        )
+        self.publish(f"{camera_name}/review_descriptions/state", payload, retain=True)

frigate/comms/mqtt.py

@@ -123,10 +123,15 @@ class MqttClient(Communicator):
                 retain=True,
             )
             self.publish(
-                f"{camera_name}/genai/state",
+                f"{camera_name}/object_descriptions/state",
                 "ON" if camera.objects.genai.enabled_in_config else "OFF",
                 retain=True,
             )
+            self.publish(
+                f"{camera_name}/review_descriptions/state",
+                "ON" if camera.review.genai.enabled_in_config else "OFF",
+                retain=True,
+            )
 
         if self.config.notifications.enabled_in_config:
             self.publish(

frigate/config/camera/review.py

@@ -63,7 +63,11 @@ class DetectionsConfig(FrigateBaseModel):
 
 
 class GenAIReviewConfig(FrigateBaseModel):
-    alerts: bool = Field(default=False, title="Enable GenAI for alerts.")
+    enabled: bool = Field(
+        default=False,
+        title="Enable GenAI descriptions for review items.",
+    )
+    alerts: bool = Field(default=True, title="Enable GenAI for alerts.")
     detections: bool = Field(default=False, title="Enable GenAI for detections.")
     debug_save_thumbnails: bool = Field(
         default=False,

frigate/config/camera/updater.py

@@ -17,13 +17,14 @@ class CameraConfigUpdateEnum(str, Enum):
     birdseye = "birdseye"
     detect = "detect"
     enabled = "enabled"
-    genai = "genai"
     motion = "motion"  # includes motion and motion masks
     notifications = "notifications"
     objects = "objects"
+    object_genai = "object_genai"
     record = "record"
     remove = "remove"  # for removing a camera
     review = "review"
+    review_genai = "review_genai"
     semantic_search = "semantic_search"  # for semantic search triggers
     snapshots = "snapshots"
     zones = "zones"
@@ -98,7 +99,7 @@ class CameraConfigUpdateSubscriber:
             config.detect = updated_config
         elif update_type == CameraConfigUpdateEnum.enabled:
             config.enabled = updated_config
-        elif update_type == CameraConfigUpdateEnum.genai:
+        elif update_type == CameraConfigUpdateEnum.object_genai:
             config.objects.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.motion:
             config.motion = updated_config
@@ -110,6 +111,8 @@ class CameraConfigUpdateSubscriber:
             config.record = updated_config
         elif update_type == CameraConfigUpdateEnum.review:
             config.review = updated_config
+        elif update_type == CameraConfigUpdateEnum.review_genai:
+            config.review.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.semantic_search:
             config.semantic_search = updated_config
         elif update_type == CameraConfigUpdateEnum.snapshots:

frigate/config/config.py

@@ -611,8 +611,7 @@ class FrigateConfig(FrigateBaseModel):
                 camera_config.objects.genai.enabled
             )
             camera_config.review.genai.enabled_in_config = (
-                camera_config.review.genai.alerts
-                or camera_config.review.genai.detections
+                camera_config.review.genai.enabled
             )
 
             # Add default filters

frigate/data_processing/post/review_descriptions.py

@@ -46,6 +46,11 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         if data_type != PostProcessDataEnum.review:
             return
 
+        camera = data["after"]["camera"]
+
+        if not self.config.cameras[camera].review.genai.enabled:
+            return
+
         id = data["after"]["id"]
 
         if data["type"] == "new" or data["type"] == "update":
@@ -91,7 +96,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
                 return
 
             final_data = data["after"]
-            camera = final_data["camera"]
 
             if (
                 final_data["severity"] == "alert"

frigate/data_processing/post/types.py

@@ -8,9 +8,8 @@ class ReviewMetadata(BaseModel):
     confidence: float = Field(
         description="A float between 0 and 1 representing your overall confidence in this analysis."
     )
-    potential_threat_level: int | None = Field(
-        default=None,
-        ge=1,
+    potential_threat_level: int = Field(
+        ge=0,
         le=3,
         description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.",
     )

frigate/embeddings/maintainer.py

@@ -102,7 +102,8 @@ class EmbeddingMaintainer(threading.Thread):
             [
                 CameraConfigUpdateEnum.add,
                 CameraConfigUpdateEnum.remove,
-                CameraConfigUpdateEnum.genai,
+                CameraConfigUpdateEnum.object_genai,
+                CameraConfigUpdateEnum.review_genai,
                 CameraConfigUpdateEnum.semantic_search,
             ],
         )

frigate/genai/__init__.py

@@ -62,10 +62,10 @@ class GenAIClient:
         - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
         - `confidence` (float): A number 0–1 for overall confidence in the analysis.
         - `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern:
+            - 0 = Normal activity is occurring
             - 1 = Unusual but not overtly threatening
             - 2 = Suspicious or potentially harmful
             - 3 = Clear and immediate threat
-            Omit this field if no concern is evident.
 
         **IMPORTANT:**
         - Values must be plain strings, floats, or integers — no nested objects, no extra commentary.