Map verified objects to their sub label directly (#20413)

* Map verified objects to their sub label directly

* Simplify access

* Cleanup

* Add protection for mismatched object and index

* Keep track of verified objects separately

frigate/data_processing/post/review_descriptions.py

@@ -255,19 +255,24 @@ def run_analysis(
     }
 
     objects = []
-    verified_objects = []
+    named_objects = []
 
-    for label in set(final_data["data"]["objects"] + final_data["data"]["sub_labels"]):
+    objects_list = final_data["data"]["objects"]
+    sub_labels_list = final_data["data"]["sub_labels"]
+
+    for label in objects_list:
         if "-verified" in label:
             continue
-
+        elif label in labelmap_objects:
-        if label in labelmap_objects:
             objects.append(label.replace("_", " ").title())
-        else:
+
-            verified_objects.append(label.replace("_", " ").title())
+    for i, verified_label in enumerate(final_data["data"]["verified_objects"]):
+        named_objects.append(
+            f"{sub_labels_list[i].replace('_', ' ').title()} ({verified_label.replace('-verified', '')})"
+        )
 
     analytics_data["objects"] = objects
-    analytics_data["recognized_objects"] = verified_objects
+    analytics_data["recognized_objects"] = named_objects
 
     metadata = genai_client.generate_review_description(
         analytics_data,

frigate/genai/__init__.py

@@ -63,6 +63,12 @@ class GenAIClient:
             else:
                 return ""
 
+        def get_verified_objects() -> str:
+            if review_data["recognized_objects"]:
+                return "  - " + "\n  - ".join(review_data["recognized_objects"])
+            else:
+                return "  None"
+
         context_prompt = f"""
 Please analyze the sequence of images ({len(thumbnails)} total) taken in chronological order from the perspective of the {review_data["camera"].replace("_", " ")} security camera.
 
@@ -102,7 +108,8 @@ Sequence details:
 - Frame 1 = earliest, Frame {len(thumbnails)} = latest
 - Activity started at {review_data["start"]} and lasted {review_data["duration"]} seconds
 - Detected objects: {", ".join(review_data["objects"])}
-- Verified recognized objects: {", ".join(review_data["recognized_objects"]) or "None"}
+- Verified recognized objects (use these names when describing these objects):
+{get_verified_objects()}
 - Zones involved: {", ".join(z.replace("_", " ").title() for z in review_data["zones"]) or "None"}
 
 **IMPORTANT:**

frigate/review/maintainer.py

@@ -147,6 +147,9 @@ class PendingReviewSegment:
                 ReviewSegment.data.name: {
                     "detections": list(set(self.detections.keys())),
                     "objects": list(set(self.detections.values())),
+                    "verified_objects": [
+                        o for o in self.detections.values() if "-verified" in o
+                    ],
                     "sub_labels": list(self.sub_labels.values()),
                     "zones": self.zones,
                     "audio": list(self.audio),