diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index 940373d3b..c985c9dc7 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -303,9 +303,6 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
             self.person_face_history[id]
         )
 
-        if len(self.person_face_history[id]) < self.face_config.min_faces:
-            weighted_sub_label = "unknown"
-
         self.requestor.send_data(
             "tracked_object_update",
             json.dumps(
@@ -460,6 +457,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
         if not results_list:
             return None, 0.0
 
+        if len(results_list) < self.face_config.min_faces:
+            return None, 0.0
+
         counts: dict[str, int] = {}
         weighted_scores: dict[str, int] = {}
         total_weights: dict[str, int] = {}