diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index 217db8e69..e3a1ff00d 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -379,6 +379,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
 
             sub_label, score = res
 
+            if score < self.face_config.unknown_score:
+                sub_label = "unknown"
+
             if self.config.face_recognition.save_attempts:
                 # write face to library
                 folder = os.path.join(FACE_DIR, "train")