diff --git a/frigate/data_processing/real_time/face_processor.py b/frigate/data_processing/real_time/face_processor.py
index ee981f5ec..086c59658 100644
--- a/frigate/data_processing/real_time/face_processor.py
+++ b/frigate/data_processing/real_time/face_processor.py
@@ -215,7 +215,7 @@ class FaceProcessor(RealTimeProcessorApi):
         if not self.landmark_detector:
             return None
 
-        if not self.recognizer:
+        if not self.label_map or not self.recognizer:
             self.__build_classifier()
 
             if not self.recognizer: