diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py
index bc0a0ff11..11f08a86c 100644
--- a/frigate/detectors/detector_config.py
+++ b/frigate/detectors/detector_config.py
@@ -157,8 +157,16 @@ class ModelConfig(BaseModel):
             self._model_hash = file_hash.hexdigest()
 
     def create_colormap(self, enabled_labels: set[str]) -> None:
-        """Get a list of colors for enabled labels."""
-        colors = generate_color_palette(len(enabled_labels))
+        """Get a list of colors for enabled labels that aren't attributes."""
+        colors = generate_color_palette(
+            len(
+                list(
+                    filter(
+                        lambda label: label not in self._all_attributes, enabled_labels
+                    )
+                )
+            )
+        )
 
         self._colormap = {label: color for label, color in zip(enabled_labels, colors)}