From e28767b107c2cddd55ff310f8fe5bb7c8d6c9a9b Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Fri, 4 Oct 2024 19:15:18 -0500
Subject: [PATCH] readd tokenizer parallelism env var

---
 docker/main/Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile
index 3e0dc83db..62effdcaf 100644
--- a/docker/main/Dockerfile
+++ b/docker/main/Dockerfile
@@ -197,6 +197,9 @@ ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility"
 
+# Disable tokenizer parallelism warning
+# https://stackoverflow.com/questions/62691279/how-to-disable-tokenizers-parallelism-true-false-warning/72926996#72926996
+ENV TOKENIZERS_PARALLELISM=true
 # https://github.com/huggingface/transformers/issues/27214
 ENV TRANSFORMERS_NO_ADVISORY_WARNINGS=1