ignore non digit GPU indexes

This commit is contained in:
jvrobert 2023-02-02 12:59:50 -07:00
parent b14d755faa
commit 9b17ca17b5

View File

@ -926,9 +926,9 @@ def get_nvidia_gpu_stats() -> dict[str, str]:
"--format=csv",
]
if ("CUDA_VISIBLE_DEVICES" in os.environ):
if ("CUDA_VISIBLE_DEVICES" in os.environ and os.environ["CUDA_VISIBLE_DEVICES"].isdigit()):
nvidia_smi_command.extend(["--id", os.environ['CUDA_VISIBLE_DEVICES']])
elif ("NVIDIA_VISIBLE_DEVICES" in os.environ):
elif ("NVIDIA_VISIBLE_DEVICES" in os.environ and os.environ["NVIDIA_VISIBLE_DEVICES"].isdigit()):
nvidia_smi_command.extend(["--id", os.environ['NVIDIA_VISIBLE_DEVICES']])
p = sp.run(