Try to limit nvidia GPU queries to included GPUs

This commit is contained in:
jvrobert 2023-02-02 12:03:06 -07:00
parent 66881eb89f
commit b14d755faa

View File

@ -926,6 +926,11 @@ def get_nvidia_gpu_stats() -> dict[str, str]:
"--format=csv",
]
if ("CUDA_VISIBLE_DEVICES" in os.environ):
nvidia_smi_command.extend(["--id", os.environ['CUDA_VISIBLE_DEVICES']])
elif ("NVIDIA_VISIBLE_DEVICES" in os.environ):
nvidia_smi_command.extend(["--id", os.environ['NVIDIA_VISIBLE_DEVICES']])
p = sp.run(
nvidia_smi_command,
encoding="ascii",