small fix for avoid errors on strange GPUs and old drivers

This commit is contained in:
Sergey Krashevich 2023-05-03 10:33:07 +03:00
parent 4eb7b31759
commit dc4b593e32
No known key found for this signature in database
GPG Key ID: 625171324E7D3856

View File

@ -930,16 +930,22 @@ def get_nvidia_gpu_stats() -> dict[str, str]:
results = {} results = {}
for i in range(deviceCount): for i in range(deviceCount):
handle = nvml.nvmlDeviceGetHandleByIndex(i) handle = nvml.nvmlDeviceGetHandleByIndex(i)
meminfo = nvml.nvmlDeviceGetMemoryInfo(handle) meminfo = try_get_info(nvml.nvmlDeviceGetMemoryInfo, handle)
util = try_get_info(nvml.nvmlDeviceGetUtilizationRates, handle) util = try_get_info(nvml.nvmlDeviceGetUtilizationRates, handle)
if util != "N/A": if util != "N/A":
gpu_util = util.gpu gpu_util = util.gpu
else: else:
gpu_util = 0 gpu_util = 0
if meminfo != "N/A":
gpu_mem_util = meminfo.used / meminfo.total * 100
else:
gpu_mem_util = -1
results[i] = { results[i] = {
"name": nvml.nvmlDeviceGetName(handle), "name": nvml.nvmlDeviceGetName(handle),
"gpu": gpu_util, "gpu": gpu_util,
"mem": meminfo.used / meminfo.total * 100, "mem": gpu_mem_util,
} }
return results return results