We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[BUG] Error will raise when CUDA driver installed but no devices detected
When ``libnvidia-ml.so`` can be detected while no devices is installed, an error will appear:

```
/usr/local/lib/python3.6/dist-packages/mars/lib/nvutils.py in get_driver_info()
    200     cuda_version = c_uint()
    201 
--> 202     _nvml_check_error(_nvml_lib.nvmlSystemGetDriverVersion(version_buf, len(version_buf)))
    203     _nvml_check_error(_nvml_lib.nvmlSystemGetCudaDriverVersion(byref(cuda_version)))
    204 

/usr/local/lib/python3.6/dist-packages/mars/lib/nvutils.py in _nvml_check_error(result)
    104         _error_str = _nvmlErrorString(result)
    105         if _error_str:
--> 106             raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))
    107         else:
    108             raise NVError('Unknown NVML API Error %d' % result)

NVError: NVML API Error 1: Uninitialized
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mars/lib/nvutils.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # Copyright 1999-2020 Alibaba Group Holding Ltd.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #      http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 
16 import logging
17 import os
18 import uuid
19 from collections import namedtuple
20 from ctypes import c_char, c_char_p, c_int, c_uint, c_ulonglong, byref,\
21     create_string_buffer, Structure, POINTER, CDLL
22 
23 logger = logging.getLogger(__name__)
24 
25 # Some constants taken from cuda.h
26 CUDA_SUCCESS = 0
27 CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT = 16
28 CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR = 39
29 CU_DEVICE_ATTRIBUTE_CLOCK_RATE = 13
30 CU_DEVICE_ATTRIBUTE_PCI_BUS_ID = 33
31 CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID = 34
32 CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE = 36
33 
34 # nvml constants
35 NVML_SUCCESS = 0
36 NVML_TEMPERATURE_GPU = 0
37 
38 
39 class _CUuuid_t(Structure):
40     _fields_ = [
41         ('bytes', c_char * 16)
42     ]
43 class _nvmlUtilization_t(Structure):
44     _fields_ = [
45         ('gpu', c_uint),
46         ('memory', c_uint),
47     ]
48 
49 class _struct_nvmlDevice_t(Structure):
50     pass  # opaque handle
51 _nvmlDevice_t = POINTER(_struct_nvmlDevice_t)
52 
53 class _nvmlBAR1Memory_t(Structure):
54     _fields_ = [
55         ('total', c_ulonglong),
56         ('free', c_ulonglong),
57         ('used', c_ulonglong),
58     ]
59 
60 
61 def _load_nv_library(*libnames):
62     for lib in libnames:
63         try:
64             return CDLL(lib)
65         except OSError:
66             continue
67 
68 
69 _cuda_lib = _nvml_lib = None
70 
71 _cu_device_info = namedtuple('_cu_device_info', 'index uuid name multiprocessors cuda_cores threads')
72 _nvml_driver_info = namedtuple('_nvml_driver_info', 'driver_version cuda_version')
73 _nvml_device_status = namedtuple(
74     '_nvml_device_status', 'gpu_util mem_util temperature fb_total_mem fb_used_mem fb_free_mem')
75 
76 
77 _init_pid = None
78 _gpu_count = None
79 _driver_info = None
80 _device_infos = dict()
81 
82 
83 class NVError(Exception):
84     pass
85 
86 
87 def _cu_check_error(result):
88     if result != CUDA_SUCCESS:
89         _error_str = c_char_p()
90         _cuda_lib.cuGetErrorString(result, byref(_error_str))
91         raise NVError('Device API Error %d: %s' % (result, _error_str.value.decode()))
92 
93 
94 _nvmlErrorString = None
95 
96 
97 def _nvml_check_error(result):
98     global _nvmlErrorString
99     if _nvmlErrorString is None:
100         _nvmlErrorString = _nvml_lib.nvmlErrorString
101         _nvmlErrorString.restype = c_char_p
102 
103     if result != NVML_SUCCESS:
104         _error_str = _nvmlErrorString(result)
105         if _error_str:
106             raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))
107         else:
108             raise NVError('Unknown NVML API Error %d' % result)
109 
110 
111 _cu_process_var_to_cores = {
112     (1, 0): 8,
113     (1, 1): 8,
114     (1, 2): 8,
115     (1, 3): 8,
116     (2, 0): 32,
117     (2, 1): 48,
118 }
119 
120 
121 def _cu_get_processor_cores(major, minor):
122     return _cu_process_var_to_cores.get((major, minor), 192)
123 
124 
125 def _init_cp():
126     global _cuda_lib
127     if _init_pid == os.getpid():
128         return
129 
130     _cuda_lib = _load_nv_library('libcuda.so', 'libcuda.dylib', 'cuda.dll')
131 
132     if _cuda_lib is None:
133         return
134     try:
135         _cu_check_error(_cuda_lib.cuInit(0))
136     except NVError:
137         logger.exception('Failed to initialize libcuda.')
138         return
139 
140 
141 def _init_nvml():
142     global _nvml_lib
143     if _init_pid == os.getpid():
144         return
145 
146     _nvml_lib = _load_nv_library('libnvidia-ml.so', 'libnvidia-ml.dylib', 'nvml.dll')
147 
148     if _nvml_lib is None:
149         return
150     try:
151         _nvml_check_error(_nvml_lib.nvmlInit_v2())
152     except NVError:
153         logger.exception('Failed to initialize libnvidia-ml.')
154         return
155 
156 
157 def _init():
158     global _init_pid
159 
160     _init_cp()
161     _init_nvml()
162 
163     if _nvml_lib is not None and _cuda_lib is not None:
164         _init_pid = os.getpid()
165 
166 
167 def get_device_count():
168     global _gpu_count
169 
170     if _gpu_count is not None:
171         return _gpu_count
172 
173     _init_nvml()
174     if _nvml_lib is None:
175         return None
176 
177     if 'CUDA_VISIBLE_DEVICES' in os.environ:
178         devices = os.environ['CUDA_VISIBLE_DEVICES'].strip()
179         if not devices:
180             _gpu_count = 0
181         else:
182             _gpu_count = len(devices.split(','))
183     else:
184         n_gpus = c_uint()
185         _cu_check_error(_nvml_lib.nvmlDeviceGetCount(byref(n_gpus)))
186         _gpu_count = n_gpus.value
187     return _gpu_count
188 
189 
190 def get_driver_info():
191     global _driver_info
192 
193     _init_nvml()
194     if _nvml_lib is None:
195         return None
196     if _driver_info is not None:
197         return _driver_info
198 
199     version_buf = create_string_buffer(100)
200     cuda_version = c_uint()
201 
202     _nvml_check_error(_nvml_lib.nvmlSystemGetDriverVersion(version_buf, len(version_buf)))
203     _nvml_check_error(_nvml_lib.nvmlSystemGetCudaDriverVersion(byref(cuda_version)))
204 
205     _driver_info = _nvml_driver_info(
206         driver_version=version_buf.value.decode(),
207         cuda_version='%d.%d' % (cuda_version.value // 1000, cuda_version.value % 1000)
208     )
209     return _driver_info
210 
211 
212 def get_device_info(dev_index):
213     try:
214         return _device_infos[dev_index]
215     except KeyError:
216         pass
217 
218     _init()
219     if _init_pid is None:
220         return None
221 
222     device = c_int()
223     name_buf = create_string_buffer(100)
224     uuid_t = _CUuuid_t()
225     cc_major = c_int()
226     cc_minor = c_int()
227     cores = c_int()
228     threads_per_core = c_int()
229 
230     _cu_check_error(_cuda_lib.cuDeviceGet(byref(device), c_int(dev_index)))
231     _cu_check_error(_cuda_lib.cuDeviceGetName(name_buf, len(name_buf), device))
232     _cu_check_error(_cuda_lib.cuDeviceGetUuid(byref(uuid_t), device))
233     _cu_check_error(_cuda_lib.cuDeviceComputeCapability(
234         byref(cc_major), byref(cc_minor), device))
235     _cu_check_error(_cuda_lib.cuDeviceGetAttribute(
236         byref(cores), CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, device))
237     _cu_check_error(_cuda_lib.cuDeviceGetAttribute(
238         byref(threads_per_core), CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR, device))
239 
240     if 'CUDA_VISIBLE_DEVICES' in os.environ:
241         real_dev_index = [int(s) for s in os.environ['CUDA_VISIBLE_DEVICES'].split(',')][dev_index]
242     else:
243         real_dev_index = dev_index
244 
245     info = _device_infos[dev_index] = _cu_device_info(
246         index=real_dev_index,
247         uuid=uuid.UUID(bytes=uuid_t.bytes),
248         name=name_buf.value.decode(),
249         multiprocessors=cores.value,
250         cuda_cores=cores.value * _cu_get_processor_cores(cc_major.value, cc_minor.value),
251         threads=cores.value * threads_per_core.value,
252     )
253     return info
254 
255 
256 def get_device_status(dev_index):
257     _init()
258     if _init_pid is None:
259         return None
260 
261     device = _nvmlDevice_t()
262     utils = _nvmlUtilization_t()
263     temperature = c_uint()
264     memory_info = _nvmlBAR1Memory_t()
265 
266     dev_uuid = get_device_info(dev_index).uuid
267 
268     uuid_str = ('GPU-' + str(dev_uuid)).encode()
269 
270     _nvml_check_error(_nvml_lib.nvmlDeviceGetHandleByUUID(uuid_str, byref(device)))
271     _nvml_check_error(_nvml_lib.nvmlDeviceGetUtilizationRates(device, byref(utils)))
272     _nvml_check_error(_nvml_lib.nvmlDeviceGetTemperature(
273         device, NVML_TEMPERATURE_GPU, byref(temperature)))
274     _nvml_check_error(_nvml_lib.nvmlDeviceGetBAR1MemoryInfo(device, byref(memory_info)))
275 
276     return _nvml_device_status(
277         gpu_util=utils.gpu,
278         mem_util=utils.memory,
279         temperature=temperature.value,
280         fb_total_mem=memory_info.total,
281         fb_free_mem=memory_info.free,
282         fb_used_mem=memory_info.used,
283     )
284
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mars/lib/nvutils.py b/mars/lib/nvutils.py
--- a/mars/lib/nvutils.py
+++ b/mars/lib/nvutils.py
@@ -31,10 +31,14 @@
 CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID = 34
 CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE = 36
 
+CU_NO_CUDA_CAPABLE_DEVICE_DETECTED = 100
+
 # nvml constants
 NVML_SUCCESS = 0
 NVML_TEMPERATURE_GPU = 0
 
+NVML_DRIVER_NOT_LOADED = 9
+
 
 class _CUuuid_t(Structure):
     _fields_ = [
@@ -73,14 +77,36 @@
 _nvml_device_status = namedtuple(
     '_nvml_device_status', 'gpu_util mem_util temperature fb_total_mem fb_used_mem fb_free_mem')
 
-
 _init_pid = None
 _gpu_count = None
 _driver_info = None
 _device_infos = dict()
 
+_no_device_warned = False
+
 
 class NVError(Exception):
+    def __init__(self, msg, *args, errno=None):
+        self._errno = errno
+        super().__init__(msg or 'Unknown error', *args)
+
+    def __str__(self):
+        return '(%s) %s' % (self._errno, super().__str__())
+
+    @property
+    def errno(self):
+        return self._errno
+
+    @property
+    def message(self):
+        return super().__str__()
+
+
+class NVDeviceAPIError(NVError):
+    pass
+
+
+class NVMLAPIError(NVError):
     pass
 
 
@@ -88,7 +114,7 @@
     if result != CUDA_SUCCESS:
         _error_str = c_char_p()
         _cuda_lib.cuGetErrorString(result, byref(_error_str))
-        raise NVError('Device API Error %d: %s' % (result, _error_str.value.decode()))
+        raise NVDeviceAPIError(_error_str.value.decode(), errno=result)
 
 
 _nvmlErrorString = None
@@ -102,10 +128,7 @@
 
     if result != NVML_SUCCESS:
         _error_str = _nvmlErrorString(result)
-        if _error_str:
-            raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))
-        else:
-            raise NVError('Unknown NVML API Error %d' % result)
+        raise NVMLAPIError(_error_str.decode(), errno=result)
 
 
 _cu_process_var_to_cores = {
@@ -123,7 +146,7 @@
 
 
 def _init_cp():
-    global _cuda_lib
+    global _cuda_lib, _no_device_warned
     if _init_pid == os.getpid():
         return
 
@@ -133,13 +156,19 @@
         return
     try:
         _cu_check_error(_cuda_lib.cuInit(0))
-    except NVError:
-        logger.exception('Failed to initialize libcuda.')
+    except NVDeviceAPIError as ex:
+        if ex.errno == CU_NO_CUDA_CAPABLE_DEVICE_DETECTED:
+            _cuda_lib = None
+            if not _no_device_warned:
+                logger.warning('No CUDA device detected')
+                _no_device_warned = True
+        else:
+            logger.exception('Failed to initialize libcuda.')
         return
 
 
 def _init_nvml():
-    global _nvml_lib
+    global _nvml_lib, _no_device_warned
     if _init_pid == os.getpid():
         return
 
@@ -149,8 +178,14 @@
         return
     try:
         _nvml_check_error(_nvml_lib.nvmlInit_v2())
-    except NVError:
-        logger.exception('Failed to initialize libnvidia-ml.')
+    except NVMLAPIError as ex:
+        if ex.errno == NVML_DRIVER_NOT_LOADED:
+            _nvml_lib = None
+            if not _no_device_warned:
+                logger.warning('Failed to load libnvidia-ml: %s, no CUDA device will be enabled', ex.message)
+                _no_device_warned = True
+        else:
+            logger.exception('Failed to initialize libnvidia-ml.')
         return

{"golden_diff": "diff --git a/mars/lib/nvutils.py b/mars/lib/nvutils.py\n--- a/mars/lib/nvutils.py\n+++ b/mars/lib/nvutils.py\n@@ -31,10 +31,14 @@\n CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID = 34\n CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE = 36\n \n+CU_NO_CUDA_CAPABLE_DEVICE_DETECTED = 100\n+\n # nvml constants\n NVML_SUCCESS = 0\n NVML_TEMPERATURE_GPU = 0\n \n+NVML_DRIVER_NOT_LOADED = 9\n+\n \n class _CUuuid_t(Structure):\n     _fields_ = [\n@@ -73,14 +77,36 @@\n _nvml_device_status = namedtuple(\n     '_nvml_device_status', 'gpu_util mem_util temperature fb_total_mem fb_used_mem fb_free_mem')\n \n-\n _init_pid = None\n _gpu_count = None\n _driver_info = None\n _device_infos = dict()\n \n+_no_device_warned = False\n+\n \n class NVError(Exception):\n+    def __init__(self, msg, *args, errno=None):\n+        self._errno = errno\n+        super().__init__(msg or 'Unknown error', *args)\n+\n+    def __str__(self):\n+        return '(%s) %s' % (self._errno, super().__str__())\n+\n+    @property\n+    def errno(self):\n+        return self._errno\n+\n+    @property\n+    def message(self):\n+        return super().__str__()\n+\n+\n+class NVDeviceAPIError(NVError):\n+    pass\n+\n+\n+class NVMLAPIError(NVError):\n     pass\n \n \n@@ -88,7 +114,7 @@\n     if result != CUDA_SUCCESS:\n         _error_str = c_char_p()\n         _cuda_lib.cuGetErrorString(result, byref(_error_str))\n-        raise NVError('Device API Error %d: %s' % (result, _error_str.value.decode()))\n+        raise NVDeviceAPIError(_error_str.value.decode(), errno=result)\n \n \n _nvmlErrorString = None\n@@ -102,10 +128,7 @@\n \n     if result != NVML_SUCCESS:\n         _error_str = _nvmlErrorString(result)\n-        if _error_str:\n-            raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))\n-        else:\n-            raise NVError('Unknown NVML API Error %d' % result)\n+        raise NVMLAPIError(_error_str.decode(), errno=result)\n \n \n _cu_process_var_to_cores = {\n@@ -123,7 +146,7 @@\n \n \n def _init_cp():\n-    global _cuda_lib\n+    global _cuda_lib, _no_device_warned\n     if _init_pid == os.getpid():\n         return\n \n@@ -133,13 +156,19 @@\n         return\n     try:\n         _cu_check_error(_cuda_lib.cuInit(0))\n-    except NVError:\n-        logger.exception('Failed to initialize libcuda.')\n+    except NVDeviceAPIError as ex:\n+        if ex.errno == CU_NO_CUDA_CAPABLE_DEVICE_DETECTED:\n+            _cuda_lib = None\n+            if not _no_device_warned:\n+                logger.warning('No CUDA device detected')\n+                _no_device_warned = True\n+        else:\n+            logger.exception('Failed to initialize libcuda.')\n         return\n \n \n def _init_nvml():\n-    global _nvml_lib\n+    global _nvml_lib, _no_device_warned\n     if _init_pid == os.getpid():\n         return\n \n@@ -149,8 +178,14 @@\n         return\n     try:\n         _nvml_check_error(_nvml_lib.nvmlInit_v2())\n-    except NVError:\n-        logger.exception('Failed to initialize libnvidia-ml.')\n+    except NVMLAPIError as ex:\n+        if ex.errno == NVML_DRIVER_NOT_LOADED:\n+            _nvml_lib = None\n+            if not _no_device_warned:\n+                logger.warning('Failed to load libnvidia-ml: %s, no CUDA device will be enabled', ex.message)\n+                _no_device_warned = True\n+        else:\n+            logger.exception('Failed to initialize libnvidia-ml.')\n         return\n", "issue": "[BUG] Error will raise when CUDA driver installed but no devices detected\nWhen ``libnvidia-ml.so`` can be detected while no devices is installed, an error will appear:\r\n\r\n```\r\n/usr/local/lib/python3.6/dist-packages/mars/lib/nvutils.py in get_driver_info()\r\n    200     cuda_version = c_uint()\r\n    201 \r\n--> 202     _nvml_check_error(_nvml_lib.nvmlSystemGetDriverVersion(version_buf, len(version_buf)))\r\n    203     _nvml_check_error(_nvml_lib.nvmlSystemGetCudaDriverVersion(byref(cuda_version)))\r\n    204 \r\n\r\n/usr/local/lib/python3.6/dist-packages/mars/lib/nvutils.py in _nvml_check_error(result)\r\n    104         _error_str = _nvmlErrorString(result)\r\n    105         if _error_str:\r\n--> 106             raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))\r\n    107         else:\r\n    108             raise NVError('Unknown NVML API Error %d' % result)\r\n\r\nNVError: NVML API Error 1: Uninitialized\r\n```\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# Copyright 1999-2020 Alibaba Group Holding Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport uuid\nfrom collections import namedtuple\nfrom ctypes import c_char, c_char_p, c_int, c_uint, c_ulonglong, byref,\\\n    create_string_buffer, Structure, POINTER, CDLL\n\nlogger = logging.getLogger(__name__)\n\n# Some constants taken from cuda.h\nCUDA_SUCCESS = 0\nCU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT = 16\nCU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR = 39\nCU_DEVICE_ATTRIBUTE_CLOCK_RATE = 13\nCU_DEVICE_ATTRIBUTE_PCI_BUS_ID = 33\nCU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID = 34\nCU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE = 36\n\n# nvml constants\nNVML_SUCCESS = 0\nNVML_TEMPERATURE_GPU = 0\n\n\nclass _CUuuid_t(Structure):\n    _fields_ = [\n        ('bytes', c_char * 16)\n    ]\nclass _nvmlUtilization_t(Structure):\n    _fields_ = [\n        ('gpu', c_uint),\n        ('memory', c_uint),\n    ]\n\nclass _struct_nvmlDevice_t(Structure):\n    pass  # opaque handle\n_nvmlDevice_t = POINTER(_struct_nvmlDevice_t)\n\nclass _nvmlBAR1Memory_t(Structure):\n    _fields_ = [\n        ('total', c_ulonglong),\n        ('free', c_ulonglong),\n        ('used', c_ulonglong),\n    ]\n\n\ndef _load_nv_library(*libnames):\n    for lib in libnames:\n        try:\n            return CDLL(lib)\n        except OSError:\n            continue\n\n\n_cuda_lib = _nvml_lib = None\n\n_cu_device_info = namedtuple('_cu_device_info', 'index uuid name multiprocessors cuda_cores threads')\n_nvml_driver_info = namedtuple('_nvml_driver_info', 'driver_version cuda_version')\n_nvml_device_status = namedtuple(\n    '_nvml_device_status', 'gpu_util mem_util temperature fb_total_mem fb_used_mem fb_free_mem')\n\n\n_init_pid = None\n_gpu_count = None\n_driver_info = None\n_device_infos = dict()\n\n\nclass NVError(Exception):\n    pass\n\n\ndef _cu_check_error(result):\n    if result != CUDA_SUCCESS:\n        _error_str = c_char_p()\n        _cuda_lib.cuGetErrorString(result, byref(_error_str))\n        raise NVError('Device API Error %d: %s' % (result, _error_str.value.decode()))\n\n\n_nvmlErrorString = None\n\n\ndef _nvml_check_error(result):\n    global _nvmlErrorString\n    if _nvmlErrorString is None:\n        _nvmlErrorString = _nvml_lib.nvmlErrorString\n        _nvmlErrorString.restype = c_char_p\n\n    if result != NVML_SUCCESS:\n        _error_str = _nvmlErrorString(result)\n        if _error_str:\n            raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))\n        else:\n            raise NVError('Unknown NVML API Error %d' % result)\n\n\n_cu_process_var_to_cores = {\n    (1, 0): 8,\n    (1, 1): 8,\n    (1, 2): 8,\n    (1, 3): 8,\n    (2, 0): 32,\n    (2, 1): 48,\n}\n\n\ndef _cu_get_processor_cores(major, minor):\n    return _cu_process_var_to_cores.get((major, minor), 192)\n\n\ndef _init_cp():\n    global _cuda_lib\n    if _init_pid == os.getpid():\n        return\n\n    _cuda_lib = _load_nv_library('libcuda.so', 'libcuda.dylib', 'cuda.dll')\n\n    if _cuda_lib is None:\n        return\n    try:\n        _cu_check_error(_cuda_lib.cuInit(0))\n    except NVError:\n        logger.exception('Failed to initialize libcuda.')\n        return\n\n\ndef _init_nvml():\n    global _nvml_lib\n    if _init_pid == os.getpid():\n        return\n\n    _nvml_lib = _load_nv_library('libnvidia-ml.so', 'libnvidia-ml.dylib', 'nvml.dll')\n\n    if _nvml_lib is None:\n        return\n    try:\n        _nvml_check_error(_nvml_lib.nvmlInit_v2())\n    except NVError:\n        logger.exception('Failed to initialize libnvidia-ml.')\n        return\n\n\ndef _init():\n    global _init_pid\n\n    _init_cp()\n    _init_nvml()\n\n    if _nvml_lib is not None and _cuda_lib is not None:\n        _init_pid = os.getpid()\n\n\ndef get_device_count():\n    global _gpu_count\n\n    if _gpu_count is not None:\n        return _gpu_count\n\n    _init_nvml()\n    if _nvml_lib is None:\n        return None\n\n    if 'CUDA_VISIBLE_DEVICES' in os.environ:\n        devices = os.environ['CUDA_VISIBLE_DEVICES'].strip()\n        if not devices:\n            _gpu_count = 0\n        else:\n            _gpu_count = len(devices.split(','))\n    else:\n        n_gpus = c_uint()\n        _cu_check_error(_nvml_lib.nvmlDeviceGetCount(byref(n_gpus)))\n        _gpu_count = n_gpus.value\n    return _gpu_count\n\n\ndef get_driver_info():\n    global _driver_info\n\n    _init_nvml()\n    if _nvml_lib is None:\n        return None\n    if _driver_info is not None:\n        return _driver_info\n\n    version_buf = create_string_buffer(100)\n    cuda_version = c_uint()\n\n    _nvml_check_error(_nvml_lib.nvmlSystemGetDriverVersion(version_buf, len(version_buf)))\n    _nvml_check_error(_nvml_lib.nvmlSystemGetCudaDriverVersion(byref(cuda_version)))\n\n    _driver_info = _nvml_driver_info(\n        driver_version=version_buf.value.decode(),\n        cuda_version='%d.%d' % (cuda_version.value // 1000, cuda_version.value % 1000)\n    )\n    return _driver_info\n\n\ndef get_device_info(dev_index):\n    try:\n        return _device_infos[dev_index]\n    except KeyError:\n        pass\n\n    _init()\n    if _init_pid is None:\n        return None\n\n    device = c_int()\n    name_buf = create_string_buffer(100)\n    uuid_t = _CUuuid_t()\n    cc_major = c_int()\n    cc_minor = c_int()\n    cores = c_int()\n    threads_per_core = c_int()\n\n    _cu_check_error(_cuda_lib.cuDeviceGet(byref(device), c_int(dev_index)))\n    _cu_check_error(_cuda_lib.cuDeviceGetName(name_buf, len(name_buf), device))\n    _cu_check_error(_cuda_lib.cuDeviceGetUuid(byref(uuid_t), device))\n    _cu_check_error(_cuda_lib.cuDeviceComputeCapability(\n        byref(cc_major), byref(cc_minor), device))\n    _cu_check_error(_cuda_lib.cuDeviceGetAttribute(\n        byref(cores), CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, device))\n    _cu_check_error(_cuda_lib.cuDeviceGetAttribute(\n        byref(threads_per_core), CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR, device))\n\n    if 'CUDA_VISIBLE_DEVICES' in os.environ:\n        real_dev_index = [int(s) for s in os.environ['CUDA_VISIBLE_DEVICES'].split(',')][dev_index]\n    else:\n        real_dev_index = dev_index\n\n    info = _device_infos[dev_index] = _cu_device_info(\n        index=real_dev_index,\n        uuid=uuid.UUID(bytes=uuid_t.bytes),\n        name=name_buf.value.decode(),\n        multiprocessors=cores.value,\n        cuda_cores=cores.value * _cu_get_processor_cores(cc_major.value, cc_minor.value),\n        threads=cores.value * threads_per_core.value,\n    )\n    return info\n\n\ndef get_device_status(dev_index):\n    _init()\n    if _init_pid is None:\n        return None\n\n    device = _nvmlDevice_t()\n    utils = _nvmlUtilization_t()\n    temperature = c_uint()\n    memory_info = _nvmlBAR1Memory_t()\n\n    dev_uuid = get_device_info(dev_index).uuid\n\n    uuid_str = ('GPU-' + str(dev_uuid)).encode()\n\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetHandleByUUID(uuid_str, byref(device)))\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetUtilizationRates(device, byref(utils)))\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetTemperature(\n        device, NVML_TEMPERATURE_GPU, byref(temperature)))\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetBAR1MemoryInfo(device, byref(memory_info)))\n\n    return _nvml_device_status(\n        gpu_util=utils.gpu,\n        mem_util=utils.memory,\n        temperature=temperature.value,\n        fb_total_mem=memory_info.total,\n        fb_free_mem=memory_info.free,\n        fb_used_mem=memory_info.used,\n    )\n", "path": "mars/lib/nvutils.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# Copyright 1999-2020 Alibaba Group Holding Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport uuid\nfrom collections import namedtuple\nfrom ctypes import c_char, c_char_p, c_int, c_uint, c_ulonglong, byref,\\\n    create_string_buffer, Structure, POINTER, CDLL\n\nlogger = logging.getLogger(__name__)\n\n# Some constants taken from cuda.h\nCUDA_SUCCESS = 0\nCU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT = 16\nCU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR = 39\nCU_DEVICE_ATTRIBUTE_CLOCK_RATE = 13\nCU_DEVICE_ATTRIBUTE_PCI_BUS_ID = 33\nCU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID = 34\nCU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE = 36\n\nCU_NO_CUDA_CAPABLE_DEVICE_DETECTED = 100\n\n# nvml constants\nNVML_SUCCESS = 0\nNVML_TEMPERATURE_GPU = 0\n\nNVML_DRIVER_NOT_LOADED = 9\n\n\nclass _CUuuid_t(Structure):\n    _fields_ = [\n        ('bytes', c_char * 16)\n    ]\nclass _nvmlUtilization_t(Structure):\n    _fields_ = [\n        ('gpu', c_uint),\n        ('memory', c_uint),\n    ]\n\nclass _struct_nvmlDevice_t(Structure):\n    pass  # opaque handle\n_nvmlDevice_t = POINTER(_struct_nvmlDevice_t)\n\nclass _nvmlBAR1Memory_t(Structure):\n    _fields_ = [\n        ('total', c_ulonglong),\n        ('free', c_ulonglong),\n        ('used', c_ulonglong),\n    ]\n\n\ndef _load_nv_library(*libnames):\n    for lib in libnames:\n        try:\n            return CDLL(lib)\n        except OSError:\n            continue\n\n\n_cuda_lib = _nvml_lib = None\n\n_cu_device_info = namedtuple('_cu_device_info', 'index uuid name multiprocessors cuda_cores threads')\n_nvml_driver_info = namedtuple('_nvml_driver_info', 'driver_version cuda_version')\n_nvml_device_status = namedtuple(\n    '_nvml_device_status', 'gpu_util mem_util temperature fb_total_mem fb_used_mem fb_free_mem')\n\n_init_pid = None\n_gpu_count = None\n_driver_info = None\n_device_infos = dict()\n\n_no_device_warned = False\n\n\nclass NVError(Exception):\n    def __init__(self, msg, *args, errno=None):\n        self._errno = errno\n        super().__init__(msg or 'Unknown error', *args)\n\n    def __str__(self):\n        return '(%s) %s' % (self._errno, super().__str__())\n\n    @property\n    def errno(self):\n        return self._errno\n\n    @property\n    def message(self):\n        return super().__str__()\n\n\nclass NVDeviceAPIError(NVError):\n    pass\n\n\nclass NVMLAPIError(NVError):\n    pass\n\n\ndef _cu_check_error(result):\n    if result != CUDA_SUCCESS:\n        _error_str = c_char_p()\n        _cuda_lib.cuGetErrorString(result, byref(_error_str))\n        raise NVDeviceAPIError(_error_str.value.decode(), errno=result)\n\n\n_nvmlErrorString = None\n\n\ndef _nvml_check_error(result):\n    global _nvmlErrorString\n    if _nvmlErrorString is None:\n        _nvmlErrorString = _nvml_lib.nvmlErrorString\n        _nvmlErrorString.restype = c_char_p\n\n    if result != NVML_SUCCESS:\n        _error_str = _nvmlErrorString(result)\n        raise NVMLAPIError(_error_str.decode(), errno=result)\n\n\n_cu_process_var_to_cores = {\n    (1, 0): 8,\n    (1, 1): 8,\n    (1, 2): 8,\n    (1, 3): 8,\n    (2, 0): 32,\n    (2, 1): 48,\n}\n\n\ndef _cu_get_processor_cores(major, minor):\n    return _cu_process_var_to_cores.get((major, minor), 192)\n\n\ndef _init_cp():\n    global _cuda_lib, _no_device_warned\n    if _init_pid == os.getpid():\n        return\n\n    _cuda_lib = _load_nv_library('libcuda.so', 'libcuda.dylib', 'cuda.dll')\n\n    if _cuda_lib is None:\n        return\n    try:\n        _cu_check_error(_cuda_lib.cuInit(0))\n    except NVDeviceAPIError as ex:\n        if ex.errno == CU_NO_CUDA_CAPABLE_DEVICE_DETECTED:\n            _cuda_lib = None\n            if not _no_device_warned:\n                logger.warning('No CUDA device detected')\n                _no_device_warned = True\n        else:\n            logger.exception('Failed to initialize libcuda.')\n        return\n\n\ndef _init_nvml():\n    global _nvml_lib, _no_device_warned\n    if _init_pid == os.getpid():\n        return\n\n    _nvml_lib = _load_nv_library('libnvidia-ml.so', 'libnvidia-ml.dylib', 'nvml.dll')\n\n    if _nvml_lib is None:\n        return\n    try:\n        _nvml_check_error(_nvml_lib.nvmlInit_v2())\n    except NVMLAPIError as ex:\n        if ex.errno == NVML_DRIVER_NOT_LOADED:\n            _nvml_lib = None\n            if not _no_device_warned:\n                logger.warning('Failed to load libnvidia-ml: %s, no CUDA device will be enabled', ex.message)\n                _no_device_warned = True\n        else:\n            logger.exception('Failed to initialize libnvidia-ml.')\n        return\n\n\ndef _init():\n    global _init_pid\n\n    _init_cp()\n    _init_nvml()\n\n    if _nvml_lib is not None and _cuda_lib is not None:\n        _init_pid = os.getpid()\n\n\ndef get_device_count():\n    global _gpu_count\n\n    if _gpu_count is not None:\n        return _gpu_count\n\n    _init_nvml()\n    if _nvml_lib is None:\n        return None\n\n    if 'CUDA_VISIBLE_DEVICES' in os.environ:\n        devices = os.environ['CUDA_VISIBLE_DEVICES'].strip()\n        if not devices:\n            _gpu_count = 0\n        else:\n            _gpu_count = len(devices.split(','))\n    else:\n        n_gpus = c_uint()\n        _cu_check_error(_nvml_lib.nvmlDeviceGetCount(byref(n_gpus)))\n        _gpu_count = n_gpus.value\n    return _gpu_count\n\n\ndef get_driver_info():\n    global _driver_info\n\n    _init_nvml()\n    if _nvml_lib is None:\n        return None\n    if _driver_info is not None:\n        return _driver_info\n\n    version_buf = create_string_buffer(100)\n    cuda_version = c_uint()\n\n    _nvml_check_error(_nvml_lib.nvmlSystemGetDriverVersion(version_buf, len(version_buf)))\n    _nvml_check_error(_nvml_lib.nvmlSystemGetCudaDriverVersion(byref(cuda_version)))\n\n    _driver_info = _nvml_driver_info(\n        driver_version=version_buf.value.decode(),\n        cuda_version='%d.%d' % (cuda_version.value // 1000, cuda_version.value % 1000)\n    )\n    return _driver_info\n\n\ndef get_device_info(dev_index):\n    try:\n        return _device_infos[dev_index]\n    except KeyError:\n        pass\n\n    _init()\n    if _init_pid is None:\n        return None\n\n    device = c_int()\n    name_buf = create_string_buffer(100)\n    uuid_t = _CUuuid_t()\n    cc_major = c_int()\n    cc_minor = c_int()\n    cores = c_int()\n    threads_per_core = c_int()\n\n    _cu_check_error(_cuda_lib.cuDeviceGet(byref(device), c_int(dev_index)))\n    _cu_check_error(_cuda_lib.cuDeviceGetName(name_buf, len(name_buf), device))\n    _cu_check_error(_cuda_lib.cuDeviceGetUuid(byref(uuid_t), device))\n    _cu_check_error(_cuda_lib.cuDeviceComputeCapability(\n        byref(cc_major), byref(cc_minor), device))\n    _cu_check_error(_cuda_lib.cuDeviceGetAttribute(\n        byref(cores), CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, device))\n    _cu_check_error(_cuda_lib.cuDeviceGetAttribute(\n        byref(threads_per_core), CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR, device))\n\n    if 'CUDA_VISIBLE_DEVICES' in os.environ:\n        real_dev_index = [int(s) for s in os.environ['CUDA_VISIBLE_DEVICES'].split(',')][dev_index]\n    else:\n        real_dev_index = dev_index\n\n    info = _device_infos[dev_index] = _cu_device_info(\n        index=real_dev_index,\n        uuid=uuid.UUID(bytes=uuid_t.bytes),\n        name=name_buf.value.decode(),\n        multiprocessors=cores.value,\n        cuda_cores=cores.value * _cu_get_processor_cores(cc_major.value, cc_minor.value),\n        threads=cores.value * threads_per_core.value,\n    )\n    return info\n\n\ndef get_device_status(dev_index):\n    _init()\n    if _init_pid is None:\n        return None\n\n    device = _nvmlDevice_t()\n    utils = _nvmlUtilization_t()\n    temperature = c_uint()\n    memory_info = _nvmlBAR1Memory_t()\n\n    dev_uuid = get_device_info(dev_index).uuid\n\n    uuid_str = ('GPU-' + str(dev_uuid)).encode()\n\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetHandleByUUID(uuid_str, byref(device)))\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetUtilizationRates(device, byref(utils)))\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetTemperature(\n        device, NVML_TEMPERATURE_GPU, byref(temperature)))\n    _nvml_check_error(_nvml_lib.nvmlDeviceGetBAR1MemoryInfo(device, byref(memory_info)))\n\n    return _nvml_device_status(\n        gpu_util=utils.gpu,\n        mem_util=utils.memory,\n        temperature=temperature.value,\n        fb_total_mem=memory_info.total,\n        fb_free_mem=memory_info.free,\n        fb_used_mem=memory_info.used,\n    )\n", "path": "mars/lib/nvutils.py"}]}