We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Python 3.8 incompatibility with nccl_built check
See: https://github.com/huge-success/sanic/issues/1774

```
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/site-packages/horovod/common/util.py:110: in wrapper
    retval = f(*args, **kwargs)
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/site-packages/horovod/common/util.py:151: in nccl_built
    result = _check_extension_lambda(
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/site-packages/horovod/common/util.py:90: in _check_extension_lambda
    p.start()
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/process.py:121: in start
    self._popen = self._Popen(self)
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/context.py:224: in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/context.py:283: in _Popen
    return Popen(process_obj)
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/popen_spawn_posix.py:32: in __init__
    super().__init__(process_obj)
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/popen_fork.py:19: in __init__
    self._launch(process_obj)
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/popen_spawn_posix.py:47: in _launch
    reduction.dump(process_obj, fp)
/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/reduction.py:60: in dump
    ForkingPickler(file, protocol).dump(obj)
E   AttributeError: Can't pickle local object '_check_extension_lambda.<locals>._target_fn'
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `horovod/common/util.py`
Content:
```
1 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2 # Modifications copyright (C) 2019 Uber Technologies, Inc.
3 # Modifications copyright Microsoft
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #     http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 # =============================================================================
17 
18 from contextlib import contextmanager
19 from multiprocessing import Process, Queue
20 import os
21 import sysconfig
22 
23 EXTENSIONS = ['tensorflow', 'torch', 'mxnet']
24 
25 
26 def get_ext_suffix():
27     """Determine library extension for various versions of Python."""
28     ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
29     if ext_suffix:
30         return ext_suffix
31 
32     ext_suffix = sysconfig.get_config_var('SO')
33     if ext_suffix:
34         return ext_suffix
35 
36     return '.so'
37 
38 
39 def get_extension_full_path(pkg_path, *args):
40     assert len(args) >= 1
41     dir_path = os.path.join(os.path.dirname(pkg_path), *args[:-1])
42     full_path = os.path.join(dir_path, args[-1] + get_ext_suffix())
43     return full_path
44 
45 
46 def check_extension(ext_name, ext_env_var, pkg_path, *args):
47     full_path = get_extension_full_path(pkg_path, *args)
48     if not os.path.exists(full_path):
49         raise ImportError(
50             'Extension %s has not been built.  If this is not expected, reinstall '
51             'Horovod with %s=1 to debug the build error.' % (ext_name, ext_env_var))
52 
53 
54 def _check_extension_lambda(ext_base_name, fn, fn_desc, verbose):
55     """
56     Tries to load the extension in a new process.  If successful, puts fn(ext)
57     to the queue or False otherwise.  Mutes all stdout/stderr.
58     """
59     def _target_fn(ext_base_name, fn, fn_desc, queue, verbose):
60         import importlib
61         import sys
62         import traceback
63 
64         if verbose:
65             print('Checking whether extension {ext_base_name} was {fn_desc}.'.format(
66                 ext_base_name=ext_base_name, fn_desc=fn_desc))
67         else:
68             # Suppress output
69             sys.stdout = open(os.devnull, 'w')
70             sys.stderr = open(os.devnull, 'w')
71 
72         try:
73             ext = importlib.import_module('.' + ext_base_name, 'horovod')
74             result = fn(ext)
75         except:
76             traceback.print_exc()
77             result = None
78 
79         if verbose:
80             print('Extension {ext_base_name} {flag} {fn_desc}.'.format(
81                 ext_base_name=ext_base_name, flag=('was' if result else 'was NOT'),
82                 fn_desc=fn_desc))
83 
84         queue.put(result)
85 
86     queue = Queue()
87     p = Process(target=_target_fn,
88                 args=(ext_base_name, fn, fn_desc, queue, verbose))
89     p.daemon = True
90     p.start()
91     p.join()
92     return queue.get_nowait()
93 
94 
95 def extension_available(ext_base_name, verbose=False):
96     available_fn = lambda ext: ext is not None
97     return _check_extension_lambda(
98         ext_base_name, available_fn, 'built', verbose) or False
99 
100 
101 def _cache(f):
102     cache = dict()
103 
104     def wrapper(*args, **kwargs):
105         key = (args, frozenset(kwargs.items()))
106 
107         if key in cache:
108             return cache[key]
109         else:
110             retval = f(*args, **kwargs)
111             cache[key] = retval
112             return retval
113 
114     return wrapper
115 
116 
117 @_cache
118 def gpu_available(ext_base_name, verbose=False):
119     available_fn = lambda ext: ext._check_has_gpu()
120     return _check_extension_lambda(
121         ext_base_name, available_fn, 'running with GPU', verbose) or False
122 
123 
124 @_cache
125 def mpi_built(verbose=False):
126     for ext_base_name in EXTENSIONS:
127         built_fn = lambda ext: ext.mpi_built()
128         result = _check_extension_lambda(
129             ext_base_name, built_fn, 'built with MPI', verbose)
130         if result is not None:
131             return result
132     return False
133 
134 
135 @_cache
136 def gloo_built(verbose=False):
137     for ext_base_name in EXTENSIONS:
138         built_fn = lambda ext: ext.gloo_built()
139         result = _check_extension_lambda(
140             ext_base_name, built_fn, 'built with Gloo', verbose)
141         if result is not None:
142             return result
143     raise RuntimeError('Failed to determine if Gloo support has been built. '
144                        'Run again with --verbose for more details.')
145 
146 
147 @_cache
148 def nccl_built(verbose=False):
149     for ext_base_name in EXTENSIONS:
150         built_fn = lambda ext: ext.nccl_built()
151         result = _check_extension_lambda(
152             ext_base_name, built_fn, 'built with NCCL', verbose)
153         if result is not None:
154             return result
155     raise RuntimeError('Failed to determine if NCCL support has been built. '
156                        'Run again with --verbose for more details.')
157 
158 
159 @_cache
160 def ddl_built(verbose=False):
161     for ext_base_name in EXTENSIONS:
162         built_fn = lambda ext: ext.ddl_built()
163         result = _check_extension_lambda(
164             ext_base_name, built_fn, 'built with DDL', verbose)
165         if result is not None:
166             return result
167     raise RuntimeError('Failed to determine if DDL support has been built. '
168                        'Run again with --verbose for more details.')
169 
170 
171 @_cache
172 def ccl_built(verbose=False):
173     for ext_base_name in EXTENSIONS:
174         built_fn = lambda ext: ext.ccl_built()
175         result = _check_extension_lambda(
176             ext_base_name, built_fn, 'built with CCL', verbose)
177         if result is not None:
178             return result
179     raise RuntimeError('Failed to determine if CCL support has been built. '
180                        'Run again with --verbose for more details.')
181 
182 
183 @contextmanager
184 def env(**kwargs):
185     # ignore args with None values
186     for k in list(kwargs.keys()):
187         if kwargs[k] is None:
188             del kwargs[k]
189 
190     # backup environment
191     backup = {}
192     for k in kwargs.keys():
193         backup[k] = os.environ.get(k)
194 
195     # set new values & yield
196     for k, v in kwargs.items():
197         os.environ[k] = v
198 
199     try:
200         yield
201     finally:
202         # restore environment
203         for k in kwargs.keys():
204             if backup[k] is not None:
205                 os.environ[k] = backup[k]
206             else:
207                 del os.environ[k]
208 
209 
210 def get_average_backwards_compatibility_fun(reduce_ops):
211     """
212     Handle backwards compatibility between the old average and the new op parameters.
213     Old code using the average parameter (e.g. hvd.allreduce(tensor, average=False))
214     gets unchanged behavior, but mixing old and new is disallowed (e.g. no
215     hvd.allreduce(tensor, average=False, op=hvd.Adasum)).
216     """
217     def impl(op, average):
218         if op != None:
219             if average != None:
220                 raise ValueError('The op parameter supersedes average. Please provide only one of them.')
221             return op
222         elif average != None:
223             return reduce_ops.Average if average else reduce_ops.Sum
224         else:
225             return reduce_ops.Average
226     return impl
227 
228 
229 def num_rank_is_power_2(num_rank):
230     """
231     Tests if the given number of ranks is of power of 2. This check is required
232     for Adasum allreduce.
233     TODO support non-power of 2 ranks.
234     """
235     return num_rank != 0 and ((num_rank & (num_rank -1)) == 0)
236
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/horovod/common/util.py b/horovod/common/util.py
--- a/horovod/common/util.py
+++ b/horovod/common/util.py
@@ -15,11 +15,15 @@
 # limitations under the License.
 # =============================================================================
 
-from contextlib import contextmanager
-from multiprocessing import Process, Queue
+import multiprocessing
 import os
+import sys
 import sysconfig
 
+from contextlib import contextmanager
+
+
+_PY3 = sys.version_info[0] == 3
 EXTENSIONS = ['tensorflow', 'torch', 'mxnet']
 
 
@@ -83,9 +87,11 @@
 
         queue.put(result)
 
-    queue = Queue()
-    p = Process(target=_target_fn,
-                args=(ext_base_name, fn, fn_desc, queue, verbose))
+    # 'fork' is required because horovodrun is a frozen executable
+    ctx = multiprocessing.get_context('fork') if _PY3 else multiprocessing
+    queue = ctx.Queue()
+    p = ctx.Process(target=_target_fn,
+                    args=(ext_base_name, fn, fn_desc, queue, verbose))
     p.daemon = True
     p.start()
     p.join()

{"golden_diff": "diff --git a/horovod/common/util.py b/horovod/common/util.py\n--- a/horovod/common/util.py\n+++ b/horovod/common/util.py\n@@ -15,11 +15,15 @@\n # limitations under the License.\n # =============================================================================\n \n-from contextlib import contextmanager\n-from multiprocessing import Process, Queue\n+import multiprocessing\n import os\n+import sys\n import sysconfig\n \n+from contextlib import contextmanager\n+\n+\n+_PY3 = sys.version_info[0] == 3\n EXTENSIONS = ['tensorflow', 'torch', 'mxnet']\n \n \n@@ -83,9 +87,11 @@\n \n         queue.put(result)\n \n-    queue = Queue()\n-    p = Process(target=_target_fn,\n-                args=(ext_base_name, fn, fn_desc, queue, verbose))\n+    # 'fork' is required because horovodrun is a frozen executable\n+    ctx = multiprocessing.get_context('fork') if _PY3 else multiprocessing\n+    queue = ctx.Queue()\n+    p = ctx.Process(target=_target_fn,\n+                    args=(ext_base_name, fn, fn_desc, queue, verbose))\n     p.daemon = True\n     p.start()\n     p.join()\n", "issue": "Python 3.8 incompatibility with nccl_built check\nSee: https://github.com/huge-success/sanic/issues/1774\r\n\r\n```\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/site-packages/horovod/common/util.py:110: in wrapper\r\n    retval = f(*args, **kwargs)\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/site-packages/horovod/common/util.py:151: in nccl_built\r\n    result = _check_extension_lambda(\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/site-packages/horovod/common/util.py:90: in _check_extension_lambda\r\n    p.start()\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/process.py:121: in start\r\n    self._popen = self._Popen(self)\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/context.py:224: in _Popen\r\n    return _default_context.get_context().Process._Popen(process_obj)\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/context.py:283: in _Popen\r\n    return Popen(process_obj)\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/popen_spawn_posix.py:32: in __init__\r\n    super().__init__(process_obj)\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/popen_fork.py:19: in __init__\r\n    self._launch(process_obj)\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/popen_spawn_posix.py:47: in _launch\r\n    reduction.dump(process_obj, fp)\r\n/Users/runner/hostedtoolcache/Python/3.8.2/x64/lib/python3.8/multiprocessing/reduction.py:60: in dump\r\n    ForkingPickler(file, protocol).dump(obj)\r\nE   AttributeError: Can't pickle local object '_check_extension_lambda.<locals>._target_fn'\r\n```\n", "before_files": [{"content": "# Copyright 2016 The TensorFlow Authors. All Rights Reserved.\n# Modifications copyright (C) 2019 Uber Technologies, Inc.\n# Modifications copyright Microsoft\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# =============================================================================\n\nfrom contextlib import contextmanager\nfrom multiprocessing import Process, Queue\nimport os\nimport sysconfig\n\nEXTENSIONS = ['tensorflow', 'torch', 'mxnet']\n\n\ndef get_ext_suffix():\n    \"\"\"Determine library extension for various versions of Python.\"\"\"\n    ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')\n    if ext_suffix:\n        return ext_suffix\n\n    ext_suffix = sysconfig.get_config_var('SO')\n    if ext_suffix:\n        return ext_suffix\n\n    return '.so'\n\n\ndef get_extension_full_path(pkg_path, *args):\n    assert len(args) >= 1\n    dir_path = os.path.join(os.path.dirname(pkg_path), *args[:-1])\n    full_path = os.path.join(dir_path, args[-1] + get_ext_suffix())\n    return full_path\n\n\ndef check_extension(ext_name, ext_env_var, pkg_path, *args):\n    full_path = get_extension_full_path(pkg_path, *args)\n    if not os.path.exists(full_path):\n        raise ImportError(\n            'Extension %s has not been built.  If this is not expected, reinstall '\n            'Horovod with %s=1 to debug the build error.' % (ext_name, ext_env_var))\n\n\ndef _check_extension_lambda(ext_base_name, fn, fn_desc, verbose):\n    \"\"\"\n    Tries to load the extension in a new process.  If successful, puts fn(ext)\n    to the queue or False otherwise.  Mutes all stdout/stderr.\n    \"\"\"\n    def _target_fn(ext_base_name, fn, fn_desc, queue, verbose):\n        import importlib\n        import sys\n        import traceback\n\n        if verbose:\n            print('Checking whether extension {ext_base_name} was {fn_desc}.'.format(\n                ext_base_name=ext_base_name, fn_desc=fn_desc))\n        else:\n            # Suppress output\n            sys.stdout = open(os.devnull, 'w')\n            sys.stderr = open(os.devnull, 'w')\n\n        try:\n            ext = importlib.import_module('.' + ext_base_name, 'horovod')\n            result = fn(ext)\n        except:\n            traceback.print_exc()\n            result = None\n\n        if verbose:\n            print('Extension {ext_base_name} {flag} {fn_desc}.'.format(\n                ext_base_name=ext_base_name, flag=('was' if result else 'was NOT'),\n                fn_desc=fn_desc))\n\n        queue.put(result)\n\n    queue = Queue()\n    p = Process(target=_target_fn,\n                args=(ext_base_name, fn, fn_desc, queue, verbose))\n    p.daemon = True\n    p.start()\n    p.join()\n    return queue.get_nowait()\n\n\ndef extension_available(ext_base_name, verbose=False):\n    available_fn = lambda ext: ext is not None\n    return _check_extension_lambda(\n        ext_base_name, available_fn, 'built', verbose) or False\n\n\ndef _cache(f):\n    cache = dict()\n\n    def wrapper(*args, **kwargs):\n        key = (args, frozenset(kwargs.items()))\n\n        if key in cache:\n            return cache[key]\n        else:\n            retval = f(*args, **kwargs)\n            cache[key] = retval\n            return retval\n\n    return wrapper\n\n\n@_cache\ndef gpu_available(ext_base_name, verbose=False):\n    available_fn = lambda ext: ext._check_has_gpu()\n    return _check_extension_lambda(\n        ext_base_name, available_fn, 'running with GPU', verbose) or False\n\n\n@_cache\ndef mpi_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.mpi_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with MPI', verbose)\n        if result is not None:\n            return result\n    return False\n\n\n@_cache\ndef gloo_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.gloo_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with Gloo', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if Gloo support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@_cache\ndef nccl_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.nccl_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with NCCL', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if NCCL support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@_cache\ndef ddl_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.ddl_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with DDL', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if DDL support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@_cache\ndef ccl_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.ccl_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with CCL', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if CCL support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@contextmanager\ndef env(**kwargs):\n    # ignore args with None values\n    for k in list(kwargs.keys()):\n        if kwargs[k] is None:\n            del kwargs[k]\n\n    # backup environment\n    backup = {}\n    for k in kwargs.keys():\n        backup[k] = os.environ.get(k)\n\n    # set new values & yield\n    for k, v in kwargs.items():\n        os.environ[k] = v\n\n    try:\n        yield\n    finally:\n        # restore environment\n        for k in kwargs.keys():\n            if backup[k] is not None:\n                os.environ[k] = backup[k]\n            else:\n                del os.environ[k]\n\n\ndef get_average_backwards_compatibility_fun(reduce_ops):\n    \"\"\"\n    Handle backwards compatibility between the old average and the new op parameters.\n    Old code using the average parameter (e.g. hvd.allreduce(tensor, average=False))\n    gets unchanged behavior, but mixing old and new is disallowed (e.g. no\n    hvd.allreduce(tensor, average=False, op=hvd.Adasum)).\n    \"\"\"\n    def impl(op, average):\n        if op != None:\n            if average != None:\n                raise ValueError('The op parameter supersedes average. Please provide only one of them.')\n            return op\n        elif average != None:\n            return reduce_ops.Average if average else reduce_ops.Sum\n        else:\n            return reduce_ops.Average\n    return impl\n\n\ndef num_rank_is_power_2(num_rank):\n    \"\"\"\n    Tests if the given number of ranks is of power of 2. This check is required\n    for Adasum allreduce.\n    TODO support non-power of 2 ranks.\n    \"\"\"\n    return num_rank != 0 and ((num_rank & (num_rank -1)) == 0)\n", "path": "horovod/common/util.py"}], "after_files": [{"content": "# Copyright 2016 The TensorFlow Authors. All Rights Reserved.\n# Modifications copyright (C) 2019 Uber Technologies, Inc.\n# Modifications copyright Microsoft\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# =============================================================================\n\nimport multiprocessing\nimport os\nimport sys\nimport sysconfig\n\nfrom contextlib import contextmanager\n\n\n_PY3 = sys.version_info[0] == 3\nEXTENSIONS = ['tensorflow', 'torch', 'mxnet']\n\n\ndef get_ext_suffix():\n    \"\"\"Determine library extension for various versions of Python.\"\"\"\n    ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')\n    if ext_suffix:\n        return ext_suffix\n\n    ext_suffix = sysconfig.get_config_var('SO')\n    if ext_suffix:\n        return ext_suffix\n\n    return '.so'\n\n\ndef get_extension_full_path(pkg_path, *args):\n    assert len(args) >= 1\n    dir_path = os.path.join(os.path.dirname(pkg_path), *args[:-1])\n    full_path = os.path.join(dir_path, args[-1] + get_ext_suffix())\n    return full_path\n\n\ndef check_extension(ext_name, ext_env_var, pkg_path, *args):\n    full_path = get_extension_full_path(pkg_path, *args)\n    if not os.path.exists(full_path):\n        raise ImportError(\n            'Extension %s has not been built.  If this is not expected, reinstall '\n            'Horovod with %s=1 to debug the build error.' % (ext_name, ext_env_var))\n\n\ndef _check_extension_lambda(ext_base_name, fn, fn_desc, verbose):\n    \"\"\"\n    Tries to load the extension in a new process.  If successful, puts fn(ext)\n    to the queue or False otherwise.  Mutes all stdout/stderr.\n    \"\"\"\n    def _target_fn(ext_base_name, fn, fn_desc, queue, verbose):\n        import importlib\n        import sys\n        import traceback\n\n        if verbose:\n            print('Checking whether extension {ext_base_name} was {fn_desc}.'.format(\n                ext_base_name=ext_base_name, fn_desc=fn_desc))\n        else:\n            # Suppress output\n            sys.stdout = open(os.devnull, 'w')\n            sys.stderr = open(os.devnull, 'w')\n\n        try:\n            ext = importlib.import_module('.' + ext_base_name, 'horovod')\n            result = fn(ext)\n        except:\n            traceback.print_exc()\n            result = None\n\n        if verbose:\n            print('Extension {ext_base_name} {flag} {fn_desc}.'.format(\n                ext_base_name=ext_base_name, flag=('was' if result else 'was NOT'),\n                fn_desc=fn_desc))\n\n        queue.put(result)\n\n    # 'fork' is required because horovodrun is a frozen executable\n    ctx = multiprocessing.get_context('fork') if _PY3 else multiprocessing\n    queue = ctx.Queue()\n    p = ctx.Process(target=_target_fn,\n                    args=(ext_base_name, fn, fn_desc, queue, verbose))\n    p.daemon = True\n    p.start()\n    p.join()\n    return queue.get_nowait()\n\n\ndef extension_available(ext_base_name, verbose=False):\n    available_fn = lambda ext: ext is not None\n    return _check_extension_lambda(\n        ext_base_name, available_fn, 'built', verbose) or False\n\n\ndef _cache(f):\n    cache = dict()\n\n    def wrapper(*args, **kwargs):\n        key = (args, frozenset(kwargs.items()))\n\n        if key in cache:\n            return cache[key]\n        else:\n            retval = f(*args, **kwargs)\n            cache[key] = retval\n            return retval\n\n    return wrapper\n\n\n@_cache\ndef gpu_available(ext_base_name, verbose=False):\n    available_fn = lambda ext: ext._check_has_gpu()\n    return _check_extension_lambda(\n        ext_base_name, available_fn, 'running with GPU', verbose) or False\n\n\n@_cache\ndef mpi_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.mpi_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with MPI', verbose)\n        if result is not None:\n            return result\n    return False\n\n\n@_cache\ndef gloo_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.gloo_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with Gloo', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if Gloo support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@_cache\ndef nccl_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.nccl_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with NCCL', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if NCCL support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@_cache\ndef ddl_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.ddl_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with DDL', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if DDL support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@_cache\ndef ccl_built(verbose=False):\n    for ext_base_name in EXTENSIONS:\n        built_fn = lambda ext: ext.ccl_built()\n        result = _check_extension_lambda(\n            ext_base_name, built_fn, 'built with CCL', verbose)\n        if result is not None:\n            return result\n    raise RuntimeError('Failed to determine if CCL support has been built. '\n                       'Run again with --verbose for more details.')\n\n\n@contextmanager\ndef env(**kwargs):\n    # ignore args with None values\n    for k in list(kwargs.keys()):\n        if kwargs[k] is None:\n            del kwargs[k]\n\n    # backup environment\n    backup = {}\n    for k in kwargs.keys():\n        backup[k] = os.environ.get(k)\n\n    # set new values & yield\n    for k, v in kwargs.items():\n        os.environ[k] = v\n\n    try:\n        yield\n    finally:\n        # restore environment\n        for k in kwargs.keys():\n            if backup[k] is not None:\n                os.environ[k] = backup[k]\n            else:\n                del os.environ[k]\n\n\ndef get_average_backwards_compatibility_fun(reduce_ops):\n    \"\"\"\n    Handle backwards compatibility between the old average and the new op parameters.\n    Old code using the average parameter (e.g. hvd.allreduce(tensor, average=False))\n    gets unchanged behavior, but mixing old and new is disallowed (e.g. no\n    hvd.allreduce(tensor, average=False, op=hvd.Adasum)).\n    \"\"\"\n    def impl(op, average):\n        if op != None:\n            if average != None:\n                raise ValueError('The op parameter supersedes average. Please provide only one of them.')\n            return op\n        elif average != None:\n            return reduce_ops.Average if average else reduce_ops.Sum\n        else:\n            return reduce_ops.Average\n    return impl\n\n\ndef num_rank_is_power_2(num_rank):\n    \"\"\"\n    Tests if the given number of ranks is of power of 2. This check is required\n    for Adasum allreduce.\n    TODO support non-power of 2 ranks.\n    \"\"\"\n    return num_rank != 0 and ((num_rank & (num_rank -1)) == 0)\n", "path": "horovod/common/util.py"}]}