We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
terminate called after throwing an instance of 'gloo::EnforceNotMet'   ifa != nullptr. Unable to find address for: eth0
hvd.init() throws following error
```
>>> hvd.init()
terminate called after throwing an instance of 'gloo::EnforceNotMet'
  what():  [enforce fail at /tmp/pip-install-3dvzdqhs/horovod/third_party/gloo/gloo/transport/tcp/device.cc:88] ifa != nullptr. Unable to find address for: eth0
Aborted
```
What I understand is that gloo is hardcoded to check for eth0 and in my case (Centos 7.6) I have em1 and em2. I came acrros solution to export the below env variables but i am still seeing the issue
```
export GLOO_SOCKET_IFNAME=em2
export NCCL_SOCKET_IFNAME=em2
```
How can I get pass this?

**Environment:**
Framework:  Keras
Tensorflow Version 1.13.1
Keras Version 2.2.4-tf
Horovod version: horovod==0.17.0.post1
Python version: 3.6


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `horovod/run/gloo_run.py`
Content:
```
1 # Copyright 2019 Uber Technologies, Inc. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 # ==============================================================================
15 
16 import os
17 import collections
18 import threading
19 import signal
20 
21 from horovod.run.rendezvous.http_server import RendezvousServer
22 from horovod.run.common.util import env as env_util, safe_shell_exec
23 from horovod.run.util import threads
24 from psutil import net_if_addrs
25 from socket import AF_INET
26 
27 try:
28     from shlex import quote
29 except ImportError:
30     from pipes import quote
31 
32 
33 class HostInfo:
34     def __init__(self, host_item):
35         hostname, slots = host_item.strip().split(':')
36         self.hostname = hostname
37         self.slots = int(slots)
38 
39 
40 class SlotInfo:
41     def __init__(self, hostname, rank, local_rank, cross_rank, size):
42         self.hostname = hostname
43         self.rank = rank
44         self.size = size
45         self.local_rank = local_rank
46         self.local_size = None
47         self.cross_rank = cross_rank
48         self.cross_size = None
49 
50 
51 def _allocate(hosts, np):
52     """
53     Find the allocation of processes on hosts, this function will try to
54     allocate as many as possible processes on the same host to leverage
55     local network.
56     :param hosts: list of addresses and number of processes on each host.
57     For example,
58         'worker-0:2,worker-1:2'
59         '10.11.11.11:4,10.11.11.12,4'
60     :type hosts: string
61     :param np: total number of processes to be allocated
62     :type np: int
63     :return: a list of the allocation of process on hosts in a AllocInfo object.
64             Members in the object include: hostname, rank, local_rank, cross_rank,
65             total_size, local_size, cross_size
66     :rtype: list[dict()]
67     """
68 
69     host_list = []
70     # split the host string to host list
71     for host_item in hosts.split(','):
72         host_list.append(HostInfo(host_item))
73 
74     rank = 0
75     alloc_list = []
76 
77     # key: local_rank; value: cross_size for this local_rank
78     local_sizes = collections.defaultdict(int)
79     # key: cross_rank; value: local_size for this cross_rank
80     cross_sizes = collections.defaultdict(int)
81 
82     # allocate processes into slots
83     for host_idx, host_info in enumerate(host_list):
84         for local_rank in range(host_info.slots):
85             if rank == np:
86                 break
87             cross_rank = host_idx
88             alloc_list.append(
89                 SlotInfo(
90                     host_info.hostname,
91                     rank,
92                     local_rank,
93                     cross_rank,
94                     np))
95             cross_sizes[local_rank] += 1
96             local_sizes[cross_rank] += 1
97             rank += 1
98 
99     if rank < np:
100         raise ValueError("Process number should not be larger than "
101                          "total available slots.")
102 
103     # Fill in the local_size and cross_size because we can only know these number after
104     # allocation is done.
105     for alloc_item in alloc_list:
106         alloc_item.local_size = local_sizes[alloc_item.cross_rank]
107         alloc_item.cross_size = cross_sizes[alloc_item.local_rank]
108 
109     return alloc_list
110 
111 
112 def _launch_jobs(settings, host_alloc_plan, remote_host_names, _run_command):
113     """
114     executes the jobs defined by run command on hosts.
115     :param hosts_alloc: list of dict indicating the allocating info.
116     For example,
117         [{'Hostname':'worker-0', 'Rank': 0, 'Local_rank': 0, 'Cross_rank':0,
118             'Size':2, 'Local_size':1, 'Cross_size':2},
119         {'Hostname':'worker-1', 'Rank': 1, 'Local_rank': 0, 'Cross_rank':1,
120             'Size':2, 'Local_size':1, 'Cross_size':2}
121         ]
122     :type hosts_alloc: list(dict)
123     :param remote_host_names: names that are resolved to one of the addresses
124     of remote hosts interfaces.
125     :type remote_host_names: set
126     :param _run_command: command to execute
127     :type _run_command: string
128     :return:
129     :rtype:
130     """
131 
132     def _exec_command(_command, _index, event_):
133         if settings.verbose:
134             print(_command)
135         try:
136             exit_code = safe_shell_exec.execute(_command, index=_index, event=event_)
137             if exit_code != 0:
138                 print('Process {idx} exit with status code {ec}.'.format(idx=_index, ec=exit_code))
139         except Exception as e:
140             print('Exception happened during safe_shell_exec, exception '
141                   'message: {message}'.format(message=e))
142         return 0
143 
144     ssh_port_arg = '-p {ssh_port}'.format(ssh_port=settings.ssh_port) if settings.ssh_port else ''
145 
146     # Create a event for communication between threads
147     event = threading.Event()
148 
149     def set_event_on_sigterm(signum, frame):
150         event.set()
151 
152     signal.signal(signal.SIGINT, set_event_on_sigterm)
153     signal.signal(signal.SIGTERM, set_event_on_sigterm)
154 
155     args_list = []
156     for alloc_info in host_alloc_plan:
157         # generate env for rendezvous
158         horovod_rendez_env = 'HOROVOD_RANK={rank} HOROVOD_SIZE={size} ' \
159                              'HOROVOD_LOCAL_RANK={local_rank} HOROVOD_LOCAL_SIZE={local_size} ' \
160                              'HOROVOD_CROSS_RANK={cross_rank} HOROVOD_CROSS_SIZE={cross_size} ' \
161             .format(rank=alloc_info.rank, size=alloc_info.size,
162                     local_rank=alloc_info.local_rank, local_size=alloc_info.local_size,
163                     cross_rank=alloc_info.cross_rank, cross_size=alloc_info.cross_size)
164 
165         host_name = alloc_info.hostname
166 
167         env = os.environ.copy()
168         # TODO: Workaround for over-buffered outputs. Investigate how mpirun avoids this problem.
169         env['PYTHONUNBUFFERED'] = '1'
170         local_command = '{horovod_env} {env} {run_command}' .format(
171             horovod_env=horovod_rendez_env,
172             env=' '.join(['%s=%s' % (key, quote(value)) for key, value in env.items()
173                           if env_util.is_exportable(key)]),
174             run_command=_run_command)
175 
176         if host_name not in remote_host_names:
177             command = local_command
178         else:
179             command = 'ssh -o StrictHostKeyChecking=no {host} {ssh_port_arg} ' \
180                 '{local_command}'.format(
181                     host=host_name,
182                     ssh_port_arg=ssh_port_arg,
183                     local_command=quote('cd {pwd} >& /dev/null ; {local_command}'
184                                         .format(pwd=os.getcwd(), local_command=local_command))
185                 )
186         args_list.append([command, alloc_info.rank, event])
187 
188     # Each thread will use ssh command to launch the job on each remote host. If an
189     # error occurs in one thread, entire process will be terminated. Otherwise,
190     # threads will keep running and ssh session. In case, the main thread receives
191     # a SIGINT, the event will be set and the spawned threads will kill their
192     # corresponding middleman processes and thus the jobs will be killed as
193     # well.
194     threads.execute_function_multithreaded(_exec_command,
195                                            args_list,
196                                            block_until_all_done=True)
197 
198 
199 def gloo_run(settings, remote_host_names, common_intfs):
200     # allocate processes into slots
201     host_alloc_plan = _allocate(settings.hosts, settings.num_proc)
202 
203     # create global rendezvous server
204     global_rendezv = RendezvousServer(settings.verbose)
205     # Start rendezvous server and get port that it is listening
206     global_rendezv_port = global_rendezv.start_server(host_alloc_plan)
207 
208     # get the server IPv4 address
209     iface = list(common_intfs)[0]
210     server_ip = None
211     for addr in net_if_addrs()[iface]:
212         if addr.family == AF_INET:
213             server_ip = addr.address
214 
215     if not server_ip:
216         raise RuntimeError(
217             'Cannot find an IPv4 address of the common interface.')
218 
219     run_command = (
220         'HOROVOD_GLOO_RENDEZVOUS_ADDR={addr} '
221         'HOROVOD_GLOO_RENDEZVOUS_PORT={port} '
222         'HOROVOD_CONTROLLER=gloo '
223         'HOROVOD_CPU_OPERATIONS=gloo '
224         'HOROVOD_IFACE={iface} '
225         'NCCL_SOCKET_IFNAME={common_intfs} '
226         '{command}'  # expect a lot of environment variables
227         .format(addr=server_ip,
228                 port=global_rendezv_port,
229                 iface=iface,  # TODO: add multiple ifaces in future
230                 common_intfs=','.join(common_intfs),
231                 command=' '.join(quote(par) for par in settings.command)))
232 
233     _launch_jobs(settings, host_alloc_plan, remote_host_names, run_command)
234     return
235
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/horovod/run/gloo_run.py b/horovod/run/gloo_run.py
--- a/horovod/run/gloo_run.py
+++ b/horovod/run/gloo_run.py
@@ -221,7 +221,7 @@
         'HOROVOD_GLOO_RENDEZVOUS_PORT={port} '
         'HOROVOD_CONTROLLER=gloo '
         'HOROVOD_CPU_OPERATIONS=gloo '
-        'HOROVOD_IFACE={iface} '
+        'HOROVOD_GLOO_IFACE={iface} '
         'NCCL_SOCKET_IFNAME={common_intfs} '
         '{command}'  # expect a lot of environment variables
         .format(addr=server_ip,

{"golden_diff": "diff --git a/horovod/run/gloo_run.py b/horovod/run/gloo_run.py\n--- a/horovod/run/gloo_run.py\n+++ b/horovod/run/gloo_run.py\n@@ -221,7 +221,7 @@\n         'HOROVOD_GLOO_RENDEZVOUS_PORT={port} '\n         'HOROVOD_CONTROLLER=gloo '\n         'HOROVOD_CPU_OPERATIONS=gloo '\n-        'HOROVOD_IFACE={iface} '\n+        'HOROVOD_GLOO_IFACE={iface} '\n         'NCCL_SOCKET_IFNAME={common_intfs} '\n         '{command}'  # expect a lot of environment variables\n         .format(addr=server_ip,\n", "issue": "terminate called after throwing an instance of 'gloo::EnforceNotMet'   ifa != nullptr. Unable to find address for: eth0\nhvd.init() throws following error\r\n```\r\n>>> hvd.init()\r\nterminate called after throwing an instance of 'gloo::EnforceNotMet'\r\n  what():  [enforce fail at /tmp/pip-install-3dvzdqhs/horovod/third_party/gloo/gloo/transport/tcp/device.cc:88] ifa != nullptr. Unable to find address for: eth0\r\nAborted\r\n```\r\nWhat I understand is that gloo is hardcoded to check for eth0 and in my case (Centos 7.6) I have em1 and em2. I came acrros solution to export the below env variables but i am still seeing the issue\r\n```\r\nexport GLOO_SOCKET_IFNAME=em2\r\nexport NCCL_SOCKET_IFNAME=em2\r\n```\r\nHow can I get pass this?\r\n\r\n**Environment:**\r\nFramework:  Keras\r\nTensorflow Version 1.13.1\r\nKeras Version 2.2.4-tf\r\nHorovod version: horovod==0.17.0.post1\r\nPython version: 3.6\r\n\n", "before_files": [{"content": "# Copyright 2019 Uber Technologies, Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\nimport os\nimport collections\nimport threading\nimport signal\n\nfrom horovod.run.rendezvous.http_server import RendezvousServer\nfrom horovod.run.common.util import env as env_util, safe_shell_exec\nfrom horovod.run.util import threads\nfrom psutil import net_if_addrs\nfrom socket import AF_INET\n\ntry:\n    from shlex import quote\nexcept ImportError:\n    from pipes import quote\n\n\nclass HostInfo:\n    def __init__(self, host_item):\n        hostname, slots = host_item.strip().split(':')\n        self.hostname = hostname\n        self.slots = int(slots)\n\n\nclass SlotInfo:\n    def __init__(self, hostname, rank, local_rank, cross_rank, size):\n        self.hostname = hostname\n        self.rank = rank\n        self.size = size\n        self.local_rank = local_rank\n        self.local_size = None\n        self.cross_rank = cross_rank\n        self.cross_size = None\n\n\ndef _allocate(hosts, np):\n    \"\"\"\n    Find the allocation of processes on hosts, this function will try to\n    allocate as many as possible processes on the same host to leverage\n    local network.\n    :param hosts: list of addresses and number of processes on each host.\n    For example,\n        'worker-0:2,worker-1:2'\n        '10.11.11.11:4,10.11.11.12,4'\n    :type hosts: string\n    :param np: total number of processes to be allocated\n    :type np: int\n    :return: a list of the allocation of process on hosts in a AllocInfo object.\n            Members in the object include: hostname, rank, local_rank, cross_rank,\n            total_size, local_size, cross_size\n    :rtype: list[dict()]\n    \"\"\"\n\n    host_list = []\n    # split the host string to host list\n    for host_item in hosts.split(','):\n        host_list.append(HostInfo(host_item))\n\n    rank = 0\n    alloc_list = []\n\n    # key: local_rank; value: cross_size for this local_rank\n    local_sizes = collections.defaultdict(int)\n    # key: cross_rank; value: local_size for this cross_rank\n    cross_sizes = collections.defaultdict(int)\n\n    # allocate processes into slots\n    for host_idx, host_info in enumerate(host_list):\n        for local_rank in range(host_info.slots):\n            if rank == np:\n                break\n            cross_rank = host_idx\n            alloc_list.append(\n                SlotInfo(\n                    host_info.hostname,\n                    rank,\n                    local_rank,\n                    cross_rank,\n                    np))\n            cross_sizes[local_rank] += 1\n            local_sizes[cross_rank] += 1\n            rank += 1\n\n    if rank < np:\n        raise ValueError(\"Process number should not be larger than \"\n                         \"total available slots.\")\n\n    # Fill in the local_size and cross_size because we can only know these number after\n    # allocation is done.\n    for alloc_item in alloc_list:\n        alloc_item.local_size = local_sizes[alloc_item.cross_rank]\n        alloc_item.cross_size = cross_sizes[alloc_item.local_rank]\n\n    return alloc_list\n\n\ndef _launch_jobs(settings, host_alloc_plan, remote_host_names, _run_command):\n    \"\"\"\n    executes the jobs defined by run command on hosts.\n    :param hosts_alloc: list of dict indicating the allocating info.\n    For example,\n        [{'Hostname':'worker-0', 'Rank': 0, 'Local_rank': 0, 'Cross_rank':0,\n            'Size':2, 'Local_size':1, 'Cross_size':2},\n        {'Hostname':'worker-1', 'Rank': 1, 'Local_rank': 0, 'Cross_rank':1,\n            'Size':2, 'Local_size':1, 'Cross_size':2}\n        ]\n    :type hosts_alloc: list(dict)\n    :param remote_host_names: names that are resolved to one of the addresses\n    of remote hosts interfaces.\n    :type remote_host_names: set\n    :param _run_command: command to execute\n    :type _run_command: string\n    :return:\n    :rtype:\n    \"\"\"\n\n    def _exec_command(_command, _index, event_):\n        if settings.verbose:\n            print(_command)\n        try:\n            exit_code = safe_shell_exec.execute(_command, index=_index, event=event_)\n            if exit_code != 0:\n                print('Process {idx} exit with status code {ec}.'.format(idx=_index, ec=exit_code))\n        except Exception as e:\n            print('Exception happened during safe_shell_exec, exception '\n                  'message: {message}'.format(message=e))\n        return 0\n\n    ssh_port_arg = '-p {ssh_port}'.format(ssh_port=settings.ssh_port) if settings.ssh_port else ''\n\n    # Create a event for communication between threads\n    event = threading.Event()\n\n    def set_event_on_sigterm(signum, frame):\n        event.set()\n\n    signal.signal(signal.SIGINT, set_event_on_sigterm)\n    signal.signal(signal.SIGTERM, set_event_on_sigterm)\n\n    args_list = []\n    for alloc_info in host_alloc_plan:\n        # generate env for rendezvous\n        horovod_rendez_env = 'HOROVOD_RANK={rank} HOROVOD_SIZE={size} ' \\\n                             'HOROVOD_LOCAL_RANK={local_rank} HOROVOD_LOCAL_SIZE={local_size} ' \\\n                             'HOROVOD_CROSS_RANK={cross_rank} HOROVOD_CROSS_SIZE={cross_size} ' \\\n            .format(rank=alloc_info.rank, size=alloc_info.size,\n                    local_rank=alloc_info.local_rank, local_size=alloc_info.local_size,\n                    cross_rank=alloc_info.cross_rank, cross_size=alloc_info.cross_size)\n\n        host_name = alloc_info.hostname\n\n        env = os.environ.copy()\n        # TODO: Workaround for over-buffered outputs. Investigate how mpirun avoids this problem.\n        env['PYTHONUNBUFFERED'] = '1'\n        local_command = '{horovod_env} {env} {run_command}' .format(\n            horovod_env=horovod_rendez_env,\n            env=' '.join(['%s=%s' % (key, quote(value)) for key, value in env.items()\n                          if env_util.is_exportable(key)]),\n            run_command=_run_command)\n\n        if host_name not in remote_host_names:\n            command = local_command\n        else:\n            command = 'ssh -o StrictHostKeyChecking=no {host} {ssh_port_arg} ' \\\n                '{local_command}'.format(\n                    host=host_name,\n                    ssh_port_arg=ssh_port_arg,\n                    local_command=quote('cd {pwd} >& /dev/null ; {local_command}'\n                                        .format(pwd=os.getcwd(), local_command=local_command))\n                )\n        args_list.append([command, alloc_info.rank, event])\n\n    # Each thread will use ssh command to launch the job on each remote host. If an\n    # error occurs in one thread, entire process will be terminated. Otherwise,\n    # threads will keep running and ssh session. In case, the main thread receives\n    # a SIGINT, the event will be set and the spawned threads will kill their\n    # corresponding middleman processes and thus the jobs will be killed as\n    # well.\n    threads.execute_function_multithreaded(_exec_command,\n                                           args_list,\n                                           block_until_all_done=True)\n\n\ndef gloo_run(settings, remote_host_names, common_intfs):\n    # allocate processes into slots\n    host_alloc_plan = _allocate(settings.hosts, settings.num_proc)\n\n    # create global rendezvous server\n    global_rendezv = RendezvousServer(settings.verbose)\n    # Start rendezvous server and get port that it is listening\n    global_rendezv_port = global_rendezv.start_server(host_alloc_plan)\n\n    # get the server IPv4 address\n    iface = list(common_intfs)[0]\n    server_ip = None\n    for addr in net_if_addrs()[iface]:\n        if addr.family == AF_INET:\n            server_ip = addr.address\n\n    if not server_ip:\n        raise RuntimeError(\n            'Cannot find an IPv4 address of the common interface.')\n\n    run_command = (\n        'HOROVOD_GLOO_RENDEZVOUS_ADDR={addr} '\n        'HOROVOD_GLOO_RENDEZVOUS_PORT={port} '\n        'HOROVOD_CONTROLLER=gloo '\n        'HOROVOD_CPU_OPERATIONS=gloo '\n        'HOROVOD_IFACE={iface} '\n        'NCCL_SOCKET_IFNAME={common_intfs} '\n        '{command}'  # expect a lot of environment variables\n        .format(addr=server_ip,\n                port=global_rendezv_port,\n                iface=iface,  # TODO: add multiple ifaces in future\n                common_intfs=','.join(common_intfs),\n                command=' '.join(quote(par) for par in settings.command)))\n\n    _launch_jobs(settings, host_alloc_plan, remote_host_names, run_command)\n    return\n", "path": "horovod/run/gloo_run.py"}], "after_files": [{"content": "# Copyright 2019 Uber Technologies, Inc. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n\nimport os\nimport collections\nimport threading\nimport signal\n\nfrom horovod.run.rendezvous.http_server import RendezvousServer\nfrom horovod.run.common.util import env as env_util, safe_shell_exec\nfrom horovod.run.util import threads\nfrom psutil import net_if_addrs\nfrom socket import AF_INET\n\ntry:\n    from shlex import quote\nexcept ImportError:\n    from pipes import quote\n\n\nclass HostInfo:\n    def __init__(self, host_item):\n        hostname, slots = host_item.strip().split(':')\n        self.hostname = hostname\n        self.slots = int(slots)\n\n\nclass SlotInfo:\n    def __init__(self, hostname, rank, local_rank, cross_rank, size):\n        self.hostname = hostname\n        self.rank = rank\n        self.size = size\n        self.local_rank = local_rank\n        self.local_size = None\n        self.cross_rank = cross_rank\n        self.cross_size = None\n\n\ndef _allocate(hosts, np):\n    \"\"\"\n    Find the allocation of processes on hosts, this function will try to\n    allocate as many as possible processes on the same host to leverage\n    local network.\n    :param hosts: list of addresses and number of processes on each host.\n    For example,\n        'worker-0:2,worker-1:2'\n        '10.11.11.11:4,10.11.11.12,4'\n    :type hosts: string\n    :param np: total number of processes to be allocated\n    :type np: int\n    :return: a list of the allocation of process on hosts in a AllocInfo object.\n            Members in the object include: hostname, rank, local_rank, cross_rank,\n            total_size, local_size, cross_size\n    :rtype: list[dict()]\n    \"\"\"\n\n    host_list = []\n    # split the host string to host list\n    for host_item in hosts.split(','):\n        host_list.append(HostInfo(host_item))\n\n    rank = 0\n    alloc_list = []\n\n    # key: local_rank; value: cross_size for this local_rank\n    local_sizes = collections.defaultdict(int)\n    # key: cross_rank; value: local_size for this cross_rank\n    cross_sizes = collections.defaultdict(int)\n\n    # allocate processes into slots\n    for host_idx, host_info in enumerate(host_list):\n        for local_rank in range(host_info.slots):\n            if rank == np:\n                break\n            cross_rank = host_idx\n            alloc_list.append(\n                SlotInfo(\n                    host_info.hostname,\n                    rank,\n                    local_rank,\n                    cross_rank,\n                    np))\n            cross_sizes[local_rank] += 1\n            local_sizes[cross_rank] += 1\n            rank += 1\n\n    if rank < np:\n        raise ValueError(\"Process number should not be larger than \"\n                         \"total available slots.\")\n\n    # Fill in the local_size and cross_size because we can only know these number after\n    # allocation is done.\n    for alloc_item in alloc_list:\n        alloc_item.local_size = local_sizes[alloc_item.cross_rank]\n        alloc_item.cross_size = cross_sizes[alloc_item.local_rank]\n\n    return alloc_list\n\n\ndef _launch_jobs(settings, host_alloc_plan, remote_host_names, _run_command):\n    \"\"\"\n    executes the jobs defined by run command on hosts.\n    :param hosts_alloc: list of dict indicating the allocating info.\n    For example,\n        [{'Hostname':'worker-0', 'Rank': 0, 'Local_rank': 0, 'Cross_rank':0,\n            'Size':2, 'Local_size':1, 'Cross_size':2},\n        {'Hostname':'worker-1', 'Rank': 1, 'Local_rank': 0, 'Cross_rank':1,\n            'Size':2, 'Local_size':1, 'Cross_size':2}\n        ]\n    :type hosts_alloc: list(dict)\n    :param remote_host_names: names that are resolved to one of the addresses\n    of remote hosts interfaces.\n    :type remote_host_names: set\n    :param _run_command: command to execute\n    :type _run_command: string\n    :return:\n    :rtype:\n    \"\"\"\n\n    def _exec_command(_command, _index, event_):\n        if settings.verbose:\n            print(_command)\n        try:\n            exit_code = safe_shell_exec.execute(_command, index=_index, event=event_)\n            if exit_code != 0:\n                print('Process {idx} exit with status code {ec}.'.format(idx=_index, ec=exit_code))\n        except Exception as e:\n            print('Exception happened during safe_shell_exec, exception '\n                  'message: {message}'.format(message=e))\n        return 0\n\n    ssh_port_arg = '-p {ssh_port}'.format(ssh_port=settings.ssh_port) if settings.ssh_port else ''\n\n    # Create a event for communication between threads\n    event = threading.Event()\n\n    def set_event_on_sigterm(signum, frame):\n        event.set()\n\n    signal.signal(signal.SIGINT, set_event_on_sigterm)\n    signal.signal(signal.SIGTERM, set_event_on_sigterm)\n\n    args_list = []\n    for alloc_info in host_alloc_plan:\n        # generate env for rendezvous\n        horovod_rendez_env = 'HOROVOD_RANK={rank} HOROVOD_SIZE={size} ' \\\n                             'HOROVOD_LOCAL_RANK={local_rank} HOROVOD_LOCAL_SIZE={local_size} ' \\\n                             'HOROVOD_CROSS_RANK={cross_rank} HOROVOD_CROSS_SIZE={cross_size} ' \\\n            .format(rank=alloc_info.rank, size=alloc_info.size,\n                    local_rank=alloc_info.local_rank, local_size=alloc_info.local_size,\n                    cross_rank=alloc_info.cross_rank, cross_size=alloc_info.cross_size)\n\n        host_name = alloc_info.hostname\n\n        env = os.environ.copy()\n        # TODO: Workaround for over-buffered outputs. Investigate how mpirun avoids this problem.\n        env['PYTHONUNBUFFERED'] = '1'\n        local_command = '{horovod_env} {env} {run_command}' .format(\n            horovod_env=horovod_rendez_env,\n            env=' '.join(['%s=%s' % (key, quote(value)) for key, value in env.items()\n                          if env_util.is_exportable(key)]),\n            run_command=_run_command)\n\n        if host_name not in remote_host_names:\n            command = local_command\n        else:\n            command = 'ssh -o StrictHostKeyChecking=no {host} {ssh_port_arg} ' \\\n                '{local_command}'.format(\n                    host=host_name,\n                    ssh_port_arg=ssh_port_arg,\n                    local_command=quote('cd {pwd} >& /dev/null ; {local_command}'\n                                        .format(pwd=os.getcwd(), local_command=local_command))\n                )\n        args_list.append([command, alloc_info.rank, event])\n\n    # Each thread will use ssh command to launch the job on each remote host. If an\n    # error occurs in one thread, entire process will be terminated. Otherwise,\n    # threads will keep running and ssh session. In case, the main thread receives\n    # a SIGINT, the event will be set and the spawned threads will kill their\n    # corresponding middleman processes and thus the jobs will be killed as\n    # well.\n    threads.execute_function_multithreaded(_exec_command,\n                                           args_list,\n                                           block_until_all_done=True)\n\n\ndef gloo_run(settings, remote_host_names, common_intfs):\n    # allocate processes into slots\n    host_alloc_plan = _allocate(settings.hosts, settings.num_proc)\n\n    # create global rendezvous server\n    global_rendezv = RendezvousServer(settings.verbose)\n    # Start rendezvous server and get port that it is listening\n    global_rendezv_port = global_rendezv.start_server(host_alloc_plan)\n\n    # get the server IPv4 address\n    iface = list(common_intfs)[0]\n    server_ip = None\n    for addr in net_if_addrs()[iface]:\n        if addr.family == AF_INET:\n            server_ip = addr.address\n\n    if not server_ip:\n        raise RuntimeError(\n            'Cannot find an IPv4 address of the common interface.')\n\n    run_command = (\n        'HOROVOD_GLOO_RENDEZVOUS_ADDR={addr} '\n        'HOROVOD_GLOO_RENDEZVOUS_PORT={port} '\n        'HOROVOD_CONTROLLER=gloo '\n        'HOROVOD_CPU_OPERATIONS=gloo '\n        'HOROVOD_GLOO_IFACE={iface} '\n        'NCCL_SOCKET_IFNAME={common_intfs} '\n        '{command}'  # expect a lot of environment variables\n        .format(addr=server_ip,\n                port=global_rendezv_port,\n                iface=iface,  # TODO: add multiple ifaces in future\n                common_intfs=','.join(common_intfs),\n                command=' '.join(quote(par) for par in settings.command)))\n\n    _launch_jobs(settings, host_alloc_plan, remote_host_names, run_command)\n    return\n", "path": "horovod/run/gloo_run.py"}]}