We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Fixes #2942


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pytorch_lightning/accelerators/ddp_backend.py`
Content:
```
1 # Copyright The PyTorch Lightning team.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License
14 
15 import os
16 import subprocess
17 import sys
18 from os.path import abspath
19 from time import sleep
20 from typing import Optional
21 
22 import numpy as np
23 import torch
24 
25 from pytorch_lightning import _logger as log
26 from pytorch_lightning.utilities import AMPType
27 from pytorch_lightning.utilities.distributed import rank_zero_only
28 
29 try:
30     from hydra.utils import to_absolute_path, get_original_cwd
31     from hydra.core.hydra_config import HydraConfig
32 except ImportError:
33     HYDRA_AVAILABLE = False
34 else:
35     HYDRA_AVAILABLE = True
36 
37 try:
38     from apex import amp
39 except ImportError:
40     amp = None
41 
42 
43 class DDPBackend(object):
44 
45     def __init__(self, trainer):
46         self.trainer = trainer
47         self.task_idx = None
48 
49     def slurm_setup(self):
50         self.task_idx = int(os.environ['SLURM_LOCALID'])
51 
52     def torchelastic_setup(self):
53         self.task_idx = int(os.environ['LOCAL_RANK'])
54 
55     def train(self, model):
56         self.ddp_train(process_idx=self.task_idx, mp_queue=None, model=model)
57 
58     def spawn_ddp_children(self, model):
59         port = os.environ['MASTER_PORT']
60 
61         master_address = '127.0.0.1' if 'MASTER_ADDR' not in os.environ else os.environ['MASTER_ADDR']
62         os.environ['MASTER_PORT'] = f'{port}'
63         os.environ['MASTER_ADDR'] = f'{master_address}'
64 
65         # allow the user to pass the node rank
66         node_rank = '0'
67         if 'NODE_RANK' in os.environ:
68             node_rank = os.environ['NODE_RANK']
69         if 'GROUP_RANK' in os.environ:
70             node_rank = os.environ['GROUP_RANK']
71 
72         os.environ['NODE_RANK'] = node_rank
73         os.environ['LOCAL_RANK'] = '0'
74 
75         # when user is using hydra find the absolute path
76         path_lib = abspath if not HYDRA_AVAILABLE else to_absolute_path
77 
78         # pull out the commands used to run the script and resolve the abs file path
79         command = sys.argv
80         try:
81             full_path = path_lib(command[0])
82         except Exception as e:
83             full_path = abspath(command[0])
84 
85         command[0] = full_path
86         # use the same python interpreter and actually running
87         command = [sys.executable] + command
88 
89         # since this script sets the visible devices we replace the gpus flag with a number
90         num_gpus = os.environ.get('CUDA_VISIBLE_DEVICES', []).split(',').__len__()
91 
92         if '--gpus' in command:
93             gpu_flag_idx = command.index('--gpus')
94             command[gpu_flag_idx + 1] = f'{num_gpus}'
95 
96         os.environ['WORLD_SIZE'] = f'{num_gpus * self.trainer.num_nodes}'
97 
98         self.trainer.interactive_ddp_procs = []
99         for local_rank in range(1, self.trainer.num_processes):
100             env_copy = os.environ.copy()
101             env_copy['LOCAL_RANK'] = f'{local_rank}'
102 
103             # start process
104             # if hydra is available and initialized, make sure to set the cwd correctly
105             cwd: Optional[str] = None
106             if HYDRA_AVAILABLE:
107                 if HydraConfig.initialized():
108                     cwd = get_original_cwd()
109             proc = subprocess.Popen(command, env=env_copy, cwd=cwd)
110             self.trainer.interactive_ddp_procs.append(proc)
111 
112             # starting all processes at once can cause issues
113             # with dataloaders delay between 1-10 seconds
114             delay = np.random.uniform(1, 5, 1)[0]
115             sleep(delay)
116 
117         local_rank = 0
118         results = self.ddp_train(local_rank, mp_queue=None, model=model, is_master=True)
119         del os.environ['WORLD_SIZE']
120 
121         return results
122 
123     def ddp_train(self, process_idx, mp_queue, model, is_master=False, proc_offset=0):
124         """
125         Entry point for ddp
126 
127         Args:
128             process_idx:
129             mp_queue: multiprocessing queue
130             model:
131             is_master:
132             proc_offset:
133 
134         Returns:
135 
136         """
137         # offset the process id if requested
138         process_idx = process_idx + proc_offset
139 
140         # show progressbar only on progress_rank 0
141         if (self.trainer.node_rank != 0 or process_idx != 0) and self.trainer.progress_bar_callback is not None:
142             self.trainer.progress_bar_callback.disable()
143 
144         # determine which process we are and world size
145         self.trainer.local_rank = process_idx
146         self.trainer.global_rank = self.trainer.node_rank * self.trainer.num_processes + process_idx
147         self.trainer.world_size = self.trainer.num_nodes * self.trainer.num_processes
148 
149         # set warning rank
150         rank_zero_only.rank = self.trainer.global_rank
151 
152         # set up server using proc 0's ip address
153         # try to init for 20 times at max in case ports are taken
154         # where to store ip_table
155         model.trainer = self.trainer
156         model.init_ddp_connection(
157             self.trainer.global_rank,
158             self.trainer.world_size,
159             self.trainer.is_slurm_managing_tasks
160         )
161 
162         # call setup after the ddp process has connected
163         self.trainer.call_setup_hook(model)
164 
165         # on world_size=0 let everyone know training is starting
166         if self.trainer.is_global_zero:
167             log.info('-' * 100)
168             log.info(f'distributed_backend={self.trainer.distributed_backend}')
169             log.info(f'All DDP processes registered. Starting ddp with {self.trainer.world_size} processes')
170             log.info('-' * 100)
171 
172         # call sync_bn before .cuda(), configure_apex and configure_ddp
173         if self.trainer.sync_batchnorm:
174             model = model.configure_sync_batchnorm(model)
175 
176         # MODEL
177         # copy model to each gpu
178         if self.trainer.on_gpu:
179             gpu_idx = process_idx
180 
181             # when using ddp, the master process (proc 0) continues running as the main one
182             # this means that the local rank will always be 0
183             # (even if cuda visible devices has other visible gpus)
184             # this means that the master process needs to pull the 0th visible index as the device number
185             if is_master:
186                 available_gpus = os.environ['CUDA_VISIBLE_DEVICES'].split(',')
187                 gpu_idx = int(available_gpus[self.trainer.local_rank])
188 
189             self.trainer.root_gpu = gpu_idx
190             torch.cuda.set_device(self.trainer.root_gpu)
191             model.cuda(self.trainer.root_gpu)
192 
193         # CHOOSE OPTIMIZER
194         # allow for lr schedulers as well
195         optimizers, lr_schedulers, optimizer_frequencies = self.trainer.init_optimizers(model)
196         self.trainer.optimizers = optimizers
197         self.trainer.lr_schedulers = lr_schedulers
198         self.trainer.optimizer_frequencies = optimizer_frequencies
199 
200         # set model properties before going into wrapper
201         self.trainer.copy_trainer_model_properties(model)
202 
203         # AMP - run through amp wrapper before going to distributed DP
204         if self.trainer.amp_backend == AMPType.APEX:
205             model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)
206             self.trainer.optimizers = optimizers
207             self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)
208 
209         # DDP2 uses all GPUs on the machine
210         if self.trainer.distributed_backend == 'ddp' or self.trainer.distributed_backend == 'ddp_spawn':
211             device_ids = [self.trainer.root_gpu]
212         else:  # includes ddp_cpu
213             device_ids = None
214 
215         # allow user to configure ddp
216         model = model.configure_ddp(model, device_ids)
217 
218         # continue training routine
219         results = self.trainer.run_pretrain_routine(model)
220 
221         # get original model
222         model = self.trainer.get_model()
223 
224         # persist info in ddp_spawn
225         self.trainer.transfer_distrib_spawn_state_on_fit_end(model, mp_queue, results)
226 
227         # clean up memory
228         torch.cuda.empty_cache()
229 
230         if self.trainer.global_rank == 0 and self.trainer.distributed_backend not in ['ddp_spawn', 'ddp_cpu']:
231             return results
232
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pytorch_lightning/accelerators/ddp_backend.py b/pytorch_lightning/accelerators/ddp_backend.py
--- a/pytorch_lightning/accelerators/ddp_backend.py
+++ b/pytorch_lightning/accelerators/ddp_backend.py
@@ -87,7 +87,7 @@
         command = [sys.executable] + command
 
         # since this script sets the visible devices we replace the gpus flag with a number
-        num_gpus = os.environ.get('CUDA_VISIBLE_DEVICES', []).split(',').__len__()
+        num_gpus = torch.cuda.device_count()
 
         if '--gpus' in command:
             gpu_flag_idx = command.index('--gpus')

{"golden_diff": "diff --git a/pytorch_lightning/accelerators/ddp_backend.py b/pytorch_lightning/accelerators/ddp_backend.py\n--- a/pytorch_lightning/accelerators/ddp_backend.py\n+++ b/pytorch_lightning/accelerators/ddp_backend.py\n@@ -87,7 +87,7 @@\n         command = [sys.executable] + command\n \n         # since this script sets the visible devices we replace the gpus flag with a number\n-        num_gpus = os.environ.get('CUDA_VISIBLE_DEVICES', []).split(',').__len__()\n+        num_gpus = torch.cuda.device_count()\n \n         if '--gpus' in command:\n             gpu_flag_idx = command.index('--gpus')\n", "issue": "Fixes #2942\n\n", "before_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License\n\nimport os\nimport subprocess\nimport sys\nfrom os.path import abspath\nfrom time import sleep\nfrom typing import Optional\n\nimport numpy as np\nimport torch\n\nfrom pytorch_lightning import _logger as log\nfrom pytorch_lightning.utilities import AMPType\nfrom pytorch_lightning.utilities.distributed import rank_zero_only\n\ntry:\n    from hydra.utils import to_absolute_path, get_original_cwd\n    from hydra.core.hydra_config import HydraConfig\nexcept ImportError:\n    HYDRA_AVAILABLE = False\nelse:\n    HYDRA_AVAILABLE = True\n\ntry:\n    from apex import amp\nexcept ImportError:\n    amp = None\n\n\nclass DDPBackend(object):\n\n    def __init__(self, trainer):\n        self.trainer = trainer\n        self.task_idx = None\n\n    def slurm_setup(self):\n        self.task_idx = int(os.environ['SLURM_LOCALID'])\n\n    def torchelastic_setup(self):\n        self.task_idx = int(os.environ['LOCAL_RANK'])\n\n    def train(self, model):\n        self.ddp_train(process_idx=self.task_idx, mp_queue=None, model=model)\n\n    def spawn_ddp_children(self, model):\n        port = os.environ['MASTER_PORT']\n\n        master_address = '127.0.0.1' if 'MASTER_ADDR' not in os.environ else os.environ['MASTER_ADDR']\n        os.environ['MASTER_PORT'] = f'{port}'\n        os.environ['MASTER_ADDR'] = f'{master_address}'\n\n        # allow the user to pass the node rank\n        node_rank = '0'\n        if 'NODE_RANK' in os.environ:\n            node_rank = os.environ['NODE_RANK']\n        if 'GROUP_RANK' in os.environ:\n            node_rank = os.environ['GROUP_RANK']\n\n        os.environ['NODE_RANK'] = node_rank\n        os.environ['LOCAL_RANK'] = '0'\n\n        # when user is using hydra find the absolute path\n        path_lib = abspath if not HYDRA_AVAILABLE else to_absolute_path\n\n        # pull out the commands used to run the script and resolve the abs file path\n        command = sys.argv\n        try:\n            full_path = path_lib(command[0])\n        except Exception as e:\n            full_path = abspath(command[0])\n\n        command[0] = full_path\n        # use the same python interpreter and actually running\n        command = [sys.executable] + command\n\n        # since this script sets the visible devices we replace the gpus flag with a number\n        num_gpus = os.environ.get('CUDA_VISIBLE_DEVICES', []).split(',').__len__()\n\n        if '--gpus' in command:\n            gpu_flag_idx = command.index('--gpus')\n            command[gpu_flag_idx + 1] = f'{num_gpus}'\n\n        os.environ['WORLD_SIZE'] = f'{num_gpus * self.trainer.num_nodes}'\n\n        self.trainer.interactive_ddp_procs = []\n        for local_rank in range(1, self.trainer.num_processes):\n            env_copy = os.environ.copy()\n            env_copy['LOCAL_RANK'] = f'{local_rank}'\n\n            # start process\n            # if hydra is available and initialized, make sure to set the cwd correctly\n            cwd: Optional[str] = None\n            if HYDRA_AVAILABLE:\n                if HydraConfig.initialized():\n                    cwd = get_original_cwd()\n            proc = subprocess.Popen(command, env=env_copy, cwd=cwd)\n            self.trainer.interactive_ddp_procs.append(proc)\n\n            # starting all processes at once can cause issues\n            # with dataloaders delay between 1-10 seconds\n            delay = np.random.uniform(1, 5, 1)[0]\n            sleep(delay)\n\n        local_rank = 0\n        results = self.ddp_train(local_rank, mp_queue=None, model=model, is_master=True)\n        del os.environ['WORLD_SIZE']\n\n        return results\n\n    def ddp_train(self, process_idx, mp_queue, model, is_master=False, proc_offset=0):\n        \"\"\"\n        Entry point for ddp\n\n        Args:\n            process_idx:\n            mp_queue: multiprocessing queue\n            model:\n            is_master:\n            proc_offset:\n\n        Returns:\n\n        \"\"\"\n        # offset the process id if requested\n        process_idx = process_idx + proc_offset\n\n        # show progressbar only on progress_rank 0\n        if (self.trainer.node_rank != 0 or process_idx != 0) and self.trainer.progress_bar_callback is not None:\n            self.trainer.progress_bar_callback.disable()\n\n        # determine which process we are and world size\n        self.trainer.local_rank = process_idx\n        self.trainer.global_rank = self.trainer.node_rank * self.trainer.num_processes + process_idx\n        self.trainer.world_size = self.trainer.num_nodes * self.trainer.num_processes\n\n        # set warning rank\n        rank_zero_only.rank = self.trainer.global_rank\n\n        # set up server using proc 0's ip address\n        # try to init for 20 times at max in case ports are taken\n        # where to store ip_table\n        model.trainer = self.trainer\n        model.init_ddp_connection(\n            self.trainer.global_rank,\n            self.trainer.world_size,\n            self.trainer.is_slurm_managing_tasks\n        )\n\n        # call setup after the ddp process has connected\n        self.trainer.call_setup_hook(model)\n\n        # on world_size=0 let everyone know training is starting\n        if self.trainer.is_global_zero:\n            log.info('-' * 100)\n            log.info(f'distributed_backend={self.trainer.distributed_backend}')\n            log.info(f'All DDP processes registered. Starting ddp with {self.trainer.world_size} processes')\n            log.info('-' * 100)\n\n        # call sync_bn before .cuda(), configure_apex and configure_ddp\n        if self.trainer.sync_batchnorm:\n            model = model.configure_sync_batchnorm(model)\n\n        # MODEL\n        # copy model to each gpu\n        if self.trainer.on_gpu:\n            gpu_idx = process_idx\n\n            # when using ddp, the master process (proc 0) continues running as the main one\n            # this means that the local rank will always be 0\n            # (even if cuda visible devices has other visible gpus)\n            # this means that the master process needs to pull the 0th visible index as the device number\n            if is_master:\n                available_gpus = os.environ['CUDA_VISIBLE_DEVICES'].split(',')\n                gpu_idx = int(available_gpus[self.trainer.local_rank])\n\n            self.trainer.root_gpu = gpu_idx\n            torch.cuda.set_device(self.trainer.root_gpu)\n            model.cuda(self.trainer.root_gpu)\n\n        # CHOOSE OPTIMIZER\n        # allow for lr schedulers as well\n        optimizers, lr_schedulers, optimizer_frequencies = self.trainer.init_optimizers(model)\n        self.trainer.optimizers = optimizers\n        self.trainer.lr_schedulers = lr_schedulers\n        self.trainer.optimizer_frequencies = optimizer_frequencies\n\n        # set model properties before going into wrapper\n        self.trainer.copy_trainer_model_properties(model)\n\n        # AMP - run through amp wrapper before going to distributed DP\n        if self.trainer.amp_backend == AMPType.APEX:\n            model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)\n            self.trainer.optimizers = optimizers\n            self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)\n\n        # DDP2 uses all GPUs on the machine\n        if self.trainer.distributed_backend == 'ddp' or self.trainer.distributed_backend == 'ddp_spawn':\n            device_ids = [self.trainer.root_gpu]\n        else:  # includes ddp_cpu\n            device_ids = None\n\n        # allow user to configure ddp\n        model = model.configure_ddp(model, device_ids)\n\n        # continue training routine\n        results = self.trainer.run_pretrain_routine(model)\n\n        # get original model\n        model = self.trainer.get_model()\n\n        # persist info in ddp_spawn\n        self.trainer.transfer_distrib_spawn_state_on_fit_end(model, mp_queue, results)\n\n        # clean up memory\n        torch.cuda.empty_cache()\n\n        if self.trainer.global_rank == 0 and self.trainer.distributed_backend not in ['ddp_spawn', 'ddp_cpu']:\n            return results\n", "path": "pytorch_lightning/accelerators/ddp_backend.py"}], "after_files": [{"content": "# Copyright The PyTorch Lightning team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License\n\nimport os\nimport subprocess\nimport sys\nfrom os.path import abspath\nfrom time import sleep\nfrom typing import Optional\n\nimport numpy as np\nimport torch\n\nfrom pytorch_lightning import _logger as log\nfrom pytorch_lightning.utilities import AMPType\nfrom pytorch_lightning.utilities.distributed import rank_zero_only\n\ntry:\n    from hydra.utils import to_absolute_path, get_original_cwd\n    from hydra.core.hydra_config import HydraConfig\nexcept ImportError:\n    HYDRA_AVAILABLE = False\nelse:\n    HYDRA_AVAILABLE = True\n\ntry:\n    from apex import amp\nexcept ImportError:\n    amp = None\n\n\nclass DDPBackend(object):\n\n    def __init__(self, trainer):\n        self.trainer = trainer\n        self.task_idx = None\n\n    def slurm_setup(self):\n        self.task_idx = int(os.environ['SLURM_LOCALID'])\n\n    def torchelastic_setup(self):\n        self.task_idx = int(os.environ['LOCAL_RANK'])\n\n    def train(self, model):\n        self.ddp_train(process_idx=self.task_idx, mp_queue=None, model=model)\n\n    def spawn_ddp_children(self, model):\n        port = os.environ['MASTER_PORT']\n\n        master_address = '127.0.0.1' if 'MASTER_ADDR' not in os.environ else os.environ['MASTER_ADDR']\n        os.environ['MASTER_PORT'] = f'{port}'\n        os.environ['MASTER_ADDR'] = f'{master_address}'\n\n        # allow the user to pass the node rank\n        node_rank = '0'\n        if 'NODE_RANK' in os.environ:\n            node_rank = os.environ['NODE_RANK']\n        if 'GROUP_RANK' in os.environ:\n            node_rank = os.environ['GROUP_RANK']\n\n        os.environ['NODE_RANK'] = node_rank\n        os.environ['LOCAL_RANK'] = '0'\n\n        # when user is using hydra find the absolute path\n        path_lib = abspath if not HYDRA_AVAILABLE else to_absolute_path\n\n        # pull out the commands used to run the script and resolve the abs file path\n        command = sys.argv\n        try:\n            full_path = path_lib(command[0])\n        except Exception as e:\n            full_path = abspath(command[0])\n\n        command[0] = full_path\n        # use the same python interpreter and actually running\n        command = [sys.executable] + command\n\n        # since this script sets the visible devices we replace the gpus flag with a number\n        num_gpus = torch.cuda.device_count()\n\n        if '--gpus' in command:\n            gpu_flag_idx = command.index('--gpus')\n            command[gpu_flag_idx + 1] = f'{num_gpus}'\n\n        os.environ['WORLD_SIZE'] = f'{num_gpus * self.trainer.num_nodes}'\n\n        self.trainer.interactive_ddp_procs = []\n        for local_rank in range(1, self.trainer.num_processes):\n            env_copy = os.environ.copy()\n            env_copy['LOCAL_RANK'] = f'{local_rank}'\n\n            # start process\n            # if hydra is available and initialized, make sure to set the cwd correctly\n            cwd: Optional[str] = None\n            if HYDRA_AVAILABLE:\n                if HydraConfig.initialized():\n                    cwd = get_original_cwd()\n            proc = subprocess.Popen(command, env=env_copy, cwd=cwd)\n            self.trainer.interactive_ddp_procs.append(proc)\n\n            # starting all processes at once can cause issues\n            # with dataloaders delay between 1-10 seconds\n            delay = np.random.uniform(1, 5, 1)[0]\n            sleep(delay)\n\n        local_rank = 0\n        results = self.ddp_train(local_rank, mp_queue=None, model=model, is_master=True)\n        del os.environ['WORLD_SIZE']\n\n        return results\n\n    def ddp_train(self, process_idx, mp_queue, model, is_master=False, proc_offset=0):\n        \"\"\"\n        Entry point for ddp\n\n        Args:\n            process_idx:\n            mp_queue: multiprocessing queue\n            model:\n            is_master:\n            proc_offset:\n\n        Returns:\n\n        \"\"\"\n        # offset the process id if requested\n        process_idx = process_idx + proc_offset\n\n        # show progressbar only on progress_rank 0\n        if (self.trainer.node_rank != 0 or process_idx != 0) and self.trainer.progress_bar_callback is not None:\n            self.trainer.progress_bar_callback.disable()\n\n        # determine which process we are and world size\n        self.trainer.local_rank = process_idx\n        self.trainer.global_rank = self.trainer.node_rank * self.trainer.num_processes + process_idx\n        self.trainer.world_size = self.trainer.num_nodes * self.trainer.num_processes\n\n        # set warning rank\n        rank_zero_only.rank = self.trainer.global_rank\n\n        # set up server using proc 0's ip address\n        # try to init for 20 times at max in case ports are taken\n        # where to store ip_table\n        model.trainer = self.trainer\n        model.init_ddp_connection(\n            self.trainer.global_rank,\n            self.trainer.world_size,\n            self.trainer.is_slurm_managing_tasks\n        )\n\n        # call setup after the ddp process has connected\n        self.trainer.call_setup_hook(model)\n\n        # on world_size=0 let everyone know training is starting\n        if self.trainer.is_global_zero:\n            log.info('-' * 100)\n            log.info(f'distributed_backend={self.trainer.distributed_backend}')\n            log.info(f'All DDP processes registered. Starting ddp with {self.trainer.world_size} processes')\n            log.info('-' * 100)\n\n        # call sync_bn before .cuda(), configure_apex and configure_ddp\n        if self.trainer.sync_batchnorm:\n            model = model.configure_sync_batchnorm(model)\n\n        # MODEL\n        # copy model to each gpu\n        if self.trainer.on_gpu:\n            gpu_idx = process_idx\n\n            # when using ddp, the master process (proc 0) continues running as the main one\n            # this means that the local rank will always be 0\n            # (even if cuda visible devices has other visible gpus)\n            # this means that the master process needs to pull the 0th visible index as the device number\n            if is_master:\n                available_gpus = os.environ['CUDA_VISIBLE_DEVICES'].split(',')\n                gpu_idx = int(available_gpus[self.trainer.local_rank])\n\n            self.trainer.root_gpu = gpu_idx\n            torch.cuda.set_device(self.trainer.root_gpu)\n            model.cuda(self.trainer.root_gpu)\n\n        # CHOOSE OPTIMIZER\n        # allow for lr schedulers as well\n        optimizers, lr_schedulers, optimizer_frequencies = self.trainer.init_optimizers(model)\n        self.trainer.optimizers = optimizers\n        self.trainer.lr_schedulers = lr_schedulers\n        self.trainer.optimizer_frequencies = optimizer_frequencies\n\n        # set model properties before going into wrapper\n        self.trainer.copy_trainer_model_properties(model)\n\n        # AMP - run through amp wrapper before going to distributed DP\n        if self.trainer.amp_backend == AMPType.APEX:\n            model, optimizers = model.configure_apex(amp, model, self.trainer.optimizers, self.trainer.amp_level)\n            self.trainer.optimizers = optimizers\n            self.trainer.reinit_scheduler_properties(self.trainer.optimizers, self.trainer.lr_schedulers)\n\n        # DDP2 uses all GPUs on the machine\n        if self.trainer.distributed_backend == 'ddp' or self.trainer.distributed_backend == 'ddp_spawn':\n            device_ids = [self.trainer.root_gpu]\n        else:  # includes ddp_cpu\n            device_ids = None\n\n        # allow user to configure ddp\n        model = model.configure_ddp(model, device_ids)\n\n        # continue training routine\n        results = self.trainer.run_pretrain_routine(model)\n\n        # get original model\n        model = self.trainer.get_model()\n\n        # persist info in ddp_spawn\n        self.trainer.transfer_distrib_spawn_state_on_fit_end(model, mp_queue, results)\n\n        # clean up memory\n        torch.cuda.empty_cache()\n\n        if self.trainer.global_rank == 0 and self.trainer.distributed_backend not in ['ddp_spawn', 'ddp_cpu']:\n            return results\n", "path": "pytorch_lightning/accelerators/ddp_backend.py"}]}