prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  torch.nn.DataParallel supporting unequal sizes
  As documented [here](http://pytorch.org/docs/master/_modules/torch/nn/parallel/data_parallel.html):
  ```
  The batch size should be larger than the number of GPUs used. It should
      also be an integer multiple of the number of GPUs so that each chunk is the
      same size (so that each GPU processes the same number of samples).
  ```
  To use `torch.nn.DataParallel`, people should carefully set the batch size according to the number of gpus they plan to use, otherwise it will pop up errors. 
  
  This issue becomes more subtle when using `torch.utils.data.DataLoader` with `drop_last=False` by default. As the total number of training/validation samples varies with the dataset, the size of the last batch of data loaded by `torch.utils.data.DataLoader` is easy to become indivisible by the number of GPUs (e.g., 2,3,4,8,...).
  
  A feature request would be:
  supporting `torch.nn.DataParallel` with batch size indivisible by the number of GPUs used.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `torch/nn/parallel/data_parallel.py`
  Content:
  ```
  1 import torch
  2 from ..modules import Module
  3 from .scatter_gather import scatter_kwargs, gather
  4 from .replicate import replicate
  5 from .parallel_apply import parallel_apply
  6 
  7 
  8 class DataParallel(Module):
  9     r"""Implements data parallelism at the module level.
  10 
  11     This container parallelizes the application of the given module by
  12     splitting the input across the specified devices by chunking in the batch
  13     dimension. In the forward pass, the module is replicated on each device,
  14     and each replica handles a portion of the input. During the backwards
  15     pass, gradients from each replica are summed into the original module.
  16 
  17     The batch size should be larger than the number of GPUs used. It should
  18     also be an integer multiple of the number of GPUs so that each chunk is the
  19     same size (so that each GPU processes the same number of samples).
  20 
  21     See also: :ref:`cuda-nn-dataparallel-instead`
  22 
  23     Arbitrary positional and keyword inputs are allowed to be passed into
  24     DataParallel EXCEPT Tensors. All variables will be scattered on dim
  25     specified (default 0). Primitive types will be broadcasted, but all
  26     other types will be a shallow copy and can be corrupted if written to in
  27     the model's forward pass.
  28 
  29     .. warning::
  30         Forward and backwrad hooks defined on :attr:`module` and its submodules
  31         won't be invoked anymore, unless the hooks are initialized in the
  32         :meth:`forward` method.
  33 
  34     Args:
  35         module: module to be parallelized
  36         device_ids: CUDA devices (default: all devices)
  37         output_device: device location of output (default: device_ids[0])
  38 
  39     Example::
  40 
  41         >>> net = torch.nn.DataParallel(model, device_ids=[0, 1, 2])
  42         >>> output = net(input_var)
  43     """
  44 
  45     # TODO: update notes/cuda.rst when this class handles 8+ GPUs well
  46 
  47     def __init__(self, module, device_ids=None, output_device=None, dim=0):
  48         super(DataParallel, self).__init__()
  49 
  50         if not torch.cuda.is_available():
  51             self.module = module
  52             self.device_ids = []
  53             return
  54 
  55         if device_ids is None:
  56             device_ids = list(range(torch.cuda.device_count()))
  57         if output_device is None:
  58             output_device = device_ids[0]
  59         self.dim = dim
  60         self.module = module
  61         self.device_ids = device_ids
  62         self.output_device = output_device
  63         if len(self.device_ids) == 1:
  64             self.module.cuda(device_ids[0])
  65 
  66     def forward(self, *inputs, **kwargs):
  67         if not self.device_ids:
  68             return self.module(*inputs, **kwargs)
  69         inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
  70         if len(self.device_ids) == 1:
  71             return self.module(*inputs[0], **kwargs[0])
  72         replicas = self.replicate(self.module, self.device_ids[:len(inputs)])
  73         outputs = self.parallel_apply(replicas, inputs, kwargs)
  74         return self.gather(outputs, self.output_device)
  75 
  76     def replicate(self, module, device_ids):
  77         return replicate(module, device_ids)
  78 
  79     def scatter(self, inputs, kwargs, device_ids):
  80         return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)
  81 
  82     def parallel_apply(self, replicas, inputs, kwargs):
  83         return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
  84 
  85     def gather(self, outputs, output_device):
  86         return gather(outputs, output_device, dim=self.dim)
  87 
  88 
  89 def data_parallel(module, inputs, device_ids=None, output_device=None, dim=0, module_kwargs=None):
  90     r"""Evaluates module(input) in parallel across the GPUs given in device_ids.
  91 
  92     This is the functional version of the DataParallel module.
  93 
  94     Args:
  95         module: the module to evaluate in parallel
  96         inputs: inputs to the module
  97         device_ids: GPU ids on which to replicate module
  98         output_device: GPU location of the output  Use -1 to indicate the CPU.
  99             (default: device_ids[0])
  100     Returns:
  101         a Variable containing the result of module(input) located on
  102         output_device
  103     """
  104     if not isinstance(inputs, tuple):
  105         inputs = (inputs,)
  106 
  107     if device_ids is None:
  108         device_ids = list(range(torch.cuda.device_count()))
  109 
  110     if output_device is None:
  111         output_device = device_ids[0]
  112 
  113     inputs, module_kwargs = scatter_kwargs(inputs, module_kwargs, device_ids, dim)
  114     if len(device_ids) == 1:
  115         return module(*inputs[0], **module_kwargs[0])
  116     used_device_ids = device_ids[:len(inputs)]
  117     replicas = replicate(module, used_device_ids)
  118     outputs = parallel_apply(replicas, inputs, module_kwargs, used_device_ids)
  119     return gather(outputs, output_device, dim)
  120
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/torch/nn/parallel/data_parallel.py b/torch/nn/parallel/data_parallel.py
  --- a/torch/nn/parallel/data_parallel.py
  +++ b/torch/nn/parallel/data_parallel.py
  @@ -14,9 +14,7 @@
       and each replica handles a portion of the input. During the backwards
       pass, gradients from each replica are summed into the original module.
   
  -    The batch size should be larger than the number of GPUs used. It should
  -    also be an integer multiple of the number of GPUs so that each chunk is the
  -    same size (so that each GPU processes the same number of samples).
  +    The batch size should be larger than the number of GPUs used.
   
       See also: :ref:`cuda-nn-dataparallel-instead`
  
verification_info: {"golden_diff": "diff --git a/torch/nn/parallel/data_parallel.py b/torch/nn/parallel/data_parallel.py\n--- a/torch/nn/parallel/data_parallel.py\n+++ b/torch/nn/parallel/data_parallel.py\n@@ -14,9 +14,7 @@\n     and each replica handles a portion of the input. During the backwards\n     pass, gradients from each replica are summed into the original module.\n \n-    The batch size should be larger than the number of GPUs used. It should\n-    also be an integer multiple of the number of GPUs so that each chunk is the\n-    same size (so that each GPU processes the same number of samples).\n+    The batch size should be larger than the number of GPUs used.\n \n     See also: :ref:`cuda-nn-dataparallel-instead`\n", "issue": "torch.nn.DataParallel supporting unequal sizes\nAs documented [here](http://pytorch.org/docs/master/_modules/torch/nn/parallel/data_parallel.html):\r\n```\r\nThe batch size should be larger than the number of GPUs used. It should\r\n    also be an integer multiple of the number of GPUs so that each chunk is the\r\n    same size (so that each GPU processes the same number of samples).\r\n```\r\nTo use `torch.nn.DataParallel`, people should carefully set the batch size according to the number of gpus they plan to use, otherwise it will pop up errors. \r\n\r\nThis issue becomes more subtle when using `torch.utils.data.DataLoader` with `drop_last=False` by default. As the total number of training/validation samples varies with the dataset, the size of the last batch of data loaded by `torch.utils.data.DataLoader` is easy to become indivisible by the number of GPUs (e.g., 2,3,4,8,...).\r\n\r\nA feature request would be:\r\nsupporting `torch.nn.DataParallel` with batch size indivisible by the number of GPUs used.\n", "before_files": [{"content": "import torch\nfrom ..modules import Module\nfrom .scatter_gather import scatter_kwargs, gather\nfrom .replicate import replicate\nfrom .parallel_apply import parallel_apply\n\n\nclass DataParallel(Module):\n    r\"\"\"Implements data parallelism at the module level.\n\n    This container parallelizes the application of the given module by\n    splitting the input across the specified devices by chunking in the batch\n    dimension. In the forward pass, the module is replicated on each device,\n    and each replica handles a portion of the input. During the backwards\n    pass, gradients from each replica are summed into the original module.\n\n    The batch size should be larger than the number of GPUs used. It should\n    also be an integer multiple of the number of GPUs so that each chunk is the\n    same size (so that each GPU processes the same number of samples).\n\n    See also: :ref:`cuda-nn-dataparallel-instead`\n\n    Arbitrary positional and keyword inputs are allowed to be passed into\n    DataParallel EXCEPT Tensors. All variables will be scattered on dim\n    specified (default 0). Primitive types will be broadcasted, but all\n    other types will be a shallow copy and can be corrupted if written to in\n    the model's forward pass.\n\n    .. warning::\n        Forward and backwrad hooks defined on :attr:`module` and its submodules\n        won't be invoked anymore, unless the hooks are initialized in the\n        :meth:`forward` method.\n\n    Args:\n        module: module to be parallelized\n        device_ids: CUDA devices (default: all devices)\n        output_device: device location of output (default: device_ids[0])\n\n    Example::\n\n        >>> net = torch.nn.DataParallel(model, device_ids=[0, 1, 2])\n        >>> output = net(input_var)\n    \"\"\"\n\n    # TODO: update notes/cuda.rst when this class handles 8+ GPUs well\n\n    def __init__(self, module, device_ids=None, output_device=None, dim=0):\n        super(DataParallel, self).__init__()\n\n        if not torch.cuda.is_available():\n            self.module = module\n            self.device_ids = []\n            return\n\n        if device_ids is None:\n            device_ids = list(range(torch.cuda.device_count()))\n        if output_device is None:\n            output_device = device_ids[0]\n        self.dim = dim\n        self.module = module\n        self.device_ids = device_ids\n        self.output_device = output_device\n        if len(self.device_ids) == 1:\n            self.module.cuda(device_ids[0])\n\n    def forward(self, *inputs, **kwargs):\n        if not self.device_ids:\n            return self.module(*inputs, **kwargs)\n        inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)\n        if len(self.device_ids) == 1:\n            return self.module(*inputs[0], **kwargs[0])\n        replicas = self.replicate(self.module, self.device_ids[:len(inputs)])\n        outputs = self.parallel_apply(replicas, inputs, kwargs)\n        return self.gather(outputs, self.output_device)\n\n    def replicate(self, module, device_ids):\n        return replicate(module, device_ids)\n\n    def scatter(self, inputs, kwargs, device_ids):\n        return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)\n\n    def parallel_apply(self, replicas, inputs, kwargs):\n        return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])\n\n    def gather(self, outputs, output_device):\n        return gather(outputs, output_device, dim=self.dim)\n\n\ndef data_parallel(module, inputs, device_ids=None, output_device=None, dim=0, module_kwargs=None):\n    r\"\"\"Evaluates module(input) in parallel across the GPUs given in device_ids.\n\n    This is the functional version of the DataParallel module.\n\n    Args:\n        module: the module to evaluate in parallel\n        inputs: inputs to the module\n        device_ids: GPU ids on which to replicate module\n        output_device: GPU location of the output  Use -1 to indicate the CPU.\n            (default: device_ids[0])\n    Returns:\n        a Variable containing the result of module(input) located on\n        output_device\n    \"\"\"\n    if not isinstance(inputs, tuple):\n        inputs = (inputs,)\n\n    if device_ids is None:\n        device_ids = list(range(torch.cuda.device_count()))\n\n    if output_device is None:\n        output_device = device_ids[0]\n\n    inputs, module_kwargs = scatter_kwargs(inputs, module_kwargs, device_ids, dim)\n    if len(device_ids) == 1:\n        return module(*inputs[0], **module_kwargs[0])\n    used_device_ids = device_ids[:len(inputs)]\n    replicas = replicate(module, used_device_ids)\n    outputs = parallel_apply(replicas, inputs, module_kwargs, used_device_ids)\n    return gather(outputs, output_device, dim)\n", "path": "torch/nn/parallel/data_parallel.py"}], "after_files": [{"content": "import torch\nfrom ..modules import Module\nfrom .scatter_gather import scatter_kwargs, gather\nfrom .replicate import replicate\nfrom .parallel_apply import parallel_apply\n\n\nclass DataParallel(Module):\n    r\"\"\"Implements data parallelism at the module level.\n\n    This container parallelizes the application of the given module by\n    splitting the input across the specified devices by chunking in the batch\n    dimension. In the forward pass, the module is replicated on each device,\n    and each replica handles a portion of the input. During the backwards\n    pass, gradients from each replica are summed into the original module.\n\n    The batch size should be larger than the number of GPUs used.\n\n    See also: :ref:`cuda-nn-dataparallel-instead`\n\n    Arbitrary positional and keyword inputs are allowed to be passed into\n    DataParallel EXCEPT Tensors. All variables will be scattered on dim\n    specified (default 0). Primitive types will be broadcasted, but all\n    other types will be a shallow copy and can be corrupted if written to in\n    the model's forward pass.\n\n    .. warning::\n        Forward and backwrad hooks defined on :attr:`module` and its submodules\n        won't be invoked anymore, unless the hooks are initialized in the\n        :meth:`forward` method.\n\n    Args:\n        module: module to be parallelized\n        device_ids: CUDA devices (default: all devices)\n        output_device: device location of output (default: device_ids[0])\n\n    Example::\n\n        >>> net = torch.nn.DataParallel(model, device_ids=[0, 1, 2])\n        >>> output = net(input_var)\n    \"\"\"\n\n    # TODO: update notes/cuda.rst when this class handles 8+ GPUs well\n\n    def __init__(self, module, device_ids=None, output_device=None, dim=0):\n        super(DataParallel, self).__init__()\n\n        if not torch.cuda.is_available():\n            self.module = module\n            self.device_ids = []\n            return\n\n        if device_ids is None:\n            device_ids = list(range(torch.cuda.device_count()))\n        if output_device is None:\n            output_device = device_ids[0]\n        self.dim = dim\n        self.module = module\n        self.device_ids = device_ids\n        self.output_device = output_device\n        if len(self.device_ids) == 1:\n            self.module.cuda(device_ids[0])\n\n    def forward(self, *inputs, **kwargs):\n        if not self.device_ids:\n            return self.module(*inputs, **kwargs)\n        inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)\n        if len(self.device_ids) == 1:\n            return self.module(*inputs[0], **kwargs[0])\n        replicas = self.replicate(self.module, self.device_ids[:len(inputs)])\n        outputs = self.parallel_apply(replicas, inputs, kwargs)\n        return self.gather(outputs, self.output_device)\n\n    def replicate(self, module, device_ids):\n        return replicate(module, device_ids)\n\n    def scatter(self, inputs, kwargs, device_ids):\n        return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)\n\n    def parallel_apply(self, replicas, inputs, kwargs):\n        return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])\n\n    def gather(self, outputs, output_device):\n        return gather(outputs, output_device, dim=self.dim)\n\n\ndef data_parallel(module, inputs, device_ids=None, output_device=None, dim=0, module_kwargs=None):\n    r\"\"\"Evaluates module(input) in parallel across the GPUs given in device_ids.\n\n    This is the functional version of the DataParallel module.\n\n    Args:\n        module: the module to evaluate in parallel\n        inputs: inputs to the module\n        device_ids: GPU ids on which to replicate module\n        output_device: GPU location of the output  Use -1 to indicate the CPU.\n            (default: device_ids[0])\n    Returns:\n        a Variable containing the result of module(input) located on\n        output_device\n    \"\"\"\n    if not isinstance(inputs, tuple):\n        inputs = (inputs,)\n\n    if device_ids is None:\n        device_ids = list(range(torch.cuda.device_count()))\n\n    if output_device is None:\n        output_device = device_ids[0]\n\n    inputs, module_kwargs = scatter_kwargs(inputs, module_kwargs, device_ids, dim)\n    if len(device_ids) == 1:\n        return module(*inputs[0], **module_kwargs[0])\n    used_device_ids = device_ids[:len(inputs)]\n    replicas = replicate(module, used_device_ids)\n    outputs = parallel_apply(replicas, inputs, module_kwargs, used_device_ids)\n    return gather(outputs, output_device, dim)\n", "path": "torch/nn/parallel/data_parallel.py"}]}