text: """
  Copyright 2020 The OneFlow Authors. All rights reserved.
  
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
  
      http://www.apache.org/licenses/LICENSE-2.0
  
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
  """
  import math
  import os
  
  import oneflow as flow
  from oneflow.nn import init
  from oneflow.nn.common_types import _size_1_t, _size_2_t, _size_3_t
  from oneflow.nn.module import Module
  from oneflow.nn.modules.utils import _pair, _single, _triple
  
  
  def slice(x, begin, size):
      ndim = len(x.shape)
      if not isinstance(begin, (list, tuple)) or len(begin) != ndim:
          raise ValueError(
              "begin must be a list/tuple with the same length as input tensor's number of dimensions"
          )
      if not all((isinstance(b, int) or b is None for b in begin)):
          raise ValueError("element of begin must be a int or None")
      if not isinstance(size, (list, tuple)) or len(size) != ndim:
          raise ValueError(
              "size must be a list/tuple with the same length as input tensor's number of dimensions."
          )
      if not all((isinstance(s, int) or s is None for s in size)):
          raise ValueError("element of size must be a int or None")
      slice_tup_list = []
      for (b, s, dim_size) in zip(begin, size, x.shape):
          (start, stop, step) = (None, None, 1)
          if b is not None:
              if b < -dim_size or b >= dim_size:
                  raise ValueError("element of begin is out of range")
              start = b
          if s is not None:
              if s == -1:
                  stop = dim_size
              else:
                  if s <= 0 or s > dim_size:
                      raise ValueError("element of size is invalid")
                  if b + s < dim_size:
                      stop = b + s
          slice_tup_list.append((start, stop, step))
      return flow.slice(x, slice_tup_list)
  
  
  class ConvUtil(object):
      @classmethod
      def split(cls, x, axis, split_num):
          split_len = x.shape[axis] // split_num
          result_list = []
          slice_begin = [0] * len(x.shape)
          slice_size = [-1] * len(x.shape)
          slice_size[axis] = split_len
          for i in range(split_num):
              slice_begin[axis] = i * split_len
              result = slice(x, slice_begin, slice_size)
              result_list.append(result)
          return result_list
  
  
  class Conv1d(Module):
      """The interface is consistent with PyTorch.    
      The documentation is referenced from: https://pytorch.org/docs/master/generated/torch.nn.Conv1d.html#conv1d
      
      Applies a 1D convolution over an input signal composed of several input
      planes.
  
      In the simplest case, the output value of the layer with input size
      :math:`(N, C_{\\text{in}}, L)` and output :math:`(N, C_{\\text{out}}, L_{\\text{out}})` can be
      precisely described as:
  
      .. math::
          \\text{out}(N_i, C_{\\text{out}_j}) = \\text{bias}(C_{\\text{out}_j}) +
          \\sum_{k = 0}^{C_{in} - 1} \\text{weight}(C_{\\text{out}_j}, k)
          \\star \\text{input}(N_i, k)
  
      where :math:`\\star` is the valid `cross-correlation`_ operator,
      :math:`N` is a batch size, :math:`C` denotes a number of channels,
      :math:`L` is a length of signal sequence.
  
      * :attr:`stride` controls the stride for the cross-correlation, a single
        number or a one-element tuple.
  
      * :attr:`padding` controls the amount of padding applied to the input. It
        can be either a string {{'valid', 'same'}} or a tuple of ints giving the
        amount of implicit padding applied on both sides.
  
      * :attr:`dilation` controls the spacing between the kernel points; also
        known as the à trous algorithm. It is harder to describe, but this `link`_
        has a nice visualization of what :attr:`dilation` does.
  
      Note:
          ``padding='valid'`` is the same as no padding. ``padding='same'`` pads
          the input so the output has the shape as the input. However, this mode
          doesn't support any stride values other than 1.
  
      Args:
          in_channels (int): Number of channels in the input image
          out_channels (int): Number of channels produced by the convolution
          kernel_size (int or tuple): Size of the convolving kernel
          stride (int or tuple, optional): Stride of the convolution. Default: 1
          padding (int, tuple or str, optional): Padding added to both sides of
              the input. Default: 0
          padding_mode (string, optional): ``'zeros'``. Default: ``'zeros'``
          dilation (int or tuple, optional): Spacing between kernel
              elements. Default: 1
          groups (int, optional): Number of blocked connections from input
              channels to output channels. Default: 1
          bias (bool, optional): If ``True``, adds a learnable bias to the
              output. Default: ``True``
  
      Shape:
          - Input: :math:`(N, C_{in}, L_{in})`
          - Output: :math:`(N, C_{out}, L_{out})` where
  
            .. math::
                L_{out} = \\left\\lfloor\\frac{L_{in} + 2 \\times \\text{padding} - \\text{dilation}
                          \\times (\\text{kernel\\_size} - 1) - 1}{\\text{stride}} + 1\\right\\rfloor
  
      Attributes:
          weight (Tensor): the learnable weights of the module of shape
              :math:`(\\text{out\\_channels},
              \\frac{\\text{in\\_channels}}{\\text{groups}}, \\text{kernel\\_size})`.
              The values of these weights are sampled from
              :math:`\\mathcal{U}(-\\sqrt{k}, \\sqrt{k})` where
              :math:`k = \\frac{groups}{C_\\text{in} * \\text{kernel\\_size}}`
          bias (Tensor):   the learnable bias of the module of shape
              (out_channels). If :attr:`bias` is ``True``, then the values of these weights are
              sampled from :math:`\\mathcal{U}(-\\sqrt{k}, \\sqrt{k})` where
              :math:`k = \\frac{groups}{C_\\text{in} * \\text{kernel\\_size}}`
  
      For example: 
  
      .. code-block:: python
  
          >>> import numpy as np
          >>> import oneflow as flow
          >>> import oneflow.nn as nn
          
          >>> arr = np.random.randn(20, 16, 50)
          >>> input = flow.Tensor(arr)
          >>> m = nn.Conv1d(16, 33, 3, stride=2)
          >>> output = m(input)
  
      .. _cross-correlation:
          https://en.wikipedia.org/wiki/Cross-correlation
  
      .. _link:
          https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
      """
  
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          kernel_size: _size_1_t,
          stride: _size_1_t = 1,
          padding: _size_1_t = 0,
          dilation: _size_1_t = 1,
          groups: int = 1,
          bias: bool = True,
          padding_mode: str = "zeros",
      ):
          super().__init__()
          assert padding_mode == "zeros"
          self.padding_mode = padding_mode
          self.kernel_size = _single(kernel_size)
          self.stride = _single(stride)
          self.padding = _single(padding)
          self.dilation = _single(dilation)
          self.groups = groups
          self.channel_pos = "channels_first"
          assert in_channels % groups == 0
          assert out_channels % groups == 0
          self.in_channels = in_channels
          self.out_channels = out_channels
          self.weight = flow.nn.Parameter(
              flow.Tensor(out_channels, in_channels // groups, *self.kernel_size)
          )
          self.out_channel_groups = out_channels // groups
          self.bias = None
          if bias:
              self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
          self.reset_parameters()
  
      def reset_parameters(self) -> None:
          init.kaiming_uniform_(self.weight, a=math.sqrt(5))
          if self.bias is not None:
              (fan_in, _) = init._calculate_fan_in_and_fan_out(self.weight)
              bound = 1 / math.sqrt(fan_in)
              init.uniform_(self.bias, -bound, bound)
  
      def forward(self, x):
          return flow._C.conv1d(
              x,
              self.weight,
              self.bias,
              stride=self.stride,
              padding=self.padding,
              dilation=self.dilation,
              groups=self.groups,
              channel_pos=self.channel_pos,
          )
  
      def extra_repr(self):
          s = "{in_channels}, {out_channels}, kernel_size={kernel_size}, stride={stride}"
          if self.padding != (0,) * len(self.padding):
              s += ", padding={padding}"
          if self.dilation != (1,) * len(self.dilation):
              s += ", dilation={dilation}"
          if self.groups != 1:
              s += ", groups={groups}"
          if self.bias is None:
              s += ", bias=False"
          if self.padding_mode != "zeros":
              s += ", padding_mode={padding_mode}"
          return s.format(**self.__dict__)
  
  
  class Conv2d(Module):
      """The interface is consistent with PyTorch.    
      The documentation is referenced from: https://pytorch.org/docs/master/generated/torch.nn.Conv2d.html#conv2d
      
      Applies a 2D convolution over an input signal composed of several input
      planes.
  
      In the simplest case, the output value of the layer with input size
      :math:`(N, C_{\\text{in}}, H, W)` and output :math:`(N, C_{\\text{out}}, H_{\\text{out}}, W_{\\text{out}})`
      can be precisely described as:
  
      .. math::
          \\text{out}(N_i, C_{\\text{out}_j}) = \\text{bias}(C_{\\text{out}_j}) +
          \\sum_{k = 0}^{C_{\\text{in}} - 1} \\text{weight}(C_{\\text{out}_j}, k) \\star \\text{input}(N_i, k)
  
  
      where :math:`\\star` is the valid 2D `cross-correlation`_ operator,
      :math:`N` is a batch size, :math:`C` denotes a number of channels,
      :math:`H` is a height of input planes in pixels, and :math:`W` is
      width in pixels.
  
  
      * :attr:`stride` controls the stride for the cross-correlation, a single
        number or a tuple.
      * :attr:`padding` controls the amount of implicit padding on both
        sides for :attr:`padding` number of points for each dimension.
      * :attr:`dilation` controls the spacing between the kernel points; also
        known as the à trous algorithm. It is harder to describe, but this `link`_
        has a nice visualization of what :attr:`dilation` does.
      * :attr:`groups` controls the connections between inputs and outputs.
        :attr:`in_channels` and :attr:`out_channels` must both be divisible by
        :attr:`groups`. For example,
  
          * At groups=1, all inputs are convolved to all outputs.
          * At groups=2, the operation becomes equivalent to having two conv
            layers side by side, each seeing half the input channels
            and producing half the output channels, and both subsequently
            concatenated.
          * At groups= :attr:`in_channels`, each input channel is convolved with
            its own set of filters (of size
            :math:`\\frac{\\text{out_channels}}{\\text{in_channels}}`).,
  
      The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`dilation` can either be:
  
          - a single ``int`` -- in which case the same value is used for the height and width dimension
          - a ``tuple`` of two ints -- in which case, the first `int` is used for the height dimension,
            and the second `int` for the width dimension
  
      Note:
          When `groups == in_channels` and `out_channels == K * in_channels`,
          where `K` is a positive integer, this operation is also known as a "depthwise convolution".
  
          In other words, for an input of size :math:`(N, C_{in}, L_{in})`,
          a depthwise convolution with a depthwise multiplier `K` can be performed with the arguments
          :math:`(C_\\text{in}=C_\\text{in}, C_\\text{out}=C_\\text{in} \\times \\text{K}, ..., \\text{groups}=C_\\text{in})`.
  
  
      Args:
          in_channels (int): Number of channels in the input image
          out_channels (int): Number of channels produced by the convolution
          kernel_size (int or tuple): Size of the convolving kernel
          stride (int or tuple, optional): Stride of the convolution. Default: 1
          padding (int or tuple, optional): Zero-padding added to both sides of
              the input. Default: 0
          padding_mode (string, optional): ``'zeros'``. Default: ``'zeros'``
          dilation (int or tuple, optional): Spacing between kernel elements. Default: 1
          groups (int, optional): Number of blocked connections from input
              channels to output channels. Default: 1
          bias (bool, optional): If ``True``, adds a learnable bias to the
              output. Default: ``True``
  
      Shape:
          - Input: :math:`(N, C_{in}, H_{in}, W_{in})`
          - Output: :math:`(N, C_{out}, H_{out}, W_{out})` where
  
            .. math::
                H_{out} = \\left\\lfloor\\frac{H_{in}  + 2 \\times \\text{padding}[0] - \\text{dilation}[0]
                          \\times (\\text{kernel_size}[0] - 1) - 1}{\\text{stride}[0]} + 1\\right\\rfloor
  
            .. math::
                W_{out} = \\left\\lfloor\\frac{W_{in}  + 2 \\times \\text{padding}[1] - \\text{dilation}[1]
                          \\times (\\text{kernel_size}[1] - 1) - 1}{\\text{stride}[1]} + 1\\right\\rfloor
  
      Attr:
          - weight (Tensor): the learnable weights of the module of shape
              :math:`(\\text{out_channels}, \\frac{\\text{in_channels}}{\\text{groups}},`
              :math:`\\text{kernel_size[0]}, \\text{kernel_size[1]})`.
              The values of these weights are sampled from
              :math:`\\mathcal{U}(-\\sqrt{k}, \\sqrt{k})` where
              :math:`k = \\frac{groups}{C_\\text{in} * \\prod_{i=0}^{1}\\text{kernel_size}[i]}`
  
          - bias (Tensor):   the learnable bias of the module of shape
              (out_channels). If :attr:`bias` is ``True``,
              then the values of these weights are
              sampled from :math:`\\mathcal{U}(-\\sqrt{k}, \\sqrt{k})` where
              :math:`k = \\frac{groups}{C_\\text{in} * \\prod_{i=0}^{1}\\text{kernel_size}[i]}`
  
      For example: 
  
      .. code-block:: python
  
          >>> import numpy as np
          >>> import oneflow as flow
          >>> import oneflow.nn as nn
          
          >>> arr = np.random.randn(20, 16, 50, 100)
          >>> input = flow.Tensor(arr)
          >>> m = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2), dilation=(3, 1))
          >>> output = m(input)
  
      .. _cross-correlation:
          https://en.wikipedia.org/wiki/Cross-correlation
  
      .. _link:
          https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
      """
  
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          kernel_size: _size_2_t,
          stride: _size_2_t = 1,
          padding: _size_2_t = 0,
          dilation: _size_2_t = 1,
          groups: int = 1,
          bias: bool = True,
          padding_mode: str = "zeros",
      ):
          super().__init__()
          assert padding_mode == "zeros"
          self.padding_mode = padding_mode
          self.kernel_size = _pair(kernel_size)
          self.stride = _pair(stride)
          self.padding = _pair(padding)
          self.dilation = _pair(dilation)
          self.groups = groups
  
          if os.getenv("ONEFLOW_ENABLE_NHWC") == "1":
              self.channel_pos = "channels_last"
          else:
              self.channel_pos = "channels_first"
  
          assert in_channels % groups == 0
          assert out_channels % groups == 0
          self.in_channels = in_channels
          self.out_channels = out_channels
          if self.channel_pos == "channels_first":
              self.weight = flow.nn.Parameter(
                  flow.Tensor(out_channels, in_channels // groups, *self.kernel_size)
              )
          else:
              self.weight = flow.nn.Parameter(
                  flow.Tensor(out_channels, *self.kernel_size, in_channels // groups)
              )
  
          self.out_channel_groups = out_channels // groups
          self.bias = None
          if bias:
              self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
          self.reset_parameters()
  
      def reset_parameters(self) -> None:
          init.kaiming_uniform_(self.weight, a=math.sqrt(5))
          if self.bias is not None:
              (fan_in, _) = init._calculate_fan_in_and_fan_out(self.weight)
              bound = 1 / math.sqrt(fan_in)
              init.uniform_(self.bias, -bound, bound)
  
      def forward(self, x):
          if self.channel_pos == "channels_first":
              in_channel_axis = 1
          else:
              in_channel_axis = 3
          if x.shape[in_channel_axis] != self.in_channels:
              raise ValueError(
                  f"The input channels {x.shape[in_channel_axis]} should be equal to self.in_channels {self.in_channels}."
              )
          return flow._C.conv2d(
              x,
              self.weight,
              self.bias,
              stride=self.stride,
              padding=self.padding,
              dilation=self.dilation,
              groups=self.groups,
              channel_pos=self.channel_pos,
          )
  
      def extra_repr(self):
          s = "{in_channels}, {out_channels}, kernel_size={kernel_size}, stride={stride}"
          if self.padding != (0,) * len(self.padding):
              s += ", padding={padding}"
          if self.dilation != (1,) * len(self.dilation):
              s += ", dilation={dilation}"
          if self.groups != 1:
              s += ", groups={groups}"
          if self.bias is None:
              s += ", bias=False"
          if self.padding_mode != "zeros":
              s += ", padding_mode={padding_mode}"
          return s.format(**self.__dict__)
  
  
  class Conv3d(Module):
      r"""The interface is consistent with PyTorch.    
      The documentation is referenced from: https://pytorch.org/docs/master/generated/torch.nn.Conv3d.html#conv3d
      
      Applies a 3D convolution over an input signal composed of several input
      planes.
  
      In the simplest case, the output value of the layer with input size :math:`(N, C_{in}, D, H, W)`
      and output :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` can be precisely described as:
  
      .. math::
          out(N_i, C_{out_j}) = bias(C_{out_j}) +
                                  \sum_{k = 0}^{C_{in} - 1} weight(C_{out_j}, k) \star input(N_i, k)
  
      where :math:`\star` is the valid 3D `cross-correlation`_ operator
  
      * :attr:`stride` controls the stride for the cross-correlation.
  
      * :attr:`padding` controls the amount of padding applied to the input. It
        can be either a string {{'valid', 'same'}} or a tuple of ints giving the
        amount of implicit padding applied on both sides.
  
      * :attr:`dilation` controls the spacing between the kernel points; also known as the à trous algorithm.
        It is harder to describe, but this `link`_ has a nice visualization of what :attr:`dilation` does.
  
      The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`dilation` can either be:
  
          - a single ``int`` -- in which case the same value is used for the depth, height and width dimension
          - a ``tuple`` of three ints -- in which case, the first `int` is used for the depth dimension,
            the second `int` for the height dimension and the third `int` for the width dimension
  
      Note:
          ``padding='valid'`` is the same as no padding. ``padding='same'`` pads
          the input so the output has the shape as the input. However, this mode
          doesn't support any stride values other than 1.
  
      Args:
          in_channels (int): Number of channels in the input image
          out_channels (int): Number of channels produced by the convolution
          kernel_size (int or tuple): Size of the convolving kernel
          stride (int or tuple, optional): Stride of the convolution. Default: 1
          padding (int, tuple or str, optional): Padding added to all six sides of
              the input. Default: 0
          padding_mode (string, optional): ``'zeros'``. Default: ``'zeros'``
          dilation (int or tuple, optional): Spacing between kernel elements. Default: 1
          groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1
          bias (bool, optional): If ``True``, adds a learnable bias to the output. Default: ``True``
      
      Shape:
          - Input: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`
          - Output: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` where
  
            .. math::
                D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] - \text{dilation}[0]
                      \times (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor
  
            .. math::
                H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] - \text{dilation}[1]
                      \times (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor
  
            .. math::
                W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] - \text{dilation}[2]
                      \times (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor
  
      Attributes:
          weight (Tensor): the learnable weights of the module of shape
                           :math:`(\text{out\_channels}, \frac{\text{in\_channels}}{\text{groups}},`
                           :math:`\text{kernel\_size[0]}, \text{kernel\_size[1]}, \text{kernel\_size[2]})`.
                           The values of these weights are sampled from
                           :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where
                           :math:`k = \frac{groups}{C_\text{in} * \prod_{i=0}^{2}\text{kernel\_size}[i]}`
          bias (Tensor):   the learnable bias of the module of shape (out_channels). If :attr:`bias` is ``True``,
                           then the values of these weights are
                           sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where
                           :math:`k = \frac{groups}{C_\text{in} * \prod_{i=0}^{2}\text{kernel\_size}[i]}`
  
      For example: 
  
      .. code-block:: python
  
          >>> import numpy as np
          >>> import oneflow as flow
          >>> import oneflow.nn as nn
  
          >>> arr = np.random.randn(1, 2, 5, 5, 5)
          >>> input = flow.Tensor(arr)
          >>> m = nn.Conv3d(2, 4, kernel_size=3, stride=1)
          >>> output = m(input)
          
      .. _cross-correlation:
          https://en.wikipedia.org/wiki/Cross-correlation
      .. _link:
          https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
      """
  
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          kernel_size: _size_3_t,
          stride: _size_3_t = 1,
          padding: _size_3_t = 0,
          dilation: _size_3_t = 1,
          groups: int = 1,
          bias: bool = True,
          padding_mode: str = "zeros",  # TODO: refine this type
      ):
          super().__init__()
  
          assert padding_mode == "zeros"
          self.padding_mode = padding_mode
          self.kernel_size = _triple(kernel_size)
          self.stride = _triple(stride)
          self.padding = _triple(padding)
          self.dilation = _triple(dilation)
          self.groups = groups
          self.channel_pos = "channels_first"
          assert in_channels % groups == 0, "in_channels must be divisible by groups"
          assert out_channels % groups == 0, "out_channels must be divisible by groups"
          self.in_channels = in_channels
          self.out_channels = out_channels
          self.weight = flow.nn.Parameter(
              flow.Tensor(out_channels, in_channels // groups, *self.kernel_size)
          )
          self.out_channel_groups = out_channels // groups
          self.bias = None
          if bias:
              self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
          self.reset_parameters()
  
      def reset_parameters(self) -> None:
          init.kaiming_uniform_(self.weight, a=math.sqrt(5))
          if self.bias is not None:
              fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
              bound = 1 / math.sqrt(fan_in)
              init.uniform_(self.bias, -bound, bound)
  
      def forward(self, x):
          if x.shape[1] != self.in_channels:
              raise ValueError("The input channels should be equal to self.in_channels")
          return flow._C.conv3d(
              x,
              self.weight,
              self.bias,
              stride=self.stride,
              padding=self.padding,
              dilation=self.dilation,
              groups=self.groups,
              channel_pos=self.channel_pos,
          )
  
      def extra_repr(self):
          s = "{in_channels}, {out_channels}, kernel_size={kernel_size}, stride={stride}"
          if self.padding != (0,) * len(self.padding):
              s += ", padding={padding}"
          if self.dilation != (1,) * len(self.dilation):
              s += ", dilation={dilation}"
          if self.groups != 1:
              s += ", groups={groups}"
          if self.bias is None:
              s += ", bias=False"
          if self.padding_mode != "zeros":
              s += ", padding_mode={padding_mode}"
          return s.format(**self.__dict__)
  
  
  class ConvTranspose1d(Module):
      r"""Applies a 1D transposed convolution operator over an input image
      composed of several input planes.
  
      This module can be seen as the gradient of Conv1d with respect to its input.
      It is also known as a fractionally-strided convolution or
      a deconvolution (although it is not an actual deconvolution operation).
  
      This module supports TensorFloat32.
  
      * :attr:`stride` controls the stride for the cross-correlation.
  
      * :attr:`padding` controls the amount of implicit zero padding on both
        sides for ``dilation * (kernel_size - 1) - padding`` number of points. See note
        below for details.
  
      * :attr:`output_padding` controls the additional size added to one side
        of the output shape. See note below for details.
  
      * :attr:`dilation` controls the spacing between the kernel points; also known as the à trous algorithm.
        It is harder to describe, but this `link`_ has a nice visualization of what :attr:`dilation` does.
  
      Note:
          The :attr:`padding` argument effectively adds ``dilation * (kernel_size - 1) - padding``
          amount of zero padding to both sizes of the input. This is set so that
          when a :class:`~torch.nn.Conv1d` and a :class:`~torch.nn.ConvTranspose1d`
          are initialized with same parameters, they are inverses of each other in
          regard to the input and output shapes. However, when ``stride > 1``,
          :class:`~torch.nn.Conv1d` maps multiple input shapes to the same output
          shape. :attr:`output_padding` is provided to resolve this ambiguity by
          effectively increasing the calculated output shape on one side. Note
          that :attr:`output_padding` is only used to find output shape, but does
          not actually add zero-padding to output.
  
      Note:
          In some circumstances when using the CUDA backend with CuDNN, this operator
          may select a nondeterministic algorithm to increase performance. If this is
          undesirable, you can try to make the operation deterministic (potentially at
          a performance cost) by setting ``torch.backends.cudnn.deterministic =
          True``.
          Please see the notes on randomness for background.
  
  
      Args:
          in_channels (int): Number of channels in the input image
          out_channels (int): Number of channels produced by the convolution
          kernel_size (int or tuple): Size of the convolving kernel
          stride (int or tuple, optional): Stride of the convolution. Default: 1
          padding (int or tuple, optional): ``dilation * (kernel_size - 1) - padding`` zero-padding
              will be added to both sides of the input. Default: 0
          output_padding (int or tuple, optional): Additional size added to one side
              of the output shape. Default: 0
          groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1
          bias (bool, optional): If ``True``, adds a learnable bias to the output. Default: ``True``
          dilation (int or tuple, optional): Spacing between kernel elements. Default: 1
  
      Shape:
          - Input: :math:`(N, C_{in}, L_{in})`
          - Output: :math:`(N, C_{out}, L_{out})` where
  
            .. math::
                L_{out} = (L_{in} - 1) \times \text{stride} - 2 \times \text{padding} + \text{dilation}
                          \times (\text{kernel_size} - 1) + \text{output_padding} + 1
  
      Attributes:
          weight (Tensor): the learnable weights of the module of shape
                           :math:`(\\text{in\_channels}, \frac{\\text{out\\_channels}}{\text{groups}},`
                           :math:`\\text{kernel\\_size})`.
                           The values of these weights are sampled from
                           :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where
                           :math:`k = \frac{groups}{C_\text{out} * \\text{kernel\\_size}}`
          bias (Tensor):   the learnable bias of the module of shape (out_channels).
                           If :attr:`bias` is ``True``, then the values of these weights are
                           sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where
                           :math:`k = \frac{groups}{C_\text{out} * \\text{kernel\\_size}}`
      
      .. _cross-correlation:
          https://en.wikipedia.org/wiki/Cross-correlation
  
      .. _link:
          https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
      """
  
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          kernel_size: _size_1_t,
          stride: _size_1_t = 1,
          padding: _size_1_t = 0,
          output_padding: _size_1_t = 0,
          groups: int = 1,
          bias: bool = True,
          dilation: _size_1_t = 1,
          padding_mode: str = "zeros",
      ) -> None:
          super().__init__()
          assert (
              padding_mode == "zeros"
          ), "Only `zeros` padding mode is supported for ConvTranspose1d"
          self.kernel_size = _single(kernel_size)
          self.stride = _single(stride)
          self.padding = _single(padding)
          self.dilation = _single(dilation)
          self.output_padding = _single(output_padding)
          self.groups = groups
          assert in_channels % groups == 0
          assert out_channels % groups == 0
          self.weight = flow.nn.Parameter(
              flow.Tensor(in_channels, out_channels // groups, *self.kernel_size)
          )
          self.filters = out_channels
          self.bias = None
          self._bias_add_op = None
          if bias:
              self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
          self.reset_parameters()
  
      def reset_parameters(self) -> None:
          init.kaiming_uniform_(self.weight, a=math.sqrt(5))
          if self.bias is not None:
              (fan_in, _) = init._calculate_fan_in_and_fan_out(self.weight)
              bound = 1 / math.sqrt(fan_in)
              init.uniform_(self.bias, -bound, bound)
  
      def forward(self, x):
          return flow._C.deconv1d(
              x,
              self.weight,
              self.bias,
              self.filters,
              self.padding,
              "channels_first",
              self.kernel_size,
              self.output_padding,
              self.stride,
              self.dilation,
              self.groups,
          )
  
  
  class ConvTranspose2d(Module):
      """
      
      Applies a 2D transposed convolution operator over an input image composed of several input planes.
  
      This module can be seen as the gradient of Conv2d with respect to its input.
      It is also known as a fractionally-strided convolution or
      a deconvolution (although it is not an actual deconvolution operation).
  
      Args:  
          in_channels (int): Number of channels in the input image
          out_channels (int): Number of channels produced by the convolution
          kernel_size (int or tuple): Size of the convolving kernel
          stride (int or tuple, optional): Stride of the convolution. Default: 1
          padding (int or tuple, optional): ``dilation * (kernel_size - 1) - padding`` zero-padding
              will be added to both sides of each dimension in the input. Default: 0
          output_padding (int or tuple, optional): Additional size added to one side
              of each dimension in the output shape. Default: 0
          groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1
          bias (bool, optional): If ``True``, adds a learnable bias to the output. Default: ``True``
          dilation (int or tuple, optional): Spacing between kernel elements. Default: 1
  
      Shape:
          - Input: :math:`(N, C_{in}, H_{in}, W_{in})`
          - Output: :math:`(N, C_{out}, H_{out}, W_{out})` where
  
          .. math::
                H_{out} = (H_{in} - 1) \\times \\text{stride}[0] - 2 \\times \\text{padding}[0] + \\text{dilation}[0] 
  
                          \\times (\\text{kernel_size}[0] - 1) + \\text{output_padding}[0] + 1
          .. math::
                W_{out} = (W_{in} - 1) \\times \\text{stride}[1] - 2 \\times \\text{padding}[1] + \\text{dilation}[1]
                
                          \\times (\\text{kernel_size}[1] - 1) + \\text{output_padding}[1] + 1
  
      Attributes:
          ConvTranspose2d.weight (Tensor): the learnable weights of the module of shape
                           :math:`(\\text{in_channels}, \\frac{\\text{out_channels}}{\\text{groups}},`
                           :math:`\\text{kernel_size[0]}, \\text{kernel_size[1]})`.
                           The values of these weights are sampled from
                           :math:`\\mathcal{U}(-\\sqrt{k}, \\sqrt{k})` where
                           :math:`k = \\frac{groups}{C_\\text{out} * \\prod_{i=0}^{1}\\text{kernel_size}[i]}`
          ConvTranspose2d.bias (Tensor): the learnable bias of the module of shape (out_channels)
                           If :attr:`bias` is ``True``, then the values of these weights are
                           sampled from :math:`\\mathcal{U}(-\\sqrt{k}, \\sqrt{k})` where
                           :math:`k = \\frac{groups}{C_\\text{out} * \\prod_{i=0}^{1}\\text{kernel_size}[i]}`
  
      Examples::
  
          >>> import numpy as np
          >>> import oneflow as flow
          >>> import oneflow.nn as nn
          
          >>> m = nn.ConvTranspose2d(16, 33, 3, stride=2)
          >>> # non-square kernels and unequal stride and with padding
          >>> m = nn.ConvTranspose2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2))
          >>> m = m.to("cuda")
          >>> input = flow.Tensor(np.random.randn(20, 16, 50, 100), device=flow.device("cuda"))
          >>> output = m(input)
          >>> output.size()
          oneflow.Size([20, 33, 93, 100])
  
      .. _cross-correlation:
          https://en.wikipedia.org/wiki/Cross-correlation
  
      .. _link:
          https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
      """
  
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          kernel_size: _size_2_t,
          stride: _size_2_t = 1,
          padding: _size_2_t = 0,
          output_padding: _size_2_t = 0,
          groups: int = 1,
          bias: bool = True,
          dilation: int = 1,
          padding_mode: str = "zeros",
      ) -> None:
          super().__init__()
          assert padding_mode == "zeros"
          self.kernel_size = _pair(kernel_size)
          self.stride = _pair(stride)
          self.padding = _pair(padding)
          self.output_padding = _pair(output_padding)
          self.dilation = _pair(dilation)
          self.groups = groups
          assert in_channels % groups == 0
          assert out_channels % groups == 0
          self.weight = flow.nn.Parameter(
              flow.Tensor(in_channels, out_channels // groups, *self.kernel_size)
          )
          self.in_channel_groups = in_channels // groups
          self.filters = out_channels
          self.bias = None
          self._bias_add_op = None
          if bias:
              self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
  
          self.reset_parameters()
  
      def reset_parameters(self) -> None:
          init.kaiming_uniform_(self.weight, a=math.sqrt(5))
          if self.bias is not None:
              (fan_in, _) = init._calculate_fan_in_and_fan_out(self.weight)
              bound = 1 / math.sqrt(fan_in)
              init.uniform_(self.bias, -bound, bound)
  
      def forward(self, x):
          res = flow._C.deconv2d(
              x,
              self.weight,
              self.bias,
              self.filters,
              self.padding,
              "channels_first",
              self.kernel_size,
              self.output_padding,
              self.stride,
              self.dilation,
              self.groups,
          )
          return res
  
  
  class ConvTranspose3d(Module):
      r"""
      Applies a 3D transposed convolution operator over an input image composed of several input
      planes.
      The transposed convolution operator multiplies each input value element-wise by a learnable kernel,
      and sums over the outputs from all input feature planes.
  
      This module can be seen as the gradient of Conv3d with respect to its input.
      It is also known as a fractionally-strided convolution or
      a deconvolution (although it is not an actual deconvolution operation).
  
      This module supports TensorFloat32.
  
      * :attr:`stride` controls the stride for the cross-correlation.
  
      * :attr:`padding` controls the amount of implicit zero padding on both
        sides for ``dilation * (kernel_size - 1) - padding`` number of points. See note
        below for details.
  
      * :attr:`output_padding` controls the additional size added to one side
        of the output shape. See note below for details.
  
      * :attr:`dilation` controls the spacing between the kernel points; also known as the à trous algorithm.
        It is harder to describe, but this `link`_ has a nice visualization of what :attr:`dilation` does.
  
  
      The parameters :attr:`kernel_size`, :attr:`stride`, :attr:`padding`, :attr:`output_padding`
      can either be:
  
          - a single ``int`` -- in which case the same value is used for the depth, height and width dimensions
          - a ``tuple`` of three ints -- in which case, the first `int` is used for the depth dimension,
            the second `int` for the height dimension and the third `int` for the width dimension
  
      Note:
          The :attr:`padding` argument effectively adds ``dilation * (kernel_size - 1) - padding``
          amount of zero padding to both sizes of the input. This is set so that
          when a :class:`~torch.nn.Conv3d` and a :class:`~torch.nn.ConvTranspose3d`
          are initialized with same parameters, they are inverses of each other in
          regard to the input and output shapes. However, when ``stride > 1``,
          :class:`~torch.nn.Conv3d` maps multiple input shapes to the same output
          shape. :attr:`output_padding` is provided to resolve this ambiguity by
          effectively increasing the calculated output shape on one side. Note
          that :attr:`output_padding` is only used to find output shape, but does
          not actually add zero-padding to output.
  
  
      Args:
          in_channels (int): Number of channels in the input image
          out_channels (int): Number of channels produced by the convolution
          kernel_size (int or tuple): Size of the convolving kernel
          stride (int or tuple, optional): Stride of the convolution. Default: 1
          padding (int or tuple, optional): ``dilation * (kernel_size - 1) - padding`` zero-padding
              will be added to both sides of each dimension in the input. Default: 0
          output_padding (int or tuple, optional): Additional size added to one side
              of each dimension in the output shape. Default: 0
          groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1
          bias (bool, optional): If ``True``, adds a learnable bias to the output. Default: ``True``
          dilation (int or tuple, optional): Spacing between kernel elements. Default: 1
      
  
      Shape:
          - Input: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})`
          - Output: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` where
  
          .. math::
                D_{out} = (D_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] + \text{dilation}[0]
                          \times (\text{kernel_size}[0] - 1) + \text{output_padding}[0] + 1
          .. math::
                H_{out} = (H_{in} - 1) \times \text{stride}[1] - 2 \times \text{padding}[1] + \text{dilation}[1]
                          \times (\text{kernel_size}[1] - 1) + \text{output_padding}[1] + 1
          .. math::
                W_{out} = (W_{in} - 1) \times \text{stride}[2] - 2 \times \text{padding}[2] + \text{dilation}[2]
                          \times (\text{kernel_size}[2] - 1) + \text{output_padding}[2] + 1
  
  
      Attributes:
          weight (Tensor): the learnable weights of the module of shape
                           :math:`(\text{in_channels}, \frac{\text{out_channels}}{\text{groups}},`
                           :math:`\text{kernel_size[0]}, \text{kernel_size[1]}, \text{kernel_size[2]})`.
                           The values of these weights are sampled from
                           :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where
                           :math:`k = \frac{groups}{C_\text{out} * \prod_{i=0}^{2}\text{kernel_size}[i]}`
          bias (Tensor):   the learnable bias of the module of shape (out_channels)
                           If :attr:`bias` is ``True``, then the values of these weights are
                           sampled from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where
                           :math:`k = \frac{groups}{C_\text{out} * \prod_{i=0}^{2}\text{kernel_size}[i]}`
  
      Examples::
  
          >>> import oneflow as flow
          >>> import oneflow.nn as nn
  
          >>> # With square kernels and equal stride
          >>> m = nn.ConvTranspose3d(16, 33, 3, stride=2)
          >>> # non-square kernels and unequal stride and with padding
          >>> m = nn.ConvTranspose3d(16, 33, (3, 5, 2), stride=(2, 1, 1), padding=(0, 4, 2))
          >>> input = flow.randn(20, 16, 10, 50, 100)
          >>> output = m(input)
  
      .. _cross-correlation:
          https://en.wikipedia.org/wiki/Cross-correlation
  
      .. _link:
          https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md
      """
  
      def __init__(
          self,
          in_channels: int,
          out_channels: int,
          kernel_size: _size_3_t,
          stride: _size_3_t = 1,
          padding: _size_3_t = 0,
          output_padding: _size_3_t = 0,
          groups: int = 1,
          bias: bool = True,
          dilation: _size_3_t = 1,
          padding_mode: str = "zeros",
      ) -> None:
          super().__init__()
          assert padding_mode == "zeros", "Only `zeros` padding mode is supported"
          self.kernel_size = _triple(kernel_size)
          self.stride = _triple(stride)
          self.padding = _triple(padding)
          self.dilation = _triple(dilation)
          self.output_padding = _triple(output_padding)
          self.groups = groups
          assert in_channels % groups == 0
          assert out_channels % groups == 0
          self.weight = flow.nn.Parameter(
              flow.Tensor(in_channels, out_channels // groups, *self.kernel_size)
          )
          self.filters = out_channels
          self.bias = None
          self._bias_add_op = None
          if bias:
              self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
          self.reset_parameters()
  
      def reset_parameters(self) -> None:
          init.kaiming_uniform_(self.weight, a=math.sqrt(5))
          if self.bias is not None:
              (fan_in, _) = init._calculate_fan_in_and_fan_out(self.weight)
              bound = 1 / math.sqrt(fan_in)
              init.uniform_(self.bias, -bound, bound)
  
      def forward(self, x):
          return flow._C.deconv3d(
              x,
              self.weight,
              self.bias,
              self.filters,
              self.padding,
              "channels_first",
              self.kernel_size,
              self.output_padding,
              self.stride,
              self.dilation,
              self.groups,
          )
  
  
  if __name__ == "__main__":
      import doctest
  
      doctest.testmod(raise_on_error=True)
  