text: """ Operations """
  import torch
  import torch.nn as nn
  # from models.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d
  # BatchNorm2d = SynchronizedBatchNorm2d
  # from apex.parallel import SyncBatchNorm
  # BatchNorm2d = SyncBatchNorm
  BatchNorm2d = nn.BatchNorm2d
  
  OPS = {
      'none': lambda C, stride, affine: Zero(stride),
      'avg_pool_3x3': lambda C, stride, affine: Pool('avg', C, 3, stride, 1, affine=affine),
      'max_pool_3x3': lambda C, stride, affine: Pool('max', C, 3, stride, 1, affine=affine),
      'skip_connect': lambda C, stride, affine: \
          Identity() if stride == 1 else FactorizedReduce(C, C, affine=affine),
      'sep_conv_3x3': lambda C, stride, affine: SepConv(C, C, 3, stride, 1, affine=affine),
      'sep_conv_5x5': lambda C, stride, affine: SepConv(C, C, 5, stride, 2, affine=affine),
      'sep_conv_7x7': lambda C, stride, affine: SepConv(C, C, 7, stride, 3, affine=affine),
      'dil_conv_3x3': lambda C, stride, affine: DilConv(C, C, 3, stride, 2, 2, affine=affine), # 5x5
      'dil_conv_5x5': lambda C, stride, affine: DilConv(C, C, 5, stride, 4, 2, affine=affine), # 9x9
      'conv_7x1_1x7': lambda C, stride, affine: FacConv(C, C, 7, stride, 3, affine=affine)
  }
  
  PRIMITIVES = [
      'max_pool_3x3',
      'avg_pool_3x3',
      'skip_connect', # identity
      'sep_conv_3x3',
      'sep_conv_5x5',
      'dil_conv_3x3',
      'dil_conv_5x5',
      'none'
  ]
  
  def channel_shuffle(x, groups):
      batchsize, num_channels, height, width = x.data.size()
  
      channels_per_group = num_channels // groups
      
      # reshape
      x = x.view(batchsize, groups, 
          channels_per_group, height, width)
  
      x = torch.transpose(x, 1, 2).contiguous()
  
      # flatten
      x = x.view(batchsize, -1, height, width)
  
      return x
      
  def drop_path_(x, drop_prob, training):
      if training and drop_prob > 0.:
          keep_prob = 1. - drop_prob
          # per data point mask; assuming x in cuda.
          mask = torch.cuda.FloatTensor(x.size(0), 1, 1, 1).bernoulli_(keep_prob)
          x.div_(keep_prob).mul_(mask)
  
      return x
  
  
  class DropPath_(nn.Module):
      def __init__(self, p=0.):
          """ [!] DropPath is inplace module
          Args:
              p: probability of an path to be zeroed.
          """
          super().__init__()
          self.p = p
  
      def extra_repr(self):
          return 'p={}, inplace'.format(self.p)
  
      def forward(self, x):
          drop_path_(x, self.p, self.training)
  
          return x
  
  
  class PoolBN(nn.Module):
      """
      AvgPool or MaxPool - BN
      """
      def __init__(self, pool_type, C, kernel_size, stride, padding, affine=True):
          """
          Args:
              pool_type: 'max' or 'avg'
          """
          super().__init__()
          if pool_type.lower() == 'max':
              self.pool = nn.MaxPool2d(kernel_size, stride, padding)
          elif pool_type.lower() == 'avg':
              self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False)
          else:
              raise ValueError()
  
          self.bn = BatchNorm2d(C, affine=affine)
  
      def forward(self, x):
          out = self.pool(x)
          out = self.bn(out)
          return out
  
  class Pool(nn.Module):
      """
      AvgPool or MaxPool
      """
      def __init__(self, pool_type, C, kernel_size, stride, padding, affine=True):
          """
          Args:
              pool_type: 'max' or 'avg'
          """
          super().__init__()
          if pool_type.lower() == 'max':
              self.pool = nn.MaxPool2d(kernel_size, stride, padding)
          elif pool_type.lower() == 'avg':
              self.pool = nn.AvgPool2d(kernel_size, stride, padding, count_include_pad=False)
          else:
              raise ValueError()
  
      def forward(self, x):
          out = self.pool(x)
          return out
  
  class StdConv(nn.Module):
      """ Standard conv
      ReLU - Conv - BN
      """
      def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True):
          super().__init__()
          self.net = nn.Sequential(
              nn.ReLU(),
              nn.Conv2d(C_in, C_out, kernel_size, stride, padding, bias=False),
              BatchNorm2d(C_out, affine=affine)
          )
  
      def forward(self, x):
          return self.net(x)
  
  
  class FacConv(nn.Module):
      """ Factorized conv
      ReLU - Conv(Kx1) - Conv(1xK) - BN
      """
      def __init__(self, C_in, C_out, kernel_length, stride, padding, affine=True):
          super().__init__()
          self.net = nn.Sequential(
              nn.ReLU(),
              nn.Conv2d(C_in, C_in, (kernel_length, 1), stride, padding, bias=False),
              nn.Conv2d(C_in, C_out, (1, kernel_length), stride, padding, bias=False),
              BatchNorm2d(C_out, affine=affine)
          )
  
      def forward(self, x):
          return self.net(x)
  
  
  class DilConv(nn.Module):
      """ (Dilated) depthwise separable conv
      ReLU - (Dilated) depthwise separable - Pointwise - BN
  
      If dilation == 2, 3x3 conv => 5x5 receptive field
                        5x5 conv => 9x9 receptive field
      """
      def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine=True):
          super().__init__()
          self.net = nn.Sequential(
              nn.ReLU(),
              nn.Conv2d(C_in, C_in, kernel_size, stride, padding, dilation=dilation, groups=C_in,
                        bias=False),
              nn.Conv2d(C_in, C_out, 1, stride=1, padding=0, bias=False),
              BatchNorm2d(C_out, affine=affine)
          )
  
      def forward(self, x):
          return self.net(x)
  
  
  class SepConv(nn.Module):
      """ Depthwise separable conv
      DilConv(dilation=1) * 2
      """
      def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True):
          super().__init__()
          self.net = nn.Sequential(
              DilConv(C_in, C_in, kernel_size, stride, padding, dilation=1, affine=affine),
              DilConv(C_in, C_out, kernel_size, 1, padding, dilation=1, affine=affine)
          )
  
      def forward(self, x):
          return self.net(x)
  
  
  class Identity(nn.Module):
      def __init__(self):
          super().__init__()
  
      def forward(self, x):
          return x
  
  
  class Zero(nn.Module):
      def __init__(self, stride):
          super().__init__()
          self.stride = stride
  
      def forward(self, x):
          if self.stride == 1:
              return x * 0.
  
          # re-sizing by stride
          return x[:, :, ::self.stride, ::self.stride] * 0.
  
  
  class FactorizedReduce(nn.Module):
      """
      Reduce feature map size by factorized pointwise(stride=2).
      """
      def __init__(self, C_in, C_out, affine=True):
          super().__init__()
          self.relu = nn.ReLU()
          self.conv1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False)
          self.conv2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False)
          self.bn = BatchNorm2d(C_out, affine=affine)
  
      def forward(self, x):
          x = self.relu(x)
          out = torch.cat([self.conv1(x), self.conv2(x[:, :, 1:, 1:])], dim=1)
          out = self.bn(out)
          # print ("conv1 param: {}".format(self.conv1.weight))
          # print ("bn mean: {}".format(self.bn.running_mean))
          # print ("bn var: {}".format(self.bn.running_var))
          return out
  
  
  def channel_shuffle(x, groups):
      batchsize, num_channels, height, width = x.data.size()
  
      channels_per_group = num_channels // groups
      
      # reshape
      x = x.view(batchsize, groups, 
          channels_per_group, height, width)
  
      x = torch.transpose(x, 1, 2).contiguous()
  
      # flatten
      x = x.view(batchsize, -1, height, width)
  
      return x
  
  
  class MixedOp(nn.Module):
      """ Mixed operation """
      def __init__(self, C, stride, is_slim=False, k=4):
          super().__init__()
          self._ops = nn.ModuleList()
          self.is_slim = is_slim
  
          for primitive in PRIMITIVES:
              op = OPS[primitive](C, stride, False)
              self._ops.append(op)
  
  
      def forward(self, x, weights):
          if self.is_slim:
              if torch.sum(weights) == 0:
                  return 0.
              else:
                  index = torch.argmax(weights).item()
                  return self._ops[index](x)
          else:
              return sum(w * op(x) for w, op in zip(weights, self._ops))
  