text: import numpy as np
  import torch
  from torch import nn
  from torch.autograd import Variable
  from torch.nn import functional as F
  
  
  class SELayer(nn.Module):
      def __init__(self, channel, reduction=16):
          super(SELayer, self).__init__()
          self.fc = nn.Sequential(
              nn.Linear(channel, channel // reduction, bias=False),
              nn.ReLU(inplace=True),
              nn.Linear(channel // reduction, channel, bias=False),
              nn.Sigmoid(),
          )
  
      def forward(self, x):
          y = self.fc(x)
          return x * y
  
  
  class CSE(nn.Module):
      def __init__(self, in_ch, r):
          super(CSE, self).__init__()
  
          self.linear_1 = nn.Linear(in_ch, in_ch // r)
          self.linear_2 = nn.Linear(in_ch // r, in_ch)
  
      def forward(self, x):
          input_x = x
  
          x = x.view(*(x.shape[:-2]), -1).mean(-1)
          x = F.relu(self.linear_1(x), inplace=True)
          x = self.linear_2(x)
          x = x.unsqueeze(-1).unsqueeze(-1)
          x = F.sigmoid(x)
  
          x = input_x * x
  
          return x
  
  
  class SSE(nn.Module):
      def __init__(self, in_ch):
          super(SSE, self).__init__()
  
          self.conv = nn.Conv2d(in_ch, in_ch, kernel_size=1, stride=1)
  
      def forward(self, x):
          input_x = x
  
          x = self.conv(x)
          x = F.sigmoid(x)
  
          x = input_x * x
  
          return x
  
  
  class SCSE(nn.Module):
      def __init__(self, in_ch, r=8):
          super(SCSE, self).__init__()
  
          self.cSE = CSE(in_ch, r)
          self.sSE = SSE(in_ch)
  
      def forward(self, x):
          cSE = self.cSE(x)
          sSE = self.sSE(x)
  
          x = cSE + sSE
  
          return x
  
  
  class SEBlock(nn.Module):
      def __init__(self, in_ch, r=8):
          super(SEBlock, self).__init__()
  
          self.linear_1 = nn.Linear(in_ch, in_ch // r)
          self.linear_2 = nn.Linear(in_ch // r, in_ch)
  
      def forward(self, x):
          input_x = x
  
          x = F.relu(self.linear_1(x), inplace=True)
          x = self.linear_2(x)
          x = F.sigmoid(x)
  
          x = input_x * x
  
          return x
  
  
  def get_sinusoid_encoding_table(n_position, d_hid, padding_idx=None):
      """ Sinusoid position encoding table """
  
      def cal_angle(position, hid_idx):
          return position / np.power(10000, 2 * (hid_idx // 2) / d_hid)
  
      def get_posi_angle_vec(position):
          return [cal_angle(position, hid_j) for hid_j in range(d_hid)]
  
      sinusoid_table = np.array(
          [get_posi_angle_vec(pos_i) for pos_i in range(n_position)]
      )
  
      sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i
      sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1
  
      if padding_idx is not None:
          # zero vector for padding dimension
          sinusoid_table[padding_idx] = 0.0
  
      return sinusoid_table
  
  
  def get_sinusoid_encoding_table_2d(H, W, d_hid):
      """ Sinusoid position encoding table """
      n_position = H * W
      sinusoid_table = get_sinusoid_encoding_table(n_position, d_hid)
      sinusoid_table = sinusoid_table.reshape(H, W, d_hid)
      return sinusoid_table
  
  
  class CBAM_Module(nn.Module):
      def __init__(
          self,
          channels,
          reduction=4,
          attention_kernel_size=3,
          position_encode=False,
      ):
          super(CBAM_Module, self).__init__()
          self.position_encode = position_encode
          self.avg_pool = nn.AdaptiveAvgPool2d(1)
          self.max_pool = nn.AdaptiveMaxPool2d(1)
          self.fc1 = nn.Conv2d(
              channels, channels // reduction, kernel_size=1, padding=0
          )
          self.relu = nn.ReLU(inplace=True)
          self.fc2 = nn.Conv2d(
              channels // reduction, channels, kernel_size=1, padding=0
          )
          self.sigmoid_channel = nn.Sigmoid()
          if self.position_encode:
              k = 3
          else:
              k = 2
          self.conv_after_concat = nn.Conv2d(
              k,
              1,
              kernel_size=attention_kernel_size,
              stride=1,
              padding=attention_kernel_size // 2,
          )
          self.sigmoid_spatial = nn.Sigmoid()
          self.position_encoded = None
  
      def forward(self, x):
          # Channel attention module
          module_input = x
          avg = self.avg_pool(x)
          mx = self.max_pool(x)
          avg = self.fc1(avg)
          mx = self.fc1(mx)
          avg = self.relu(avg)
          mx = self.relu(mx)
          avg = self.fc2(avg)
          mx = self.fc2(mx)
          x = avg + mx
          x = self.sigmoid_channel(x)
          # Spatial attention module
          x = module_input * x
          module_input = x
          b, c, h, w = x.size()
          if self.position_encode:
              if self.position_encoded is None:
  
                  pos_enc = get_sinusoid_encoding_table(h, w)
                  pos_enc = Variable(
                      torch.FloatTensor(pos_enc), requires_grad=False
                  )
                  if x.is_cuda:
                      pos_enc = pos_enc.cuda()
                  self.position_encoded = pos_enc
          avg = torch.mean(x, 1, True)
          mx, _ = torch.max(x, 1, True)
          if self.position_encode:
              pos_enc = self.position_encoded
              pos_enc = pos_enc.view(1, 1, h, w).repeat(b, 1, 1, 1)
              x = torch.cat((avg, mx, pos_enc), 1)
          else:
              x = torch.cat((avg, mx), 1)
          x = self.conv_after_concat(x)
          x = self.sigmoid_spatial(x)
          x = module_input * x
          return x
  
  
  def init_layer(layer):
      nn.init.xavier_uniform_(layer.weight)
  
      if hasattr(layer, "bias"):
          if layer.bias is not None:
              layer.bias.data.fill_(0.0)
  
  
  class AttBlockV2(nn.Module):
      def __init__(
          self, in_features: int, out_features: int, activation="linear"
      ):
          super().__init__()
  
          self.activation = activation
          self.att = nn.Conv1d(
              in_channels=in_features,
              out_channels=out_features,
              kernel_size=1,
              stride=1,
              padding=0,
              bias=True,
          )
          self.cla = nn.Conv1d(
              in_channels=in_features,
              out_channels=out_features,
              kernel_size=1,
              stride=1,
              padding=0,
              bias=True,
          )
  
          self.init_weights()
  
      def init_weights(self):
          init_layer(self.att)
          init_layer(self.cla)
  
      def forward(self, x):
          # x: (n_samples, n_in, n_time)
          norm_att = torch.softmax(torch.tanh(self.att(x)), dim=-1)
          cla = self.nonlinear_transform(self.cla(x))
          x = torch.sum(norm_att * cla, dim=2)
          return x, norm_att, cla
  
      def nonlinear_transform(self, x):
          if self.activation == "linear":
              return x
          elif self.activation == "sigmoid":
              return torch.sigmoid(x)
  
  
  class AttentionBlock(nn.Module):
      def __init__(self, in_features, middle_features, out_features=1):
          super().__init__()
          self.in_features = in_features
          self.middle_features = middle_features
          self.out_features = out_features
          self.W = nn.Linear(in_features, middle_features)
          self.V = nn.Linear(middle_features, out_features)
  
      def forward(self, features):
          att = torch.tanh(self.W(features))
          score = self.V(att)
          attention_weights = torch.softmax(score, dim=1)
          context_vector = attention_weights * features
          context_vector = torch.sum(context_vector, dim=1)
          return context_vector
  