code: """ Define the sublayers in encoder/decoder layer """
  import numpy as np
  import torch
  import torch.nn as nn
  import torch.nn.functional as F
  
  
  class ScaledDotProductAttention(nn.Module):
      """ Scaled Dot-Product Attention """
  
      def __init__(self, temperature, attn_dropout=0.1):
          super().__init__()
          self.temperature = temperature
  
      def forward(self, q, k, v, mask=None):
          # Scale based on the current shape
          attn = torch.matmul(q / (q.shape[-1] ** 0.5), k.transpose(2, 3))
          if mask is not None:
              attn = attn.masked_fill(mask == 0, -1e9)
          attn = F.softmax(attn, dim=-1)
          output = torch.matmul(attn, v)
  
          return output, attn
  
  
  class MultiHeadAttention(nn.Module):
      """ Multi-Head Attention module """
  
      def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):
          super().__init__()
  
          self.n_head = n_head
          self.d_k = d_k
          self.d_v = d_v
  
          self.w_qs = nn.Linear(d_model, n_head * d_k, bias=False)
          self.w_ks = nn.Linear(d_model, n_head * d_k, bias=False)
          self.w_vs = nn.Linear(d_model, n_head * d_v, bias=False)
          self.fc = nn.Linear(n_head * d_v, d_model, bias=False)
  
          self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5)
  
          self.dropout = nn.Dropout(dropout)
          self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
  
      def forward(self, q, k, v, mask=None):
  
          d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
          sz_b, len_q, len_k, len_v = q.size(0), q.size(1), k.size(1), v.size(1)
  
          residual = q
  
          # Pass through the pre-attention projection: b x lq x (n*dv)
          # Separate different heads: b x lq x n x dv
          q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)
          k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)
          v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)
  
          # Transpose for attention dot product: b x n x lq x dv
          q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
  
          if mask is not None:
              mask = mask.unsqueeze(1)  # For head axis broadcasting.
  
          q, attn = self.attention(q, k, v, mask=mask)
  
          # Transpose to move the head dimension back: b x lq x n x dv
          # Combine the last two dimensions to concatenate all the heads together: b x lq x (n*dv)
          q = q.transpose(1, 2).contiguous().view(sz_b, len_q, -1)
          q = self.dropout(self.fc(q))
          q += residual
  
          q = self.layer_norm(q)
  
          return q, attn
  
  
  class PositionwiseFeedForward(nn.Module):
      """ A two-feed-forward-layer module """
  
      def __init__(self, d_in, d_hid, dropout=0.1):
          super().__init__()
          self.w_1 = nn.Linear(d_in, d_hid)  # position-wise
          self.w_2 = nn.Linear(d_hid, d_in)  # position-wise
          self.layer_norm = nn.LayerNorm(d_in, eps=1e-6)
          self.dropout = nn.Dropout(dropout)
  
      def forward(self, x):
  
          residual = x
  
          x = self.w_2(F.relu(self.w_1(x)))
          x = self.dropout(x)
          x += residual
  
          x = self.layer_norm(x)
  
          return x
   import torch
  
  
  class Attack(object):
      r"""
      Base class for all attacks.
  
      .. note::
          It automatically set device to the device where given model is.
          It temporarily changes the original model's training mode to `test`
          by `.eval()` only during an attack process.
      """
  
      def __init__(self, name, model):
          r"""
          Initializes internal attack state.
  
          Arguments:
              name (str) : name of an attack.
              model (torch.nn.Module): model to attack.
          """
  
          self.attack = name
          self.model = model
          self.model_name = str(model).split("(")[0]
  
          self.training = model.training
          self.device = next(model.parameters()).device
  
          self._targeted = 1
          self._attack_mode = "original"
          self._return_type = "float"
  
      def forward(self, *input):
          r"""
          It defines the computation performed at every call.
          Should be overridden by all subclasses.
          """
          raise NotImplementedError
  
      def set_attack_mode(self, mode):
          r"""
          Set the attack mode.
  
          Arguments:
              mode (str) : 'original' (DEFAULT)
                           'targeted' - Use input labels as targeted labels.
                           'least_likely' - Use least likely labels as targeted labels.
  
          """
          if self._attack_mode is "only_original":
              raise ValueError(
                  "Changing attack mode is not supported in this attack method."
              )
  
          if mode == "original":
              self._attack_mode = "original"
              self._targeted = 1
              self._transform_label = self._get_label
          elif mode == "targeted":
              self._attack_mode = "targeted"
              self._targeted = -1
              self._transform_label = self._get_label
          elif mode == "least_likely":
              self._attack_mode = "least_likely"
              self._targeted = -1
              self._transform_label = self._get_least_likely_label
          else:
              raise ValueError(
                  mode
                  + " is not a valid mode. [Options : original, targeted, least_likely]"
              )
  
      def set_return_type(self, type):
          r"""
          Set the return type of adversarial images: `int` or `float`.
  
          Arguments:
              type (str) : 'float' or 'int'. (DEFAULT : 'float')
  
          """
          if type == "float":
              self._return_type = "float"
          elif type == "int":
              self._return_type = "int"
          else:
              raise ValueError(type + " is not a valid type. [Options : float, int]")
  
      def save(self, save_path, data_loader, verbose=True):
          r"""
          Save adversarial images as torch.tensor from given torch.utils.data.DataLoader.
  
          Arguments:
              save_path (str) : save_path.
              data_loader (torch.utils.data.DataLoader) : data loader.
              verbose (bool) : True for displaying detailed information. (DEFAULT : True)
  
          """
          self.model.eval()
  
          image_list = []
          label_list = []
  
          correct = 0
          total = 0
  
          total_batch = len(data_loader)
  
          for step, (images, labels) in enumerate(data_loader):
              adv_images = self.__call__(images, labels)
  
              image_list.append(adv_images.cpu())
              label_list.append(labels.cpu())
  
              if self._return_type == "int":
                  adv_images = adv_images.float() / 255
  
              if verbose:
                  outputs = self.model(adv_images)
                  _, predicted = torch.max(outputs.data, 1)
                  total += labels.size(0)
                  correct += (predicted == labels.to(self.device)).sum()
  
                  acc = 100 * float(correct) / total
                  print(
                      "- Save Progress : %2.2f %% / Accuracy : %2.2f %%"
                      % ((step + 1) / total_batch * 100, acc),
                      end="\r",
                  )
  
          x = torch.cat(image_list, 0)
          y = torch.cat(label_list, 0)
          torch.save((x, y), save_path)
          print("\n- Save Complete!")
  
          self._switch_model()
  
      def _transform_label(self, images, labels):
          r"""
          Function for changing the attack mode.
          """
          return labels
  
      def _get_label(self, images, labels):
          r"""
          Function for changing the attack mode.
          Return input labels.
          """
          return labels
  
      def _get_least_likely_label(self, images, labels):
          r"""
          Function for changing the attack mode.
          Return least likely labels.
          """
          outputs = self.model(images)
          _, labels = torch.min(outputs.data, 1)
          labels = labels.detach_()
          return labels
  
      def _to_uint(self, images):
          r"""
          Function for changing the return type.
          Return images as int.
          """
          return (images * 255).type(torch.uint8)
  
      def _switch_model(self):
          r"""
          Function for changing the training mode of the model.
          """
          if self.training:
              self.model.train()
          else:
              self.model.eval()
  
      def __str__(self):
          info = self.__dict__.copy()
  
          del_keys = ["model", "attack"]
  
          for key in info.keys():
              if key[0] == "_":
                  del_keys.append(key)
  
          for key in del_keys:
              del info[key]
  
          info["attack_mode"] = self._attack_mode
          if info["attack_mode"] == "only_original":
              info["attack_mode"] = "original"
  
          info["return_type"] = self._return_type
  
          return (
              self.attack
              + "("
              + ", ".join("{}={}".format(key, val) for key, val in info.items())
              + ")"
          )
  
      def __call__(self, *input, **kwargs):
          self.model.eval()
          images = self.forward(*input, **kwargs)
          self._switch_model()
  
          if self._return_type == "int":
              images = self._to_uint(images)
  
          return images
  
apis: torch.nn.Linear torch.nn.functional.softmax torch.nn.LayerNorm torch.nn.Dropout torch.matmul
apis: torch.min torch.save torch.cat torch.max