class_docstr: Multihead Attention with extra flags on the q/k/v and out projections.
class_docstr_tok: ["Multihead","Attention","with","extra","flags","on","the","q\/k\/v","and","out","projections","."]
code: def __init__(
      self,
      *args,
      add_zero_attn=None,
      rpe=False,
      window_size=7,
      **kwargs,
  ):
      super(Attention, self).__init__(*args, **kwargs)
  
      self.add_zero_attn = add_zero_attn
      self.rpe = rpe
      if rpe:
          self.window_size = [window_size] * 2
          # define a parameter table of relative position bias
          parameter_value = paddle.zeros(
              [
                  (2 * self.window_size[0] - 1)
                  * (2 * self.window_size[1] - 1),
                  self.num_heads,
              ]
          )
          self.relative_position_bias_table = self.create_parameter(
              shape=parameter_value.shape,
              dtype=str(parameter_value.numpy().dtype),
              default_initializer=nn.initializer.Assign(
                  parameter_value
              ),
          )
  
          # get pair-wise relative position index for each token inside the window
          coords_h = paddle.arange(self.window_size[0])
          coords_w = paddle.arange(self.window_size[1])
          coords = paddle.stack(
              paddle.meshgrid([coords_h, coords_w])
          )  # 2, Wh, Ww
          coords_flatten = paddle.flatten(coords, 1)  # 2, Wh*Ww
          relative_coords = (
              coords_flatten[:, :, None] - coords_flatten[:, None, :]
          )  # 2, Wh*Ww, Wh*Ww
          relative_coords = relative_coords.transpose(
              [1, 2, 0]
          )  # Wh*Ww, Wh*Ww, 2
          relative_coords[:, :, 0] += (
              self.window_size[0] - 1
          )  # shift to start from 0
          relative_coords[:, :, 1] += self.window_size[1] - 1
          relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
          relative_position_index = relative_coords.sum(
              -1
          )  # Wh*Ww, Wh*Ww
          self.register_buffer(
              "relative_position_index", relative_position_index
          )
          trunc_normal_(self.relative_position_bias_table)
  
imports: import paddle
  import paddle.nn
  import paddle.nn.functional
  from paddleseg.cvlibs import manager, param_init
  from paddleseg.models import layers
  from paddleseg.utils import utils, logger
  from paddleseg.models.backbones.transformer_utils import *
label_desc: Use image node_id 1 to create a new Attention object from inherited base classes: nn with example: obj = Attention()