class_docstr: Window based multi-head self-attention (W-MSA) module with relative
  position bias.
  
  Args:
      embed_dims (int): Number of input channels.
      num_heads (int): Number of attention heads.
      window_size (tuple[int]): The height and width of the window.
      qkv_bias (bool, optional):  If True, add a learnable bias to q, k, v.
          Default: True.
      qk_scale (float | None, optional): Override default qk scale of
          head_dim ** -0.5 if set. Default: None.
      attn_drop_rate (float, optional): Dropout ratio of attention weight.
          Default: 0.0
      proj_drop_rate (float, optional): Dropout ratio of output. Default: 0.
      init_cfg (dict | None, optional): The Config for initialization.
          Default: None.
class_docstr_tok: ["Window","based","multi-head","self-attention","(","W-MSA",")","module","with","relative","position","bias",".","Args",":","embed_dims","(","int",")",":","Number","of","input","channels",".","num_heads","(","int",")",":","Number","of","attention","heads",".","window_size","(","tuple","[","int","]",")",":","The","height","and","width","of","the","window",".","qkv_bias","(","bool",",","optional",")",":","If","True",",","add","a","learnable","bias","to","q",",","k",",","v.","Default",":","True",".","qk_scale","(","float","|","None",",","optional",")",":","Override","default","qk","scale","of","head_dim","*","*","-0.5","if","set",".","Default",":","None",".","attn_drop_rate","(","float",",","optional",")",":","Dropout","ratio","of","attention","weight",".","Default",":","0.0","proj_drop_rate","(","float",",","optional",")",":","Dropout","ratio","of","output",".","Default",":","0.","init_cfg","(","dict","|","None",",","optional",")",":","The","Config","for","initialization",".","Default",":","None","."]
code: def __init__(
      self,
      embed_dims,
      num_heads,
      window_size,
      qkv_bias=True,
      qk_scale=None,
      attn_drop_rate=0.0,
      proj_drop_rate=0.0,
      init_cfg=None,
  ):
      super().__init__()
      self.embed_dims = embed_dims
      self.window_size = window_size  # Wh, Ww
      self.num_heads = num_heads
      head_embed_dims = embed_dims // num_heads
      self.scale = qk_scale or head_embed_dims**-0.5
      self.init_cfg = init_cfg
  
      # define a parameter table of relative position bias
      self.relative_position_bias_table = nn.Parameter(
          torch.zeros(
              (2 * window_size[0] - 1) * (2 * window_size[1] - 1),
              num_heads,
          )
      )  # 2*Wh-1 * 2*Ww-1, nH
  
      # About 2x faster than original impl
      Wh, Ww = self.window_size
      rel_index_coords = self.double_step_seq(2 * Ww - 1, Wh, 1, Ww)
      rel_position_index = rel_index_coords + rel_index_coords.T
      rel_position_index = rel_position_index.flip(1).contiguous()
      self.register_buffer(
          "relative_position_index", rel_position_index
      )
  
      self.qkv = nn.Linear(embed_dims, embed_dims * 3, bias=qkv_bias)
      self.attn_drop = nn.Dropout(attn_drop_rate)
      self.proj = nn.Linear(embed_dims, embed_dims)
      self.proj_drop = nn.Dropout(proj_drop_rate)
  
      self.softmax = nn.Softmax(dim=-1)
  
imports: import warnings
  from collections import OrderedDict
  from copy import deepcopy
  import torch
  import torch.nn
  import torch.nn.functional
  import torch.utils.checkpoint
  from mmcv.cnn import build_norm_layer
  from mmcv.cnn.bricks.transformer import FFN, build_dropout
  from mmengine.logging import MMLogger
  from mmengine.model import BaseModule, ModuleList
  from mmengine.model.weight_init import constant_init, trunc_normal_, trunc_normal_init
  from mmengine.runner.checkpoint import CheckpointLoader
  from mmengine.utils import to_2tuple
  from mmdet.registry import MODELS
  from ..layers import PatchEmbed, PatchMerging
label_desc: Use image node_id 1 to create a new WindowMSA object from inherited base classes: BaseModule with example: obj = WindowMSA(embed_dims, num_heads, window_size, qkv_bias, qk_scale, attn_drop_rate, proj_drop_rate, init_cfg)