class_docstr: Window based multi-head self-attention (W-MSA) module with relative
  position bias.
  
  Args:
      embed_dims (int): Number of input channels.
      num_heads (int): Number of attention heads.
      window_size (tuple[int]): The height and width of the window.
      qkv_bias (bool, optional):  If True, add a learnable bias to q, k, v.
          Default: True.
      qk_scale (float | None, optional): Override default qk scale of
          head_dim ** -0.5 if set. Default: None.
      attn_drop_rate (float, optional): Dropout ratio of attention weight.
          Default: 0.0
      proj_drop_rate (float, optional): Dropout ratio of output. Default: 0.
      init_cfg (dict | None, optional): The Config for initialization.
          Default: None.
class_docstr_tok: ["Window","based","multi-head","self-attention","(","W-MSA",")","module","with","relative","position","bias",".","Args",":","embed_dims","(","int",")",":","Number","of","input","channels",".","num_heads","(","int",")",":","Number","of","attention","heads",".","window_size","(","tuple","[","int","]",")",":","The","height","and","width","of","the","window",".","qkv_bias","(","bool",",","optional",")",":","If","True",",","add","a","learnable","bias","to","q",",","k",",","v.","Default",":","True",".","qk_scale","(","float","|","None",",","optional",")",":","Override","default","qk","scale","of","head_dim","*","*","-0.5","if","set",".","Default",":","None",".","attn_drop_rate","(","float",",","optional",")",":","Dropout","ratio","of","attention","weight",".","Default",":","0.0","proj_drop_rate","(","float",",","optional",")",":","Dropout","ratio","of","output",".","Default",":","0.","init_cfg","(","dict","|","None",",","optional",")",":","The","Config","for","initialization",".","Default",":","None","."]
docstr: Args:
  
      x (tensor): input features with shape of (num_windows*B, N, C)
      mask (tensor | None, Optional): mask with shape of (num_windows,
          Wh*Ww, Wh*Ww), value should be between (-inf, 0].
code: def forward(self, x, mask=None):
      """
      Args:
  
          x (tensor): input features with shape of (num_windows*B, N, C)
          mask (tensor | None, Optional): mask with shape of (num_windows,
              Wh*Ww, Wh*Ww), value should be between (-inf, 0].
      """
      B, N, C = x.shape
      qkv = (
          self.qkv(x)
          .reshape(B, N, 3, self.num_heads, C // self.num_heads)
          .permute(2, 0, 3, 1, 4)
      )
      # make torchscript happy (cannot use tensor as tuple)
      q, k, v = qkv[0], qkv[1], qkv[2]
  
      q = q * self.scale
      attn = q @ k.transpose(-2, -1)
  
      relative_position_bias = self.relative_position_bias_table[
          self.relative_position_index.view(-1)
      ].view(
          self.window_size[0] * self.window_size[1],
          self.window_size[0] * self.window_size[1],
          -1,
      )  # Wh*Ww,Wh*Ww,nH
      relative_position_bias = relative_position_bias.permute(
          2, 0, 1
      ).contiguous()  # nH, Wh*Ww, Wh*Ww
      attn = attn + relative_position_bias.unsqueeze(0)
  
      if mask is not None:
          nW = mask.shape[0]
          attn = attn.view(
              B // nW, nW, self.num_heads, N, N
          ) + mask.unsqueeze(1).unsqueeze(0)
          attn = attn.view(-1, self.num_heads, N, N)
      attn = self.softmax(attn)
  
      attn = self.attn_drop(attn)
  
      x = (attn @ v).transpose(1, 2).reshape(B, N, C)
      x = self.proj(x)
      x = self.proj_drop(x)
      return x
  
imports: import warnings
  from collections import OrderedDict
  from copy import deepcopy
  import torch
  import torch.nn
  import torch.nn.functional
  import torch.utils.checkpoint
  from mmcv.cnn import build_norm_layer
  from mmcv.cnn.bricks.transformer import FFN, build_dropout
  from mmengine.logging import MMLogger
  from mmengine.model import BaseModule, ModuleList
  from mmengine.model.weight_init import constant_init, trunc_normal_, trunc_normal_init
  from mmengine.runner.checkpoint import CheckpointLoader
  from mmengine.utils import to_2tuple
  from mmdet.registry import MODELS
  from ..layers import PatchEmbed, PatchMerging
label_desc: Use image node_id 3 for calling the WindowMSA obj's underlying member method code with example usage: obj.forward(x, mask) and returns: x