class_docstr: Window based multi-head self attention (W-MSA) module with relative position bias.
  It supports shifted and non-shifted windows.
class_docstr_tok: ["Window","based","multi-head","self","attention","(","W-MSA",")","module","with","relative","position","bias",".","It","supports","shifted","and","non-shifted","windows","."]
docstr: Args:
      dim: Number of input channels.
      num_heads: Number of attention heads.
      head_dim: Number of channels per head (dim // num_heads if not set)
      window_size: The height and width of the window.
      qkv_bias:  If True, add a learnable bias to query, key, value.
      attn_drop: Dropout ratio of attention weight.
      proj_drop: Dropout ratio of output.
docstr_tok: ["Args",":","dim",":","Number","of","input","channels",".","num_heads",":","Number","of","attention","heads",".","head_dim",":","Number","of","channels","per","head","(","dim","\/\/","num_heads","if","not","set",")","window_size",":","The","height","and","width","of","the","window",".","qkv_bias",":","If","True",",","add","a","learnable","bias","to","query",",","key",",","value",".","attn_drop",":","Dropout","ratio","of","attention","weight",".","proj_drop",":","Dropout","ratio","of","output","."]
code: def __init__(
      self,
      dim: int,
      num_heads: int,
      head_dim: Optional[int] = None,
      window_size: _int_or_tuple_2_t = 7,
      qkv_bias: bool = True,
      attn_drop: float = 0.0,
      proj_drop: float = 0.0,
  ):
      """
      Args:
          dim: Number of input channels.
          num_heads: Number of attention heads.
          head_dim: Number of channels per head (dim // num_heads if not set)
          window_size: The height and width of the window.
          qkv_bias:  If True, add a learnable bias to query, key, value.
          attn_drop: Dropout ratio of attention weight.
          proj_drop: Dropout ratio of output.
      """
      super().__init__()
      self.dim = dim
      self.window_size = to_2tuple(window_size)  # Wh, Ww
      win_h, win_w = self.window_size
      self.window_area = win_h * win_w
      self.num_heads = num_heads
      head_dim = head_dim or dim // num_heads
      attn_dim = head_dim * num_heads
      self.scale = head_dim**-0.5
      self.fused_attn = use_fused_attn(
          experimental=True
      )  # NOTE not tested for prime-time yet
  
      # define a parameter table of relative position bias, shape: 2*Wh-1 * 2*Ww-1, nH
      self.relative_position_bias_table = nn.Parameter(
          torch.zeros((2 * win_h - 1) * (2 * win_w - 1), num_heads)
      )
  
      # get pair-wise relative position index for each token inside the window
      self.register_buffer(
          "relative_position_index",
          get_relative_position_index(win_h, win_w),
          persistent=False,
      )
  
      self.qkv = nn.Linear(dim, attn_dim * 3, bias=qkv_bias)
      self.attn_drop = nn.Dropout(attn_drop)
      self.proj = nn.Linear(attn_dim, dim)
      self.proj_drop = nn.Dropout(proj_drop)
  
      trunc_normal_(self.relative_position_bias_table, std=0.02)
      self.softmax = nn.Softmax(dim=-1)
  
imports: import logging
  import math
  from typing import Callable, List, Optional, Tuple, Union
  import torch
  import torch.nn
  from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
  from timm.layers import PatchEmbed, Mlp, DropPath, ClassifierHead, to_2tuple, to_ntuple, trunc_normal_, _assert, use_fused_attn, resize_rel_pos_bias_table, resample_patch_embed
  from ._builder import build_model_with_cfg
  from ._features_fx import register_notrace_function
  from ._manipulate import checkpoint_seq, named_apply
  from ._registry import generate_default_cfgs, register_model, register_model_deprecations
  from .vision_transformer import get_init_weights_vit
label_desc: Use image node_id 1 to create a new WindowAttention object from inherited base classes: nn with example: obj = WindowAttention(dim, num_heads, head_dim, window_size, qkv_bias, attn_drop, proj_drop)