class_docstr: Transformer blocks with support of window attention and residual propagation blocks
docstr: Args:
      dim (int): Number of input channels.
      num_heads (int): Number of attention heads in each ViT block.
      mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
      qkv_bias (bool): If True, add a learnable bias to query, key, value.
      norm_layer (nn.Module): Normalization layer.
      act_layer (nn.Module): Activation layer.
      use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
      rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
      window_size (int): Window size for window attention blocks. If it equals 0, then
          use global attention.
      input_size (tuple(int, int) or None): Input resolution for calculating the relative
          positional parameter size.
docstr_tok: ["Args",":","dim","(","int",")",":","Number","of","input","channels",".","num_heads","(","int",")",":","Number","of","attention","heads","in","each","ViT","block",".","mlp_ratio","(","float",")",":","Ratio","of","mlp","hidden","dim","to","embedding","dim",".","qkv_bias","(","bool",")",":","If","True",",","add","a","learnable","bias","to","query",",","key",",","value",".","norm_layer","(","nn.Module",")",":","Normalization","layer",".","act_layer","(","nn.Module",")",":","Activation","layer",".","use_rel_pos","(","bool",")",":","If","True",",","add","relative","positional","embeddings","to","the","attention","map",".","rel_pos_zero_init","(","bool",")",":","If","True",",","zero","initialize","relative","positional","parameters",".","window_size","(","int",")",":","Window","size","for","window","attention","blocks",".","If","it","equals","0",",","then","use","global","attention",".","input_size","(","tuple","(","int",",","int",")","or","None",")",":","Input","resolution","for","calculating","the","relative","positional","parameter","size","."]
code: def __init__(
      self,
      dim: int,
      num_heads: int,
      mlp_ratio: float = 4.0,
      qkv_bias: bool = True,
      norm_layer: Type[nn.Module] = nn.LayerNorm,
      act_layer: Type[nn.Module] = nn.GELU,
      use_rel_pos: bool = False,
      rel_pos_zero_init: bool = True,
      window_size: int = 0,
      input_size: Optional[Tuple[int, int]] = None,
  ) -> None:
      """
      Args:
          dim (int): Number of input channels.
          num_heads (int): Number of attention heads in each ViT block.
          mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
          qkv_bias (bool): If True, add a learnable bias to query, key, value.
          norm_layer (nn.Module): Normalization layer.
          act_layer (nn.Module): Activation layer.
          use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
          rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
          window_size (int): Window size for window attention blocks. If it equals 0, then
              use global attention.
          input_size (tuple(int, int) or None): Input resolution for calculating the relative
              positional parameter size.
      """
      super().__init__()
      self.norm1 = norm_layer(dim)
      self.attn = Attention(
          dim,
          num_heads=num_heads,
          qkv_bias=qkv_bias,
          use_rel_pos=use_rel_pos,
          rel_pos_zero_init=rel_pos_zero_init,
          input_size=input_size
          if window_size == 0
          else (window_size, window_size),
      )
  
      self.norm2 = norm_layer(dim)
      self.mlp = MLPBlock(
          embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer
      )
  
      self.window_size = window_size
  
imports: import torch
  import torch.nn
  import torch.nn.functional
  from typing import Optional, Tuple, Type
  from .common import LayerNorm2d, MLPBlock
label_desc: Use image node_id 1 to create a new Block object from inherited base classes: nn with example: obj = Block(dim, num_heads, mlp_ratio, qkv_bias, norm_layer, act_layer, use_rel_pos, rel_pos_zero_init, window_size, input_size)