class_docstr: Vision Transformer
  A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`  -
      https://arxiv.org/abs/2010.11929
class_docstr_tok: ["Vision","Transformer","A","PyTorch","impl","of",":","`","An","Image","is","Worth","16x16","Words",":","Transformers","for","Image","Recognition","at","Scale","`","-","https",":","\/\/arxiv.org\/abs\/2010.11929"]
docstr: Args:
      layer (list): number of block in each layer
      img_size (int, tuple): input image size
      in_chans (int): number of input channels
      num_classes (int): number of classes for classification head
      embed_dim (int): embedding dimension
      head_dim (int): dimension of attention heads
      mlp_ratio (int): ratio of mlp hidden dim to embedding dim
      qkv_bias (bool): enable bias for qkv if True
      qk_scale (float): override default qk scale of head_dim ** -0.5 if set
      representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set
      drop_rate (float): dropout rate
      attn_drop_rate (float): attention dropout rate
      drop_path_rate (float): stochastic depth rate
      norm_layer (nn.Module): normalization layer
      pretrained_path (str): path of pretrained model
      use_checkpoint (bool): whether use checkpoint
      checkpoint_num (list): index for using checkpoint in every stage
      windows (bool): whether use window MHRA
      hybrid (bool): whether use hybrid MHRA
      window_size (int): size of window (>14)
docstr_tok: ["Args",":","layer","(","list",")",":","number","of","block","in","each","layer","img_size","(","int",",","tuple",")",":","input","image","size","in_chans","(","int",")",":","number","of","input","channels","num_classes","(","int",")",":","number","of","classes","for","classification","head","embed_dim","(","int",")",":","embedding","dimension","head_dim","(","int",")",":","dimension","of","attention","heads","mlp_ratio","(","int",")",":","ratio","of","mlp","hidden","dim","to","embedding","dim","qkv_bias","(","bool",")",":","enable","bias","for","qkv","if","True","qk_scale","(","float",")",":","override","default","qk","scale","of","head_dim","*","*","-0.5","if","set","representation_size","(","Optional","[","int","]",")",":","enable","and","set","representation","layer","(","pre-logits",")","to","this","value","if","set","drop_rate","(","float",")",":","dropout","rate","attn_drop_rate","(","float",")",":","attention","dropout","rate","drop_path_rate","(","float",")",":","stochastic","depth","rate","norm_layer","(","nn.Module",")",":","normalization","layer","pretrained_path","(","str",")",":","path","of","pretrained","model","use_checkpoint","(","bool",")",":","whether","use","checkpoint","checkpoint_num","(","list",")",":","index","for","using","checkpoint","in","every","stage","windows","(","bool",")",":","whether","use","window","MHRA","hybrid","(","bool",")",":","whether","use","hybrid","MHRA","window_size","(","int",")",":","size","of","window","(",">","14",")"]
code: def __init__(
      self,
      layers=[3, 4, 8, 3],
      img_size=224,
      in_chans=3,
      num_classes=80,
      embed_dim=[64, 128, 320, 512],
      head_dim=64,
      mlp_ratio=4.0,
      qkv_bias=True,
      qk_scale=None,
      representation_size=None,
      drop_rate=0.0,
      attn_drop_rate=0.0,
      drop_path_rate=0.0,
      norm_layer=partial(nn.LayerNorm, eps=1e-6),
      pretrained_path=None,
      use_checkpoint=False,
      checkpoint_num=[0, 0, 0, 0],
      windows=False,
      hybrid=False,
      window_size=14,
  ):
      """
      Args:
          layer (list): number of block in each layer
          img_size (int, tuple): input image size
          in_chans (int): number of input channels
          num_classes (int): number of classes for classification head
          embed_dim (int): embedding dimension
          head_dim (int): dimension of attention heads
          mlp_ratio (int): ratio of mlp hidden dim to embedding dim
          qkv_bias (bool): enable bias for qkv if True
          qk_scale (float): override default qk scale of head_dim ** -0.5 if set
          representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set
          drop_rate (float): dropout rate
          attn_drop_rate (float): attention dropout rate
          drop_path_rate (float): stochastic depth rate
          norm_layer (nn.Module): normalization layer
          pretrained_path (str): path of pretrained model
          use_checkpoint (bool): whether use checkpoint
          checkpoint_num (list): index for using checkpoint in every stage
          windows (bool): whether use window MHRA
          hybrid (bool): whether use hybrid MHRA
          window_size (int): size of window (>14)
      """
      super().__init__()
      self.num_classes = num_classes
      self.use_checkpoint = use_checkpoint
      self.checkpoint_num = checkpoint_num
      self.windows = windows
      print(f"Use Checkpoint: {self.use_checkpoint}")
      print(f"Checkpoint Number: {self.checkpoint_num}")
      self.num_features = (
          self.embed_dim
      ) = embed_dim  # num_features for consistency with other models
      norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
  
      self.patch_embed1 = PatchEmbed(
          img_size=img_size,
          patch_size=4,
          in_chans=in_chans,
          embed_dim=embed_dim[0],
      )
      self.patch_embed2 = PatchEmbed(
          img_size=img_size // 4,
          patch_size=2,
          in_chans=embed_dim[0],
          embed_dim=embed_dim[1],
      )
      self.patch_embed3 = PatchEmbed(
          img_size=img_size // 8,
          patch_size=2,
          in_chans=embed_dim[1],
          embed_dim=embed_dim[2],
      )
      self.patch_embed4 = PatchEmbed(
          img_size=img_size // 16,
          patch_size=2,
          in_chans=embed_dim[2],
          embed_dim=embed_dim[3],
      )
  
      self.pos_drop = nn.Dropout(p=drop_rate)
      dpr = [
          x.item()
          for x in torch.linspace(0, drop_path_rate, sum(layers))
      ]  # stochastic depth decay rule
      num_heads = [dim // head_dim for dim in embed_dim]
      self.blocks1 = nn.ModuleList(
          [
              CBlock(
                  dim=embed_dim[0],
                  num_heads=num_heads[0],
                  mlp_ratio=mlp_ratio,
                  qkv_bias=qkv_bias,
                  qk_scale=qk_scale,
                  drop=drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i],
                  norm_layer=norm_layer,
              )
              for i in range(layers[0])
          ]
      )
      self.norm1 = norm_layer(embed_dim[0])
      self.blocks2 = nn.ModuleList(
          [
              CBlock(
                  dim=embed_dim[1],
                  num_heads=num_heads[1],
                  mlp_ratio=mlp_ratio,
                  qkv_bias=qkv_bias,
                  qk_scale=qk_scale,
                  drop=drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i + layers[0]],
                  norm_layer=norm_layer,
              )
              for i in range(layers[1])
          ]
      )
      self.norm2 = norm_layer(embed_dim[1])
      if self.windows:
          print("Use local window for all blocks in stage3")
          self.blocks3 = nn.ModuleList(
              [
                  SABlock_Windows(
                      dim=embed_dim[2],
                      num_heads=num_heads[2],
                      window_size=window_size,
                      mlp_ratio=mlp_ratio,
                      qkv_bias=qkv_bias,
                      qk_scale=qk_scale,
                      drop=drop_rate,
                      attn_drop=attn_drop_rate,
                      drop_path=dpr[i + layers[0] + layers[1]],
                      norm_layer=norm_layer,
                  )
                  for i in range(layers[2])
              ]
          )
      elif hybrid:
          print("Use hybrid window for blocks in stage3")
          block3 = []
          for i in range(layers[2]):
              if (i + 1) % 4 == 0:
                  block3.append(
                      SABlock(
                          dim=embed_dim[2],
                          num_heads=num_heads[2],
                          mlp_ratio=mlp_ratio,
                          qkv_bias=qkv_bias,
                          qk_scale=qk_scale,
                          drop=drop_rate,
                          attn_drop=attn_drop_rate,
                          drop_path=dpr[i + layers[0] + layers[1]],
                          norm_layer=norm_layer,
                      )
                  )
              else:
                  block3.append(
                      SABlock_Windows(
                          dim=embed_dim[2],
                          num_heads=num_heads[2],
                          window_size=window_size,
                          mlp_ratio=mlp_ratio,
                          qkv_bias=qkv_bias,
                          qk_scale=qk_scale,
                          drop=drop_rate,
                          attn_drop=attn_drop_rate,
                          drop_path=dpr[i + layers[0] + layers[1]],
                          norm_layer=norm_layer,
                      )
                  )
          self.blocks3 = nn.ModuleList(block3)
      else:
          print("Use global window for all blocks in stage3")
          self.blocks3 = nn.ModuleList(
              [
                  SABlock(
                      dim=embed_dim[2],
                      num_heads=num_heads[2],
                      mlp_ratio=mlp_ratio,
                      qkv_bias=qkv_bias,
                      qk_scale=qk_scale,
                      drop=drop_rate,
                      attn_drop=attn_drop_rate,
                      drop_path=dpr[i + layers[0] + layers[1]],
                      norm_layer=norm_layer,
                  )
                  for i in range(layers[2])
              ]
          )
      self.norm3 = norm_layer(embed_dim[2])
      self.blocks4 = nn.ModuleList(
          [
              SABlock(
                  dim=embed_dim[3],
                  num_heads=num_heads[3],
                  mlp_ratio=mlp_ratio,
                  qkv_bias=qkv_bias,
                  qk_scale=qk_scale,
                  drop=drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i + layers[0] + layers[1] + layers[2]],
                  norm_layer=norm_layer,
              )
              for i in range(layers[3])
          ]
      )
      self.norm4 = norm_layer(embed_dim[3])
  
      # Representation layer
      if representation_size:
          self.num_features = representation_size
          self.pre_logits = nn.Sequential(
              OrderedDict(
                  [
                      ("fc", nn.Linear(embed_dim, representation_size)),
                      ("act", nn.Tanh()),
                  ]
              )
          )
      else:
          self.pre_logits = nn.Identity()
  
      self.apply(self._init_weights)
      self.init_weights(pretrained=pretrained_path)
  
imports: from collections import OrderedDict
  import math
  from functools import partial
  import torch
  import torch.nn
  import torch.nn.functional
  import torch.utils.checkpoint
  import numpy
  from timm.models.layers import DropPath, to_2tuple, trunc_normal_
  from annotator.uniformer.mmcv_custom import load_checkpoint
  from annotator.uniformer.mmseg.utils import get_root_logger
  from ..builder import BACKBONES
label_desc: Use image node_id 1 to create a new UniFormer object from inherited base classes: nn with example: obj = UniFormer(layers, img_size, in_chans, num_classes, embed_dim, head_dim, mlp_ratio, qkv_bias, qk_scale, representation_size, drop_rate, attn_drop_rate, drop_path_rate, norm_layer, pretrained_path, use_checkpoint, checkpoint_num, windows, hybrid, window_size)