class_docstr: Vision Transformer
  A PyTorch impl of :
      `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
       https://arxiv.org/abs/2010.11929
class_docstr_tok: ["Vision","Transformer","A","PyTorch","impl","of",":","`","An","Image","is","Worth","16x16","Words",":","Transformers","for","Image","Recognition","at","Scale","`","https",":","\/\/arxiv.org\/abs\/2010.11929"]
docstr: Args:
      img_size (int, tuple): input image size
      patch_size (int, tuple): patch size
      in_chans (int): number of input channels
      num_classes (int): number of classes for classification head
      embed_dim (int): embedding dimension
      depth (int): depth of transformer
      num_heads (int): number of attention heads
      mlp_ratio (int): ratio of mlp hidden dim to embedding dim
      qkv_bias (bool): enable bias for qkv if True
      qk_scale (float): override default qk scale of head_dim ** -0.5 if set
      representation_size (Optional[int]): enable and set representation layer
                                           (pre-logits) to this value if set
      drop_rate (float): dropout rate
      attn_drop_rate (float): attention dropout rate
      drop_path_rate (float): stochastic depth rate
      norm_layer: (nn.Module): normalization layer
docstr_tok: ["Args",":","img_size","(","int",",","tuple",")",":","input","image","size","patch_size","(","int",",","tuple",")",":","patch","size","in_chans","(","int",")",":","number","of","input","channels","num_classes","(","int",")",":","number","of","classes","for","classification","head","embed_dim","(","int",")",":","embedding","dimension","depth","(","int",")",":","depth","of","transformer","num_heads","(","int",")",":","number","of","attention","heads","mlp_ratio","(","int",")",":","ratio","of","mlp","hidden","dim","to","embedding","dim","qkv_bias","(","bool",")",":","enable","bias","for","qkv","if","True","qk_scale","(","float",")",":","override","default","qk","scale","of","head_dim","*","*","-0.5","if","set","representation_size","(","Optional","[","int","]",")",":","enable","and","set","representation","layer","(","pre-logits",")","to","this","value","if","set","drop_rate","(","float",")",":","dropout","rate","attn_drop_rate","(","float",")",":","attention","dropout","rate","drop_path_rate","(","float",")",":","stochastic","depth","rate","norm_layer",":","(","nn.Module",")",":","normalization","layer"]
code: def __init__(self, config: omegaconf.DictConfig):
      """
      Args:
          img_size (int, tuple): input image size
          patch_size (int, tuple): patch size
          in_chans (int): number of input channels
          num_classes (int): number of classes for classification head
          embed_dim (int): embedding dimension
          depth (int): depth of transformer
          num_heads (int): number of attention heads
          mlp_ratio (int): ratio of mlp hidden dim to embedding dim
          qkv_bias (bool): enable bias for qkv if True
          qk_scale (float): override default qk scale of head_dim ** -0.5 if set
          representation_size (Optional[int]): enable and set representation layer
                                               (pre-logits) to this value if set
          drop_rate (float): dropout rate
          attn_drop_rate (float): attention dropout rate
          drop_path_rate (float): stochastic depth rate
          norm_layer: (nn.Module): normalization layer
      """
      super().__init__()
      self.img_size = config.get("img_size", 224)
      self.patch_size = config.get("patch_size", 16)
      self.in_chans = config.get("in_chans", 3)
      self.num_classes = config.get("num_classes", 1000)
      self.embed_dim = config.get("embed_dim", 768)
      self.depth = config.get("depth", 12)
      self.num_heads = config.get("num_heads", 12)
      self.mlp_ratio = config.get("mlp_ratio", 4.0)
      self.qkv_bias = config.get("qkv_bias", True)
      self.qk_scale = config.get("qk_scale", None)
      self.representation_size = config.get("representation_size", None)
      self.drop_rate = config.get("drop_rate", 0.0)
      self.attn_drop_rate = config.get("attn_drop_rate", 0.0)
      self.drop_path_rate = config.get("drop_path_rate", 0.0)
      self.norm_layer = config.get("norm_layer", None)
  
      self.num_features = (
          self.embed_dim
      )  # num_features for consistency with other models
      norm_layer = self.norm_layer or partial(nn.LayerNorm, eps=1e-6)
  
      self.patch_embed = PatchEmbed(
          img_size=self.img_size,
          patch_size=self.patch_size,
          in_chans=self.in_chans,
          embed_dim=self.embed_dim,
      )
      num_patches = self.patch_embed.num_patches
  
      self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
      self.pos_embed = nn.Parameter(
          torch.zeros(1, num_patches + 1, self.embed_dim)
      )
      self.pos_drop = nn.Dropout(p=self.drop_rate)
  
      dpr = [
          x.item()
          for x in torch.linspace(0, self.drop_path_rate, self.depth)
      ]  # stochastic depth decay rule
      self.blocks = nn.ModuleList(
          [
              Block(
                  dim=self.embed_dim,
                  num_heads=self.num_heads,
                  mlp_ratio=self.mlp_ratio,
                  qkv_bias=self.qkv_bias,
                  qk_scale=self.qk_scale,
                  drop=self.drop_rate,
                  attn_drop=self.attn_drop_rate,
                  drop_path=dpr[i],
                  norm_layer=norm_layer,
              )
              for i in range(self.depth)
          ]
      )
      self.norm = norm_layer(self.embed_dim)
  
      trunc_normal_(self.pos_embed, std=0.02)
      trunc_normal_(self.cls_token, std=0.02)
      self.apply(self._init_weights)
  
imports: import collections.abc
  import math
  import warnings
  from dataclasses import dataclass
  from functools import partial
  from itertools import repeat
  import omegaconf
  import torch
  import torch.nn
  import torch.nn.functional
  from mmf.common.registry import registry
  from mmf.modules.encoders import Encoder
label_desc: Use image node_id 1 to create a new VisionTransformer object from inherited base classes: nn with example: obj = VisionTransformer(config)