class_docstr: Vision Transformer.
  
  A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
      - https://arxiv.org/abs/2010.11929
  
  Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
      - https://arxiv.org/abs/2012.12877
class_docstr_tok: ["Vision","Transformer",".","A","PyTorch","impl","of",":","`","An","Image","is","Worth","16x16","Words",":","Transformers","for","Image","Recognition","at","Scale","`","-","https",":","\/\/arxiv.org\/abs\/2010.11929","Includes","distillation","token","&","head","support","for","`","DeiT",":","Data-efficient","Image","Transformers","`","-","https",":","\/\/arxiv.org\/abs\/2012.12877"]
docstr: Args:
      img_size (int, tuple): input image size
      patch_size (int, tuple): patch size
      in_chans (int): number of input channels
      num_classes (int): number of classes for classification head
      embed_dim (int): embedding dimension
      depth (int): depth of transformer
      num_heads (int): number of attention heads
      mlp_ratio (int): ratio of mlp hidden dim to embedding dim
      qkv_bias (bool): enable bias for qkv if True
      drop_rate (float): dropout rate
      attn_drop_rate (float): attention dropout rate
      drop_path_rate (float): stochastic depth rate
      embed_layer (nn.Module): patch embedding layer
      norm_layer: (nn.Module): normalization layer
      pretrained: (str): pretrained path
docstr_tok: ["Args",":","img_size","(","int",",","tuple",")",":","input","image","size","patch_size","(","int",",","tuple",")",":","patch","size","in_chans","(","int",")",":","number","of","input","channels","num_classes","(","int",")",":","number","of","classes","for","classification","head","embed_dim","(","int",")",":","embedding","dimension","depth","(","int",")",":","depth","of","transformer","num_heads","(","int",")",":","number","of","attention","heads","mlp_ratio","(","int",")",":","ratio","of","mlp","hidden","dim","to","embedding","dim","qkv_bias","(","bool",")",":","enable","bias","for","qkv","if","True","drop_rate","(","float",")",":","dropout","rate","attn_drop_rate","(","float",")",":","attention","dropout","rate","drop_path_rate","(","float",")",":","stochastic","depth","rate","embed_layer","(","nn.Module",")",":","patch","embedding","layer","norm_layer",":","(","nn.Module",")",":","normalization","layer","pretrained",":","(","str",")",":","pretrained","path"]
code: def __init__(
      self,
      img_size=224,
      patch_size=16,
      in_chans=3,
      num_classes=1000,
      embed_dim=768,
      depth=12,
      num_heads=12,
      mlp_ratio=4.0,
      qkv_bias=True,
      drop_rate=0.0,
      attn_drop_rate=0.0,
      drop_path_rate=0.0,
      layer_scale=True,
      embed_layer=PatchEmbed,
      norm_layer=partial(nn.LayerNorm, eps=1e-6),
      act_layer=nn.GELU,
      window_attn=False,
      window_size=14,
      pretrained=None,
      with_cp=False,
      pre_norm=False,
      ffn_type="mlp",
      memeff=False,
  ):
      """
      Args:
          img_size (int, tuple): input image size
          patch_size (int, tuple): patch size
          in_chans (int): number of input channels
          num_classes (int): number of classes for classification head
          embed_dim (int): embedding dimension
          depth (int): depth of transformer
          num_heads (int): number of attention heads
          mlp_ratio (int): ratio of mlp hidden dim to embedding dim
          qkv_bias (bool): enable bias for qkv if True
          drop_rate (float): dropout rate
          attn_drop_rate (float): attention dropout rate
          drop_path_rate (float): stochastic depth rate
          embed_layer (nn.Module): patch embedding layer
          norm_layer: (nn.Module): normalization layer
          pretrained: (str): pretrained path
      """
      super().__init__()
      self.num_classes = num_classes
      self.num_features = (
          self.embed_dim
      ) = embed_dim  # num_features for consistency with other models
      self.num_tokens = 1
      norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
      act_layer = act_layer or nn.GELU
      self.norm_layer = norm_layer
      self.act_layer = act_layer
      self.pretrain_size = img_size
      self.drop_path_rate = drop_path_rate
      self.drop_rate = drop_rate
      self.patch_size = patch_size
  
      window_attn = (
          [window_attn] * depth
          if not isinstance(window_attn, list)
          else window_attn
      )
      window_size = (
          [window_size] * depth
          if not isinstance(window_size, list)
          else window_size
      )
      logging.info("window attention:", window_attn)
      logging.info("window size:", window_size)
      logging.info("layer scale:", layer_scale)
  
      self.patch_embed = embed_layer(
          img_size=img_size,
          patch_size=patch_size,
          in_chans=in_chans,
          embed_dim=embed_dim,
          bias=not pre_norm,
      )
      num_patches = self.patch_embed.num_patches
  
      self.pos_embed = nn.Parameter(
          torch.zeros(1, num_patches + self.num_tokens, embed_dim)
      )
      self.pos_drop = nn.Dropout(p=drop_rate)
  
      ffn_types = {"mlp": Mlp, "swiglu": SwiGLUFFN}
  
      dpr = [
          x.item() for x in torch.linspace(0, drop_path_rate, depth)
      ]  # stochastic depth decay rule
      self.blocks = nn.Sequential(
          *[
              Block(
                  dim=embed_dim,
                  num_heads=num_heads,
                  mlp_ratio=mlp_ratio,
                  qkv_bias=qkv_bias,
                  drop=drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i],
                  norm_layer=norm_layer,
                  act_layer=act_layer,
                  windowed=window_attn[i],
                  window_size=window_size[i],
                  layer_scale=layer_scale,
                  with_cp=with_cp,
                  ffn_layer=ffn_types[ffn_type],
                  memeff=memeff,
              )
              for i in range(depth)
          ]
      )
  
      # self.norm = norm_layer(embed_dim)
      self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
      # For CLIP
      if pre_norm:
          norm_pre = norm_layer(embed_dim)
          self.norm_pre = norm_pre
      else:
          self.norm_pre = nn.Identity()
      self.init_weights(pretrained)
  
imports: import logging
  import math
  from functools import partial
  from itertools import repeat
  from typing import Callable, Optional
  import torch
  import torch.nn
  import torch.nn.functional
  import torch.utils.checkpoint
  from mmcv.runner import BaseModule, load_checkpoint
  from mmseg.ops import resize
  from mmseg.utils import get_root_logger
  from torch import Tensor
  from .drop_path import DropPath
label_desc: Use image node_id 1 to create a new TIMMVisionTransformer object from inherited base classes: BaseModule with example: obj = TIMMVisionTransformer(img_size, patch_size, in_chans, num_classes, embed_dim, depth, num_heads, mlp_ratio, qkv_bias, drop_rate, attn_drop_rate, drop_path_rate, layer_scale, embed_layer, norm_layer, act_layer, window_attn, window_size, pretrained, with_cp, pre_norm, ffn_type, memeff)