class_docstr: Vision Transformer w/ Relative Position Bias
  
  Differing from classic vit, this impl
    * uses relative position index (swin v1 / beit) or relative log coord + mlp (swin v2) pos embed
    * defaults to no class token (can be enabled)
    * defaults to global avg pool for head (can be changed)
    * layer-scale (residual branch gain) enabled
class_docstr_tok: ["Vision","Transformer","w\/","Relative","Position","Bias","Differing","from","classic","vit",",","this","impl","*","uses","relative","position","index","(","swin","v1","\/","beit",")","or","relative","log","coord","+","mlp","(","swin","v2",")","pos","embed","*","defaults","to","no","class","token","(","can","be","enabled",")","*","defaults","to","global","avg","pool","for","head","(","can","be","changed",")","*","layer-scale","(","residual","branch","gain",")","enabled"]
docstr: Args:
      img_size (int, tuple): input image size
      patch_size (int, tuple): patch size
      in_chans (int): number of input channels
      num_classes (int): number of classes for classification head
      global_pool (str): type of global pooling for final sequence (default: 'avg')
      embed_dim (int): embedding dimension
      depth (int): depth of transformer
      num_heads (int): number of attention heads
      mlp_ratio (int): ratio of mlp hidden dim to embedding dim
      qkv_bias (bool): enable bias for qkv if True
      qk_norm (bool): Enable normalization of query and key in attention
      init_values: (float): layer-scale init values
      class_token (bool): use class token (default: False)
      fc_norm (bool): use pre classifier norm instead of pre-pool
      rel_pos_ty pe (str): type of relative position
      shared_rel_pos (bool): share relative pos across all blocks
      drop_rate (float): dropout rate
      proj_drop_rate (float): projection dropout rate
      attn_drop_rate (float): attention dropout rate
      drop_path_rate (float): stochastic depth rate
      weight_init (str): weight init scheme
      embed_layer (nn.Module): patch embedding layer
      norm_layer: (nn.Module): normalization layer
      act_layer: (nn.Module): MLP activation layer
code: def __init__(
      self,
      img_size=224,
      patch_size=16,
      in_chans=3,
      num_classes=1000,
      global_pool="avg",
      embed_dim=768,
      depth=12,
      num_heads=12,
      mlp_ratio=4.0,
      qkv_bias=True,
      qk_norm=False,
      init_values=1e-6,
      class_token=False,
      fc_norm=False,
      rel_pos_type="mlp",
      rel_pos_dim=None,
      shared_rel_pos=False,
      drop_rate=0.0,
      proj_drop_rate=0.0,
      attn_drop_rate=0.0,
      drop_path_rate=0.0,
      weight_init="skip",
      embed_layer=PatchEmbed,
      norm_layer=None,
      act_layer=None,
      block_fn=RelPosBlock,
  ):
      """
      Args:
          img_size (int, tuple): input image size
          patch_size (int, tuple): patch size
          in_chans (int): number of input channels
          num_classes (int): number of classes for classification head
          global_pool (str): type of global pooling for final sequence (default: 'avg')
          embed_dim (int): embedding dimension
          depth (int): depth of transformer
          num_heads (int): number of attention heads
          mlp_ratio (int): ratio of mlp hidden dim to embedding dim
          qkv_bias (bool): enable bias for qkv if True
          qk_norm (bool): Enable normalization of query and key in attention
          init_values: (float): layer-scale init values
          class_token (bool): use class token (default: False)
          fc_norm (bool): use pre classifier norm instead of pre-pool
          rel_pos_ty pe (str): type of relative position
          shared_rel_pos (bool): share relative pos across all blocks
          drop_rate (float): dropout rate
          proj_drop_rate (float): projection dropout rate
          attn_drop_rate (float): attention dropout rate
          drop_path_rate (float): stochastic depth rate
          weight_init (str): weight init scheme
          embed_layer (nn.Module): patch embedding layer
          norm_layer: (nn.Module): normalization layer
          act_layer: (nn.Module): MLP activation layer
      """
      super().__init__()
      assert global_pool in ("", "avg", "token")
      assert class_token or global_pool != "token"
      norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
      act_layer = act_layer or nn.GELU
  
      self.num_classes = num_classes
      self.global_pool = global_pool
      self.num_features = (
          self.embed_dim
      ) = embed_dim  # num_features for consistency with other models
      self.num_prefix_tokens = 1 if class_token else 0
      self.grad_checkpointing = False
  
      self.patch_embed = embed_layer(
          img_size=img_size,
          patch_size=patch_size,
          in_chans=in_chans,
          embed_dim=embed_dim,
      )
      feat_size = self.patch_embed.grid_size
  
      rel_pos_args = dict(
          window_size=feat_size, prefix_tokens=self.num_prefix_tokens
      )
      if rel_pos_type.startswith("mlp"):
          if rel_pos_dim:
              rel_pos_args["hidden_dim"] = rel_pos_dim
          if "swin" in rel_pos_type:
              rel_pos_args["mode"] = "swin"
          rel_pos_cls = partial(RelPosMlp, **rel_pos_args)
      else:
          rel_pos_cls = partial(RelPosBias, **rel_pos_args)
      self.shared_rel_pos = None
      if shared_rel_pos:
          self.shared_rel_pos = rel_pos_cls(num_heads=num_heads)
          # NOTE shared rel pos currently mutually exclusive w/ per-block, but could support both...
          rel_pos_cls = None
  
      self.cls_token = (
          nn.Parameter(
              torch.zeros(1, self.num_prefix_tokens, embed_dim)
          )
          if class_token
          else None
      )
  
      dpr = [
          x.item() for x in torch.linspace(0, drop_path_rate, depth)
      ]  # stochastic depth decay rule
      self.blocks = nn.ModuleList(
          [
              block_fn(
                  dim=embed_dim,
                  num_heads=num_heads,
                  mlp_ratio=mlp_ratio,
                  qkv_bias=qkv_bias,
                  qk_norm=qk_norm,
                  rel_pos_cls=rel_pos_cls,
                  init_values=init_values,
                  proj_drop=proj_drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i],
                  norm_layer=norm_layer,
                  act_layer=act_layer,
              )
              for i in range(depth)
          ]
      )
      self.norm = (
          norm_layer(embed_dim) if not fc_norm else nn.Identity()
      )
  
      # Classifier Head
      self.fc_norm = norm_layer(embed_dim) if fc_norm else nn.Identity()
      self.head_drop = nn.Dropout(drop_rate)
      self.head = (
          nn.Linear(self.embed_dim, num_classes)
          if num_classes > 0
          else nn.Identity()
      )
  
      if weight_init != "skip":
          self.init_weights(weight_init)
  
imports: import logging
  import math
  from functools import partial
  from typing import Optional, Tuple
  import torch
  import torch.nn
  from torch.jit import Final
  from torch.utils.checkpoint import checkpoint
  from timm.data import IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
  from timm.layers import PatchEmbed, Mlp, DropPath, RelPosMlp, RelPosBias, use_fused_attn
  from ._builder import build_model_with_cfg
  from ._registry import generate_default_cfgs, register_model
label_desc: Use image node_id 1 to create a new VisionTransformerRelPos object from inherited base classes: nn with example: obj = VisionTransformerRelPos(img_size, patch_size, in_chans, num_classes, global_pool, embed_dim, depth, num_heads, mlp_ratio, qkv_bias, qk_norm, init_values, class_token, fc_norm, rel_pos_type, rel_pos_dim, shared_rel_pos, drop_rate, proj_drop_rate, attn_drop_rate, drop_path_rate, weight_init, embed_layer, norm_layer, act_layer, block_fn)