class_docstr: Eva Vision Transformer w/ Abs & Rotary Pos Embed
  
  This class implements the EVA and EVA02 models that were based on the BEiT ViT variant
    * EVA - abs pos embed, global avg pool
    * EVA02 - abs + rope pos embed, global avg pool, SwiGLU, scale Norm in MLP (ala normformer)
class_docstr_tok: ["Eva","Vision","Transformer","w\/","Abs","&","Rotary","Pos","Embed","This","class","implements","the","EVA","and","EVA02","models","that","were","based","on","the","BEiT","ViT","variant","*","EVA","-","abs","pos","embed",",","global","avg","pool","*","EVA02","-","abs","+","rope","pos","embed",",","global","avg","pool",",","SwiGLU",",","scale","Norm","in","MLP","(","ala","normformer",")"]
docstr: Args:
      img_size:
      patch_size:
      in_chans:
      num_classes:
      global_pool:
      embed_dim:
      depth:
      num_heads:
      qkv_bias:
      qkv_fused:
      mlp_ratio:
      swiglu_mlp:
      scale_mlp:
      scale_attn_inner:
      drop_rate:
      pos_drop_rate:
      proj_drop_rate:
      attn_drop_rate:
      drop_path_rate:
      norm_layer:
      init_values:
      class_token:
      use_abs_pos_emb:
      use_rot_pos_emb:
      use_post_norm:
      ref_feat_shape:
      head_init_scale:
code: def __init__(
      self,
      img_size: Union[int, Tuple[int, int]] = 224,
      patch_size: Union[int, Tuple[int, int]] = 16,
      in_chans: int = 3,
      num_classes: int = 1000,
      global_pool: str = "avg",
      embed_dim: int = 768,
      depth: int = 12,
      num_heads: int = 12,
      qkv_bias: bool = True,
      qkv_fused: bool = True,
      mlp_ratio: float = 4.0,
      swiglu_mlp: bool = False,
      scale_mlp: bool = False,
      scale_attn_inner: bool = False,
      drop_rate: float = 0.0,
      pos_drop_rate: float = 0.0,
      patch_drop_rate: float = 0.0,
      proj_drop_rate: float = 0.0,
      attn_drop_rate: float = 0.0,
      drop_path_rate: float = 0.0,
      norm_layer: Callable = LayerNorm,
      init_values: Optional[float] = None,
      class_token: bool = True,
      use_abs_pos_emb: bool = True,
      use_rot_pos_emb: bool = False,
      use_post_norm: bool = False,
      dynamic_img_size: bool = False,
      dynamic_img_pad: bool = False,
      ref_feat_shape: Optional[Union[Tuple[int, int], int]] = None,
      head_init_scale: float = 0.001,
  ):
      """
  
      Args:
          img_size:
          patch_size:
          in_chans:
          num_classes:
          global_pool:
          embed_dim:
          depth:
          num_heads:
          qkv_bias:
          qkv_fused:
          mlp_ratio:
          swiglu_mlp:
          scale_mlp:
          scale_attn_inner:
          drop_rate:
          pos_drop_rate:
          proj_drop_rate:
          attn_drop_rate:
          drop_path_rate:
          norm_layer:
          init_values:
          class_token:
          use_abs_pos_emb:
          use_rot_pos_emb:
          use_post_norm:
          ref_feat_shape:
          head_init_scale:
      """
      super().__init__()
      self.num_classes = num_classes
      self.global_pool = global_pool
      self.num_features = (
          self.embed_dim
      ) = embed_dim  # num_features for consistency with other models
      self.num_prefix_tokens = 1 if class_token else 0
      self.dynamic_img_size = dynamic_img_size
      self.grad_checkpointing = False
  
      embed_args = {}
      if dynamic_img_size:
          # flatten deferred until after pos embed
          embed_args.update(
              dict(strict_img_size=False, output_fmt="NHWC")
          )
      self.patch_embed = PatchEmbed(
          img_size=img_size,
          patch_size=patch_size,
          in_chans=in_chans,
          embed_dim=embed_dim,
          dynamic_img_pad=dynamic_img_pad,
          **embed_args,
      )
      num_patches = self.patch_embed.num_patches
  
      self.cls_token = (
          nn.Parameter(torch.zeros(1, 1, embed_dim))
          if class_token
          else None
      )
  
      self.pos_embed = (
          nn.Parameter(
              torch.zeros(
                  1, num_patches + self.num_prefix_tokens, embed_dim
              )
          )
          if use_abs_pos_emb
          else None
      )
      self.pos_drop = nn.Dropout(p=pos_drop_rate)
      if patch_drop_rate > 0:
          self.patch_drop = PatchDropout(
              patch_drop_rate,
              num_prefix_tokens=self.num_prefix_tokens,
              return_indices=True,
          )
      else:
          self.patch_drop = None
  
      if use_rot_pos_emb:
          ref_feat_shape = (
              to_2tuple(ref_feat_shape)
              if ref_feat_shape is not None
              else None
          )
          self.rope = RotaryEmbeddingCat(
              embed_dim // num_heads,
              in_pixels=False,
              feat_shape=None
              if dynamic_img_size
              else self.patch_embed.grid_size,
              ref_feat_shape=ref_feat_shape,
          )
      else:
          self.rope = None
  
      dpr = [
          x.item() for x in torch.linspace(0, drop_path_rate, depth)
      ]  # stochastic depth decay rule
      block_fn = EvaBlockPostNorm if use_post_norm else EvaBlock
      self.blocks = nn.ModuleList(
          [
              block_fn(
                  dim=embed_dim,
                  num_heads=num_heads,
                  qkv_bias=qkv_bias,
                  qkv_fused=qkv_fused,
                  mlp_ratio=mlp_ratio,
                  swiglu_mlp=swiglu_mlp,
                  scale_mlp=scale_mlp,
                  scale_attn_inner=scale_attn_inner,
                  proj_drop=proj_drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i],
                  norm_layer=norm_layer,
                  init_values=init_values,
              )
              for i in range(depth)
          ]
      )
  
      use_fc_norm = self.global_pool == "avg"
      self.norm = (
          nn.Identity() if use_fc_norm else norm_layer(embed_dim)
      )
      self.fc_norm = (
          norm_layer(embed_dim) if use_fc_norm else nn.Identity()
      )
      self.head_drop = nn.Dropout(drop_rate)
      self.head = (
          nn.Linear(embed_dim, num_classes)
          if num_classes > 0
          else nn.Identity()
      )
  
      self.apply(self._init_weights)
      if self.pos_embed is not None:
          trunc_normal_(self.pos_embed, std=0.02)
      if self.cls_token is not None:
          trunc_normal_(self.cls_token, std=0.02)
  
      self.fix_init_weight()
      if isinstance(self.head, nn.Linear):
          trunc_normal_(self.head.weight, std=0.02)
          self.head.weight.data.mul_(head_init_scale)
          self.head.bias.data.mul_(head_init_scale)
  
imports: import math
  from typing import Callable, Optional, Tuple, Union
  import torch
  import torch.nn
  import torch.nn.functional
  from torch.utils.checkpoint import checkpoint
  from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
  from timm.layers import PatchEmbed, Mlp, GluMlp, SwiGLU, LayerNorm, DropPath, PatchDropout, RotaryEmbeddingCat, apply_rot_embed_cat, apply_keep_indices_nlc, trunc_normal_, resample_patch_embed, resample_abs_pos_embed, to_2tuple, use_fused_attn
  from ._builder import build_model_with_cfg
  from ._registry import generate_default_cfgs, register_model
label_desc: Use image node_id 1 to create a new Eva object from inherited base classes: nn with example: obj = Eva(img_size, patch_size, in_chans, num_classes, global_pool, embed_dim, depth, num_heads, qkv_bias, qkv_fused, mlp_ratio, swiglu_mlp, scale_mlp, scale_attn_inner, drop_rate, pos_drop_rate, patch_drop_rate, proj_drop_rate, attn_drop_rate, drop_path_rate, norm_layer, init_values, class_token, use_abs_pos_emb, use_rot_pos_emb, use_post_norm, dynamic_img_size, dynamic_img_pad, ref_feat_shape, head_init_scale)