class_docstr: Code for "UniFormer: Unified Transformer for Efficient Spatiotemporal Representation Learning"
  arXiv:2201.04676
  Kunchang Li, Yali Wang, Peng Gao, Guanglu Song, Yu Liu, Hongsheng Li, Yu Qiao
  
  Args:
      depth (`tuple[int]`)
          Depth. Default: (5, 8, 20, 7).
      num_classes (`int`):
          dimension of head. Default: 400.
      img_size (`int`):
          The spatial crop size for training. Default: 224.
      in_chans (`int`):
          Image input channels num. Default: 3.
      embed_dim (`tuple[int]`)
          Embedding dimension. Default: (64, 128, 320, 512).
      head_dim (`int`):
          Dimension of head. Default: 64.
      mlp_ratio (`int`):
          Ratio of mlp hidden dim to embedding dim. Default: 4.
      qkv_bias (`bool`):
          Enable bias for qkv if True. Default: True.
      qk_scale (`float`):
          Override default qk scale of head_dim ** -0.5 if set. Default: None.
      representation_size (`int`):
          Enable and set representation layer (pre-logits) to this value if set. Default: None.
      drop_rate (`float`):
          Dropout rate. Default: 0.
      attn_drop_rate (`float`):
          Attention dropout rate. Default: 0.
      drop_path_rate (`float`):
          Stochastic depth rate. Default: 0.1.
      split (`bool`):
          Whether use split attention. Default: False.
      std (`bool`):
          Spatial-temporal downsample. Default: False.
      use_checkpoint (`bool`):
          Whether use checkpoint. Default: False.
      checkpoint_num (`tuple[int]`)
          Checkpoint num. Default: (0, 0, 0, 0).
      pretrain_name (`string`):
          Pretrained name. Default: None.
class_docstr_tok: ["Code","for","``","UniFormer",":","Unified","Transformer","for","Efficient","Spatiotemporal","Representation","Learning","''","arXiv:2201.04676","Kunchang","Li",",","Yali","Wang",",","Peng","Gao",",","Guanglu","Song",",","Yu","Liu",",","Hongsheng","Li",",","Yu","Qiao","Args",":","depth","(","`","tuple","[","int","]","`",")","Depth",".","Default",":","(","5",",","8",",","20",",","7",")",".","num_classes","(","`","int","`",")",":","dimension","of","head",".","Default",":","400.","img_size","(","`","int","`",")",":","The","spatial","crop","size","for","training",".","Default",":","224.","in_chans","(","`","int","`",")",":","Image","input","channels","num",".","Default",":","3.","embed_dim","(","`","tuple","[","int","]","`",")","Embedding","dimension",".","Default",":","(","64",",","128",",","320",",","512",")",".","head_dim","(","`","int","`",")",":","Dimension","of","head",".","Default",":","64.","mlp_ratio","(","`","int","`",")",":","Ratio","of","mlp","hidden","dim","to","embedding","dim",".","Default",":","4.","qkv_bias","(","`","bool","`",")",":","Enable","bias","for","qkv","if","True",".","Default",":","True",".","qk_scale","(","`","float","`",")",":","Override","default","qk","scale","of","head_dim","*","*","-0.5","if","set",".","Default",":","None",".","representation_size","(","`","int","`",")",":","Enable","and","set","representation","layer","(","pre-logits",")","to","this","value","if","set",".","Default",":","None",".","drop_rate","(","`","float","`",")",":","Dropout","rate",".","Default",":","0.","attn_drop_rate","(","`","float","`",")",":","Attention","dropout","rate",".","Default",":","0.","drop_path_rate","(","`","float","`",")",":","Stochastic","depth","rate",".","Default",":","0.1.","split","(","`","bool","`",")",":","Whether","use","split","attention",".","Default",":","False",".","std","(","`","bool","`",")",":","Spatial-temporal","downsample",".","Default",":","False",".","use_checkpoint","(","`","bool","`",")",":","Whether","use","checkpoint",".","Default",":","False",".","checkpoint_num","(","`","tuple","[","int","]","`",")","Checkpoint","num",".","Default",":","(","0",",","0",",","0",",","0",")",".","pretrain_name","(","`","string","`",")",":","Pretrained","name",".","Default",":","None","."]
code: def __init__(
      self,
      depth=(5, 8, 20, 7),
      num_classes=400,
      img_size=32,
      in_chans=3,
      embed_dim=(64, 128, 320, 512),
      head_dim=64,
      mlp_ratio=4,
      qkv_bias=True,
      qk_scale=None,
      representation_size=None,
      drop_rate=0,
      attn_drop_rate=0,
      drop_path_rate=0.1,
      split=False,
      std=False,
      use_checkpoint=False,
      checkpoint_num=(0, 0, 0, 0),
  ):
      super().__init__()
      self.use_checkpoint = use_checkpoint
      self.checkpoint_num = checkpoint_num
      self.num_classes = num_classes
      self.num_features = (
          self.embed_dim
      ) = embed_dim  # num_features for consistency with other models
      norm_layer = partial(nn.LayerNorm, eps=1e-6)
  
      self.patch_embed1 = SpeicalPatchEmbed(
          img_size=img_size,
          patch_size=4,
          in_chans=in_chans,
          embed_dim=embed_dim[0],
      )
      self.patch_embed2 = PatchEmbed(
          img_size=img_size // 4,
          patch_size=2,
          in_chans=embed_dim[0],
          embed_dim=embed_dim[1],
          std=std,
      )
      self.patch_embed3 = PatchEmbed(
          img_size=img_size // 8,
          patch_size=2,
          in_chans=embed_dim[1],
          embed_dim=embed_dim[2],
          std=std,
      )
      self.patch_embed4 = PatchEmbed(
          img_size=img_size // 16,
          patch_size=2,
          in_chans=embed_dim[2],
          embed_dim=embed_dim[3],
          std=std,
      )
  
      self.pos_drop = nn.Dropout(p=drop_rate)
      dpr = [
          x.item()
          for x in torch.linspace(0, drop_path_rate, sum(depth))
      ]  # stochastic depth decay rule
      num_heads = [dim // head_dim for dim in embed_dim]
      self.blocks1 = nn.ModuleList(
          [
              CBlock(
                  dim=embed_dim[0],
                  num_heads=num_heads[0],
                  mlp_ratio=mlp_ratio,
                  qkv_bias=qkv_bias,
                  qk_scale=qk_scale,
                  drop=drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i],
                  norm_layer=norm_layer,
              )
              for i in range(depth[0])
          ]
      )
      self.blocks2 = nn.ModuleList(
          [
              CBlock(
                  dim=embed_dim[1],
                  num_heads=num_heads[1],
                  mlp_ratio=mlp_ratio,
                  qkv_bias=qkv_bias,
                  qk_scale=qk_scale,
                  drop=drop_rate,
                  attn_drop=attn_drop_rate,
                  drop_path=dpr[i + depth[0]],
                  norm_layer=norm_layer,
              )
              for i in range(depth[1])
          ]
      )
      if split:
          self.blocks3 = nn.ModuleList(
              [
                  SplitSABlock(
                      dim=embed_dim[2],
                      num_heads=num_heads[2],
                      mlp_ratio=mlp_ratio,
                      qkv_bias=qkv_bias,
                      qk_scale=qk_scale,
                      drop=drop_rate,
                      attn_drop=attn_drop_rate,
                      drop_path=dpr[i + depth[0] + depth[1]],
                      norm_layer=norm_layer,
                  )
                  for i in range(depth[2])
              ]
          )
          self.blocks4 = nn.ModuleList(
              [
                  SplitSABlock(
                      dim=embed_dim[3],
                      num_heads=num_heads[3],
                      mlp_ratio=mlp_ratio,
                      qkv_bias=qkv_bias,
                      qk_scale=qk_scale,
                      drop=drop_rate,
                      attn_drop=attn_drop_rate,
                      drop_path=dpr[i + depth[0] + depth[1] + depth[2]],
                      norm_layer=norm_layer,
                  )
                  for i in range(depth[3])
              ]
          )
      else:
          self.blocks3 = nn.ModuleList(
              [
                  SABlock(
                      dim=embed_dim[2],
                      num_heads=num_heads[2],
                      mlp_ratio=mlp_ratio,
                      qkv_bias=qkv_bias,
                      qk_scale=qk_scale,
                      drop=drop_rate,
                      attn_drop=attn_drop_rate,
                      drop_path=dpr[i + depth[0] + depth[1]],
                      norm_layer=norm_layer,
                  )
                  for i in range(depth[2])
              ]
          )
          self.blocks4 = nn.ModuleList(
              [
                  SABlock(
                      dim=embed_dim[3],
                      num_heads=num_heads[3],
                      mlp_ratio=mlp_ratio,
                      qkv_bias=qkv_bias,
                      qk_scale=qk_scale,
                      drop=drop_rate,
                      attn_drop=attn_drop_rate,
                      drop_path=dpr[i + depth[0] + depth[1] + depth[2]],
                      norm_layer=norm_layer,
                  )
                  for i in range(depth[3])
              ]
          )
      self.norm = bn_3d(embed_dim[-1])
  
      # Representation layer
      if representation_size:
          self.num_features = representation_size
          self.pre_logits = nn.Sequential(
              OrderedDict(
                  [
                      ("fc", nn.Linear(embed_dim, representation_size)),
                      ("act", nn.Tanh()),
                  ]
              )
          )
      else:
          self.pre_logits = nn.Identity()
  
      # Classifier head
      self.head = (
          nn.Linear(embed_dim[-1], num_classes)
          if num_classes > 0
          else nn.Identity()
      )
  
      self.apply(self._init_weights)
  
      for name, p in self.named_parameters():
          # fill proj weight with 1 here to improve training dynamics. Otherwise temporal attention inputs
          # are multiplied by 0*0, which is hard for the model to move out of.
          if "t_attn.qkv.weight" in name:
              nn.init.constant_(p, 0)
          if "t_attn.qkv.bias" in name:
              nn.init.constant_(p, 0)
          if "t_attn.proj.weight" in name:
              nn.init.constant_(p, 1)
          if "t_attn.proj.bias" in name:
              nn.init.constant_(p, 0)
  
imports: from collections import OrderedDict
  import torch
  from torch import nn
  from torch.utils import checkpoint
  from functools import partial
  from towhee.models.utils.general_utils import to_2tuple
  from towhee.models.utils.weight_init import trunc_normal_
  from towhee.models.layers.droppath import DropPath
  from towhee.models.layers.mlp import Mlp
  from towhee.models.layers.attention import MultiHeadAttention
  from towhee.models.uniformer.config import _C
  import os
label_desc: Use image node_id 1 to create a new Uniformer object from inherited base classes: nn with example: obj = Uniformer(depth, num_classes, img_size, in_chans, embed_dim, head_dim, mlp_ratio, qkv_bias, qk_scale, representation_size, drop_rate, attn_drop_rate, drop_path_rate, split, std, use_checkpoint, checkpoint_num)