class_docstr: Implementation of a multiscale vision transformer block. Each block contains a
  multiscale attention layer and a Mlp layer.
  
  ::
  
  
                                    Input
                                      |-------------------+
                                      ↓                   |
                                     Norm                 |
                                      ↓                   |
                              MultiScaleAttention        Pool
                                      ↓                   |
                                   DropPath               |
                                      ↓                   |
                                  Summation ←-------------+
                                      |
                                      |-------------------+
                                      ↓                   |
                                     Norm                 |
                                      ↓                   |
                                     Mlp                 Proj
                                      ↓                   |
                                   DropPath               |
                                      ↓                   |
                                  Summation  ←------------+
docstr: Args:
      dim (int): Input feature dimension.
      dim_out (int): Output feature dimension.
      num_heads (int): Number of heads in the attention layer.
      mlp_ratio (float): Mlp ratio which controls the feature dimension in the
          hidden layer of the Mlp block.
      qkv_bias (bool): If set to False, the qkv layer will not learn an additive
          bias. Default: False.
      dropout_rate (float): DropOut rate. If set to 0, DropOut is disabled.
      droppath_rate (float): DropPath rate. If set to 0, DropPath is disabled.
      act_layer (nn.Module): Activation layer used in the Mlp layer.
      norm_layer (nn.Module): Normalization layer.
      attn_norm_layer (nn.Module): Normalization layer in the attention module.
      kernel_q (_size_3_t): Pooling kernel size for q. If pooling kernel size is
          1 for all the dimensions, pooling is not used (by default).
      kernel_kv (_size_3_t): Pooling kernel size for kv. If pooling kernel size
          is 1 for all the dimensions, pooling is not used. By default, pooling
          is disabled.
      stride_q (_size_3_t): Pooling kernel stride for q.
      stride_kv (_size_3_t): Pooling kernel stride for kv.
      pool_mode (str): Pooling mode. Option includes "conv" (learned pooling), "avg"
          (average pooling), and "max" (max pooling).
      has_cls_embed (bool): If set to True, the first token of the input tensor
          should be a cls token. Otherwise, the input tensor does not contain a
          cls token. Pooling is not applied to the cls token.
      pool_first (bool): If set to True, pool is applied before qkv projection.
          Otherwise, pool is applied after qkv projection. Default: False.
      residual_pool (bool): If set to True, use Improved Multiscale Vision
          Transformer's pooling residual connection.
      depthwise_conv (bool): Whether use depthwise or full convolution for pooling.
      bias_on (bool): Whether use biases for linear layers.
      separate_qkv (bool): Whether to use separate or one layer for qkv projections.
docstr_tok: ["Args",":","dim","(","int",")",":","Input","feature","dimension",".","dim_out","(","int",")",":","Output","feature","dimension",".","num_heads","(","int",")",":","Number","of","heads","in","the","attention","layer",".","mlp_ratio","(","float",")",":","Mlp","ratio","which","controls","the","feature","dimension","in","the","hidden","layer","of","the","Mlp","block",".","qkv_bias","(","bool",")",":","If","set","to","False",",","the","qkv","layer","will","not","learn","an","additive","bias",".","Default",":","False",".","dropout_rate","(","float",")",":","DropOut","rate",".","If","set","to","0",",","DropOut","is","disabled",".","droppath_rate","(","float",")",":","DropPath","rate",".","If","set","to","0",",","DropPath","is","disabled",".","act_layer","(","nn.Module",")",":","Activation","layer","used","in","the","Mlp","layer",".","norm_layer","(","nn.Module",")",":","Normalization","layer",".","attn_norm_layer","(","nn.Module",")",":","Normalization","layer","in","the","attention","module",".","kernel_q","(","_size_3_t",")",":","Pooling","kernel","size","for","q",".","If","pooling","kernel","size","is","1","for","all","the","dimensions",",","pooling","is","not","used","(","by","default",")",".","kernel_kv","(","_size_3_t",")",":","Pooling","kernel","size","for","kv",".","If","pooling","kernel","size","is","1","for","all","the","dimensions",",","pooling","is","not","used",".","By","default",",","pooling","is","disabled",".","stride_q","(","_size_3_t",")",":","Pooling","kernel","stride","for","q.","stride_kv","(","_size_3_t",")",":","Pooling","kernel","stride","for","kv",".","pool_mode","(","str",")",":","Pooling","mode",".","Option","includes","``","conv","''","(","learned","pooling",")",",","``","avg","''","(","average","pooling",")",",","and","``","max","''","(","max","pooling",")",".","has_cls_embed","(","bool",")",":","If","set","to","True",",","the","first","token","of","the","input","tensor","should","be","a","cls","token",".","Otherwise",",","the","input","tensor","does","not","contain","a","cls","token",".","Pooling","is","not","applied","to","the","cls","token",".","pool_first","(","bool",")",":","If","set","to","True",",","pool","is","applied","before","qkv","projection",".","Otherwise",",","pool","is","applied","after","qkv","projection",".","Default",":","False",".","residual_pool","(","bool",")",":","If","set","to","True",",","use","Improved","Multiscale","Vision","Transformer","'s","pooling","residual","connection",".","depthwise_conv","(","bool",")",":","Whether","use","depthwise","or","full","convolution","for","pooling",".","bias_on","(","bool",")",":","Whether","use","biases","for","linear","layers",".","separate_qkv","(","bool",")",":","Whether","to","use","separate","or","one","layer","for","qkv","projections","."]
code: def __init__(
      self,
      dim: int,
      dim_out: int,
      num_heads: int,
      mlp_ratio: float = 4.0,
      qkv_bias: bool = False,
      dropout_rate: float = 0.0,
      droppath_rate: float = 0.0,
      act_layer: nn.Module = nn.GELU,
      norm_layer: nn.Module = nn.LayerNorm,
      attn_norm_layer: nn.Module = nn.LayerNorm,
      kernel_q: _size_3_t = (1, 1, 1),
      kernel_kv: _size_3_t = (1, 1, 1),
      stride_q: _size_3_t = (1, 1, 1),
      stride_kv: _size_3_t = (1, 1, 1),
      pool_mode: str = "conv",
      has_cls_embed: bool = True,
      pool_first: bool = False,
      residual_pool: bool = False,
      depthwise_conv: bool = True,
      bias_on: bool = True,
      separate_qkv: bool = True,
  ) -> None:
      """
      Args:
          dim (int): Input feature dimension.
          dim_out (int): Output feature dimension.
          num_heads (int): Number of heads in the attention layer.
          mlp_ratio (float): Mlp ratio which controls the feature dimension in the
              hidden layer of the Mlp block.
          qkv_bias (bool): If set to False, the qkv layer will not learn an additive
              bias. Default: False.
          dropout_rate (float): DropOut rate. If set to 0, DropOut is disabled.
          droppath_rate (float): DropPath rate. If set to 0, DropPath is disabled.
          act_layer (nn.Module): Activation layer used in the Mlp layer.
          norm_layer (nn.Module): Normalization layer.
          attn_norm_layer (nn.Module): Normalization layer in the attention module.
          kernel_q (_size_3_t): Pooling kernel size for q. If pooling kernel size is
              1 for all the dimensions, pooling is not used (by default).
          kernel_kv (_size_3_t): Pooling kernel size for kv. If pooling kernel size
              is 1 for all the dimensions, pooling is not used. By default, pooling
              is disabled.
          stride_q (_size_3_t): Pooling kernel stride for q.
          stride_kv (_size_3_t): Pooling kernel stride for kv.
          pool_mode (str): Pooling mode. Option includes "conv" (learned pooling), "avg"
              (average pooling), and "max" (max pooling).
          has_cls_embed (bool): If set to True, the first token of the input tensor
              should be a cls token. Otherwise, the input tensor does not contain a
              cls token. Pooling is not applied to the cls token.
          pool_first (bool): If set to True, pool is applied before qkv projection.
              Otherwise, pool is applied after qkv projection. Default: False.
          residual_pool (bool): If set to True, use Improved Multiscale Vision
              Transformer's pooling residual connection.
          depthwise_conv (bool): Whether use depthwise or full convolution for pooling.
          bias_on (bool): Whether use biases for linear layers.
          separate_qkv (bool): Whether to use separate or one layer for qkv projections.
      """
      super().__init__()
      assert not pool_first
      assert not separate_qkv
      self.dim = dim
      self.dim_out = dim_out
      kernel_skip = [s + 1 if s > 1 else s for s in stride_q]
      stride_skip = stride_q
      padding_skip = [int(skip // 2) for skip in kernel_skip]
      self.attn = MultiScaleAttention(
          dim,
          num_heads=num_heads,
          qkv_bias=qkv_bias,
          dropout_rate=dropout_rate,
          kernel_q=kernel_q,
          kernel_kv=kernel_kv,
          stride_q=stride_q,
          stride_kv=stride_kv,
          norm_layer=attn_norm_layer,
          has_cls_embed=has_cls_embed,
          pool_mode=pool_mode,
          pool_first=pool_first,
          residual_pool=residual_pool,
          bias_on=bias_on,
          depthwise_conv=depthwise_conv,
          separate_qkv=separate_qkv,
      )
      self.drop_path = (
          DropPath(droppath_rate)
          if droppath_rate > 0.0
          else nn.Identity()
      )
      mlp_hidden_dim = int(dim * mlp_ratio)
      self.has_cls_embed = has_cls_embed
      self.mlp = Mlp(
          in_features=dim,
          hidden_features=mlp_hidden_dim,
          out_features=dim_out,
          act_layer=act_layer,
          dropout_rate=dropout_rate,
          bias_on=bias_on,
      )
      self.proj = (
          nn.Linear(dim, dim_out, bias=bias_on)
          if dim != dim_out
          else nn.Identity()
      )
  
      self.pool_skip = (
          nn.MaxPool3d(
              kernel_skip, stride_skip, padding_skip, ceil_mode=False
          )
          if len(stride_skip) > 0 and numpy.prod(stride_skip) > 1
          else None
      )
  
imports: from typing import List, Optional, Tuple
  import numpy
  import torch
  import torch.fx
  import torch.nn
  from torch.nn.common_types import _size_3_t
  from .drop_path import DropPath
label_desc: Use image node_id 1 to create a new ScriptableMultiScaleBlock object from inherited base classes: nn with example: obj = ScriptableMultiScaleBlock(dim, dim_out, num_heads, mlp_ratio, qkv_bias, dropout_rate, droppath_rate, act_layer, norm_layer, attn_norm_layer, kernel_q, kernel_kv, stride_q, stride_kv, pool_mode, has_cls_embed, pool_first, residual_pool, depthwise_conv, bias_on, separate_qkv)