code: def __init__(
      self,
      dim,
      dim_out,
      input_size,
      num_heads=8,
      qkv_bias=False,
      drop_rate=0.0,
      kernel_q=(1, 1, 1),
      kernel_kv=(1, 1, 1),
      stride_q=(1, 1, 1),
      stride_kv=(1, 1, 1),
      norm_layer=nn.LayerNorm,
      has_cls_embed=True,
      # Options include `conv`, `avg`, and `max`.
      mode="conv",
      # If True, perform pool before projection.
      pool_first=False,
      rel_pos_spatial=False,
      rel_pos_temporal=False,
      rel_pos_zero_init=False,
      residual_pooling=False,
      separate_qkv=False,
  ):
      super().__init__()
      self.pool_first = pool_first
      self.separate_qkv = separate_qkv
      self.drop_rate = drop_rate
      self.num_heads = num_heads
      self.dim_out = dim_out
      head_dim = dim_out // num_heads
      self.scale = head_dim**-0.5
      self.has_cls_embed = has_cls_embed
      self.mode = mode
      padding_q = [int(q // 2) for q in kernel_q]
      padding_kv = [int(kv // 2) for kv in kernel_kv]
  
      if pool_first or separate_qkv:
          self.q = nn.Linear(dim, dim_out, bias=qkv_bias)
          self.k = nn.Linear(dim, dim_out, bias=qkv_bias)
          self.v = nn.Linear(dim, dim_out, bias=qkv_bias)
      else:
          self.qkv = nn.Linear(dim, dim_out * 3, bias=qkv_bias)
  
      self.proj = nn.Linear(dim_out, dim_out)
      if drop_rate > 0.0:
          self.proj_drop = nn.Dropout(drop_rate)
  
      # Skip pooling with kernel and stride size of (1, 1, 1).
      if numpy.prod(kernel_q) == 1 and numpy.prod(stride_q) == 1:
          kernel_q = ()
      if numpy.prod(kernel_kv) == 1 and numpy.prod(stride_kv) == 1:
          kernel_kv = ()
  
      if mode in ("avg", "max"):
          pool_op = nn.MaxPool3d if mode == "max" else nn.AvgPool3d
          self.pool_q = (
              pool_op(kernel_q, stride_q, padding_q, ceil_mode=False)
              if len(kernel_q) > 0
              else None
          )
          self.pool_k = (
              pool_op(kernel_kv, stride_kv, padding_kv, ceil_mode=False)
              if len(kernel_kv) > 0
              else None
          )
          self.pool_v = (
              pool_op(kernel_kv, stride_kv, padding_kv, ceil_mode=False)
              if len(kernel_kv) > 0
              else None
          )
      elif mode == "conv" or mode == "conv_unshared":
          if pool_first:
              dim_conv = dim // num_heads if mode == "conv" else dim
          else:
              dim_conv = (
                  dim_out // num_heads if mode == "conv" else dim_out
              )
          self.pool_q = (
              nn.Conv3d(
                  dim_conv,
                  dim_conv,
                  kernel_q,
                  stride=stride_q,
                  padding=padding_q,
                  groups=dim_conv,
                  bias=False,
              )
              if len(kernel_q) > 0
              else None
          )
          self.norm_q = (
              norm_layer(dim_conv) if len(kernel_q) > 0 else None
          )
          self.pool_k = (
              nn.Conv3d(
                  dim_conv,
                  dim_conv,
                  kernel_kv,
                  stride=stride_kv,
                  padding=padding_kv,
                  groups=dim_conv,
                  bias=False,
              )
              if len(kernel_kv) > 0
              else None
          )
          self.norm_k = (
              norm_layer(dim_conv) if len(kernel_kv) > 0 else None
          )
          self.pool_v = (
              nn.Conv3d(
                  dim_conv,
                  dim_conv,
                  kernel_kv,
                  stride=stride_kv,
                  padding=padding_kv,
                  groups=dim_conv,
                  bias=False,
              )
              if len(kernel_kv) > 0
              else None
          )
          self.norm_v = (
              norm_layer(dim_conv) if len(kernel_kv) > 0 else None
          )
      else:
          raise NotImplementedError(f"Unsupported model {mode}")
  
      self.rel_pos_spatial = rel_pos_spatial
      self.rel_pos_temporal = rel_pos_temporal
      if self.rel_pos_spatial:
          assert input_size[1] == input_size[2]
          size = input_size[1]
          q_size = size // stride_q[1] if len(stride_q) > 0 else size
          kv_size = size // stride_kv[1] if len(stride_kv) > 0 else size
          rel_sp_dim = 2 * max(q_size, kv_size) - 1
  
          self.rel_pos_h = nn.Parameter(
              torch.zeros(rel_sp_dim, head_dim)
          )
          self.rel_pos_w = nn.Parameter(
              torch.zeros(rel_sp_dim, head_dim)
          )
          if not rel_pos_zero_init:
              trunc_normal_(self.rel_pos_h, std=0.02)
              trunc_normal_(self.rel_pos_w, std=0.02)
      if self.rel_pos_temporal:
          self.rel_pos_t = nn.Parameter(
              torch.zeros(2 * input_size[0] - 1, head_dim)
          )
          if not rel_pos_zero_init:
              trunc_normal_(self.rel_pos_t, std=0.02)
  
      self.residual_pooling = residual_pooling
  
file_path: SlowFast/slowfast/models/attention.py
imports: import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from torch.nn.init import trunc_normal_
  from slowfast.models.common import DropPath, Mlp
label_desc: Use image node_id 1 to create a new MultiScaleAttention object from inherited base classes: nn with example: obj = MultiScaleAttention(dim, dim_out, input_size, num_heads, qkv_bias, drop_rate, kernel_q, kernel_kv, stride_q, stride_kv, norm_layer, has_cls_embed, mode, pool_first, rel_pos_spatial, rel_pos_temporal, rel_pos_zero_init, residual_pooling, separate_qkv)