class_docstr: Attention Layer as used in Vision Transformer.
class_docstr_tok: ["Attention","Layer","as","used","in","Vision","Transformer","."]
code: def __init__(
      self,
      dim,
      num_heads=8,
      qkv_bias=False,
      qk_scale=None,
      attn_drop=0.0,
      proj_drop=0.0,
  ):
      super().__init__()
      self.num_heads = num_heads
      head_dim = dim // num_heads
      # NOTE scale factor was wrong in my original version,
      # can set manually to be compat with prev weights
      self.scale = qk_scale or head_dim**-0.5
      self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
      self.attn_drop = nn.Dropout(attn_drop)
      self.proj = nn.Linear(dim, dim)
      self.proj_drop = nn.Dropout(proj_drop)
      self.attn_gradients = None
      self.attention_map = None
  
imports: import collections.abc
  import math
  import warnings
  from dataclasses import dataclass
  from functools import partial
  from itertools import repeat
  import omegaconf
  import torch
  import torch.nn
  import torch.nn.functional
  from mmf.common.registry import registry
  from mmf.modules.encoders import Encoder
label_desc: Use image node_id 1 to create a new Attention object from inherited base classes: nn with example: obj = Attention(dim, num_heads, qkv_bias, qk_scale, attn_drop, proj_drop)