class_docstr: Multi-headed attention from 'Attention Is All You Need' paper
code: def __init__(
      self,
      embed_dim: int,
      num_heads: int,
      heavy_ratio: float,
      recent_ratio: float,
      dropout: float = 0.0,
      is_decoder: bool = False,
      bias: bool = True,
  ):
      super().__init__()
      self.embed_dim = embed_dim
      self.num_heads = num_heads
      self.dropout = dropout
      self.head_dim = embed_dim // num_heads
  
      if (self.head_dim * num_heads) != self.embed_dim:
          raise ValueError(
              f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
              f" and `num_heads`: {num_heads})."
          )
      self.scaling = self.head_dim**-0.5
      self.is_decoder = is_decoder
  
      self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
  
      self.heavy_budget_ratio = heavy_ratio
      self.recent_budget_ratio = recent_ratio
  
imports: import os
  import pdb
  import copy
  import math
  import numpy
  from dataclasses import dataclass
  from typing import Optional, Tuple, Union
  import torch
  from torch import nn
  import torch.utils.checkpoint
  import torch.nn.functional
  from torch.cuda.amp import autocast
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
  from transformers.models.opt.modeling_opt import OPTAttention
label_desc: Use image node_id 1 to create a new OPTAttention_Mask object from inherited base classes: nn with example: obj = OPTAttention_Mask(embed_dim, num_heads, heavy_ratio, recent_ratio, dropout, is_decoder, bias)