class_docstr: Multi-headed attention from 'Attention Is All You Need' paper
code: def __init__(self, config: LlamaConfig):
      super().__init__()
      self.config = config
      self.hidden_size = config.hidden_size
      self.num_heads = config.num_attention_heads
      self.head_dim = self.hidden_size // self.num_heads
      self.max_position_embeddings = config.max_position_embeddings
  
      if (self.head_dim * self.num_heads) != self.hidden_size:
          raise ValueError(
              f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
              f" and `num_heads`: {self.num_heads})."
          )
      self.q_proj = nn.Linear(
          self.hidden_size, self.num_heads * self.head_dim, bias=False
      )
      self.k_proj = nn.Linear(
          self.hidden_size, self.num_heads * self.head_dim, bias=False
      )
      self.v_proj = nn.Linear(
          self.hidden_size, self.num_heads * self.head_dim, bias=False
      )
      self.o_proj = nn.Linear(
          self.num_heads * self.head_dim, self.hidden_size, bias=False
      )
      self.rotary_emb = LlamaRotaryEmbedding(
          self.head_dim,
          max_position_embeddings=self.max_position_embeddings,
      )
  
      self.heavy_budget_ratio = config.heavy_ratio
      self.recent_budget_ratio = config.recent_ratio
  
imports: import os
  import pdb
  import copy
  import math
  import numpy
  from dataclasses import dataclass
  from typing import Optional, Tuple, Union
  import torch
  from torch import nn
  import torch.utils.checkpoint
  import torch.nn.functional
  from torch.cuda.amp import autocast
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
  from transformers.models.llama.configuration_llama import LlamaConfig
  from transformers.models.llama.modeling_llama import LlamaRotaryEmbedding, LlamaAttention, apply_rotary_pos_emb
label_desc: Use image node_id 1 to create a new LlamaAttention_heavy_hitter object from inherited base classes: nn with example: obj = LlamaAttention_heavy_hitter(config)