class_docstr: Multi-headed attention from 'Attention Is All You Need' paper
class_docstr_tok: ["Multi-headed","attention","from","'Attention","Is","All","You","Need","'","paper"]
code: def _init_rope(self):
      if self.config.rope_scaling is None:
          self.rotary_emb = LlamaRotaryEmbedding(
              self.head_dim,
              max_position_embeddings=self.max_position_embeddings,
              base=self.rope_theta,
          )
      else:
          scaling_type = self.config.rope_scaling["type"]
          scaling_factor = self.config.rope_scaling["factor"]
          if scaling_type == "linear":
              self.rotary_emb = LlamaLinearScalingRotaryEmbedding(
                  self.head_dim,
                  max_position_embeddings=self.max_position_embeddings,
                  scaling_factor=scaling_factor,
                  base=self.rope_theta,
              )
          elif scaling_type == "dynamic":
              self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding(
                  self.head_dim,
                  max_position_embeddings=self.max_position_embeddings,
                  scaling_factor=scaling_factor,
                  base=self.rope_theta,
              )
          else:
              raise ValueError(
                  f"Unknown RoPE scaling type {scaling_type}"
              )
  
imports: import math
  from typing import Optional, Tuple
  import pdb
  import torch
  from torch import nn
  import torch.utils.checkpoint
  import torch.nn.functional
  from transformers.models.llama.modeling_llama import LlamaAttention, rotate_half, apply_rotary_pos_emb, LlamaRotaryEmbedding, apply_rotary_pos_emb, LlamaForCausalLM
  import types
  from transformers.configuration_utils import PretrainedConfig
label_desc: Use image node_id 2 for calling the H2OLlamaAttention obj's underlying member method code with example usage: obj._init_rope() without return types