class_docstr: ## Rotary Positional Embeddings
  
  GPT-NeoX uses [rotary positional embeddings (RoPE)](https://arxiv.org/abs/2104.09864).
  
  WE have annotated implementation of RoPE [here](https://nn.labml.ai/transformers/rope/index.html)
  with more notes the theory.
class_docstr_tok: ["#","#","Rotary","Positional","Embeddings","GPT-NeoX","uses","[","rotary","positional","embeddings","(","RoPE",")","]","(","https",":","\/\/arxiv.org\/abs\/2104.09864",")",".","WE","have","annotated","implementation","of","RoPE","[","here","]","(","https",":","\/\/nn.labml.ai\/transformers\/rope\/index.html",")","with","more","notes","the","theory","."]
docstr: :param d_rope: is the number of features for RoPE embeddings
  :param base: is the base for $  heta_i = 10000^{rac{2(i-1)}{d}}$, which defaults to $10000$
code: def __init__(self, d_rope: int, base: float = 10_000.0):
      """
      :param d_rope: is the number of features for RoPE embeddings
      :param base: is the base for $\theta_i = 10000^{\frac{2(i-1)}{d}}$, which defaults to $10000$
      """
      super().__init__()
  
      # To store $\theta_i$ for the features
      self.theta = None
      # Cache $\cos m\theta_i$ and $\sin m\theta_i$
      self.cos_cached = None
      self.sin_cached = None
  
      # Base for $\theta_i = 10000^{\frac{2(i-1)}{d}}$
      self.base = base
      # Number of features for RoPE
      self.d_rope = d_rope
  
imports: import copy
  import math
  from typing import Dict, Optional, Set, Callable, Any, Generator, Tuple
  import torch
  from torch import nn
  from torch.cuda.amp import autocast
  from labml import monit, logger
  from labml.logger import Text
  from labml_nn.neox import checkpoint
  from labml_nn.neox.utils.cache import get_cache
label_desc: Use image node_id 1 to create a new RoPE object from inherited base classes: nn with example: obj = RoPE(d_rope, base)