class_docstr: ## Relative Multi-Head Attention Module
  
  We override [Multi-Head Attention](mha.html) module so we only need to 
  write the `get_scores` method.
code: def __init__(
      self, heads: int, d_model: int, dropout_prob: float = 0.1
  ):
      # The linear transformations do not need a bias since we
      # explicitly include it when calculating scores.
      # However having a bias for `value` might make sense.
      super().__init__(heads, d_model, dropout_prob, bias=False)
  
      # Number of relative positions
      self.P = 2**12
  
      # Relative positional embeddings for key relative to the query.
      # We need $2P$ embeddings because the keys can be before or after the query.
      self.key_pos_embeddings = nn.Parameter(
          torch.zeros((self.P * 2, heads, self.d_k)), requires_grad=True
      )
      # Relative positional embedding bias for key relative to the query.
      self.key_pos_bias = nn.Parameter(
          torch.zeros((self.P * 2, heads)), requires_grad=True
      )
      # Positional embeddings for the query is independent of the position of the query
      self.query_pos_bias = nn.Parameter(
          torch.zeros((heads, self.d_k)), requires_grad=True
      )
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/transformers/xl/relative_mha.py
imports: import torch
  from torch import nn
  from labml.logger import inspect
  from labml_nn.transformers.mha import MultiHeadAttention
label_desc: Use image node_id 1 to create a new RelativeMultiHeadAttention object from inherited base classes: MultiHeadAttention with example: obj = RelativeMultiHeadAttention(heads, d_model, dropout_prob)