class_docstr: Multi-head self attention.
  Using torch or triton attention implemetation enables user to also use additive bias.
class_docstr_tok: ["Multi-head","self","attention",".","Using","torch","or","triton","attention","implemetation","enables","user","to","also","use","additive","bias","."]
code: def __init__(self, config: MptConfig):
      super().__init__()
      self.hidden_size = config.hidden_size
      self.n_heads = config.n_heads
      self.max_seq_length = config.max_seq_len
      self.head_dim = self.hidden_size // self.n_heads
      self.softmax_scale = config.attn_config.softmax_scale
      if self.softmax_scale is None:
          self.softmax_scale = 1 / math.sqrt(
              self.hidden_size / self.n_heads
          )
  
      self.attn_dropout_p = config.attn_config.attn_pdrop
      self.Wqkv = nn.Linear(
          self.hidden_size, 3 * self.hidden_size, bias=False
      )
      self.out_proj = nn.Linear(
          self.hidden_size, self.hidden_size, bias=False
      )
  
imports: import math
  from typing import Optional, Tuple, Union
  import torch
  import torch.utils.checkpoint
  from torch import nn
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss
  from torch.nn import functional
  from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
  from ...modeling_attn_mask_utils import _prepare_4d_causal_attention_mask
  from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions, QuestionAnsweringModelOutput, SequenceClassifierOutputWithPast, TokenClassifierOutput
  from ...modeling_utils import PreTrainedModel
  from ...utils import logging
  from .configuration_mpt import MptConfig
label_desc: Use image node_id 1 to create a new MptAttention object from inherited base classes: nn with example: obj = MptAttention(config)