class_docstr: This is an implementation of transformer language model.
  
  The architecture is based on the paper "Attention Is All You Need": https://arxiv.org/pdf/1706.03762.pdf
  
  Arguments
  ----------
  d_model : int
      The number of expected features in the encoder/decoder inputs (default=512).
  nhead : int
      The number of heads in the multiheadattention models (default=8).
  num_encoder_layers : int
      The number of sub-encoder-layers in the encoder (default=6).
  num_decoder_layers : int
      The number of sub-decoder-layers in the decoder (default=6).
  dim_ffn : int
      The dimension of the feedforward network model (default=2048).
  dropout : int
      The dropout value (default=0.1).
  activation: torch class
      The activation function of encoder/decoder intermediate layer, relu or gelu (default=relu).
  decoder_use_memory: bool
      whether to use the hidden state in the decoder
  
  Example
  -------
  >>> src = torch.randint(0, 720, [8, 120])
  >>> net = TransformerLM(720, 512, 8, 1, 0, 1024, activation=torch.nn.GELU)
  >>> enc_out = net.forward(src)
  >>> print(enc_out.shape)
  torch.Size([8, 120, 720])
code: def __init__(
      self,
      vocab,
      d_model=512,
      nhead=8,
      num_encoder_layers=12,
      num_decoder_layers=0,
      d_ffn=2048,
      dropout=0.1,
      activation=nn.ReLU,
      positional_encoding="fixed_abs_sine",
      normalize_before=False,
      d_embedding=None,
      max_length=2500,
      causal=True,
      attention_type="regularMHA",
      decoder_use_memory=False,
  ):
      super().__init__(
          d_model=d_model,
          nhead=nhead,
          num_encoder_layers=num_encoder_layers,
          num_decoder_layers=num_decoder_layers,
          d_ffn=d_ffn,
          dropout=dropout,
          activation=activation,
          positional_encoding=positional_encoding,
          normalize_before=normalize_before,
          max_length=max_length,
          causal=causal,
          attention_type=attention_type,
      )
  
      self.d_embedding = d_embedding
      if d_embedding is None:
          self.d_embedding = d_model
  
      self.custom_src_module = NormalizedEmbedding(
          self.d_embedding, vocab
      )
  
      self.embedding_proj = None
      if d_embedding is not None:
          self.embedding_proj = Linear(
              input_size=self.d_embedding, n_neurons=d_model
          )
  
      self.output_proj = ModuleList(
          Linear(input_size=d_model, n_neurons=d_model),
          LayerNorm(d_model, eps=1e-6),
          Linear(input_size=d_model, n_neurons=vocab),
      )
  
      self.num_encoder_layers = num_encoder_layers
      self.num_decoder_layers = num_decoder_layers
      self.decoder_use_memory = decoder_use_memory
  
      # reset the params of the transformer model
      self._reset_params()
  
file_path: speechbrain/speechbrain/lobes/models/transformer/TransformerLM.py
imports: import torch
  from torch import nn
  from speechbrain.nnet.linear import Linear
  from speechbrain.nnet.normalization import LayerNorm
  from speechbrain.nnet.containers import ModuleList
  from speechbrain.lobes.models.transformer.Transformer import TransformerInterface, get_lookahead_mask, get_key_padding_mask, NormalizedEmbedding
label_desc: Use image node_id 1 to create a new TransformerLM object from inherited base classes: TransformerInterface with example: obj = TransformerLM(vocab, d_model, nhead, num_encoder_layers, num_decoder_layers, d_ffn, dropout, activation, positional_encoding, normalize_before, d_embedding, max_length, causal, attention_type, decoder_use_memory)