class_docstr: The Transformer encoder from "Attention is All You Need"
  :cite:`DBLP:journals/corr/VaswaniSPUJGKP17`
  
  Args:
      num_layers (int): number of encoder layers
      d_model (int): size of the model
      heads (int): number of heads
      d_ff (int): size of the inner FF layer
      dropout (float): dropout parameters
      embeddings (onmt.modules.Embeddings):
        embeddings to use, should have positional encodings
      pos_ffn_activation_fn (ActivationFunction):
          activation function choice for PositionwiseFeedForward layer
  
  Returns:
      (torch.FloatTensor, torch.FloatTensor):
  
      * enc_out ``(batch_size, src_len, model_dim)``
      * encoder final state: None in the case of Transformer
      * src_len ``(batch_size)``
class_docstr_tok: ["The","Transformer","encoder","from","``","Attention","is","All","You","Need","''",":","cite",":","`","DBLP",":","journals\/corr\/VaswaniSPUJGKP17","`","Args",":","num_layers","(","int",")",":","number","of","encoder","layers","d_model","(","int",")",":","size","of","the","model","heads","(","int",")",":","number","of","heads","d_ff","(","int",")",":","size","of","the","inner","FF","layer","dropout","(","float",")",":","dropout","parameters","embeddings","(","onmt.modules.Embeddings",")",":","embeddings","to","use",",","should","have","positional","encodings","pos_ffn_activation_fn","(","ActivationFunction",")",":","activation","function","choice","for","PositionwiseFeedForward","layer","Returns",":","(","torch.FloatTensor",",","torch.FloatTensor",")",":","*","enc_out","``","(","batch_size",",","src_len",",","model_dim",")","``","*","encoder","final","state",":","None","in","the","case","of","Transformer","*","src_len","``","(","batch_size",")","``"]
code: def __init__(
      self,
      num_layers,
      d_model,
      heads,
      d_ff,
      dropout,
      attention_dropout,
      embeddings,
      max_relative_positions,
      relative_positions_buckets,
      pos_ffn_activation_fn=ActivationFunction.relu,
      add_qkvbias=False,
      num_kv=0,
      add_ffnbias=True,
      parallel_residual=False,
      layer_norm="standard",
      norm_eps=1e-6,
      use_ckpting=[],
      parallel_gpu=1,
      rotary_interleave=True,
      rotary_theta=1e4,
      rotary_dim=0,
  ):
      super(TransformerEncoder, self).__init__()
  
      self.embeddings = embeddings
      self.transformer = nn.ModuleList(
          [
              TransformerEncoderLayer(
                  d_model,
                  heads,
                  d_ff,
                  dropout,
                  attention_dropout,
                  max_relative_positions=max_relative_positions,
                  relative_positions_buckets=relative_positions_buckets,
                  pos_ffn_activation_fn=pos_ffn_activation_fn,
                  add_qkvbias=add_qkvbias,
                  num_kv=num_kv,
                  add_ffnbias=add_ffnbias,
                  parallel_residual=parallel_residual,
                  layer_norm=layer_norm,
                  norm_eps=norm_eps,
                  use_ckpting=use_ckpting,
                  parallel_gpu=parallel_gpu,
                  rotary_interleave=rotary_interleave,
                  rotary_theta=rotary_theta,
                  rotary_dim=rotary_dim,
              )
              for i in range(num_layers)
          ]
      )
      if layer_norm == "standard":
          self.layer_norm = nn.LayerNorm(d_model, eps=norm_eps)
      elif layer_norm == "rms":
          self.layer_norm = RMSNorm(d_model, eps=norm_eps)
      else:
          raise ValueError(
              f"{layer_norm} layer norm type is not supported"
          )
  
file_path: OpenNMT-py/onmt/encoders/transformer.py
imports: import torch.nn
  from onmt.encoders.encoder import EncoderBase
  from onmt.modules import MultiHeadedAttention
  from onmt.modules.position_ffn import PositionwiseFeedForward
  from onmt.modules.position_ffn import ActivationFunction
  from onmt.utils.misc import sequence_mask
label_desc: Use image node_id 1 to create a new TransformerEncoder object from inherited base classes: EncoderBase with example: obj = TransformerEncoder(num_layers, d_model, heads, d_ff, dropout, attention_dropout, embeddings, max_relative_positions, relative_positions_buckets, pos_ffn_activation_fn, add_qkvbias, num_kv, add_ffnbias, parallel_residual, layer_norm, norm_eps, use_ckpting, parallel_gpu, rotary_interleave, rotary_theta, rotary_dim)