class_docstr: Implementation for a Bi-directional Luna based Sentence Encoder used
  in masked pre-trained language models.
  This first computes the token embedding using the token embedding matrix,
  position embeddings (if specified) and segment embeddings
  (if specified). After applying the specified number of
  TransformerEncoderLayers, it outputs all the internal states of the
  encoder as well as the final representation associated with the first
  token (usually CLS token).
  Input:
      - tokens: B x T matrix representing sentences
      - segment_labels: B x T matrix representing segment label for tokens
  Output:
      - a tuple of the following:
          - a list of internal model states used to compute the
            predictions where each tensor has shape T x B x C
          - sentence representation associated with first input token
            in format B x C.
class_docstr_tok: ["Implementation","for","a","Bi-directional","Luna","based","Sentence","Encoder","used","in","masked","pre-trained","language","models",".","This","first","computes","the","token","embedding","using","the","token","embedding","matrix",",","position","embeddings","(","if","specified",")","and","segment","embeddings","(","if","specified",")",".","After","applying","the","specified","number","of","TransformerEncoderLayers",",","it","outputs","all","the","internal","states","of","the","encoder","as","well","as","the","final","representation","associated","with","the","first","token","(","usually","CLS","token",")",".","Input",":","-","tokens",":","B","x","T","matrix","representing","sentences","-","segment_labels",":","B","x","T","matrix","representing","segment","label","for","tokens","Output",":","-","a","tuple","of","the","following",":","-","a","list","of","internal","model","states","used","to","compute","the","predictions","where","each","tensor","has","shape","T","x","B","x","C","-","sentence","representation","associated","with","first","input","token","in","format","B","x","C","."]
code: def __init__(
      self,
      padding_idx: int,
      vocab_size: int,
      projection_length: int = 128,
      num_encoder_layers: int = 12,
      embedding_dim: int = 768,
      ffn_embedding_dim: int = 3072,
      num_attention_heads: int = 12,
      num_projected_attention_heads: int = 12,
      dropout: float = 0.1,
      attention_dropout: float = 0.1,
      activation_dropout: float = 0.1,
      layerdrop: float = 0.0,
      max_seq_len: int = 512,
      num_segments: int = 0,
      use_position_embeddings: bool = True,
      offset_positions_by_padding: bool = True,
      layernorm_embedding: bool = False,
      normalize_before: bool = False,
      dynamic_projection: bool = True,
      tie_kv=False,
      apply_bert_init: bool = False,
      activation_fn: str = "gelu",
      learned_pos_embedding: bool = True,
      embed_scale: float = None,
      freeze_embeddings: bool = False,
      n_trans_layers_to_freeze: int = 0,
      export: bool = False,
      traceable: bool = False,
  ) -> None:
      super().__init__()
      self.padding_idx = padding_idx
      self.vocab_size = vocab_size
      self.proj_len = projection_length
      self.dynamic_projection = dynamic_projection
      self.dropout_module = FairseqDropout(
          dropout, module_name=self.__class__.__name__
      )
      self.layerdrop = layerdrop
      self.max_seq_len = max_seq_len
      self.embedding_dim = embedding_dim
      self.num_segments = num_segments
      self.use_position_embeddings = use_position_embeddings
      self.apply_bert_init = apply_bert_init
      self.learned_pos_embedding = learned_pos_embedding
      self.traceable = traceable
      self.tpu = False  # whether we're on TPU
  
      self.embed_tokens = self.build_embedding(
          self.vocab_size, self.embedding_dim, self.padding_idx
      )
      self.embed_scale = embed_scale
  
      if self.num_segments > 0:
          self.segment_embeddings = nn.Embedding(
              self.num_segments, self.embedding_dim, padding_idx=None
          )
          nn.init.normal_(
              self.segment_embeddings.weight,
              mean=0.0,
              std=self.embedding_dim**-0.5,
          )
      else:
          self.segment_embeddings = None
  
      self.embed_positions = (
          PositionalEmbedding(
              self.max_seq_len,
              self.embedding_dim,
              padding_idx=(
                  self.padding_idx
                  if offset_positions_by_padding
                  else None
              ),
              learned=self.learned_pos_embedding,
          )
          if self.use_position_embeddings
          else None
      )
  
      self.projected_embeddings = Parameter(
          torch.Tensor(self.proj_len, self.embedding_dim)
      )
      nn.init.normal_(
          self.projected_embeddings,
          mean=0.0,
          std=self.embedding_dim**-0.5,
      )
      if (
          self.use_position_embeddings
          and not self.learned_pos_embedding
      ):
          projected_positions = get_sinusoidal_positional_embedding(
              self.proj_len, self.embedding_dim
          )
          if self.embed_scale is None:
              self.embed_scale = math.sqrt(self.embedding_dim)
      else:
          projected_positions = None
      self.register_buffer("projected_positions", projected_positions)
  
      if self.layerdrop > 0.0:
          self.layers = LayerDropModuleList(p=self.layerdrop)
      else:
          self.layers = nn.ModuleList([])
  
      self.layers.extend(
          [
              self.build_luna_sentence_encoder_layer(
                  embedding_dim=self.embedding_dim,
                  ffn_embedding_dim=ffn_embedding_dim,
                  num_attention_heads=num_attention_heads,
                  num_projected_attention_heads=num_projected_attention_heads,
                  dropout=self.dropout_module.p,
                  attention_dropout=attention_dropout,
                  activation_dropout=activation_dropout,
                  activation_fn=activation_fn,
                  normalize_before=normalize_before,
                  tie_kv=tie_kv,
                  export=export,
              )
              for _ in range(num_encoder_layers)
          ]
      )
  
      assert not layernorm_embedding or not normalize_before
  
      if layernorm_embedding:
          self.emb_layer_norm = LayerNorm(
              self.embedding_dim, export=export
          )
          self.proj_emb_layer_norm = LayerNorm(
              self.embedding_dim, export=export
          )
      else:
          self.emb_layer_norm = None
          self.proj_emb_layer_norm = None
  
      if normalize_before:
          self.layer_norm = LayerNorm(self.embedding_dim, export=export)
          self.proj_layer_norm = LayerNorm(
              self.embedding_dim, export=export
          )
      else:
          self.layer_norm = None
          self.proj_layer_norm = None
  
      # Apply initialization of model params after building the model
      if self.apply_bert_init:
          self.apply(init_bert_params)
  
      def freeze_module_params(m):
          if m is not None:
              for p in m.parameters():
                  p.requires_grad = False
  
      if freeze_embeddings:
          self.projected_embeddings.requires_grad = False
          freeze_module_params(self.embed_tokens)
          freeze_module_params(self.segment_embeddings)
          freeze_module_params(self.embed_positions)
          freeze_module_params(self.emb_layer_norm)
          freeze_module_params(self.proj_emb_layer_norm)
  
      for layer in range(n_trans_layers_to_freeze):
          freeze_module_params(self.layers[layer])
  
      log_class_usage(__class__)
  
imports: import math
  from typing import Callable, Optional
  import torch
  from fairseq import utils
  from fairseq.modules import LayerDropModuleList, LayerNorm, PositionalEmbedding
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from pytext.utils.usage import log_class_usage
  from torch import nn
  from torch.nn import Parameter
  from .None import LunarMultiheadAttention
label_desc: Use image node_id 1 to create a new LunaSentenceEncoder object from inherited base classes: nn with example: obj = LunaSentenceEncoder(padding_idx, vocab_size, projection_length, num_encoder_layers, embedding_dim, ffn_embedding_dim, num_attention_heads, num_projected_attention_heads, dropout, attention_dropout, activation_dropout, layerdrop, max_seq_len, num_segments, use_position_embeddings, offset_positions_by_padding, layernorm_embedding, normalize_before, dynamic_projection, tie_kv, apply_bert_init, activation_fn, learned_pos_embedding, embed_scale, freeze_embeddings, n_trans_layers_to_freeze, export, traceable)