class_docstr: Implements a Luna Encoder Layer used in masked pre-trained language models
  and fine-tuned classication/regression mdoel
class_docstr_tok: ["Implements","a","Luna","Encoder","Layer","used","in","masked","pre-trained","language","models","and","fine-tuned","classication\/regression","mdoel"]
code: def __init__(
      self,
      embedding_dim: int = 768,
      ffn_embedding_dim: int = 3072,
      num_attention_heads: int = 8,
      num_projected_attention_heads: int = 8,
      dropout: float = 0.1,
      attention_dropout: float = 0.1,
      activation_dropout: float = 0.1,
      activation_fn: str = "relu",
      normalize_before: bool = False,
      tie_kv=True,
      export: bool = False,
      init_fn: Callable = None,
  ) -> None:
      super().__init__()
  
      if init_fn is not None:
          init_fn()
  
      # Initialize parameters
      self.embedding_dim = embedding_dim
      self.dropout_module = FairseqDropout(
          dropout, module_name=self.__class__.__name__
      )
      self.activation_dropout_module = FairseqDropout(
          activation_dropout, module_name=self.__class__.__name__
      )
  
      # Initialize blocks
      self.activation_fn = utils.get_activation_fn(activation_fn)
      self.self_attn = self.build_self_attention(
          self.embedding_dim,
          num_attention_heads,
          num_projected_attention_heads,
          dropout=attention_dropout,
          tie_kv=tie_kv,
      )
  
      # layer norm associated with the self attention layer
      self.normalize_before = normalize_before
      self.self_attn_layer_norm = LayerNorm(
          self.embedding_dim, export=export
      )
      self.self_atten_proj_layer_norm = LayerNorm(
          self.embedding_dim, export=export
      )
  
      self.fc1 = self.build_fc1(
          self.embedding_dim,
          ffn_embedding_dim,
      )
      self.fc2 = self.build_fc2(
          ffn_embedding_dim,
          self.embedding_dim,
      )
  
      # layer norm associated with the position wise feed-forward NN
      self.final_layer_norm = LayerNorm(
          self.embedding_dim, export=export
      )
  
imports: import math
  from typing import Callable, Optional
  import torch
  from fairseq import utils
  from fairseq.modules import LayerDropModuleList, LayerNorm, PositionalEmbedding
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from pytext.utils.usage import log_class_usage
  from torch import nn
  from torch.nn import Parameter
  from .None import LunarMultiheadAttention
label_desc: Use image node_id 1 to create a new LunaSentenceEncoderLayer object from inherited base classes: nn with example: obj = LunaSentenceEncoderLayer(embedding_dim, ffn_embedding_dim, num_attention_heads, num_projected_attention_heads, dropout, attention_dropout, activation_dropout, activation_fn, normalize_before, tie_kv, export, init_fn)