class_docstr: A single block of the Conformer encoder.
  
  Args:
      d_model (int): input dimension of MultiheadAttentionMechanism and PositionwiseFeedForward
      d_ff (int): hidden dimension of PositionwiseFeedForward
      self_attention_model (str): type of the attention layer and positional encoding
          'rel_pos': relative positional embedding and Transformer-XL
          'rel_pos_local_attn': relative positional embedding and Transformer-XL with local attention using
              overlapping chunks. Attention context is determined by att_context_size parameter.
          'abs_pos': absolute positional embedding and Transformer
          Default is rel_pos.
      global_tokens (int): number of tokens to be used for global attention.
          Only relevant if self_attention_model is 'rel_pos_local_attn'.
          Defaults to 0.
      global_tokens_spacing (int): how far apart the global tokens are
          Defaults to 1.
      global_attn_separate (bool): whether the q, k, v layers used for global tokens should be separate.
          Defaults to False.
      n_heads (int): number of heads for multi-head attention
      conv_kernel_size (int): kernel size for depthwise convolution in convolution module
      dropout (float): dropout probabilities for linear layers
      dropout_att (float): dropout probabilities for attention distributions
class_docstr_tok: ["A","single","block","of","the","Conformer","encoder",".","Args",":","d_model","(","int",")",":","input","dimension","of","MultiheadAttentionMechanism","and","PositionwiseFeedForward","d_ff","(","int",")",":","hidden","dimension","of","PositionwiseFeedForward","self_attention_model","(","str",")",":","type","of","the","attention","layer","and","positional","encoding","'rel_pos","'",":","relative","positional","embedding","and","Transformer-XL","'rel_pos_local_attn","'",":","relative","positional","embedding","and","Transformer-XL","with","local","attention","using","overlapping","chunks",".","Attention","context","is","determined","by","att_context_size","parameter",".","'abs_pos","'",":","absolute","positional","embedding","and","Transformer","Default","is","rel_pos",".","global_tokens","(","int",")",":","number","of","tokens","to","be","used","for","global","attention",".","Only","relevant","if","self_attention_model","is","'rel_pos_local_attn","'",".","Defaults","to","0.","global_tokens_spacing","(","int",")",":","how","far","apart","the","global","tokens","are","Defaults","to","1.","global_attn_separate","(","bool",")",":","whether","the","q",",","k",",","v","layers","used","for","global","tokens","should","be","separate",".","Defaults","to","False",".","n_heads","(","int",")",":","number","of","heads","for","multi-head","attention","conv_kernel_size","(","int",")",":","kernel","size","for","depthwise","convolution","in","convolution","module","dropout","(","float",")",":","dropout","probabilities","for","linear","layers","dropout_att","(","float",")",":","dropout","probabilities","for","attention","distributions"]
code: def __init__(
      self,
      d_model,
      d_ff,
      self_attention_model="rel_pos",
      global_tokens=0,
      global_tokens_spacing=1,
      global_attn_separate=False,
      n_heads=4,
      conv_kernel_size=31,
      conv_norm_type="batch_norm",
      conv_context_size=None,
      dropout=0.1,
      dropout_att=0.1,
      pos_bias_u=None,
      pos_bias_v=None,
      att_context_size=[-1, -1],
  ):
      super(ConformerLayer, self).__init__()
  
      self.self_attention_model = self_attention_model
      self.n_heads = n_heads
      self.fc_factor = 0.5
  
      # first feed forward module
      self.norm_feed_forward1 = LayerNorm(d_model)
      self.feed_forward1 = ConformerFeedForward(
          d_model=d_model, d_ff=d_ff, dropout=dropout
      )
  
      # convolution module
      self.norm_conv = LayerNorm(d_model)
      self.conv = ConformerConvolution(
          d_model=d_model,
          kernel_size=conv_kernel_size,
          norm_type=conv_norm_type,
          conv_context_size=conv_context_size,
      )
  
      # multi-headed self-attention module
      self.norm_self_att = LayerNorm(d_model)
      MHA_max_cache_len = att_context_size[0]
  
      if self_attention_model == "rel_pos":
          self.self_attn = RelPositionMultiHeadAttention(
              n_head=n_heads,
              n_feat=d_model,
              dropout_rate=dropout_att,
              pos_bias_u=pos_bias_u,
              pos_bias_v=pos_bias_v,
              max_cache_len=MHA_max_cache_len,
          )
      elif self_attention_model == "rel_pos_local_attn":
          self.self_attn = RelPositionMultiHeadAttentionLongformer(
              n_head=n_heads,
              n_feat=d_model,
              dropout_rate=dropout_att,
              pos_bias_u=pos_bias_u,
              pos_bias_v=pos_bias_v,
              max_cache_len=MHA_max_cache_len,
              att_context_size=att_context_size,
              global_tokens=global_tokens,
              global_tokens_spacing=global_tokens_spacing,
              global_attn_separate=global_attn_separate,
          )
      elif self_attention_model == "abs_pos":
          self.self_attn = MultiHeadAttention(
              n_head=n_heads,
              n_feat=d_model,
              dropout_rate=dropout_att,
              max_cache_len=MHA_max_cache_len,
          )
      else:
          raise ValueError(
              f"'{self_attention_model}' is not not a valid value for 'self_attention_model', "
              f"valid values can be from ['rel_pos', 'rel_pos_local_attn', 'abs_pos']"
          )
  
      # second feed forward module
      self.norm_feed_forward2 = LayerNorm(d_model)
      self.feed_forward2 = ConformerFeedForward(
          d_model=d_model, d_ff=d_ff, dropout=dropout
      )
  
      self.dropout = nn.Dropout(dropout)
      self.norm_out = LayerNorm(d_model)
  
imports: import torch
  from torch import nn
  from torch.nn import LayerNorm
  from nemo.collections.asr.parts.submodules.batchnorm import FusedBatchNorm1d
  from nemo.collections.asr.parts.submodules.causal_convs import CausalConv1D
  from nemo.collections.asr.parts.submodules.multi_head_attention import MultiHeadAttention, RelPositionMultiHeadAttention, RelPositionMultiHeadAttentionLongformer
  from nemo.collections.asr.parts.utils.activations import Swish
  from nemo.collections.common.parts import adapter_modules
  from nemo.collections.common.parts.utils import activation_registry
  from nemo.core.classes.mixins import AccessMixin
  from nemo.core.classes.mixins.adapter_mixins import AdapterModuleMixin
label_desc: Use image node_id 1 to create a new ConformerLayer object from inherited base classes: torch.nn, AdapterModuleMixin, AccessMixin with example: obj = ConformerLayer(d_model, d_ff, _attention_model, global_tokens, global_tokens_spacing, global_attn_separate, n_heads, conv_kernel_size, conv_norm_type, conv_context_size, dropout, dropout_att, pos_bias_u, pos_bias_v, att_context_size)