class_docstr: This is an implementation of Branchformer encoder layer.
  
  Arguments
  ----------
  d_model : int
      The expected size of the input embedding.
  nhead : int
      Number of attention heads.
  kernel_size : int, optional
      Kernel size of convolution model.
  kdim : int, optional
      Dimension of the key.
  vdim : int, optional
      Dimension of the value.
  activation: torch.nn.Module
       Activation function used in each Conformer layer.
  dropout : int, optional
      Dropout for the encoder.
  attention_type: str, optional
      type of attention layer, e.g. regulaMHA for regular MultiHeadAttention.
  csgu_linear_units: int, optional
      Number of neurons in the hidden linear units of the CSGU Module.
  gate_activation: torch.nn.Module, optional
       Activation function used at the gate of the CSGU module.
  use_linear_after_conv: bool, optional
      If True, will apply a linear transformation of size input_size//2
  
  Example
  -------
  >>> import torch
  >>> x = torch.rand((8, 60, 512))
  >>> pos_embs = torch.rand((1, 2*60-1, 512))
  >>> net = BranchformerEncoderLayer(nhead=8, d_model=512, kernel_size=3)
  >>> output = net(x, pos_embs=pos_embs)
  >>> output[0].shape
  torch.Size([8, 60, 512])
class_docstr_tok: ["This","is","an","implementation","of","Branchformer","encoder","layer",".","Arguments","--","--","--","--","--","d_model",":","int","The","expected","size","of","the","input","embedding",".","nhead",":","int","Number","of","attention","heads",".","kernel_size",":","int",",","optional","Kernel","size","of","convolution","model",".","kdim",":","int",",","optional","Dimension","of","the","key",".","vdim",":","int",",","optional","Dimension","of","the","value",".","activation",":","torch.nn.Module","Activation","function","used","in","each","Conformer","layer",".","dropout",":","int",",","optional","Dropout","for","the","encoder",".","attention_type",":","str",",","optional","type","of","attention","layer",",","e.g",".","regulaMHA","for","regular","MultiHeadAttention",".","csgu_linear_units",":","int",",","optional","Number","of","neurons","in","the","hidden","linear","units","of","the","CSGU","Module",".","gate_activation",":","torch.nn.Module",",","optional","Activation","function","used","at","the","gate","of","the","CSGU","module",".","use_linear_after_conv",":","bool",",","optional","If","True",",","will","apply","a","linear","transformation","of","size","input_size\/\/2","Example","--","--","--","-",">",">",">","import","torch",">",">",">","x","=","torch.rand","(","(","8",",","60",",","512",")",")",">",">",">","pos_embs","=","torch.rand","(","(","1",",","2","*","60-1",",","512",")",")",">",">",">","net","=","BranchformerEncoderLayer","(","nhead=8",",","d_model=512",",","kernel_size=3",")",">",">",">","output","=","net","(","x",",","pos_embs=pos_embs",")",">",">",">","output","[","0","]",".shape","torch.Size","(","[","8",",","60",",","512","]",")"]
code: def __init__(
      self,
      d_model,
      nhead,
      kernel_size=31,
      kdim=None,
      vdim=None,
      activation=nn.GELU,
      dropout=0.0,
      attention_type="RelPosMHAXL",
      csgu_linear_units=3072,
      gate_activation=nn.Identity,
      use_linear_after_conv=False,
  ):
      super().__init__()
  
      if attention_type == "regularMHA":
          self.mha_layer = MultiheadAttention(
              nhead=nhead,
              d_model=d_model,
              dropout=dropout,
              kdim=kdim,
              vdim=vdim,
          )
      elif attention_type == "RelPosMHAXL":
          # transformerXL style positional encoding
          self.mha_layer = RelPosMHAXL(
              num_heads=nhead,
              embed_dim=d_model,
              dropout=dropout,
              mask_pos_future=False,
          )
      elif attention_type == "hypermixing":
          self.mha_layer = HyperMixing(
              input_output_dim=d_model,
              hypernet_size=d_model * 4,
              tied=False,
              num_heads=nhead,
              fix_tm_hidden_size=False,
          )
  
      self.convolution_branch = ConvolutionBranch(
          input_size=d_model,
          kernel_size=kernel_size,
          linear_units=csgu_linear_units,
          activation=activation,
          gate_activation=gate_activation,
          dropout=dropout,
          use_linear_after_conv=use_linear_after_conv,
      )
  
      self.merge_proj = torch.nn.Linear(d_model * 2, d_model)
  
      self.norm_mhsa = LayerNorm(d_model)
      self.norm_conv = LayerNorm(d_model)
      self.dropout = nn.Dropout(dropout)
  
file_path: speechbrain/speechbrain/lobes/models/transformer/Branchformer.py
imports: import torch
  import torch.nn
  from typing import Optional
  from speechbrain.nnet.attention import RelPosMHAXL, MultiheadAttention
  from speechbrain.nnet.normalization import LayerNorm
  from speechbrain.lobes.models.convolution import ConvolutionalSpatialGatingUnit
  from speechbrain.lobes.models.transformer.hypermixing import HyperMixing
label_desc: Use image node_id 1 to create a new BranchformerEncoderLayer object from inherited base classes: nn with example: obj = BranchformerEncoderLayer(d_model, nhead, kernel_size, kdim, vdim, activation, dropout, attention_type, csgu_linear_units, gate_activation, use_linear_after_conv)