class_docstr: This is an implementation of transformer model for ST.
  
  The architecture is based on the paper "Attention Is All You Need":
  https://arxiv.org/pdf/1706.03762.pdf
  
  Arguments
  ----------
  tgt_vocab: int
      Size of vocabulary.
  input_size: int
      Input feature size.
  d_model : int, optional
      Embedding dimension size.
      (default=512).
  nhead : int, optional
      The number of heads in the multi-head attention models (default=8).
  num_encoder_layers : int, optional
      The number of sub-encoder-layers in the encoder (default=6).
  num_decoder_layers : int, optional
      The number of sub-decoder-layers in the decoder (default=6).
  dim_ffn : int, optional
      The dimension of the feedforward network model (default=2048).
  dropout : int, optional
      The dropout value (default=0.1).
  activation : torch.nn.Module, optional
      The activation function of FFN layers.
      Recommended: relu or gelu (default=relu).
  positional_encoding: str, optional
      Type of positional encoding used. e.g. 'fixed_abs_sine' for fixed absolute positional encodings.
  normalize_before: bool, optional
      Whether normalization should be applied before or after MHA or FFN in Transformer layers.
      Defaults to True as this was shown to lead to better performance and training stability.
  kernel_size: int, optional
      Kernel size in convolutional layers when Conformer is used.
  bias: bool, optional
      Whether to use bias in Conformer convolutional layers.
  encoder_module: str, optional
      Choose between Conformer and Transformer for the encoder. The decoder is fixed to be a Transformer.
  conformer_activation: torch.nn.Module, optional
      Activation module used after Conformer convolutional layers. E.g. Swish, ReLU etc. it has to be a torch Module.
  attention_type: str, optional
      Type of attention layer used in all Transformer or Conformer layers.
      e.g. regularMHA or RelPosMHA.
  max_length: int, optional
      Max length for the target and source sequence in input.
      Used for positional encodings.
  causal: bool, optional
      Whether the encoder should be causal or not (the decoder is always causal).
      If causal the Conformer convolutional layer is causal.
  ctc_weight: float
      The weight of ctc for asr task
  asr_weight: float
      The weight of asr task for calculating loss
  mt_weight: float
      The weight of mt task for calculating loss
  asr_tgt_vocab: int
      The size of the asr target language
  mt_src_vocab: int
      The size of the mt source language
  Example
  -------
  >>> src = torch.rand([8, 120, 512])
  >>> tgt = torch.randint(0, 720, [8, 120])
  >>> net = TransformerST(
  ...     720, 512, 512, 8, 1, 1, 1024, activation=torch.nn.GELU,
  ...     ctc_weight=1, asr_weight=0.3,
  ... )
  >>> enc_out, dec_out = net.forward(src, tgt)
  >>> enc_out.shape
  torch.Size([8, 120, 512])
  >>> dec_out.shape
  torch.Size([8, 120, 512])
class_docstr_tok: ["This","is","an","implementation","of","transformer","model","for","ST",".","The","architecture","is","based","on","the","paper","``","Attention","Is","All","You","Need","''",":","https",":","\/\/arxiv.org\/pdf\/1706.03762.pdf","Arguments","--","--","--","--","--","tgt_vocab",":","int","Size","of","vocabulary",".","input_size",":","int","Input","feature","size",".","d_model",":","int",",","optional","Embedding","dimension","size",".","(","default=512",")",".","nhead",":","int",",","optional","The","number","of","heads","in","the","multi-head","attention","models","(","default=8",")",".","num_encoder_layers",":","int",",","optional","The","number","of","sub-encoder-layers","in","the","encoder","(","default=6",")",".","num_decoder_layers",":","int",",","optional","The","number","of","sub-decoder-layers","in","the","decoder","(","default=6",")",".","dim_ffn",":","int",",","optional","The","dimension","of","the","feedforward","network","model","(","default=2048",")",".","dropout",":","int",",","optional","The","dropout","value","(","default=0.1",")",".","activation",":","torch.nn.Module",",","optional","The","activation","function","of","FFN","layers",".","Recommended",":","relu","or","gelu","(","default=relu",")",".","positional_encoding",":","str",",","optional","Type","of","positional","encoding","used",".","e.g",".","'fixed_abs_sine","'","for","fixed","absolute","positional","encodings",".","normalize_before",":","bool",",","optional","Whether","normalization","should","be","applied","before","or","after","MHA","or","FFN","in","Transformer","layers",".","Defaults","to","True","as","this","was","shown","to","lead","to","better","performance","and","training","stability",".","kernel_size",":","int",",","optional","Kernel","size","in","convolutional","layers","when","Conformer","is","used",".","bias",":","bool",",","optional","Whether","to","use","bias","in","Conformer","convolutional","layers",".","encoder_module",":","str",",","optional","Choose","between","Conformer","and","Transformer","for","the","encoder",".","The","decoder","is","fixed","to","be","a","Transformer",".","conformer_activation",":","torch.nn.Module",",","optional","Activation","module","used","after","Conformer","convolutional","layers",".","E.g",".","Swish",",","ReLU","etc",".","it","has","to","be","a","torch","Module",".","attention_type",":","str",",","optional","Type","of","attention","layer","used","in","all","Transformer","or","Conformer","layers",".","e.g",".","regularMHA","or","RelPosMHA",".","max_length",":","int",",","optional","Max","length","for","the","target","and","source","sequence","in","input",".","Used","for","positional","encodings",".","causal",":","bool",",","optional","Whether","the","encoder","should","be","causal","or","not","(","the","decoder","is","always","causal",")",".","If","causal","the","Conformer","convolutional","layer","is","causal",".","ctc_weight",":","float","The","weight","of","ctc","for","asr","task","asr_weight",":","float","The","weight","of","asr","task","for","calculating","loss","mt_weight",":","float","The","weight","of","mt","task","for","calculating","loss","asr_tgt_vocab",":","int","The","size","of","the","asr","target","language","mt_src_vocab",":","int","The","size","of","the","mt","source","language","Example","--","--","--","-",">",">",">","src","=","torch.rand","(","[","8",",","120",",","512","]",")",">",">",">","tgt","=","torch.randint","(","0",",","720",",","[","8",",","120","]",")",">",">",">","net","=","TransformerST","(","...","720",",","512",",","512",",","8",",","1",",","1",",","1024",",","activation=torch.nn.GELU",",","...","ctc_weight=1",",","asr_weight=0.3",",","...",")",">",">",">","enc_out",",","dec_out","=","net.forward","(","src",",","tgt",")",">",">",">","enc_out.shape","torch.Size","(","[","8",",","120",",","512","]",")",">",">",">","dec_out.shape","torch.Size","(","[","8",",","120",",","512","]",")"]
docstr: This method implements a decoding step for asr task
  
  Arguments
  ----------
  encoder_out : tensor
      The representation of the encoder (required).
  tgt (transcription): tensor
      The sequence to the decoder (required).
  pad_idx : int
      The index for <pad> token (default=0).
docstr_tok: ["This","method","implements","a","decoding","step","for","asr","task","Arguments","--","--","--","--","--","encoder_out",":","tensor","The","representation","of","the","encoder","(","required",")",".","tgt","(","transcription",")",":","tensor","The","sequence","to","the","decoder","(","required",")",".","pad_idx",":","int","The","index","for","<","pad",">","token","(","default=0",")","."]
code: def forward_asr(self, encoder_out, src, tgt, wav_len, pad_idx=0):
      """This method implements a decoding step for asr task
  
      Arguments
      ----------
      encoder_out : tensor
          The representation of the encoder (required).
      tgt (transcription): tensor
          The sequence to the decoder (required).
      pad_idx : int
          The index for <pad> token (default=0).
      """
      # reshpae the src vector to [Batch, Time, Fea] is a 4d vector is given
      if src.dim() == 4:
          bz, t, ch1, ch2 = src.shape
          src = src.reshape(bz, t, ch1 * ch2)
  
      (
          src_key_padding_mask,
          tgt_key_padding_mask,
          src_mask,
          tgt_mask,
      ) = self.make_masks(src, tgt, wav_len, pad_idx=pad_idx)
  
      transcription = self.custom_asr_tgt_module(tgt)
  
      if self.attention_type == "RelPosMHAXL":
          transcription = (
              transcription
              + self.positional_encoding_decoder(transcription)
          )
      elif self.attention_type == "fixed_abs_sine":
          transcription = transcription + self.positional_encoding(
              transcription
          )
  
      asr_decoder_out, _, _ = self.asr_decoder(
          tgt=transcription,
          memory=encoder_out,
          memory_mask=src_mask,
          tgt_mask=tgt_mask,
          tgt_key_padding_mask=tgt_key_padding_mask,
          memory_key_padding_mask=src_key_padding_mask,
      )
  
      return asr_decoder_out
  
file_path: speechbrain/speechbrain/lobes/models/transformer/TransformerST.py
imports: import torch
  import logging
  from torch import nn
  from typing import Optional
  from speechbrain.nnet.containers import ModuleList
  from speechbrain.lobes.models.transformer.Transformer import get_lookahead_mask, get_key_padding_mask, NormalizedEmbedding, TransformerDecoder, TransformerEncoder
  from speechbrain.lobes.models.transformer.Conformer import ConformerEncoder
  from speechbrain.lobes.models.transformer.TransformerASR import TransformerASR
  from speechbrain.nnet.activations import Swish
label_desc: Use image node_id 2 for calling the TransformerST obj's underlying member method code with example usage: obj.forward_asr(encoder_out, src, tgt, wav_len, pad_idx) and returns: asr_decoder_out