class_docstr: CTC-attention hybrid Encoder-Decoder model
docstr: Frontend + Encoder + Decoder + Calc loss
  
  Args:
      speech: (Batch, Length, ...)
      speech_lengths: (Batch,)
      text: (Batch, Length)
      text_lengths: (Batch,)
      src_text: (Batch, length)
      src_text_lengths: (Batch,)
      kwargs: "utt_id" is among the input.
docstr_tok: ["Frontend","+","Encoder","+","Decoder","+","Calc","loss","Args",":","speech",":","(","Batch",",","Length",",","...",")","speech_lengths",":","(","Batch",",",")","text",":","(","Batch",",","Length",")","text_lengths",":","(","Batch",",",")","src_text",":","(","Batch",",","length",")","src_text_lengths",":","(","Batch",",",")","kwargs",":","``","utt_id","''","is","among","the","input","."]
code: def forward(
      self,
      speech: torch.Tensor,
      speech_lengths: torch.Tensor,
      text: torch.Tensor,
      text_lengths: torch.Tensor,
      src_text: Optional[torch.Tensor] = None,
      src_text_lengths: Optional[torch.Tensor] = None,
      **kwargs,
  ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
      """Frontend + Encoder + Decoder + Calc loss
  
      Args:
          speech: (Batch, Length, ...)
          speech_lengths: (Batch,)
          text: (Batch, Length)
          text_lengths: (Batch,)
          src_text: (Batch, length)
          src_text_lengths: (Batch,)
          kwargs: "utt_id" is among the input.
      """
      assert text_lengths.dim() == 1, text_lengths.shape
      # Check that batch_size is unified
      assert (
          speech.shape[0]
          == speech_lengths.shape[0]
          == text.shape[0]
          == text_lengths.shape[0]
      ), (
          speech.shape,
          speech_lengths.shape,
          text.shape,
          text_lengths.shape,
      )
  
      # additional checks with valid src_text
      if src_text is not None:
          assert src_text_lengths.dim() == 1, src_text_lengths.shape
          assert (
              text.shape[0]
              == src_text.shape[0]
              == src_text_lengths.shape[0]
          ), (
              text.shape,
              src_text.shape,
              src_text_lengths.shape,
          )
  
      batch_size = speech.shape[0]
  
      text[text == -1] = self.tgt_ignore_id
  
      # for data-parallel
      text = text[:, : text_lengths.max()]
      if src_text is not None:
          src_text = src_text[:, : src_text_lengths.max()]
  
      # lang id for mbart
      if (
          hasattr(self, "lang_token_id")
          and self.lang_token_id is not None
      ):
          text = torch.cat(
              [
                  self.lang_token_id.repeat(text.size(0), 1).to(
                      text.device
                  ),
                  text,
              ],
              dim=1,
          )
          text_lengths += 1
  
      # 1. Encoder
      if self.hier_encoder is not None or (
          self.postencoder is not None
          and self.postencoder.return_int_enc
      ):
          (
              st_encoder_out,
              st_encoder_out_lens,
              asr_encoder_out,
              asr_encoder_out_lens,
          ) = self.encode(speech, speech_lengths, return_int_enc=True)
      else:
          st_encoder_out, st_encoder_out_lens = self.encode(
              speech, speech_lengths
          )
          asr_encoder_out, asr_encoder_out_lens = (
              st_encoder_out,
              st_encoder_out_lens,
          )
  
      # 2a. CTC branch
      if self.asr_weight > 0:
          assert (
              src_text is not None
          ), "missing source text for asr sub-task of ST"
  
      if self.asr_weight > 0 and self.mtlalpha > 0:
          loss_asr_ctc, cer_asr_ctc = self._calc_asr_ctc_loss(
              asr_encoder_out,
              asr_encoder_out_lens,
              src_text,
              src_text_lengths,
          )
      else:
          loss_asr_ctc, cer_asr_ctc = 0.0, None
  
      if self.st_mtlalpha > 0:
          if (
              self.postencoder is not None
              and self.postencoder.return_int_enc
          ):
              # run ST CTC without post-encoder downsampling (no hier enc)
              loss_st_ctc, bleu_st_ctc = self._calc_mt_ctc_loss(
                  asr_encoder_out,
                  asr_encoder_out_lens,
                  text,
                  text_lengths,
              )
          else:
              loss_st_ctc, bleu_st_ctc = self._calc_mt_ctc_loss(
                  st_encoder_out,
                  st_encoder_out_lens,
                  text,
                  text_lengths,
              )
      else:
          loss_st_ctc, bleu_st_ctc = 0.0, None
  
      # 2b. Attention-decoder branch (extra ASR)
      if self.asr_weight > 0 and self.mtlalpha < 1.0:
          (
              loss_asr_att,
              acc_asr_att,
              cer_asr_att,
              wer_asr_att,
              hs_dec_asr,
          ) = self._calc_asr_att_loss(
              asr_encoder_out,
              asr_encoder_out_lens,
              src_text,
              src_text_lengths,
              self.use_multidecoder,
          )
      else:
          loss_asr_att, acc_asr_att, cer_asr_att, wer_asr_att = (
              0.0,
              None,
              None,
              None,
          )
  
      # 2c. Attention-decoder branch (extra MT)
      if self.mt_weight > 0:
          mt_encoder_out, mt_encoder_out_lens = self.extra_mt_encoder(
              src_text, src_text_lengths
          )
          loss_mt_att, acc_mt_att, bleu_mt_att = self._calc_mt_att_loss(
              mt_encoder_out,
              mt_encoder_out_lens,
              text,
              text_lengths,
              None,
              None,
              st=False,  # uses same decoder as ST
          )
          # loss_mt_att, acc_mt_att = self._calc_mt_att_loss(
          #     st_encoder_out, st_encoder_out_lens, text, text_lengths, st=False
          # )
      else:
          loss_mt_att, acc_mt_att = 0.0, None
  
      # 2d. Multi-Decoder encoder
      if self.use_speech_attn:
          speech_out = st_encoder_out
          speech_lens = st_encoder_out_lens
      else:
          speech_out = None
          speech_lens = None
  
      if self.use_multidecoder:
          dec_asr_lengths = src_text_lengths + 1
          st_encoder_out, st_encoder_out_lens, _ = self.md_encoder(
              hs_dec_asr, dec_asr_lengths
          )
  
      st_ctc_weight = self.st_mtlalpha
      if st_ctc_weight < 1.0:
          if self.st_use_transducer_decoder:
              # 2e. Transducer decoder branch
              (
                  loss_st_trans,
                  _,
                  _,
              ) = self._calc_st_transducer_loss(
                  st_encoder_out,
                  st_encoder_out_lens,
                  text,
              )
  
              if st_ctc_weight == 1.0:
                  loss_st = loss_st_ctc
              elif st_ctc_weight == 0.0:
                  loss_st = loss_st_trans
              else:
                  loss_st = (
                      st_ctc_weight * loss_st_ctc
                      + (1 - st_ctc_weight) * loss_st_trans
                  )
              loss_st_att = 0.0
              acc_st_att = None
              bleu_st_att = None
          else:
              # 2e. Attention-decoder branch (ST)
              (
                  loss_st_att,
                  acc_st_att,
                  bleu_st_att,
              ) = self._calc_mt_att_loss(
                  st_encoder_out,
                  st_encoder_out_lens,
                  text,
                  text_lengths,
                  speech_out,
                  speech_lens,
                  st=True,
              )
  
              if st_ctc_weight == 1.0:
                  loss_st = loss_st_ctc
              elif st_ctc_weight == 0.0:
                  loss_st = loss_st_att
              else:
                  loss_st = (
                      st_ctc_weight * loss_st_ctc
                      + (1 - st_ctc_weight) * loss_st_att
                  )
              loss_st_trans = 0.0
      else:
          loss_st = loss_st_ctc
          loss_st_att = 0.0
          acc_st_att = None
          bleu_st_att = None
          loss_st_trans = 0.0
  
      # 3. Loss computation
      asr_ctc_weight = self.mtlalpha
  
      if asr_ctc_weight == 1.0:
          loss_asr = loss_asr_ctc
      elif asr_ctc_weight == 0.0:
          loss_asr = loss_asr_att
      else:
          loss_asr = (
              asr_ctc_weight * loss_asr_ctc
              + (1 - asr_ctc_weight) * loss_asr_att
          )
      loss_mt = self.mt_weight * loss_mt_att
      loss = (
          (1 - self.asr_weight - self.mt_weight) * loss_st
          + self.asr_weight * loss_asr
          + self.mt_weight * loss_mt
      )
  
      stats = dict(
          loss=loss.detach(),
          loss_asr=loss_asr.detach()
          if type(loss_asr) is not float
          else loss_asr,
          loss_mt=loss_mt.detach()
          if type(loss_mt) is not float
          else loss_mt,
          loss_st_ctc=loss_st_ctc.detach()
          if type(loss_st_ctc) is not float
          else loss_st_ctc,
          loss_st_trans=loss_st_trans.detach()
          if type(loss_st_trans) is not float
          else loss_st_trans,
          loss_st_att=loss_st_att.detach()
          if type(loss_st_att) is not float
          else loss_st_att,
          loss_st=loss_st.detach(),
          acc_asr=acc_asr_att,
          acc_mt=acc_mt_att,
          acc=acc_st_att,
          cer_ctc=cer_asr_ctc,
          cer=cer_asr_att,
          wer=wer_asr_att,
          bleu=bleu_st_att,
          bleu_ctc=bleu_st_ctc,
      )
  
      # force_gatherable: to-device and to-tensor if scalar for DataParallel
      loss, stats, weight = force_gatherable(
          (loss, stats, batch_size), loss.device
      )
      return loss, stats, weight
  
imports: import logging
  import random
  from contextlib import contextmanager
  from itertools import groupby
  from typing import Dict, List, Optional, Tuple, Union
  import torch
  from packaging.version import parse
  from torch.nn.utils.rnn import pad_sequence
  from typeguard import check_argument_types
  from espnet2.asr.ctc import CTC
  from espnet2.asr.decoder.abs_decoder import AbsDecoder
  from espnet2.asr.encoder.abs_encoder import AbsEncoder
  from espnet2.asr.frontend.abs_frontend import AbsFrontend
  from espnet2.asr.postencoder.abs_postencoder import AbsPostEncoder
  from espnet2.asr.preencoder.abs_preencoder import AbsPreEncoder
  from espnet2.asr.specaug.abs_specaug import AbsSpecAug
  from espnet2.asr_transducer.utils import get_transducer_task_io
  from espnet2.layers.abs_normalize import AbsNormalize
  from espnet2.torch_utils.device_funcs import force_gatherable
  from espnet2.train.abs_espnet_model import AbsESPnetModel
  from espnet.nets.e2e_asr_common import ErrorCalculator
  from espnet.nets.e2e_mt_common import ErrorCalculator
  from espnet.nets.pytorch_backend.nets_utils import th_accuracy
  from espnet.nets.pytorch_backend.transformer.add_sos_eos import add_sos_eos
  from espnet.nets.pytorch_backend.transformer.label_smoothing_loss import LabelSmoothingLoss
label_desc: Use image node_id 2 for calling the ESPnetSTModel obj's underlying member method code with example usage: obj.forward(speech, speech_lengths, text, text_lengths, src_text, src_text_lengths) and returns: loss,  stats,  weight