class_docstr: Generator module in VISinger.
class_docstr_tok: ["Generator","module","in","VISinger","."]
docstr: Calculate forward propagation.
  
  Args:
      text (LongTensor): Batch of padded character ids (B, Tmax).
      text_lengths (LongTensor): Batch of lengths of each input batch (B,).
      feats (Tensor): Batch of padded target features (B, Lmax, odim).
      feats_lengths (LongTensor): Batch of the lengths of each target (B,).
      label (LongTensor): Batch of padded label ids (B, Tmax).
      label_lengths (LongTensor): Batch of the lengths of padded label ids (B, ).
      melody (LongTensor): Batch of padded midi (B, Tmax).
      gt_dur (LongTensor): Batch of padded ground truth duration (B, Tmax).
      score_dur (LongTensor): Batch of padded score duration (B, Tmax).
      pitch (FloatTensor): Batch of padded f0 (B, Tmax).
      ying (Optional[Tensor]): Batch of padded ying (B, Tmax).
      spembs (Optional[Tensor]): Batch of speaker embeddings (B, spk_embed_dim).
      sids (Optional[Tensor]): Batch of speaker IDs (B, 1).
      lids (Optional[Tensor]): Batch of language IDs (B, 1).
  
  Returns:
      Tensor: Waveform tensor (B, 1, segment_size * upsample_factor).
      Tensor: Duration negative log-likelihood (NLL) tensor (B,).
      Tensor: Monotonic attention weight tensor (B, 1, T_feats, T_text).
      Tensor: Segments start index tensor (B,).
      Tensor: Text mask tensor (B, 1, T_text).
      Tensor: Feature mask tensor (B, 1, T_feats).
      tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]:
          - Tensor: Posterior encoder hidden representation (B, H, T_feats).
          - Tensor: Flow hidden representation (B, H, T_feats).
          - Tensor: Expanded text encoder projected mean (B, H, T_feats).
          - Tensor: Expanded text encoder projected scale (B, H, T_feats).
          - Tensor: Posterior encoder projected mean (B, H, T_feats).
          - Tensor: Posterior encoder projected scale (B, H, T_feats).
docstr_tok: ["Calculate","forward","propagation",".","Args",":","text","(","LongTensor",")",":","Batch","of","padded","character","ids","(","B",",","Tmax",")",".","text_lengths","(","LongTensor",")",":","Batch","of","lengths","of","each","input","batch","(","B",",",")",".","feats","(","Tensor",")",":","Batch","of","padded","target","features","(","B",",","Lmax",",","odim",")",".","feats_lengths","(","LongTensor",")",":","Batch","of","the","lengths","of","each","target","(","B",",",")",".","label","(","LongTensor",")",":","Batch","of","padded","label","ids","(","B",",","Tmax",")",".","label_lengths","(","LongTensor",")",":","Batch","of","the","lengths","of","padded","label","ids","(","B",",",")",".","melody","(","LongTensor",")",":","Batch","of","padded","midi","(","B",",","Tmax",")",".","gt_dur","(","LongTensor",")",":","Batch","of","padded","ground","truth","duration","(","B",",","Tmax",")",".","score_dur","(","LongTensor",")",":","Batch","of","padded","score","duration","(","B",",","Tmax",")",".","pitch","(","FloatTensor",")",":","Batch","of","padded","f0","(","B",",","Tmax",")",".","ying","(","Optional","[","Tensor","]",")",":","Batch","of","padded","ying","(","B",",","Tmax",")",".","spembs","(","Optional","[","Tensor","]",")",":","Batch","of","speaker","embeddings","(","B",",","spk_embed_dim",")",".","sids","(","Optional","[","Tensor","]",")",":","Batch","of","speaker","IDs","(","B",",","1",")",".","lids","(","Optional","[","Tensor","]",")",":","Batch","of","language","IDs","(","B",",","1",")",".","Returns",":","Tensor",":","Waveform","tensor","(","B",",","1",",","segment_size","*","upsample_factor",")",".","Tensor",":","Duration","negative","log-likelihood","(","NLL",")","tensor","(","B",",",")",".","Tensor",":","Monotonic","attention","weight","tensor","(","B",",","1",",","T_feats",",","T_text",")",".","Tensor",":","Segments","start","index","tensor","(","B",",",")",".","Tensor",":","Text","mask","tensor","(","B",",","1",",","T_text",")",".","Tensor",":","Feature","mask","tensor","(","B",",","1",",","T_feats",")",".","tuple","[","Tensor",",","Tensor",",","Tensor",",","Tensor",",","Tensor",",","Tensor","]",":","-","Tensor",":","Posterior","encoder","hidden","representation","(","B",",","H",",","T_feats",")",".","-","Tensor",":","Flow","hidden","representation","(","B",",","H",",","T_feats",")",".","-","Tensor",":","Expanded","text","encoder","projected","mean","(","B",",","H",",","T_feats",")",".","-","Tensor",":","Expanded","text","encoder","projected","scale","(","B",",","H",",","T_feats",")",".","-","Tensor",":","Posterior","encoder","projected","mean","(","B",",","H",",","T_feats",")",".","-","Tensor",":","Posterior","encoder","projected","scale","(","B",",","H",",","T_feats",")","."]
code: def forward(
      self,
      text: torch.Tensor,
      text_lengths: torch.Tensor,
      feats: torch.Tensor,
      feats_lengths: torch.Tensor,
      label: torch.Tensor = None,
      label_lengths: torch.Tensor = None,
      melody: torch.Tensor = None,
      gt_dur: torch.Tensor = None,
      score_dur: torch.Tensor = None,
      slur: torch.Tensor = None,
      pitch: torch.Tensor = None,
      ying: Optional[torch.Tensor] = None,
      sids: Optional[torch.Tensor] = None,
      spembs: Optional[torch.Tensor] = None,
      lids: Optional[torch.Tensor] = None,
  ) -> Tuple[
      torch.Tensor,
      torch.Tensor,
      torch.Tensor,
      torch.Tensor,
      torch.Tensor,
      torch.Tensor,
      Tuple[
          torch.Tensor,
          torch.Tensor,
          torch.Tensor,
          torch.Tensor,
          torch.Tensor,
          torch.Tensor,
      ],
  ]:
      """Calculate forward propagation.
  
      Args:
          text (LongTensor): Batch of padded character ids (B, Tmax).
          text_lengths (LongTensor): Batch of lengths of each input batch (B,).
          feats (Tensor): Batch of padded target features (B, Lmax, odim).
          feats_lengths (LongTensor): Batch of the lengths of each target (B,).
          label (LongTensor): Batch of padded label ids (B, Tmax).
          label_lengths (LongTensor): Batch of the lengths of padded label ids (B, ).
          melody (LongTensor): Batch of padded midi (B, Tmax).
          gt_dur (LongTensor): Batch of padded ground truth duration (B, Tmax).
          score_dur (LongTensor): Batch of padded score duration (B, Tmax).
          pitch (FloatTensor): Batch of padded f0 (B, Tmax).
          ying (Optional[Tensor]): Batch of padded ying (B, Tmax).
          spembs (Optional[Tensor]): Batch of speaker embeddings (B, spk_embed_dim).
          sids (Optional[Tensor]): Batch of speaker IDs (B, 1).
          lids (Optional[Tensor]): Batch of language IDs (B, 1).
  
      Returns:
          Tensor: Waveform tensor (B, 1, segment_size * upsample_factor).
          Tensor: Duration negative log-likelihood (NLL) tensor (B,).
          Tensor: Monotonic attention weight tensor (B, 1, T_feats, T_text).
          Tensor: Segments start index tensor (B,).
          Tensor: Text mask tensor (B, 1, T_text).
          Tensor: Feature mask tensor (B, 1, T_feats).
          tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]:
              - Tensor: Posterior encoder hidden representation (B, H, T_feats).
              - Tensor: Flow hidden representation (B, H, T_feats).
              - Tensor: Expanded text encoder projected mean (B, H, T_feats).
              - Tensor: Expanded text encoder projected scale (B, H, T_feats).
              - Tensor: Posterior encoder projected mean (B, H, T_feats).
              - Tensor: Posterior encoder projected scale (B, H, T_feats).
  
      """
      # calculate global conditioning
      g = None
      if self.spks is not None:
          # speaker one-hot vector embedding: (B, global_channels, 1)
          g = self.global_emb(sids.view(-1)).unsqueeze(-1)
      if self.spk_embed_dim is not None:
          # pretreined speaker embedding, e.g., X-vector (B, global_channels, 1)
          g_ = self.spemb_proj(F.normalize(spembs)).unsqueeze(-1)
          if g is None:
              g = g_
          else:
              g = g + g_
      if self.langs is not None:
          # language one-hot vector embedding: (B, global_channels, 1)
          g_ = self.lang_emb(lids.view(-1)).unsqueeze(-1)
          if g is None:
              g = g_
          else:
              g = g + g_
  
      # forward text encoder
  
      # Encoder
      x, x_mask, dur_input, x_pitch = self.text_encoder(
          label, label_lengths, melody, score_dur, slur
      )
  
      # dur
      # Note this is different, we use frame level duration not time level
      # but it has no big difference on performance
      predict_dur = self.duration_predictor(dur_input, x_mask, g=g)
      predict_dur = (torch.exp(predict_dur) - 1) * x_mask
      predict_dur = predict_dur * self.sample_rate / self.hop_length
  
      # LR
      decoder_input, mel_len = self.lr(x, gt_dur, use_state_info=True)
      decoder_input_pitch, mel_len = self.lr(
          x_pitch, gt_dur, use_state_info=True
      )
  
      LF0 = 2595.0 * torch.log10(1.0 + pitch / 700.0)
      LF0 = LF0 / 500
      LF0 = LF0.transpose(1, 2)
  
      predict_lf0, predict_bn_mask = self.f0_decoder(
          decoder_input + decoder_input_pitch, feats_lengths, g=g
      )
      predict_lf0 = torch.max(
          predict_lf0, torch.zeros_like(predict_lf0).to(predict_lf0)
      )
  
      if self.generator_type == "visinger2":
          predict_mel, predict_bn_mask = self.mel_decoder(
              decoder_input + self.f0_prenet(LF0), feats_lengths, g=g
          )
  
          predict_energy = (
              predict_mel.detach().sum(1).unsqueeze(1)
              / self.aux_channels
          )
  
      decoder_input = decoder_input + self.f0_prenet(LF0)
      if self.generator_type == "visinger2":
          decoder_input = (
              decoder_input
              + self.energy_prenet(predict_energy)
              + self.mel_prenet(predict_mel.detach())
          )
      decoder_output, predict_bn_mask = self.prior_decoder(
          decoder_input, feats_lengths, g=g
      )
  
      prior_info = decoder_output
      prior_mean = prior_info[:, : self.hidden_channels, :]
      prior_logstd = prior_info[:, self.hidden_channels :, :]
  
      # forward posterior encoder
      (
          posterior_z,
          posterior_mean,
          posterior_logstd,
          y_mask,
      ) = self.posterior_encoder(feats, feats_lengths, g=g)
  
      if self.use_flow:
          z_flow = self.flow(posterior_z, y_mask, g=g)
      else:
          z_flow = None
  
      # phoneme predictor
      if self.use_phoneme_predictor:
          log_probs = self.phoneme_predictor(posterior_z, y_mask)
      else:
          log_probs = None
  
      p_z = posterior_z
      p_z = self.dropout(p_z)
  
      # get random segments
      z_segments, z_start_idxs = get_random_segments(
          p_z, feats_lengths, self.segment_size
      )
  
      if self.vocoder_generator_type == "uhifigan":
          # get sine wave
  
          # def plot_sine_waves(sine_waves, name):
          #     import matplotlib.pyplot as plt
  
          #     sine_waves_np = sine_waves[0].detach().cpu().numpy()
          #     plt.plot(sine_waves_np)
          #     plt.xlabel("Time (samples)")
          #     plt.ylabel("Amplitude")
          #     plt.title("Sine Wave")
          #     plt.savefig(name + ".png")
          #     plt.close()
  
          # plot_sine_waves(pitch_segments[0], "pitch_segments")
  
          pitch_segments = get_segments(
              pitch.transpose(1, 2), z_start_idxs, self.segment_size
          )
          pitch_segments_expended = expand_f0(
              pitch_segments, self.hop_length, method="repeat"
          )
  
          # plot_sine_waves(
          #     pitch_segments_expended[0].unsqueeze(0), "pitch_segments_expended"
          # )
          pitch_segments_expended = pitch_segments_expended.reshape(
              -1, pitch_segments_expended.shape[-1], 1
          )
  
          sine_waves, uv, noise = self.sine_generator(
              pitch_segments_expended
          )
  
          sine_waves = sine_waves.transpose(1, 2)
  
          wav = self.decoder(z_segments, excitation=sine_waves, g=g)
      elif self.vocoder_generator_type == "visinger2":
          pitch_ = upsample(pitch, self.hop_length)
          omega = torch.cumsum(
              2 * math.pi * pitch_ / self.sample_rate, 1
          )
          sin = torch.sin(omega).transpose(1, 2)
  
          # dsp synthesize
          pitch = pitch.transpose(1, 2)
          noise_x = self.dec_noise(posterior_z, y_mask)
          harm_x = self.dec_harm(pitch, posterior_z, y_mask)
  
          # dsp waveform
          dsp_o = torch.cat([harm_x, noise_x], axis=1)
  
          # decoder_condition = torch.cat([harm_x, noise_x, sin], axis=1)
          decoder_condition = self.sin_prenet(sin)
  
          # dsp based HiFiGAN vocoder
          F0_slice = get_segments(
              pitch, z_start_idxs, self.segment_size
          )
          dsp_slice = get_segments(
              dsp_o,
              z_start_idxs * self.hop_length,
              self.segment_size * self.hop_length,
          )
  
          condition_slice = get_segments(
              decoder_condition,
              z_start_idxs * self.hop_length,
              self.segment_size * self.hop_length,
          )
          wav = self.decoder(z_segments, condition_slice, g=g)
      else:
          wav = self.decoder(z_segments, g=g)
  
          # wav = dsp_slice.sum(1, keepdim=True)
  
      common_tuple = (
          posterior_z,
          z_flow,
          prior_mean,
          prior_logstd,
          posterior_mean,
          posterior_logstd,
          predict_lf0,
          LF0 * predict_bn_mask,
          predict_dur,
          gt_dur,
          log_probs,
      )
  
      output = (wav, z_start_idxs, x_mask, y_mask, common_tuple)
  
      if self.vocoder_generator_type == "visinger2":
          output = output + (dsp_slice.sum(1),)
      if self.generator_type == "visinger2":
          output = output + (predict_mel,)
      return output
  
imports: import math
  from typing import List, Optional, Tuple
  import numpy
  import torch
  import torch.nn.functional
  from espnet2.gan_svs.avocodo import AvocodoGenerator
  from espnet2.gan_svs.uhifigan import UHiFiGANGenerator
  from espnet2.gan_svs.uhifigan.sine_generator import SineGen
  from espnet2.gan_svs.utils.expand_f0 import expand_f0
  from espnet2.gan_svs.visinger2 import Generator_Harm, Generator_Noise, VISinger2VocoderGenerator
  from espnet2.gan_svs.visinger2.ddsp import upsample
  from espnet2.gan_svs.vits.duration_predictor import DurationPredictor
  from espnet2.gan_svs.vits.length_regulator import LengthRegulator
  from espnet2.gan_svs.vits.phoneme_predictor import PhonemePredictor
  from espnet2.gan_svs.vits.pitch_predictor import Decoder
  from espnet2.gan_svs.vits.prior_decoder import PriorDecoder
  from espnet2.gan_svs.vits.text_encoder import TextEncoder
  from espnet2.gan_tts.hifigan import HiFiGANGenerator
  from espnet2.gan_tts.utils import get_random_segments, get_segments
  from espnet2.gan_tts.vits.posterior_encoder import PosteriorEncoder
  from espnet2.gan_tts.vits.residual_coupling import ResidualAffineCouplingBlock
label_desc: Use image node_id 2 for calling the VISingerGenerator obj's underlying member method code with example usage: obj.forward(text, text_lengths, feats, feats_lengths, label, label_lengths, melody, gt_dur, score_dur, slur, pitch, ying, sids, spembs, lids) and returns: output