docstr: Pad sequences and convert them to numpy.
docstr_tok: ["Pad","sequences","and","convert","them","to","numpy","."]
code: def pad_and_convert_to_numpy(
      cls,
      output_tokens,
      masked_positions,
      masked_labels,
      sentinel_tokens,
      bos_id,
      eos_id,
      pad_id,
      max_seq_length,
      max_seq_length_dec,
      masked_spans=None,
  ):
      """Pad sequences and convert them to numpy."""
      sentinel_tokens = collections.deque(sentinel_tokens)
      t5_input = []
      (t5_decoder_in, t5_decoder_out) = ([bos_id], [])
      (start_index, end_index) = (0, None)
      if masked_spans is not None:
          for span in masked_spans:
              flag = sentinel_tokens.popleft()
  
              # Append the same tokens in decoder input and output
              t5_decoder_in.append(flag)
              t5_decoder_in.extend(span.label)
              t5_decoder_out.append(flag)
              t5_decoder_out.extend(span.label)
  
              end_index = span.index[0]
              t5_input.extend(output_tokens[start_index:end_index])
              t5_input.append(flag)
  
              # the next start index is the token after the last span token
              start_index = span.index[-1] + 1
  
      # Add <eos> token to the t5_decoder_out
      t5_decoder_out.append(eos_id)
  
      # Add the remaining tokens to the t5 input
      t5_input.extend(output_tokens[start_index:])
  
      # assert (len(t5_input) - len(masked_spans)) + \
      #        (len(t5_decoder_in) - (len(masked_spans) + 1)) == len(tokens)
  
      # Some checks.
  
      # Encoder-side padding mask.
      num_tokens = len(t5_input)
      padding_length = max_seq_length - num_tokens
      assert padding_length >= 0, padding_length
      assert len(masked_positions) == len(masked_labels)
  
      # Tokens..
      filler = [pad_id] * padding_length
      tokens_enc = np.array(t5_input + filler, dtype=np.int64)
  
      # Decoder-side padding mask.
      num_tokens_dec = len(t5_decoder_in)
      padding_length_dec = max_seq_length_dec - num_tokens_dec
      assert padding_length_dec >= 0, (
          padding_length_dec,
          max_seq_length_dec,
          num_tokens_dec,
      )
      filler_dec = [pad_id] * padding_length_dec
      tokens_dec_in = np.array(
          t5_decoder_in + filler_dec, dtype=np.int64
      )
  
      # Create attention masks
      enc_mask = (tokens_enc != pad_id).astype(np.int64)
      dec_mask = (tokens_dec_in != pad_id).astype(np.int64)
  
      # Labels mask.
      labels = t5_decoder_out + ([pad_id] * padding_length_dec)
      labels = np.array(labels, dtype=np.int64)
  
      # Loss mask
      loss_mask = ([1] * num_tokens_dec) + ([0] * padding_length_dec)
      loss_mask = np.array(loss_mask, dtype=np.int64)
  
      return (
          tokens_enc,
          tokens_dec_in,
          labels,
          enc_mask,
          dec_mask,
          loss_mask,
      )
  
imports: import collections
  import os
  import numpy
  import torch
  from nemo.collections.common.tokenizers import SentencePieceTokenizer, YouTokenToMeTokenizer
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import create_masked_lm_predictions, get_samples_mapping
  from nemo.collections.nlp.data.language_modeling.megatron.gpt_dataset import _build_index_mappings
  from nemo.core import Dataset
label_desc: Use image node_id 8 for calling the T5Dataset obj's underlying member method code with example usage: obj.pad_and_convert_to_numpy(cls, output_tokens, masked_positions, masked_labels, sentinel_tokens, bos_id, eos_id, pad_id, max_seq_length, max_seq_length_dec, masked_spans) and returns: tokens_enc,  tokens_dec_in,  labels,  enc_mask,  dec_mask,  loss_mask