class_docstr: Break a Dataset of tokens into sentence pair blocks for next sentence
     prediction as well as masked language model.
  
     High-level logics are:
     1. break input tensor to tensor blocks
     2. pair the blocks with 50% next sentence and 50% random sentence
     3. return paired blocks as well as related segment labels
  
  Args:
      dataset (~torch.utils.data.Dataset): dataset to break into blocks
      sizes: array of sentence lengths
      dictionary: dictionary for the task
      block_size: maximum block size
      break_mode: mode for breaking copurs into block pairs. currently we support
          2 modes
          doc: respect document boundaries and each part of the pair should belong to on document
          none: don't respect any boundary and cut tokens evenly
      short_seq_prob: probability for generating shorter block pairs
      doc_break_size: Size for empty line separating documents. Typically 1 if
                      the sentences have eos, 0 otherwise.
class_docstr_tok: ["Break","a","Dataset","of","tokens","into","sentence","pair","blocks","for","next","sentence","prediction","as","well","as","masked","language","model",".","High-level","logics","are",":","1.","break","input","tensor","to","tensor","blocks","2.","pair","the","blocks","with","50","%","next","sentence","and","50","%","random","sentence","3.","return","paired","blocks","as","well","as","related","segment","labels","Args",":","dataset","(","~torch.utils.data.Dataset",")",":","dataset","to","break","into","blocks","sizes",":","array","of","sentence","lengths","dictionary",":","dictionary","for","the","task","block_size",":","maximum","block","size","break_mode",":","mode","for","breaking","copurs","into","block","pairs",".","currently","we","support","2","modes","doc",":","respect","document","boundaries","and","each","part","of","the","pair","should","belong","to","on","document","none",":","do","n't","respect","any","boundary","and","cut","tokens","evenly","short_seq_prob",":","probability","for","generating","shorter","block","pairs","doc_break_size",":","Size","for","empty","line","separating","documents",".","Typically","1","if","the","sentences","have","eos",",","0","otherwise","."]
docstr: Trancate a pair of sentence to limit total length under max_num_tokens
  Logics:
      1. Truncate longer sentence
      2. Tokens to be truncated could be at the beginning or the end of the sentnce
  Returns:
      Truncated sentences represented by dataset idx
docstr_tok: ["Trancate","a","pair","of","sentence","to","limit","total","length","under","max_num_tokens","Logics",":","1",".","Truncate","longer","sentence","2",".","Tokens","to","be","truncated","could","be","at","the","beginning","or","the","end","of","the","sentnce","Returns",":","Truncated","sentences","represented","by","dataset","idx"]
code: def _truncate_sentences(self, sent_a, sent_b, max_num_tokens):
      """
      Trancate a pair of sentence to limit total length under max_num_tokens
      Logics:
          1. Truncate longer sentence
          2. Tokens to be truncated could be at the beginning or the end of the sentnce
      Returns:
          Truncated sentences represented by dataset idx
      """
      len_a, len_b = sum(self.dataset.sizes[sent_a]), sum(
          self.dataset.sizes[sent_b]
      )
      front_cut_a = front_cut_b = end_cut_a = end_cut_b = 0
  
      while True:
          total_length = (
              len_a
              + len_b
              - front_cut_a
              - front_cut_b
              - end_cut_a
              - end_cut_b
          )
          if total_length <= max_num_tokens:
              break
  
          if (
              len_a - front_cut_a - end_cut_a
              > len_b - front_cut_b - end_cut_b
          ):
              if np.random.rand() < 0.5:
                  front_cut_a += 1
              else:
                  end_cut_a += 1
          else:
              if np.random.rand() < 0.5:
                  front_cut_b += 1
              else:
                  end_cut_b += 1
  
      # calculate ds indices as well as offsets and return
      truncated_sent_a = self._cut_sentence(
          sent_a, front_cut_a, end_cut_a
      )
      truncated_sent_b = self._cut_sentence(
          sent_b, front_cut_b, end_cut_b
      )
      return truncated_sent_a, truncated_sent_b
  
file_path: fairseq/fairseq/data/legacy/block_pair_dataset.py
imports: import math
  import numpy
  import torch
  from fairseq.data import FairseqDataset
label_desc: Use image node_id 6 for calling the BlockPairDataset obj's underlying member method code with example usage: obj._truncate_sentences(sent_a, sent_b, max_num_tokens) and returns: truncated_sent_a,  truncated_sent_b