class_docstr: Break a Dataset of tokens into blocks.
  
  Args:
      dataset (~torch.utils.data.Dataset): dataset to break into blocks
      sizes (List[int]): sentence lengths (required for 'complete' and 'eos')
      block_size (int): maximum block size (ignored in 'eos' break mode)
      break_mode (str, optional): Mode used for breaking tokens. Values can
          be one of:
          - 'none': break tokens into equally sized blocks (up to block_size)
          - 'complete': break tokens into blocks (up to block_size) such that
              blocks contains complete sentences, although block_size may be
              exceeded if some sentences exceed block_size
          - 'complete_doc': similar to 'complete' mode, but do not
              cross document boundaries
          - 'eos': each block contains one sentence (block_size is ignored)
      include_targets (bool, optional): return next tokens as targets
          (default: False).
      document_sep_len (int, optional): document separator size (required for
          'complete_doc' break mode). Typically 1 if the sentences have eos
          and 0 otherwise.
class_docstr_tok: ["Break","a","Dataset","of","tokens","into","blocks",".","Args",":","dataset","(","~torch.utils.data.Dataset",")",":","dataset","to","break","into","blocks","sizes","(","List","[","int","]",")",":","sentence","lengths","(","required","for","'complete","'","and","'eos","'",")","block_size","(","int",")",":","maximum","block","size","(","ignored","in","'eos","'","break","mode",")","break_mode","(","str",",","optional",")",":","Mode","used","for","breaking","tokens",".","Values","can","be","one","of",":","-","'none","'",":","break","tokens","into","equally","sized","blocks","(","up","to","block_size",")","-","'complete","'",":","break","tokens","into","blocks","(","up","to","block_size",")","such","that","blocks","contains","complete","sentences",",","although","block_size","may","be","exceeded","if","some","sentences","exceed","block_size","-","'complete_doc","'",":","similar","to","'complete","'","mode",",","but","do","not","cross","document","boundaries","-","'eos","'",":","each","block","contains","one","sentence","(","block_size","is","ignored",")","include_targets","(","bool",",","optional",")",":","return","next","tokens","as","targets","(","default",":","False",")",".","document_sep_len","(","int",",","optional",")",":","document","separator","size","(","required","for","'complete_doc","'","break","mode",")",".","Typically","1","if","the","sentences","have","eos","and","0","otherwise","."]
docstr: Use token_block_utils_fast to build arrays for indexing into self.dataset
docstr_tok: ["Use","token_block_utils_fast","to","build","arrays","for","indexing","into","self.dataset"]
code: def _build_slice_indices(
      sizes, break_mode, document_sep_len, block_size
  ) -> Tuple[np.ndarray]:
      """Use token_block_utils_fast to build arrays for indexing into self.dataset"""
      try:
          from fairseq.data.token_block_utils_fast import (
              _get_slice_indices_fast,
              _get_block_to_dataset_index_fast,
          )
      except ImportError:
          raise ImportError(
              "Please build Cython components with: `pip install --editable .` "
              "or `python setup.py build_ext --inplace`"
          )
  
      if isinstance(sizes, list):
          sizes = np.array(sizes, dtype=np.int64)
      else:
          if torch.is_tensor(sizes):
              sizes = sizes.numpy()
          sizes = sizes.astype(np.int64)
  
      break_mode = break_mode if break_mode is not None else "none"
  
      # For "eos" break-mode, block_size is not required parameters.
      if break_mode == "eos" and block_size is None:
          block_size = 0
  
      slice_indices = _get_slice_indices_fast(
          sizes, str(break_mode), block_size, document_sep_len
      )
      _sizes = slice_indices[:, 1] - slice_indices[:, 0]
  
      # build index mapping block indices to the underlying dataset indices
      if break_mode == "eos":
          # much faster version for eos break mode
          block_to_dataset_index = np.stack(
              [
                  np.arange(len(sizes)),  # starting index in dataset
                  np.zeros(
                      len(sizes), dtype=np.compat.long
                  ),  # starting offset within starting index
                  np.arange(len(sizes)),  # ending index in dataset
              ],
              1,
          )
      else:
          block_to_dataset_index = _get_block_to_dataset_index_fast(
              sizes,
              slice_indices,
          )
      size_dtype = np.uint16 if block_size < 65535 else np.uint32
      num_tokens = slice_indices[-1].max()
      slice_indices_dtype = best_fitting_int_dtype(num_tokens)
      slice_indices = slice_indices.astype(slice_indices_dtype)
      _sizes = _sizes.astype(size_dtype)
      block_to_dataset_index = block_to_dataset_index.astype(
          slice_indices_dtype
      )
      return _sizes, block_to_dataset_index, slice_indices
  
file_path: fairseq/fairseq/data/token_block_dataset.py
imports: import numpy
  import torch
  from fairseq.data import FairseqDataset, plasma_utils
  from fairseq.data.indexed_dataset import best_fitting_int_dtype
  from typing import Tuple
label_desc: Use image node_id 2 for calling the TokenBlockDataset obj's underlying member method code with example usage: obj._build_slice_indices(sizes, break_mode, document_sep_len, block_size) and returns: _sizes,  block_to_dataset_index,  slice_indices