class_docstr: Dataset for bert pretraining when using data preprocessing including tokenization
class_docstr_tok: ["Dataset","for","bert","pretraining","when","using","data","preprocessing","including","tokenization"]
docstr: Args:
      tokenizer: tokenizer
      data_file: path to data
      max_seq_length: maximum sequence length of input tensors
      mask_probability: proability to mask token
      short_seq_prob: probability to create a sequence shorter than max_seq_length
      seq_a_ratio: ratio between lengths of first and second sequence
      sentence_idx_file: sentence indices file for caching
docstr_tok: ["Args",":","tokenizer",":","tokenizer","data_file",":","path","to","data","max_seq_length",":","maximum","sequence","length","of","input","tensors","mask_probability",":","proability","to","mask","token","short_seq_prob",":","probability","to","create","a","sequence","shorter","than","max_seq_length","seq_a_ratio",":","ratio","between","lengths","of","first","and","second","sequence","sentence_idx_file",":","sentence","indices","file","for","caching"]
code: def __init__(
      self,
      tokenizer: object,
      data_file: str,
      max_seq_length: Optional[int] = 128,
      mask_prob: Optional[float] = 0.15,
      short_seq_prob: Optional[float] = 0.1,
      seq_a_ratio: Optional[float] = 0.6,
      sentence_idx_file: Optional[str] = None,
  ):
      """
      Args:
          tokenizer: tokenizer
          data_file: path to data
          max_seq_length: maximum sequence length of input tensors
          mask_probability: proability to mask token
          short_seq_prob: probability to create a sequence shorter than max_seq_length
          seq_a_ratio: ratio between lengths of first and second sequence
          sentence_idx_file: sentence indices file for caching
      """
      self.tokenizer = tokenizer
  
      # Loading enormous datasets into RAM isn't always feasible -- for
      # example, the pubmed corpus is 200+ GB, which doesn't fit into RAM on
      # most computers. To get around this, we store the indices of newlines
      # in each file so we can seek to and retrieve sentences immediately
      # from main memory when needed during training.
  
      # Try and load sentence indices file if already exists
      sentence_indices, sentence_idx_file, data_dir = load_data_indices(
          sentence_idx_file, data_file, "sentence_indices"
      )
  
      # If sentence indices file doesn't exists, generate and store sentence indices
      if sentence_indices is None:
          sentence_indices = {}
          filenames = [data_file]
  
          for filename in tqdm(filenames):
              with open(filename, "rb") as f:
                  contents = f.read()
                  newline_indices = find_newlines(contents)
  
              if os.path.isdir(data_dir):
                  # Only keep the parts of the filepath that are invariant to
                  # the dataset's location on disk
                  filename = os.path.basename(filename)
  
              # In python, arrays are much more space-efficient than lists
              sentence_indices[filename] = array.array(
                  "I", newline_indices
              )
  
          # Save sentence indices so we don't have to do this again
          with open(sentence_idx_file, "wb") as f:
              pickle.dump(sentence_indices, f)
  
      corpus_size = 0
      empty_files = []
  
      # Find total number of newlines across entire corpus and remove files
      # without any newlines
      for filename in sentence_indices:
          if len(sentence_indices[filename]) <= 1:
              empty_files.append(filename)
          else:
              corpus_size += len(sentence_indices[filename])
  
      for filename in empty_files:
          del sentence_indices[filename]
  
      self.corpus_size = corpus_size
      self.dataset = data_dir
      self.filenames = list(sentence_indices.keys())
      self.mask_probability = mask_prob
      self.max_seq_length = max_seq_length
      self.sentence_indices = sentence_indices
      self.vocab_size = self.tokenizer.vocab_size
      self.short_seq_prob = short_seq_prob
      self.seq_a_ratio = seq_a_ratio
  
imports: import array
  import os
  import pickle
  import random
  from typing import Dict, List, Optional
  import h5py
  import numpy
  from torch.utils.data import DataLoader, DistributedSampler
  from tqdm import tqdm
  from nemo.collections.nlp.data.data_utils.data_preprocessing import find_newlines, load_data_indices
  from nemo.core.classes import Dataset
label_desc: Use image node_id 1 to create a new BertPretrainingDataset object from inherited base classes: Dataset with example: obj = BertPretrainingDataset(tokenizer, data_file, max_seq_length, mask_prob, short_seq_prob, seq_a_ratio, sentence_idx_file)