class_docstr: Dataset for bert pretraining when using data preprocessing including tokenization
class_docstr_tok: ["Dataset","for","bert","pretraining","when","using","data","preprocessing","including","tokenization"]
docstr: Args:
    ids: list of token ids representing a chunk of text
  Returns:
    masked_ids: list of input tokens with some of the entries masked
      according to the following protocol from the original BERT paper:
      each token is masked with a probability of 15% and is replaced with
      1) the [MASK] token 80% of the time,
      2) random token 10% of the time,
      3) the same token 10% of the time.
    output_mask: list of binary variables which indicate what tokens has
      been masked (to calculate the loss function for these tokens only)
docstr_tok: ["Args",":","ids",":","list","of","token","ids","representing","a","chunk","of","text","Returns",":","masked_ids",":","list","of","input","tokens","with","some","of","the","entries","masked","according","to","the","following","protocol","from","the","original","BERT","paper",":","each","token","is","masked","with","a","probability","of","15","%","and","is","replaced","with","1",")","the","[","MASK","]","token","80","%","of","the","time",",","2",")","random","token","10","%","of","the","time",",","3",")","the","same","token","10","%","of","the","time",".","output_mask",":","list","of","binary","variables","which","indicate","what","tokens","has","been","masked","(","to","calculate","the","loss","function","for","these","tokens","only",")"]
code: def mask_ids(self, ids: List[int]):
      """
      Args:
        ids: list of token ids representing a chunk of text
      Returns:
        masked_ids: list of input tokens with some of the entries masked
          according to the following protocol from the original BERT paper:
          each token is masked with a probability of 15% and is replaced with
          1) the [MASK] token 80% of the time,
          2) random token 10% of the time,
          3) the same token 10% of the time.
        output_mask: list of binary variables which indicate what tokens has
          been masked (to calculate the loss function for these tokens only)
      """
  
      # Whole-word masking by default, as it gives better performance.
      cand_indexes = [[ids[0]]]
      for tid in ids[1:]:
          token = self.tokenizer.ids_to_tokens([tid])[0]
          is_suffix = token.startswith("\u2581")
          if is_suffix:
              # group together with its previous token to form a whole-word
              cand_indexes[-1].append(tid)
          else:
              cand_indexes.append([tid])
  
      masked_ids, output_mask = [], []
      mask_id = self.tokenizer.token_to_id("[MASK]")
  
      for word_ids in cand_indexes:
          is_special = (word_ids[0] == self.tokenizer.cls_id) or (
              word_ids[0] == self.tokenizer.sep_id
          )
          if is_special or (random.random() > self.mask_probability):
              output_mask.extend([0] * len(word_ids))
              masked_ids.extend(word_ids)
          else:
              output_mask.extend([1] * len(word_ids))
              p = random.random()
              # for 80%, replace with mask
              if p < 0.8:
                  masked_ids.extend([mask_id] * len(word_ids))
              # for 10%, replace by a random token
              elif p < 0.9:
                  for _ in word_ids:
                      # randomly select a valid word
                      random_word = random.randrange(self.vocab_size)
                      while random_word in (
                          self.tokenizer.cls_id,
                          self.tokenizer.sep_id,
                      ):
                          random_word = random.randrange(
                              self.vocab_size
                          )
                      masked_ids.append(random_word)
              # for 10%, use same token
              else:
                  masked_ids.extend(word_ids)
  
      return masked_ids, output_mask
  
imports: import array
  import os
  import pickle
  import random
  from typing import Dict, List, Optional
  import h5py
  import numpy
  from torch.utils.data import DataLoader, DistributedSampler
  from tqdm import tqdm
  from nemo.collections.nlp.data.data_utils.data_preprocessing import find_newlines, load_data_indices
  from nemo.core.classes import Dataset
label_desc: Use image node_id 4 for calling the BertPretrainingDataset obj's underlying member method code with example usage: obj.mask_ids(ids) and returns: masked_ids,  output_mask