docstr: Creates the predictions for the masked LM objective.
docstr_tok: ["Creates","the","predictions","for","the","masked","LM","objective","."]
code: def create_whole_masked_lm_predictions(self, tokens):
      """Creates the predictions for the masked LM objective."""
  
      cand_indexes = []
      for i, token in enumerate(tokens):
          if token == "[CLS]" or token == "[SEP]":
              continue
          # Whole Word Masking means that if we mask all of the wordpieces
          # corresponding to an original word. When a word has been split into
          # WordPieces, the first token does not have any marker and any subsequence
          # tokens are prefixed with ##. So whenever we see the ## token, we
          # append it to the previous set of word indexes.
          #
          # Note that Whole Word Masking does *not* change the training code
          # at all -- we still predict each WordPiece independently, softmaxed
          # over the entire vocabulary.
          if (
              self.do_whole_word_mask
              and len(cand_indexes) >= 1
              and token.startswith("##")
          ):
              cand_indexes[-1].append(i)
          else:
              cand_indexes.append([i])
  
      random.shuffle(cand_indexes)
  
      output_tokens = [
          t[2:] if len(self.whole_rec.findall(t)) > 0 else t
          for t in tokens
      ]  # 去掉"##"
  
      num_to_predict = min(
          self.max_predictions_per_seq,
          max(1, int(round(len(tokens) * self.masked_lm_prob))),
      )
  
      masked_lms = []
      covered_indexes = set()
      for index_set in cand_indexes:
          if len(masked_lms) >= num_to_predict:
              break
          # If adding a whole-word mask would exceed the maximum number of
          # predictions, then just skip this candidate.
          if len(masked_lms) + len(index_set) > num_to_predict:
              continue
          is_any_index_covered = False
          for index in index_set:
              if index in covered_indexes:
                  is_any_index_covered = True
                  break
          if is_any_index_covered:
              continue
          for index in index_set:
              covered_indexes.add(index)
  
              masked_token = None
              # 80% of the time, replace with [MASK]
              if random.random() < 0.8:
                  masked_token = "[MASK]"
              else:
                  # 10% of the time, keep original
                  if random.random() < 0.5:
                      masked_token = (
                          tokens[index][2:]
                          if len(self.whole_rec.findall(tokens[index]))
                          > 0
                          else tokens[index]
                      )  # 去掉"##"
                  # 10% of the time, replace with random word
                  else:
                      masked_token = self.vocab_words[
                          random.randint(0, len(self.vocab_words) - 1)
                      ]
  
              output_tokens[index] = masked_token
  
              masked_lms.append(
                  MaskedLMInstance(
                      index=index,
                      label=tokens[index][2:]
                      if len(self.whole_rec.findall(tokens[index])) > 0
                      else tokens[index],
                  )
              )
      assert len(masked_lms) <= num_to_predict
      masked_lms = sorted(masked_lms, key=lambda x: x.index)
      masked_lm_output = [-1] * len(output_tokens)
      for p in masked_lms:
          masked_lm_output[p.index] = self.tokenizer.vocab[p.label]
  
      return (output_tokens, masked_lm_output)
  
file_path: colossalai/examples/community/roberta/preprocessing/get_mask.py
imports: import collections
  import logging
  import random
  import jieba
  import re
  import mask
  import numpy
label_desc: Use image node_id 6 for calling the PreTrainingDataset obj's underlying member method code with example usage: obj.create_whole_masked_lm_predictions(tokens) and returns: output_tokens,  masked_lm_output