def mask_tokens(self, inputs: torch.Tensor, mask_labels: torch.Tensor) -> Tuple[(torch.Tensor, torch.Tensor)]:
    "\n        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set\n        'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref.\n        "
    if (self.tokenizer.mask_token is None):
        raise ValueError('This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the --mlm flag if you want to use this tokenizer.')
    labels = inputs.clone()
    probability_matrix = mask_labels
    special_tokens_mask = [self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()]
    probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)
    if (self.tokenizer._pad_token is not None):
        padding_mask = labels.eq(self.tokenizer.pad_token_id)
        probability_matrix.masked_fill_(padding_mask, value=0.0)
    masked_indices = probability_matrix.bool()
    labels[(~ masked_indices)] = (- 100)
    indices_replaced = (torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices)
    inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
    indices_random = ((torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices) & (~ indices_replaced))
    random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
    inputs[indices_random] = random_words[indices_random]
    return (inputs, labels)
Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref.
def mask_tokens(self, inputs: torch.Tensor, mask_labels: torch.Tensor) -> Tuple[(torch.Tensor, torch.Tensor)]:
    "\n        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set\n        'mask_labels' means we use whole word mask (wwm), we directly mask idxs according to it's ref.\n        "
    if (self.tokenizer.mask_token is None):
        raise ValueError('This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the --mlm flag if you want to use this tokenizer.')
    labels = inputs.clone()
    probability_matrix = mask_labels
    special_tokens_mask = [self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()]
    probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)
    if (self.tokenizer._pad_token is not None):
        padding_mask = labels.eq(self.tokenizer.pad_token_id)
        probability_matrix.masked_fill_(padding_mask, value=0.0)
    masked_indices = probability_matrix.bool()
    labels[(~ masked_indices)] = (- 100)
    indices_replaced = (torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices)
    inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
    indices_random = ((torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices) & (~ indices_replaced))
    random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
    inputs[indices_random] = random_words[indices_random]
    return (inputs, labels)