class_docstr: Task for used with Masked language modelling.
  
  .. todo::
      Describe MLM
  
  :param mask_ratio: Fraction of tokens to mask.
  :param smart_masking: Whether to apply smart masking (use tokens from the same group when choosing randoms).
class_docstr_tok: ["Task","for","used","with","Masked","language","modelling",".","..","todo",":",":","Describe","MLM",":","param","mask_ratio",":","Fraction","of","tokens","to","mask",".",":","param","smart_masking",":","Whether","to","apply","smart","masking","(","use","tokens","from","the","same","group","when","choosing","randoms",")","."]
docstr: Mask out the tokens for mlm training
docstr_tok: ["Mask","out","the","tokens","for","mlm","training"]
code: def mlm_mask(
      self, token_ids: np.ndarray
  ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
      """Mask out the tokens for mlm training"""
  
      vocabulary = self.datamodule.vocabulary
      token2index = vocabulary.token2index
  
      unk_id = token2index["[UNK]"]
      mask_id = token2index["[MASK]"]
      sep_id = token2index["[SEP]"]
  
      # limit is length of an actual sequence
      n_tokens = len(token_ids)
  
      num_tokens_to_mask = np.floor(n_tokens * self.mask_ratio).astype(
          np.int32
      )
      # firt 10% of tokens won't be changed
      pos_unchange = np.floor(num_tokens_to_mask * 0.1).astype(np.int32)
      # last 10% of tokens would be random ; the rest will be changed
      pos_random = num_tokens_to_mask - pos_unchange
  
      # we do not mask SEP and UNK
      legal_mask = (token_ids[1:] != sep_id) & (token_ids[1:] != unk_id)
      legal_indx = np.arange(start=1, stop=n_tokens)[legal_mask]
  
      indx_to_mask = np.random.choice(
          a=legal_indx, size=num_tokens_to_mask, replace=False
      )
  
      max_masked_num = np.floor(
          self.mask_ratio * self.max_length
      ).astype(np.int32)
  
      # positions of the masked tokens
      y_indx = np.full(
          shape=max_masked_num, fill_value=int(self.max_length - 1)
      )
      y_indx[: len(indx_to_mask)] = indx_to_mask.copy()
  
      # remember the actual tokens on positions
      y_token = np.zeros(shape=max_masked_num)
      y_token[: len(indx_to_mask)] = token_ids[indx_to_mask].copy()
  
      # masked token_ids #rather change the sampling domain for accurate masking
      # ratio?
      token_ids[indx_to_mask[pos_unchange:pos_random]] = mask_id
  
      vocab_size = len(token2index)
      n_general_tokens = len(vocabulary.general_tokens)
  
      if self.smart_masking:
          smart_edge = int(pos_random + int(pos_unchange * 0.3))
  
          # Random 7% of all random cases
          token_ids[indx_to_mask[smart_edge:]] = np.random.randint(
              # low we do not mask any special tokens
              low=n_general_tokens,
              high=vocab_size,
              size=(1, len(indx_to_mask[smart_edge:])),
          )
  
          # Smart Random 3% of all the cases
          smart_values = token_ids[indx_to_mask[pos_random:smart_edge]]
  
          for i, j in self.token_groups:
              smart_values = self.smart_masked(
                  smart_values, i, j + 1
              )  # background
  
          token_ids[indx_to_mask[pos_random:smart_edge]] = smart_values
  
      else:
          token_ids[indx_to_mask[pos_random:]] = torch.randint(
              # low we do not mask any special tokens
              low=n_general_tokens,
              high=vocab_size,
              size=(1, len(indx_to_mask[pos_random:])),
          )
      return token_ids, y_indx, y_token
  
file_path: life2vec/src/tasks/mlm.py
imports: import logging
  from dataclasses import dataclass
  from functools import cached_property
  from itertools import chain
  from typing import List, Tuple, TypeVar, cast
  from xml.dom.minidom import Document
  import numpy
  import torch
  from random import shuffle
  from src.data_new.types import Background, PersonDocument, EncodedDocument
  from src.tasks.base import Task
label_desc: Use image node_id 4 for calling the MLM obj's underlying member method code with example usage: obj.mlm_mask(token_ids) and returns: token_ids,  y_indx,  y_token