docstr: Mask will be done on token level.
  
  Condition:
  * `mask_length` == subword;
  * or not specify subword marker (joiner/spacer) by `is_joiner`.
docstr_tok: ["Mask","will","be","done","on","token","level",".","Condition",":","*","`","mask_length","`","==","subword",";","*","or","not","specify","subword","marker","(","joiner\/spacer",")","by","`","is_joiner","`","."]
code: def test_token_mask(self):
      """Mask will be done on token level.
  
      Condition:
      * `mask_length` == subword;
      * or not specify subword marker (joiner/spacer) by `is_joiner`.
      """
      bart_noise = BARTNoising(
          vocab=[self.FAKE_VOCAB],
          mask_tok=self.MASK_TOK,
          mask_ratio=0.5,
          mask_length="subword",
          replace_length=0,  # 0 to drop them, 1 to replace them with MASK
          # insert_ratio=0.0,
          # random_ratio=0.0,
          # Defalt: full_stop_token=[".", "?", "!"]
      )
      tokens = ["H￭", "ell￭", "o", "world", "."]
      # all token are considered as an individual word
      self.assertTrue(all(bart_noise._is_word_start(tokens)))
      n_tokens = len(tokens)
  
      # 1. tokens are dropped when replace_length is 0
      masked = bart_noise.apply(tokens)
      n_masked = math.ceil(n_tokens * bart_noise.mask_ratio)
      # print(f"token delete: {masked} / {tokens}")
      self.assertEqual(len(masked), n_tokens - n_masked)
  
      # 2. tokens are replaced by MASK when replace_length is 1
      bart_noise.replace_length = 1
      masked = bart_noise.apply(tokens)
      n_masked = math.ceil(n_tokens * bart_noise.mask_ratio)
      # print(f"token mask: {masked} / {tokens}")
      self.assertEqual(len(masked), n_tokens)
      self.assertEqual(
          sum([1 if tok == self.MASK_TOK else 0 for tok in masked]),
          n_masked,
      )
  
imports: import unittest
  import copy
  import yaml
  import math
  from argparse import Namespace
  from onmt.transforms import get_transforms_cls, get_specials, make_transforms, TransformPipe
  from onmt.transforms.bart import BARTNoising
label_desc: Use image node_id 5 for calling the TestBARTNoising obj's underlying member method code with example usage: obj.test_token_mask() without return types