code: def test_bpe(self):
      bpe_cls = get_transforms_cls(["bpe"])["bpe"]
      opt = Namespace(**self.base_opts)
      bpe_cls._validate_options(opt)
      bpe_transform = bpe_cls(opt)
      bpe_transform.warm_up()
      ex = {
          "src": ["Hello", "world", "."],
          "tgt": ["Bonjour", "le", "monde", "."],
      }
      bpe_transform.apply(ex, is_train=True)
      ex_gold = {
          "src": ["H@@", "ell@@", "o", "world", "."],
          "tgt": [
              "B@@",
              "on@@",
              "j@@",
              "our",
              "le",
              "mon@@",
              "de",
              ".",
          ],
      }
      self.assertEqual(ex, ex_gold)
      # test BPE-dropout:
      bpe_transform.dropout["src"] = 1.0
      tokens = ["Another", "world", "."]
      gold_bpe = ["A@@", "no@@", "ther", "world", "."]
      gold_dropout = [
          "A@@",
          "n@@",
          "o@@",
          "t@@",
          "h@@",
          "e@@",
          "r",
          "w@@",
          "o@@",
          "r@@",
          "l@@",
          "d",
          ".",
      ]
      # 1. disable bpe dropout for not training example
      after_bpe = bpe_transform._tokenize(tokens, is_train=False)
      self.assertEqual(after_bpe, gold_bpe)
      # 2. enable bpe dropout for training example
      after_bpe = bpe_transform._tokenize(tokens, is_train=True)
      self.assertEqual(after_bpe, gold_dropout)
      # 3. (NOTE) disable dropout won't take effect if already seen
      # this is caused by the cache mechanism in bpe:
      # return cached subword if the original token is seen when no dropout
      after_bpe2 = bpe_transform._tokenize(tokens, is_train=False)
      self.assertEqual(after_bpe2, gold_dropout)
  
imports: import unittest
  import copy
  import yaml
  import math
  from argparse import Namespace
  from onmt.transforms import get_transforms_cls, get_specials, make_transforms, TransformPipe
  from onmt.transforms.bart import BARTNoising
label_desc: Use image node_id 2 for calling the TestSubwordTransform obj's underlying member method code with example usage: obj.test_bpe() without return types