class_docstr: OpenNMT Tokenizer transform class.
docstr: Initialize Tokenizer models.
docstr_tok: ["Initialize","Tokenizer","models","."]
code: def warm_up(self, vocabs=None):
      """Initialize Tokenizer models."""
      super().warm_up(None)
      import pyonmttok
  
      src_subword_kwargs = self._get_subword_kwargs(side="src")
      src_tokenizer = pyonmttok.Tokenizer(
          **src_subword_kwargs, **self.src_other_kwargs
      )
      tgt_subword_kwargs = self._get_subword_kwargs(side="tgt")
      _diff_vocab = src_subword_kwargs.get(
          "vocabulary_path", ""
      ) != tgt_subword_kwargs.get(
          "vocabulary_path", ""
      ) or src_subword_kwargs.get(
          "vocabulary_threshold", 0
      ) != tgt_subword_kwargs.get(
          "vocabulary_threshold", 0
      )
      if self.share_vocab and not _diff_vocab:
          self.load_models = {
              "src": src_tokenizer,
              "tgt": src_tokenizer,
          }
      else:
          tgt_subword_kwargs = self._get_subword_kwargs(side="tgt")
          tgt_tokenizer = pyonmttok.Tokenizer(
              **tgt_subword_kwargs, **self.tgt_other_kwargs
          )
          self.load_models = {
              "src": src_tokenizer,
              "tgt": tgt_tokenizer,
          }
      if self.gpt2_pretok:
          """
          Returns list of utf-8 byte and a corresponding list of unicode
          strings. The reversible bpe codes work on unicode strings.
          code taken from openai/gpt2
          """
          bs = (
              list(range(ord("!"), ord("~") + 1))
              + list(range(ord("¡"), ord("¬") + 1))
              + list(range(ord("®"), ord("ÿ") + 1))
          )
          cs = bs[:]
          n = 0
          for b in range(2**8):
              if b not in bs:
                  bs.append(b)
                  cs.append(2**8 + n)
                  n += 1
          cs = [chr(n) for n in cs]
          self.maptable = dict(zip(bs, cs))
          self.revtable = {v: k for k, v in self.maptable.items()}
  
imports: import re
  from onmt.utils.logging import logger
  from onmt.transforms import register_transform
  from .transform import Transform, ObservableStats
  from onmt.constants import DefaultTokens
label_desc: Use image node_id 8 for calling the ONMTTokenizerTransform obj's underlying member method code with example usage: obj.warm_up(vocabs) without return types