docstr: Parameters
  ----------
  model_path
      Path of the YTTM model file
  vocab_path
      Path of the vocabulary. It must be compatible to the YTTM model file.
      You are able to add new special tokens, but it must be compatible to the YTTM file.
  bpe_dropout
      The dropout probability in BPE-Dropout:
          "BPE-Dropout: Simple and Effective Subword Regularization"
  n_threads
      The number of threads for encoding
docstr_tok: ["Parameters","--","--","--","--","--","model_path","Path","of","the","YTTM","model","file","vocab_path","Path","of","the","vocabulary",".","It","must","be","compatible","to","the","YTTM","model","file",".","You","are","able","to","add","new","special","tokens",",","but","it","must","be","compatible","to","the","YTTM","file",".","bpe_dropout","The","dropout","probability","in","BPE-Dropout",":","``","BPE-Dropout",":","Simple","and","Effective","Subword","Regularization","''","n_threads","The","number","of","threads","for","encoding"]
code: def __init__(
      self,
      model_path: str,
      vocab: Optional[Union[str, Vocab]] = None,
      bpe_dropout: float = 0.0,
      n_threads: int = -1,
  ):
      """
  
      Parameters
      ----------
      model_path
          Path of the YTTM model file
      vocab_path
          Path of the vocabulary. It must be compatible to the YTTM model file.
          You are able to add new special tokens, but it must be compatible to the YTTM file.
      bpe_dropout
          The dropout probability in BPE-Dropout:
              "BPE-Dropout: Simple and Effective Subword Regularization"
      n_threads
          The number of threads for encoding
      """
      yttm = try_import_yttm()
      self._model_path = model_path
      self._bpe = yttm.BPE(model=model_path, n_threads=n_threads)
      self._bpe_dropout = bpe_dropout
      self._out_type = yttm.OutputType
      all_tokens = self._bpe.vocab()
      if vocab is not None:
          self._vocab = load_vocab(vocab)
      else:
          self._vocab = Vocab(
              all_tokens,
              unk_token="<UNK>",
              pad_token="<PAD>",
              bos_token="<BOS>",
              eos_token="<EOS>",
          )
      self._meta_symbol = (
          "▁"  # U+2581 as the symbol for the first subword token
      )
      # Assertions to verify that the vocabulary is compatible to the YTTM Model file.
      assert (
          self._vocab.unk_token == "<UNK>"
          and self._vocab.pad_token == "<PAD>"
          and self._vocab.bos_token == "<BOS>"
          and self._vocab.eos_token == "<EOS>"
      ), "Incompatible vocabulary! Received " "vocab={}".format(
          self._vocab
      )
      ocab_nonspecial_tokens_set = frozenset(
          self._vocab.non_special_tokens
      )
      for token in all_tokens:
          if token not in [
              self._vocab.unk_token,
              self._vocab.pad_token,
              self._vocab.bos_token,
              self._vocab.eos_token,
          ]:
              assert token in ocab_nonspecial_tokens_set
      for idx, token in enumerate(all_tokens):
          assert self._vocab[token] == idx
      self._first_subword_id_set = frozenset(
          [
              self._vocab[ele]
              for ele in self._vocab.all_tokens
              if ele.startswith(self._meta_symbol)
          ]
      )
  
file_path: gluon-nlp/src/gluonnlp/data/tokenizers/yttm.py
imports: from typing import Optional, Tuple, Union, List
  import os
  from .base import *
  from ..vocab import Vocab, load_vocab
  from ...utils.lazy_imports import try_import_yttm
label_desc: Use image node_id 1 to create a new YTTMTokenizer object from inherited base classes: BaseTokenizerWithVocab with example: obj = YTTMTokenizer(model_path, vocab, bpe_dropout, n_threads)