class_docstr: Apply the Sentencepiece Tokenizer, which trains subword tokenization via the
  unigram language modeling.
  
  Users of this class are required to `install sentencepiece
  <https://github.com/google/sentencepiece>`_. For example, one can use
  :samp:`pip install sentencepiece`
  
  
  Parameters
  ----------
  model_path
      Path to the pre-trained sentencepiece model.
  vocab
      Path to the vocabulary of the sentencepiece model in GluonNLP
  num_best
      A scalar for sampling subwords. If num_best = {0,1}, no sampling is performed.
      If num_best > 1, then samples from the num_best results.
      If num_best < 0, then assume that num_best is infinite and
      samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling
      algorithm.
  alpha
      A scalar for a smoothing parameter for probability rescaling.
  lowercase
      Whether to convert the input string to lower-case strings
  **kwargs
  
  Examples
  --------
  >>> from mxnet import gluon
  >>> url = 'https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/tokenizer_test_models/sentencepiece/test_ende-a9bee4.model'
  >>> model_f = gluon.utils.download(url)
  -etc-
  
  >>> tokenizer = gluonnlp.data.SentencepieceTokenizer(model_f)
  >>> sentence = 'This is a very awesome, life-changing sentence.'
  >>> tokenizer.encode(sentence)
  ['▁This', '▁is', '▁a', '▁very', '▁awesome', ',', '▁life', '-', 'ch', 'anging', '▁sentence', '.']
  
  >>> tokenizer.decode(tokenizer.encode(sentence))
  'This is a very awesome, life-changing sentence.'
  
  >>> os.remove('test_ende-a9bee4.model')
class_docstr_tok: ["Apply","the","Sentencepiece","Tokenizer",",","which","trains","subword","tokenization","via","the","unigram","language","modeling",".","Users","of","this","class","are","required","to","`","install","sentencepiece","<","https",":","\/\/github.com\/google\/sentencepiece",">","`","_",".","For","example",",","one","can","use",":","samp",":","`","pip","install","sentencepiece","`","Parameters","--","--","--","--","--","model_path","Path","to","the","pre-trained","sentencepiece","model",".","vocab","Path","to","the","vocabulary","of","the","sentencepiece","model","in","GluonNLP","num_best","A","scalar","for","sampling","subwords",".","If","num_best","=","{","0,1","}",",","no","sampling","is","performed",".","If","num_best",">","1",",","then","samples","from","the","num_best","results",".","If","num_best","<","0",",","then","assume","that","num_best","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","alpha","A","scalar","for","a","smoothing","parameter","for","probability","rescaling",".","lowercase","Whether","to","convert","the","input","string","to","lower-case","strings","*","*","kwargs","Examples","--","--","--","--",">",">",">","from","mxnet","import","gluon",">",">",">","url","=","'https",":","\/\/gluonnlp-numpy-data.s3-accelerate.amazonaws.com\/tokenizer_test_models\/sentencepiece\/test_ende-a9bee4.model'",">",">",">","model_f","=","gluon.utils.download","(","url",")","-etc-",">",">",">","tokenizer","=","gluonnlp.data.SentencepieceTokenizer","(","model_f",")",">",">",">","sentence","=","'This","is","a","very","awesome",",","life-changing","sentence",".","'",">",">",">","tokenizer.encode","(","sentence",")","[","'\u2581This","'",",","'\u2581is","'",",","'\u2581a","'",",","'\u2581very","'",",","'\u2581awesome","'",",","'",",","'",",","'\u2581life","'",",","'-","'",",","'ch","'",",","'anging","'",",","'\u2581sentence","'",",","'",".","'","]",">",">",">","tokenizer.decode","(","tokenizer.encode","(","sentence",")",")","'This","is","a","very","awesome",",","life-changing","sentence",".","'",">",">",">","os.remove","(","'test_ende-a9bee4.model","'",")"]
code: def __init__(
      self,
      model_path: Optional[str] = None,
      vocab: Optional[Union[str, Vocab]] = None,
      nbest: int = 0,
      alpha: float = 0.0,
      lowercase=False,
      **kwargs,
  ):
      self._model_path = model_path
      sentencepiece = try_import_sentencepiece()
      from ...third_party.sentencepiece_pb2 import SentencePieceText
  
      self._spt_cls = SentencePieceText
      self._sp_model = sentencepiece.SentencePieceProcessor()
      self._sp_model.load(model_path)
      self._nbest = nbest
      self._alpha = alpha
      self._lowercase = lowercase
      self._meta_symbol = "▁"
      sp_model_all_tokens = [
          self._sp_model.id_to_piece(i)
          for i in range(len(self._sp_model))
      ]
      special_tokens_kv = dict()
      existing_control_token_ids = set()
      token_id_to_token_name = dict()
      if self._sp_model.unk_id() != -1:
          special_tokens_kv["unk_token"] = self._sp_model.id_to_piece(
              self._sp_model.unk_id()
          )
          existing_control_token_ids.add(self._sp_model.unk_id())
          token_id_to_token_name[self._sp_model.unk_id()] = "unk_token"
      if self._sp_model.pad_id() != -1:
          special_tokens_kv["pad_token"] = self._sp_model.id_to_piece(
              self._sp_model.pad_id()
          )
          existing_control_token_ids.add(self._sp_model.pad_id())
          token_id_to_token_name[self._sp_model.pad_id()] = "pad_token"
      if self._sp_model.bos_id() != -1:
          special_tokens_kv["bos_token"] = self._sp_model.id_to_piece(
              self._sp_model.bos_id()
          )
          existing_control_token_ids.add(self._sp_model.bos_id())
          token_id_to_token_name[self._sp_model.bos_id()] = "bos_token"
      if self._sp_model.eos_id() != -1:
          special_tokens_kv["eos_token"] = self._sp_model.id_to_piece(
              self._sp_model.eos_id()
          )
          existing_control_token_ids.add(self._sp_model.eos_id())
          token_id_to_token_name[self._sp_model.eos_id()] = "eos_token"
      existing_control_tokens = set(
          [
              self._sp_model.id_to_piece(ele)
              for ele in existing_control_token_ids
          ]
      )
      other_control_tokens_ids = [
          i
          for i in range(len(self._sp_model))
          if self._sp_model.is_control(i)
          and i not in existing_control_token_ids
      ]
      other_control_tokens = set(
          [
              self._sp_model.id_to_piece(ele)
              for ele in other_control_tokens_ids
          ]
      )
      matched_other_control_tokens = dict()
      for k, v in kwargs.items():
          if k in special_tokens_kv:
              if v != special_tokens_kv[k]:
                  raise ValueError(
                      '"vocab.{}" is already set to "{}" in the sentencepiece model. '
                      'Cannot reset it to "{}"'.format(
                          k, special_tokens_kv[k], v
                      )
                  )
              continue
          if v in existing_control_tokens:
              if k != token_id_to_token_name[v]:
                  raise ValueError(
                      '"{}" is already registered as "vocab.{}". '
                      'We cannot rename it to "vocab.{}".'.format(
                          v, token_id_to_token_name[v], k
                      )
                  )
              continue
          if v in other_control_tokens:
              if v in matched_other_control_tokens:
                  raise ValueError(
                      '"{}" has already been registered as "vocab.{}", '
                      'we cannot register it again as "vocab.{}".'.format(
                          v, matched_other_control_tokens[v], k
                      )
                  )
              matched_other_control_tokens[v] = k
              special_tokens_kv[k] = v
          else:
              raise ValueError(
                  "Mismatch vocabulary! All special tokens specified "
                  "must be control tokens in the sentencepiece vocabulary."
              )
      if vocab is None:
          if len(matched_other_control_tokens) < len(
              other_control_tokens
          ):
              for i, token in enumerate(
                  other_control_tokens.difference(
                      set(matched_other_control_tokens.keys())
                  )
              ):
                  token_key = "other{}_token".format(i)
                  assert token_key not in special_tokens_kv
                  special_tokens_kv[token_key] = token
          self._vocab = Vocab(sp_model_all_tokens, **special_tokens_kv)
      else:
          self._vocab = load_vocab(vocab)
      # Sanity check
      assert len(self._vocab.all_tokens) >= len(sp_model_all_tokens)
      assert (
          self._vocab.all_tokens[: len(sp_model_all_tokens)]
          == sp_model_all_tokens
      )
      for token in self._vocab.special_tokens:
          piece_id = self._sp_model.piece_to_id(token)
          if not self._sp_model.is_unknown(piece_id):
              assert self._sp_model.is_control(piece_id), (
                  'Vocab mismatch! "{}" is a special token in the given vocab but not in the '
                  "sentencepiece model!".format(token)
              )
      self._first_subword_id_set = frozenset(
          [
              self._vocab[ele]
              for ele in sp_model_all_tokens
              if ele.startswith(self._meta_symbol)
          ]
      )
  
file_path: gluon-nlp/src/gluonnlp/data/tokenizers/sentencepiece.py
imports: import os
  from typing import Optional, Union, List
  from .base import *
  from ..vocab import Vocab, load_vocab
  from ...utils.lazy_imports import try_import_sentencepiece
label_desc: Use image node_id 1 to create a new SentencepieceTokenizer object from inherited base classes: BaseTokenizerWithVocab with example: obj = SentencepieceTokenizer(model_path, vocab, nbest, alpha, lowercase)