class_docstr: Constructs a tokenizer based on [SentencePiece](https://github.com/google/sentencepiece).
  
  Args:
      vocab_file (`str`):
          [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
          contains the vocabulary necessary to instantiate a tokenizer.
      sp_model_kwargs (`dict`, *optional*):
          Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
          SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
          to set:
  
          - `enable_sampling`: Enable subword regularization.
          - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
  
            - `nbest_size = {0,1}`: No sampling is performed.
            - `nbest_size > 1`: samples from the nbest_size results.
            - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
              using forward-filtering-and-backward-sampling algorithm.
  
          - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
            BPE-dropout.
class_docstr_tok: ["Constructs","a","tokenizer","based","on","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")",".","Args",":","vocab_file","(","`","str","`",")",":","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")","file","(","generally","has","a","*",".spm","*","extension",")","that","contains","the","vocabulary","necessary","to","instantiate","a","tokenizer",".","sp_model_kwargs","(","`","dict","`",",","*","optional","*",")",":","Will","be","passed","to","the","`","SentencePieceProcessor.__init__","(",")","`","method",".","The","[","Python","wrapper","for","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece\/tree\/master\/python",")","can","be","used",",","among","other","things",",","to","set",":","-","`","enable_sampling","`",":","Enable","subword","regularization",".","-","`","nbest_size","`",":","Sampling","parameters","for","unigram",".","Invalid","for","BPE-Dropout",".","-","`","nbest_size","=","{","0,1","}","`",":","No","sampling","is","performed",".","-","`","nbest_size",">","1","`",":","samples","from","the","nbest_size","results",".","-","`","nbest_size","<","0","`",":","assuming","that","nbest_size","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","-","`","alpha","`",":","Smoothing","parameter","for","unigram","sampling",",","and","dropout","probability","of","merge","operations","for","BPE-dropout","."]
code: def decode(self, tokens, start=-1, end=-1, raw_text=None):
      if raw_text is None:
          current_sub_tokens = []
          out_string = ""
          prev_is_special = False
          for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.special_tokens:
                  if not prev_is_special:
                      out_string += " "
                  out_string += (
                      self.spm.decode_pieces(current_sub_tokens) + token
                  )
                  prev_is_special = True
                  current_sub_tokens = []
              else:
                  current_sub_tokens.append(token)
                  prev_is_special = False
          out_string += self.spm.decode_pieces(current_sub_tokens)
          return out_string.strip()
      else:
          words = self.split_to_words(raw_text)
          word_tokens = [self.tokenize(w) for w in words]
          token2words = [0] * len(tokens)
          tid = 0
          for i, w in enumerate(word_tokens):
              for k, t in enumerate(w):
                  token2words[tid] = i
                  tid += 1
          word_start = token2words[start]
          word_end = (
              token2words[end] if end < len(tokens) else len(words)
          )
          text = "".join(words[word_start:word_end])
          return text
  
imports: import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 6 for calling the SPMTokenizer obj's underlying member method code with example usage: obj.decode(tokens, start, end, raw_text) and returns: out_string, text