class_docstr: Apply the Sentencepiece Tokenizer, which trains subword tokenization via the
  unigram language modeling.
  
  Users of this class are required to `install sentencepiece
  <https://github.com/google/sentencepiece>`_. For example, one can use
  :samp:`pip install sentencepiece`
  
  
  Parameters
  ----------
  model_path
      Path to the pre-trained sentencepiece model.
  vocab
      Path to the vocabulary of the sentencepiece model in GluonNLP
  num_best
      A scalar for sampling subwords. If num_best = {0,1}, no sampling is performed.
      If num_best > 1, then samples from the num_best results.
      If num_best < 0, then assume that num_best is infinite and
      samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling
      algorithm.
  alpha
      A scalar for a smoothing parameter for probability rescaling.
  lowercase
      Whether to convert the input string to lower-case strings
  **kwargs
  
  Examples
  --------
  >>> from mxnet import gluon
  >>> url = 'https://gluonnlp-numpy-data.s3-accelerate.amazonaws.com/tokenizer_test_models/sentencepiece/test_ende-a9bee4.model'
  >>> model_f = gluon.utils.download(url)
  -etc-
  
  >>> tokenizer = gluonnlp.data.SentencepieceTokenizer(model_f)
  >>> sentence = 'This is a very awesome, life-changing sentence.'
  >>> tokenizer.encode(sentence)
  ['▁This', '▁is', '▁a', '▁very', '▁awesome', ',', '▁life', '-', 'ch', 'anging', '▁sentence', '.']
  
  >>> tokenizer.decode(tokenizer.encode(sentence))
  'This is a very awesome, life-changing sentence.'
  
  >>> os.remove('test_ende-a9bee4.model')
class_docstr_tok: ["Apply","the","Sentencepiece","Tokenizer",",","which","trains","subword","tokenization","via","the","unigram","language","modeling",".","Users","of","this","class","are","required","to","`","install","sentencepiece","<","https",":","\/\/github.com\/google\/sentencepiece",">","`","_",".","For","example",",","one","can","use",":","samp",":","`","pip","install","sentencepiece","`","Parameters","--","--","--","--","--","model_path","Path","to","the","pre-trained","sentencepiece","model",".","vocab","Path","to","the","vocabulary","of","the","sentencepiece","model","in","GluonNLP","num_best","A","scalar","for","sampling","subwords",".","If","num_best","=","{","0,1","}",",","no","sampling","is","performed",".","If","num_best",">","1",",","then","samples","from","the","num_best","results",".","If","num_best","<","0",",","then","assume","that","num_best","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","alpha","A","scalar","for","a","smoothing","parameter","for","probability","rescaling",".","lowercase","Whether","to","convert","the","input","string","to","lower-case","strings","*","*","kwargs","Examples","--","--","--","--",">",">",">","from","mxnet","import","gluon",">",">",">","url","=","'https",":","\/\/gluonnlp-numpy-data.s3-accelerate.amazonaws.com\/tokenizer_test_models\/sentencepiece\/test_ende-a9bee4.model'",">",">",">","model_f","=","gluon.utils.download","(","url",")","-etc-",">",">",">","tokenizer","=","gluonnlp.data.SentencepieceTokenizer","(","model_f",")",">",">",">","sentence","=","'This","is","a","very","awesome",",","life-changing","sentence",".","'",">",">",">","tokenizer.encode","(","sentence",")","[","'\u2581This","'",",","'\u2581is","'",",","'\u2581a","'",",","'\u2581very","'",",","'\u2581awesome","'",",","'",",","'",",","'\u2581life","'",",","'-","'",",","'ch","'",",","'anging","'",",","'\u2581sentence","'",",","'",".","'","]",">",">",">","tokenizer.decode","(","tokenizer.encode","(","sentence",")",")","'This","is","a","very","awesome",",","life-changing","sentence",".","'",">",">",">","os.remove","(","'test_ende-a9bee4.model","'",")"]
docstr: Whether the token is the first subword token. This can be used to implement
  whole-word masking.
  
  Parameters
  ----------
  tokens
      The input tokens
  
  Returns
  -------
  ret
      Whether the token is the first subword token in the list of subwords
docstr_tok: ["Whether","the","token","is","the","first","subword","token",".","This","can","be","used","to","implement","whole-word","masking",".","Parameters","--","--","--","--","--","tokens","The","input","tokens","Returns","--","--","--","-","ret","Whether","the","token","is","the","first","subword","token","in","the","list","of","subwords"]
code: def is_first_subword(
      self, tokens: Union[str, int, List[str], List[int]]
  ) -> Union[bool, List[bool]]:
      """Whether the token is the first subword token. This can be used to implement
      whole-word masking.
  
      Parameters
      ----------
      tokens
          The input tokens
  
      Returns
      -------
      ret
          Whether the token is the first subword token in the list of subwords
      """
      if isinstance(tokens, str):
          return tokens.startswith(self._meta_symbol)
      elif isinstance(tokens, int):
          return tokens in self._first_subword_id_set
      elif isinstance(tokens, list):
          if len(tokens) == 0:
              return []
          if isinstance(tokens[0], str):
              return [
                  ele.startswith(self._meta_symbol) for ele in tokens
              ]
          elif isinstance(tokens[0], int):
              return [
                  ele in self._first_subword_id_set for ele in tokens
              ]
          else:
              raise NotImplementedError
      else:
          raise NotImplementedError
  
file_path: gluon-nlp/src/gluonnlp/data/tokenizers/sentencepiece.py
imports: import os
  from typing import Optional, Union, List
  from .base import *
  from ..vocab import Vocab, load_vocab
  from ...utils.lazy_imports import try_import_sentencepiece
label_desc: Use image node_id 5 for calling the SentencepieceTokenizer obj's underlying member method code with example usage: obj.is_first_subword(tokens) and returns: tokens, unknown, list, unknown, unknown