class_docstr: Tokenizes a tensor of UTF-8 strings.
  
  SentencePiece is an unsupervised text tokenizer and detokenizer. It is used
  mainly for Neural Network-based text generation systems where the vocabulary
  size is predetermined prior to the neural model training. SentencePiece
  implements subword units with the extension of direct training from raw
  sentences.
  
  Before using the tokenizer, you will need to train a vocabulary and build a
  model configuration for it. Please visit the [Sentencepiece
  repository](https://github.com/google/sentencepiece#train-sentencepiece-model)
  for the most up-to-date instructions on this process.
docstr: Creates & initializes a Sentencepiece processor.
  
  Args:
    model: The sentencepiece model serialized proto.
    out_type: output type. tf.int32 or tf.string (Default = tf.int32) Setting
      tf.int32 directly encodes the string into an id sequence.
    nbest_size: A scalar for sampling.
      * `nbest_size = {0,1}`: No sampling is performed. (default)
      * `nbest_size > 1`: samples from the nbest_size results.
      * `nbest_size < 0`: assuming that nbest_size is infinite and samples
          from the all hypothesis (lattice) using
          forward-filtering-and-backward-sampling algorithm.
    alpha: A scalar for a smoothing parameter. Inverse temperature for
      probability rescaling.
    reverse: Reverses the tokenized sequence (Default = false)
    add_bos: Add beginning of sentence token to the result (Default = false)
    add_eos: Add end of sentence token to the result (Default = false). When
      `reverse=True` beginning/end of sentence tokens are added after
      reversing.
    return_nbest: If True requires that `nbest_size` is a scalar and `> 1`.
      Returns the `nbest_size` best tokenizations for each sentence instead
      of a single one. The returned tensor has shape
      `[batch * nbest, (tokens)]`.
    name: The name argument that is passed to the op function.
  
  Returns:
    pieces: A SentencepieceTokenizer.
docstr_tok: ["Creates","&","initializes","a","Sentencepiece","processor",".","Args",":","model",":","The","sentencepiece","model","serialized","proto",".","out_type",":","output","type",".","tf.int32","or","tf.string","(","Default","=","tf.int32",")","Setting","tf.int32","directly","encodes","the","string","into","an","id","sequence",".","nbest_size",":","A","scalar","for","sampling",".","*","`","nbest_size","=","{","0,1","}","`",":","No","sampling","is","performed",".","(","default",")","*","`","nbest_size",">","1","`",":","samples","from","the","nbest_size","results",".","*","`","nbest_size","<","0","`",":","assuming","that","nbest_size","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","alpha",":","A","scalar","for","a","smoothing","parameter",".","Inverse","temperature","for","probability","rescaling",".","reverse",":","Reverses","the","tokenized","sequence","(","Default","=","false",")","add_bos",":","Add","beginning","of","sentence","token","to","the","result","(","Default","=","false",")","add_eos",":","Add","end","of","sentence","token","to","the","result","(","Default","=","false",")",".","When","`","reverse=True","`","beginning\/end","of","sentence","tokens","are","added","after","reversing",".","return_nbest",":","If","True","requires","that","`","nbest_size","`","is","a","scalar","and","`",">","1","`",".","Returns","the","`","nbest_size","`","best","tokenizations","for","each","sentence","instead","of","a","single","one",".","The","returned","tensor","has","shape","`","[","batch","*","nbest",",","(","tokens",")","]","`",".","name",":","The","name","argument","that","is","passed","to","the","op","function",".","Returns",":","pieces",":","A","SentencepieceTokenizer","."]
code: def __init__(
      self,
      model=None,
      out_type=dtypes.int32,
      nbest_size=0,
      alpha=1.0,
      reverse=False,
      add_bos=False,
      add_eos=False,
      return_nbest=False,
      name=None,
  ):
      """Creates & initializes a Sentencepiece processor.
  
      Args:
        model: The sentencepiece model serialized proto.
        out_type: output type. tf.int32 or tf.string (Default = tf.int32) Setting
          tf.int32 directly encodes the string into an id sequence.
        nbest_size: A scalar for sampling.
          * `nbest_size = {0,1}`: No sampling is performed. (default)
          * `nbest_size > 1`: samples from the nbest_size results.
          * `nbest_size < 0`: assuming that nbest_size is infinite and samples
              from the all hypothesis (lattice) using
              forward-filtering-and-backward-sampling algorithm.
        alpha: A scalar for a smoothing parameter. Inverse temperature for
          probability rescaling.
        reverse: Reverses the tokenized sequence (Default = false)
        add_bos: Add beginning of sentence token to the result (Default = false)
        add_eos: Add end of sentence token to the result (Default = false). When
          `reverse=True` beginning/end of sentence tokens are added after
          reversing.
        return_nbest: If True requires that `nbest_size` is a scalar and `> 1`.
          Returns the `nbest_size` best tokenizations for each sentence instead
          of a single one. The returned tensor has shape
          `[batch * nbest, (tokens)]`.
        name: The name argument that is passed to the op function.
  
      Returns:
        pieces: A SentencepieceTokenizer.
      """
      super(SentencepieceTokenizer, self).__init__()
      _tf_text_sentencepiece_tokenizer_op_create_counter.get_cell().increase_by(
          1
      )
      self.nbest_size = nbest_size
      self.alpha = alpha
      self.out_type = out_type
      self.reverse = reverse
      self.add_bos = add_bos
      self.add_eos = add_eos
      self.return_nbest = return_nbest
      self._model_resource = _SentencepieceModelResource(model, name)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops.ragged import ragged_conversion_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow.python.trackable import resource
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 1 to create a new SentencepieceTokenizer object from inherited base classes: TokenizerWithOffsets, Detokenizer with example: obj = SentencepieceTokenizer(model, out_type, nbest_size, alpha, reverse, add_bos, add_eos, return_nbest, name)