class_docstr: Runs sentencepiece tokenization. Based on transformers.models.albert.tokenization_albert.AlbertTokenizer.
class_docstr_tok: ["Runs","sentencepiece","tokenization",".","Based","on","transformers.models.albert.tokenization_albert.AlbertTokenizer","."]
docstr: Tokenizes text by sentencepiece. Based on [SentencePiece](https://github.com/google/sentencepiece).
  Tokenization needs the given vocabulary.
  
  Args:
      text: A string needs to be tokenized.
  
  Returns:
      A list of sentencepiece tokens.
docstr_tok: ["Tokenizes","text","by","sentencepiece",".","Based","on","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")",".","Tokenization","needs","the","given","vocabulary",".","Args",":","text",":","A","string","needs","to","be","tokenized",".","Returns",":","A","list","of","sentencepiece","tokens","."]
code: def tokenize(self, text):
      """
      Tokenizes text by sentencepiece. Based on [SentencePiece](https://github.com/google/sentencepiece).
      Tokenization needs the given vocabulary.
  
      Args:
          text: A string needs to be tokenized.
  
      Returns:
          A list of sentencepiece tokens.
      """
      text = self.preprocess_text(text)
      pieces = self.sp_model.encode(text, out_type=str)
      new_pieces = []
      for piece in pieces:
          if (
              len(piece) > 1
              and piece[-1] == str(",")
              and piece[-2].isdigit()
          ):
              cur_pieces = self.sp_model.EncodeAsPieces(
                  piece[:-1].replace(SPIECE_UNDERLINE, "")
              )
              if (
                  piece[0] != SPIECE_UNDERLINE
                  and cur_pieces[0][0] == SPIECE_UNDERLINE
              ):
                  if len(cur_pieces[0]) == 1:
                      cur_pieces = cur_pieces[1:]
                  else:
                      cur_pieces[0] = cur_pieces[0][1:]
              cur_pieces.append(piece[-1])
              new_pieces.extend(cur_pieces)
          else:
              new_pieces.append(piece)
  
      return new_pieces
  
imports: import collections
  import copy
  import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
label_desc: Use image node_id 3 for calling the SentencepieceTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: new_pieces