class_docstr: Sentencepiecetokenizer https://github.com/google/sentencepiece.
  
      Args:
      model_path: path to sentence piece tokenizer model. To create the model use create_spt_model()
      special_tokens: either list of special tokens or dictionary of token name to token value
      legacy: when set to True, the previous behavior of the SentecePiece wrapper will be restored,
          including the possibility to add special tokens inside wrapper.
class_docstr_tok: ["Sentencepiecetokenizer","https",":","\/\/github.com\/google\/sentencepiece",".","Args",":","model_path",":","path","to","sentence","piece","tokenizer","model",".","To","create","the","model","use","create_spt_model","(",")","special_tokens",":","either","list","of","special","tokens","or","dictionary","of","token","name","to","token","value","legacy",":","when","set","to","True",",","the","previous","behavior","of","the","SentecePiece","wrapper","will","be","restored",",","including","the","possibility","to","add","special","tokens","inside","wrapper","."]
code: def text_to_ids(self, text):
      if self.legacy:
          ids = []
          idx = 0
          last_idx = 0
  
          while 1:
              indices = {}
  
              for token in self.special_token_to_id:
                  try:
                      indices[token] = text[idx:].index(token)
                  except ValueError:
                      continue
  
              if len(indices) == 0:
                  break
  
              next_token = min(indices, key=indices.get)
              next_idx = idx + indices[next_token]
  
              ids.extend(
                  self.tokenizer.encode_as_ids(text[idx:next_idx])
              )
              ids.append(self.special_token_to_id[next_token])
              idx = next_idx + len(next_token)
  
          ids.extend(self.tokenizer.encode_as_ids(text[idx:]))
          return ids
  
      return self.tokenizer.encode_as_ids(text)
  
imports: import os
  from typing import Dict, List, Optional, Union
  import numpy
  import sentencepiece
  from nemo.collections.common.parts.utils import if_exist
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.utils import logging
label_desc: Use image node_id 3 for calling the SentencePieceTokenizer obj's underlying member method code with example usage: obj.text_to_ids(text) and returns: self, ids