class_docstr: Pre-trained XLNet Tokenizer.
  
  Args:
      pretrained_model_name (optional): a `str`, the name of
          pre-trained model (e.g., `xlnet-base-uncased`). Please refer to
          :class:`~texar.torch.modules.PretrainedXLNetMixin` for
          all supported models.
          If None, the model name in :attr:`hparams` is used.
      cache_dir (optional): the path to a folder in which the
          pre-trained models will be cached. If `None` (default),
          a default directory (``texar_data`` folder under user's home
          directory) will be used.
      hparams (dict or HParams, optional): Hyperparameters. Missing
          hyperparameter will be set to default values. See
          :meth:`default_hparams` for the hyperparameter structure
          and default values.
class_docstr_tok: ["Pre-trained","XLNet","Tokenizer",".","Args",":","pretrained_model_name","(","optional",")",":","a","`","str","`",",","the","name","of","pre-trained","model","(","e.g.",",","`","xlnet-base-uncased","`",")",".","Please","refer","to",":","class",":","`","~texar.torch.modules.PretrainedXLNetMixin","`","for","all","supported","models",".","If","None",",","the","model","name","in",":","attr",":","`","hparams","`","is","used",".","cache_dir","(","optional",")",":","the","path","to","a","folder","in","which","the","pre-trained","models","will","be","cached",".","If","`","None","`","(","default",")",",","a","default","directory","(","``","texar_data","``","folder","under","user","'s","home","directory",")","will","be","used",".","hparams","(","dict","or","HParams",",","optional",")",":","Hyperparameters",".","Missing","hyperparameter","will","be","set","to","default","values",".","See",":","meth",":","`","default_hparams","`","for","the","hyperparameter","structure","and","default","values","."]
code: def _map_text_to_token(
      self, text: str, sample: bool = False  # type: ignore
  ) -> List[str]:
      text = self._preprocess_text(text)
      if not sample:
          pieces = self.sp_model.EncodeAsPieces(text)
      else:
          pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1)
  
      new_pieces: List[str] = []
      for piece in pieces:
          if (
              len(piece) > 1
              and piece[-1] == ","
              and piece[-2].isdigit()
          ):
              cur_pieces = self.sp_model.EncodeAsPieces(
                  piece[:-1].replace(SPIECE_UNDERLINE, "")
              )
              if (
                  piece[0] != SPIECE_UNDERLINE
                  and cur_pieces[0][0] == SPIECE_UNDERLINE
              ):
                  if len(cur_pieces[0]) == 1:
                      cur_pieces = cur_pieces[1:]
                  else:
                      cur_pieces[0] = cur_pieces[0][1:]
              cur_pieces.append(piece[-1])
              new_pieces.extend(cur_pieces)
          else:
              new_pieces.append(piece)
  
      return new_pieces
  
imports: from typing import Any, Dict, List, Optional, Tuple
  import os
  import unicodedata
  from shutil import copyfile
  import sentencepiece
  from texar.tf.modules.pretrained.xlnet import PretrainedXLNetMixin
  from texar.tf.data.tokenizers.tokenizer_base import TokenizerBase
  from texar.tf.utils.utils import truncate_seq_pair
label_desc: Use image node_id 5 for calling the XLNetTokenizer obj's underlying member method code with example usage: obj._map_text_to_token(text, sample) and returns: new_pieces