class_docstr: Pre-trained XLNet Tokenizer.
  
  Args:
      pretrained_model_name (optional): a `str`, the name of
          pre-trained model (e.g., `xlnet-base-uncased`). Please refer to
          :class:`~texar.torch.modules.PretrainedXLNetMixin` for
          all supported models.
          If None, the model name in :attr:`hparams` is used.
      cache_dir (optional): the path to a folder in which the
          pre-trained models will be cached. If `None` (default),
          a default directory (``texar_data`` folder under user's home
          directory) will be used.
      hparams (dict or HParams, optional): Hyperparameters. Missing
          hyperparameter will be set to default values. See
          :meth:`default_hparams` for the hyperparameter structure
          and default values.
class_docstr_tok: ["Pre-trained","XLNet","Tokenizer",".","Args",":","pretrained_model_name","(","optional",")",":","a","`","str","`",",","the","name","of","pre-trained","model","(","e.g.",",","`","xlnet-base-uncased","`",")",".","Please","refer","to",":","class",":","`","~texar.torch.modules.PretrainedXLNetMixin","`","for","all","supported","models",".","If","None",",","the","model","name","in",":","attr",":","`","hparams","`","is","used",".","cache_dir","(","optional",")",":","the","path","to","a","folder","in","which","the","pre-trained","models","will","be","cached",".","If","`","None","`","(","default",")",",","a","default","directory","(","``","texar_data","``","folder","under","user","'s","home","directory",")","will","be","used",".","hparams","(","dict","or","HParams",",","optional",")",":","Hyperparameters",".","Missing","hyperparameter","will","be","set","to","default","values",".","See",":","meth",":","`","default_hparams","`","for","the","hyperparameter","structure","and","default","values","."]
docstr: Returns a dictionary of hyperparameters with default values.
  
  * The tokenizer is determined by the constructor argument
    :attr:`pretrained_model_name` if it's specified. In this case,
    `hparams` are ignored.
  * Otherwise, the tokenizer is determined by
    `hparams['pretrained_model_name']` if it's specified. All other
    configurations in `hparams` are ignored.
  * If the above two are `None`, the tokenizer is defined by the
    configurations in `hparams`.
  
  .. code-block:: python
  
      {
          "pretrained_model_name": "xlnet-base-cased",
          "vocab_file": None,
          "max_len": None,
          "bos_token": "<s>",
          "eos_token": "</s>",
          "unk_token": "<unk>",
          "sep_token": "<sep>",
          "pad_token": "<pad>",
          "cls_token": "<cls>",
          "mask_token": "<mask>",
          "additional_special_tokens": ["<eop>", "<eod>"],
          "do_lower_case": False,
          "remove_space": True,
          "keep_accents": False,
      }
  
  Here:
  
  `"pretrained_model_name"`: str or None
      The name of the pre-trained XLNet model.
  
  `"vocab_file"`: str or None
      The path to a sentencepiece vocabulary file.
  
  `"max_len"`: int or None
      The maximum sequence length that this model might ever be used with.
  
  `"bos_token"`: str
      Beginning of sentence token.
  
  `"eos_token"`: str
      End of sentence token.
  
  `"unk_token"`: str
      Unknown token.
  
  `"sep_token"`: str
      Separation token.
  
  `"pad_token"`: str
      Padding token.
  
  `"cls_token"`: str
      Classification token.
  
  `"mask_token"`: str
      Masking token.
  
  `"additional_special_tokens"`: list
      A list of additional special tokens.
  
  `"do_lower_case"`: bool
      Whether to lower-case the text.
  
  `"remove_space"`: bool
      Whether to remove the space in the text.
  
  `"keep_accents"`: bool
      Whether to keep the accents in the text.
  
  `"name"`: str
      Name of the tokenizer.
docstr_tok: ["Returns","a","dictionary","of","hyperparameters","with","default","values",".","*","The","tokenizer","is","determined","by","the","constructor","argument",":","attr",":","`","pretrained_model_name","`","if","it","'s","specified",".","In","this","case",",","`","hparams","`","are","ignored",".","*","Otherwise",",","the","tokenizer","is","determined","by","`","hparams","[","'pretrained_model_name","'","]","`","if","it","'s","specified",".","All","other","configurations","in","`","hparams","`","are","ignored",".","*","If","the","above","two","are","`","None","`",",","the","tokenizer","is","defined","by","the","configurations","in","`","hparams","`",".","..","code-block",":",":","python","{","``","pretrained_model_name","''",":","``","xlnet-base-cased","''",",","``","vocab_file","''",":","None",",","``","max_len","''",":","None",",","``","bos_token","''",":","``","<","s",">","''",",","``","eos_token","''",":","``","<","\/s",">","''",",","``","unk_token","''",":","``","<","unk",">","''",",","``","sep_token","''",":","``","<","sep",">","''",",","``","pad_token","''",":","``","<","pad",">","''",",","``","cls_token","''",":","``","<","cls",">","''",",","``","mask_token","''",":","``","<","mask",">","''",",","``","additional_special_tokens","''",":","[","``","<","eop",">","''",",","``","<","eod",">","''","]",",","``","do_lower_case","''",":","False",",","``","remove_space","''",":","True",",","``","keep_accents","''",":","False",",","}","Here",":","`","``","pretrained_model_name","''","`",":","str","or","None","The","name","of","the","pre-trained","XLNet","model",".","`","``","vocab_file","''","`",":","str","or","None","The","path","to","a","sentencepiece","vocabulary","file",".","`","``","max_len","''","`",":","int","or","None","The","maximum","sequence","length","that","this","model","might","ever","be","used","with",".","`","``","bos_token","''","`",":","str","Beginning","of","sentence","token",".","`","``","eos_token","''","`",":","str","End","of","sentence","token",".","`","``","unk_token","''","`",":","str","Unknown","token",".","`","``","sep_token","''","`",":","str","Separation","token",".","`","``","pad_token","''","`",":","str","Padding","token",".","`","``","cls_token","''","`",":","str","Classification","token",".","`","``","mask_token","''","`",":","str","Masking","token",".","`","``","additional_special_tokens","''","`",":","list","A","list","of","additional","special","tokens",".","`","``","do_lower_case","''","`",":","bool","Whether","to","lower-case","the","text",".","`","``","remove_space","''","`",":","bool","Whether","to","remove","the","space","in","the","text",".","`","``","keep_accents","''","`",":","bool","Whether","to","keep","the","accents","in","the","text",".","`","``","name","''","`",":","str","Name","of","the","tokenizer","."]
code: def default_hparams() -> Dict[str, Any]:
      r"""Returns a dictionary of hyperparameters with default values.
  
      * The tokenizer is determined by the constructor argument
        :attr:`pretrained_model_name` if it's specified. In this case,
        `hparams` are ignored.
      * Otherwise, the tokenizer is determined by
        `hparams['pretrained_model_name']` if it's specified. All other
        configurations in `hparams` are ignored.
      * If the above two are `None`, the tokenizer is defined by the
        configurations in `hparams`.
  
      .. code-block:: python
  
          {
              "pretrained_model_name": "xlnet-base-cased",
              "vocab_file": None,
              "max_len": None,
              "bos_token": "<s>",
              "eos_token": "</s>",
              "unk_token": "<unk>",
              "sep_token": "<sep>",
              "pad_token": "<pad>",
              "cls_token": "<cls>",
              "mask_token": "<mask>",
              "additional_special_tokens": ["<eop>", "<eod>"],
              "do_lower_case": False,
              "remove_space": True,
              "keep_accents": False,
          }
  
      Here:
  
      `"pretrained_model_name"`: str or None
          The name of the pre-trained XLNet model.
  
      `"vocab_file"`: str or None
          The path to a sentencepiece vocabulary file.
  
      `"max_len"`: int or None
          The maximum sequence length that this model might ever be used with.
  
      `"bos_token"`: str
          Beginning of sentence token.
  
      `"eos_token"`: str
          End of sentence token.
  
      `"unk_token"`: str
          Unknown token.
  
      `"sep_token"`: str
          Separation token.
  
      `"pad_token"`: str
          Padding token.
  
      `"cls_token"`: str
          Classification token.
  
      `"mask_token"`: str
          Masking token.
  
      `"additional_special_tokens"`: list
          A list of additional special tokens.
  
      `"do_lower_case"`: bool
          Whether to lower-case the text.
  
      `"remove_space"`: bool
          Whether to remove the space in the text.
  
      `"keep_accents"`: bool
          Whether to keep the accents in the text.
  
      `"name"`: str
          Name of the tokenizer.
      """
      return {
          "pretrained_model_name": "xlnet-base-cased",
          "vocab_file": None,
          "max_len": None,
          "bos_token": "<s>",
          "eos_token": "</s>",
          "unk_token": "<unk>",
          "sep_token": "<sep>",
          "pad_token": "<pad>",
          "cls_token": "<cls>",
          "mask_token": "<mask>",
          "additional_special_tokens": ["<eop>", "<eod>"],
          "do_lower_case": False,
          "remove_space": True,
          "keep_accents": False,
          "name": "xlnet_tokenizer",
          "@no_typecheck": ["pretrained_model_name"],
      }
  
imports: from typing import Any, Dict, List, Optional, Tuple
  import os
  import unicodedata
  from shutil import copyfile
  import sentencepiece
  from texar.tf.modules.pretrained.xlnet import PretrainedXLNetMixin
  from texar.tf.data.tokenizers.tokenizer_base import TokenizerBase
  from texar.tf.utils.utils import truncate_seq_pair
label_desc: Use image node_id 12 for calling the XLNetTokenizer obj's underlying member method code with example usage: obj.default_hparams() and returns: dict