class_docstr: Pre-trained XLNet Tokenizer.
  
  Args:
      pretrained_model_name (optional): a `str`, the name of
          pre-trained model (e.g., `xlnet-base-uncased`). Please refer to
          :class:`~texar.torch.modules.PretrainedXLNetMixin` for
          all supported models.
          If None, the model name in :attr:`hparams` is used.
      cache_dir (optional): the path to a folder in which the
          pre-trained models will be cached. If `None` (default),
          a default directory (``texar_data`` folder under user's home
          directory) will be used.
      hparams (dict or HParams, optional): Hyperparameters. Missing
          hyperparameter will be set to default values. See
          :meth:`default_hparams` for the hyperparameter structure
          and default values.
class_docstr_tok: ["Pre-trained","XLNet","Tokenizer",".","Args",":","pretrained_model_name","(","optional",")",":","a","`","str","`",",","the","name","of","pre-trained","model","(","e.g.",",","`","xlnet-base-uncased","`",")",".","Please","refer","to",":","class",":","`","~texar.torch.modules.PretrainedXLNetMixin","`","for","all","supported","models",".","If","None",",","the","model","name","in",":","attr",":","`","hparams","`","is","used",".","cache_dir","(","optional",")",":","the","path","to","a","folder","in","which","the","pre-trained","models","will","be","cached",".","If","`","None","`","(","default",")",",","a","default","directory","(","``","texar_data","``","folder","under","user","'s","home","directory",")","will","be","used",".","hparams","(","dict","or","HParams",",","optional",")",":","Hyperparameters",".","Missing","hyperparameter","will","be","set","to","default","values",".","See",":","meth",":","`","default_hparams","`","for","the","hyperparameter","structure","and","default","values","."]
docstr: Adds special tokens to a sequence or sequence pair and computes the
  corresponding segment ids and input mask for XLNet specific tasks.
  The sequence will be truncated if its length is larger than
  ``max_seq_length``.
  
  A XLNet sequence has the following format:
  X `[sep_token]` `[cls_token]`
  
  A XLNet sequence pair has the following format:
  `[cls_token]` A `[sep_token]` B `[sep_token]`
  
  Args:
      text_a: The first input text.
      text_b: The second input text.
      max_seq_length: Maximum sequence length.
  
  Returns:
      A tuple of `(input_ids, segment_ids, input_mask)`, where
  
      - ``input_ids``: A list of input token ids with added
        special token ids.
      - ``segment_ids``: A list of segment ids.
      - ``input_mask``: A list of mask ids. The mask has 1 for real
        tokens and 0 for padding tokens. Only real tokens are
        attended to.
docstr_tok: ["Adds","special","tokens","to","a","sequence","or","sequence","pair","and","computes","the","corresponding","segment","ids","and","input","mask","for","XLNet","specific","tasks",".","The","sequence","will","be","truncated","if","its","length","is","larger","than","``","max_seq_length","``",".","A","XLNet","sequence","has","the","following","format",":","X","`","[","sep_token","]","`","`","[","cls_token","]","`","A","XLNet","sequence","pair","has","the","following","format",":","`","[","cls_token","]","`","A","`","[","sep_token","]","`","B","`","[","sep_token","]","`","Args",":","text_a",":","The","first","input","text",".","text_b",":","The","second","input","text",".","max_seq_length",":","Maximum","sequence","length",".","Returns",":","A","tuple","of","`","(","input_ids",",","segment_ids",",","input_mask",")","`",",","where","-","``","input_ids","``",":","A","list","of","input","token","ids","with","added","special","token","ids",".","-","``","segment_ids","``",":","A","list","of","segment","ids",".","-","``","input_mask","``",":","A","list","of","mask","ids",".","The","mask","has","1","for","real","tokens","and","0","for","padding","tokens",".","Only","real","tokens","are","attended","to","."]
code: def encode_text(
      self,
      text_a: str,
      text_b: Optional[str] = None,
      max_seq_length: Optional[int] = None,
  ) -> Tuple[List[int], List[int], List[int]]:
      r"""Adds special tokens to a sequence or sequence pair and computes the
      corresponding segment ids and input mask for XLNet specific tasks.
      The sequence will be truncated if its length is larger than
      ``max_seq_length``.
  
      A XLNet sequence has the following format:
      X `[sep_token]` `[cls_token]`
  
      A XLNet sequence pair has the following format:
      `[cls_token]` A `[sep_token]` B `[sep_token]`
  
      Args:
          text_a: The first input text.
          text_b: The second input text.
          max_seq_length: Maximum sequence length.
  
      Returns:
          A tuple of `(input_ids, segment_ids, input_mask)`, where
  
          - ``input_ids``: A list of input token ids with added
            special token ids.
          - ``segment_ids``: A list of segment ids.
          - ``input_mask``: A list of mask ids. The mask has 1 for real
            tokens and 0 for padding tokens. Only real tokens are
            attended to.
      """
      if max_seq_length is None:
          max_seq_length = self.max_len
  
      cls_token_id = self._map_token_to_id(self.cls_token)
      sep_token_id = self._map_token_to_id(self.sep_token)
  
      token_ids_a = self.map_text_to_id(text_a)
      assert isinstance(token_ids_a, list)
  
      token_ids_b = None
      if text_b:
          token_ids_b = self.map_text_to_id(text_b)
  
      if token_ids_b:
          assert isinstance(token_ids_b, list)
          # Modifies `token_ids_a` and `token_ids_b` in place so that the
          # total length is less than the specified length.
          # Account for [CLS], [SEP], [SEP] with "- 3"
          truncate_seq_pair(
              token_ids_a, token_ids_b, max_seq_length - 3
          )
  
          input_ids = (
              token_ids_a
              + [sep_token_id]
              + token_ids_b
              + [sep_token_id]
              + [cls_token_id]
          )
          segment_ids = (
              [SEG_ID_A] * (len(token_ids_a) + 1)
              + [SEG_ID_B] * (len(token_ids_b) + 1)
              + [SEG_ID_CLS]
          )
      else:
          # Account for [CLS] and [SEP] with "- 2"
          token_ids = token_ids_a[: max_seq_length - 2]
  
          input_ids = token_ids + [sep_token_id] + [cls_token_id]
          segment_ids = [SEG_ID_A] * (len(input_ids) - 1) + [SEG_ID_CLS]
  
      input_mask = [0] * len(input_ids)
  
      # Zero-pad up to the maximum sequence length.
      input_ids = [0] * (max_seq_length - len(input_ids)) + input_ids
      input_mask = [1] * (max_seq_length - len(input_mask)) + input_mask
      segment_ids = [SEG_ID_PAD] * (
          max_seq_length - len(segment_ids)
      ) + segment_ids
  
      assert len(input_ids) == max_seq_length
      assert len(input_mask) == max_seq_length
      assert len(segment_ids) == max_seq_length
  
      return input_ids, segment_ids, input_mask
  
imports: from typing import Any, Dict, List, Optional, Tuple
  import os
  import unicodedata
  from shutil import copyfile
  import sentencepiece
  from texar.tf.modules.pretrained.xlnet import PretrainedXLNetMixin
  from texar.tf.data.tokenizers.tokenizer_base import TokenizerBase
  from texar.tf.utils.utils import truncate_seq_pair
label_desc: Use image node_id 11 for calling the XLNetTokenizer obj's underlying member method code with example usage: obj.encode_text(text_a, text_b, max_seq_length) and returns: input_ids,  segment_ids,  input_mask