class_docstr: Constructs a BERT tokenizer. Based on WordPiece.
  Args:
      vocab_file (:obj:`string`):
          File containing the vocabulary.
      do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`):
          Whether to lowercase the input when tokenizing.
      do_basic_tokenize (:obj:`bool`, `optional`, defaults to :obj:`True`):
          Whether to do basic tokenization before WordPiece.
      never_split (:obj:`bool`, `optional`, defaults to :obj:`True`):
          List of tokens which will never be split during tokenization. Only has an effect when
          :obj:`do_basic_tokenize=True`
      unk_token (:obj:`string`, `optional`, defaults to '[UNK]'):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      sep_token (:obj:`string`, `optional`, defaults to '[SEP]'):
          The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
          for sequence classification or for a text and a question for question answering.
          It is also used as the last token of a sequence built with special tokens.
      pad_token (:obj:`string`, `optional`, defaults to '[PAD]'):
          The token used for padding, for example when batching sequences of different lengths.
      cls_token (:obj:`string`, `optional`, defaults to '[CLS]'):
          The classifier token which is used when doing sequence classification (classification of the whole
          sequence instead of per-token classification). It is the first token of the sequence when built with
          special tokens.
      mask_token (:obj:`string`, `optional`, defaults to '[MASK]'):
          The token used for masking values. This is the token used when training this model with masked language
          modeling. This is the token which the model will try to predict.
      tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
          Whether to tokenize Chinese characters.
          This should likely be deactivated for Japanese:
          see: https://github.com/huggingface/transformers/issues/328
class_docstr_tok: ["Constructs","a","BERT","tokenizer",".","Based","on","WordPiece",".","Args",":","vocab_file","(",":","obj",":","`","string","`",")",":","File","containing","the","vocabulary",".","do_lower_case","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","lowercase","the","input","when","tokenizing",".","do_basic_tokenize","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","do","basic","tokenization","before","WordPiece",".","never_split","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","List","of","tokens","which","will","never","be","split","during","tokenization",".","Only","has","an","effect","when",":","obj",":","`","do_basic_tokenize=True","`","unk_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","UNK","]","'",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","sep_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","SEP","]","'",")",":","The","separator","token",",","which","is","used","when","building","a","sequence","from","multiple","sequences",",","e.g",".","two","sequences","for","sequence","classification","or","for","a","text","and","a","question","for","question","answering",".","It","is","also","used","as","the","last","token","of","a","sequence","built","with","special","tokens",".","pad_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","PAD","]","'",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","cls_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","CLS","]","'",")",":","The","classifier","token","which","is","used","when","doing","sequence","classification","(","classification","of","the","whole","sequence","instead","of","per-token","classification",")",".","It","is","the","first","token","of","the","sequence","when","built","with","special","tokens",".","mask_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","MASK","]","'",")",":","The","token","used","for","masking","values",".","This","is","the","token","used","when","training","this","model","with","masked","language","modeling",".","This","is","the","token","which","the","model","will","try","to","predict",".","tokenize_chinese_chars","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese",":","see",":","https",":","\/\/github.com\/huggingface\/transformers\/issues\/328"]
docstr: Returns a dictionary containing the encoded sequence or sequence pair and additional information:
  the mask for sequence classification and the overflowing elements if a ``max_seq_len`` is specified.
  Args:
      text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`):
          The first sequence to be encoded. This can be a string, a list of strings (tokenized string using
          the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
          method)
      text_pair (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`, `optional`, defaults to :obj:`None`):
          Optional second sequence to be encoded. This can be a string, a list of strings (tokenized
          string using the `tokenize` method) or a list of integers (tokenized string ids using the
          `convert_tokens_to_ids` method)
      max_seq_len (:obj:`int`, `optional`, defaults to :int:`None`):
          If set to a number, will limit the total sequence returned so that it has a maximum length.
          If there are overflowing tokens, those will be added to the returned dictionary
      pad_to_max_seq_len (:obj:`bool`, `optional`, defaults to :obj:`True`):
          If set to True, the returned sequences will be padded according to the model's padding side and
          padding index, up to their max length. If no max length is specified, the padding is done up to the
          model's max length.
      truncation_strategy (:obj:`str`, `optional`, defaults to `longest_first`):
          String selected in the following options:
          - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_seq_len
            starting from the longest one at each token (when there is a pair of input sequences)
          - 'only_first': Only truncate the first sequence
          - 'only_second': Only truncate the second sequence
          - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_seq_len)
      return_position_ids (:obj:`bool`, `optional`, defaults to :obj:`False`):
          Set to True to return tokens position ids (default True).
      return_segment_ids (:obj:`bool`, `optional`, defaults to :obj:`True`):
          Whether to return token type IDs.
      return_input_mask (:obj:`bool`, `optional`, defaults to :obj:`False`):
          Whether to return the attention mask.
      return_length (:obj:`int`, defaults to :obj:`True`):
          If set the resulting dictionary will include the length of each encoded inputs
      return_overflowing_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
          Set to True to return overflowing token information (default False).
      return_special_tokens_mask (:obj:`bool`, `optional`, defaults to :obj:`False`):
          Set to True to return special tokens mask information (default False).
  Return:
      A Dictionary of shape::
          {
              input_ids: list[int],
              position_ids: list[int] if return_position_ids is True (default)
              segment_ids: list[int] if return_segment_ids is True (default)
              input_mask: list[int] if return_input_mask is True (default)
              seq_len: int if return_length is True (default)
              overflowing_tokens: list[int] if a ``max_seq_len`` is specified and return_overflowing_tokens is True
              num_truncated_tokens: int if a ``max_seq_len`` is specified and return_overflowing_tokens is True
              special_tokens_mask: list[int] if return_special_tokens_mask is True
          }
      With the fields:
      - ``input_ids``: list of token ids to be fed to a model
      - ``position_ids``: list of token position ids to be fed to a model
      - ``segment_ids``: list of token type ids to be fed to a model
      - ``input_mask``: list of indices specifying which tokens should be attended to by the model
      - ``length``: the input_ids length
      - ``overflowing_tokens``: list of overflowing tokens if a max length is specified.
      - ``num_truncated_tokens``: number of overflowing tokens a ``max_seq_len`` is specified
      - ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added
        tokens and 1 specifying sequence tokens.
docstr_tok: ["Returns","a","dictionary","containing","the","encoded","sequence","or","sequence","pair","and","additional","information",":","the","mask","for","sequence","classification","and","the","overflowing","elements","if","a","``","max_seq_len","``","is","specified",".","Args",":","text","(",":","obj",":","`","str","`",",",":","obj",":","`","List","[","str","]","`","or",":","obj",":","`","List","[","int","]","`",")",":","The","first","sequence","to","be","encoded",".","This","can","be","a","string",",","a","list","of","strings","(","tokenized","string","using","the","`","tokenize","`","method",")","or","a","list","of","integers","(","tokenized","string","ids","using","the","`","convert_tokens_to_ids","`","method",")","text_pair","(",":","obj",":","`","str","`",",",":","obj",":","`","List","[","str","]","`","or",":","obj",":","`","List","[","int","]","`",",","`","optional","`",",","defaults","to",":","obj",":","`","None","`",")",":","Optional","second","sequence","to","be","encoded",".","This","can","be","a","string",",","a","list","of","strings","(","tokenized","string","using","the","`","tokenize","`","method",")","or","a","list","of","integers","(","tokenized","string","ids","using","the","`","convert_tokens_to_ids","`","method",")","max_seq_len","(",":","obj",":","`","int","`",",","`","optional","`",",","defaults","to",":","int",":","`","None","`",")",":","If","set","to","a","number",",","will","limit","the","total","sequence","returned","so","that","it","has","a","maximum","length",".","If","there","are","overflowing","tokens",",","those","will","be","added","to","the","returned","dictionary","pad_to_max_seq_len","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","If","set","to","True",",","the","returned","sequences","will","be","padded","according","to","the","model","'s","padding","side","and","padding","index",",","up","to","their","max","length",".","If","no","max","length","is","specified",",","the","padding","is","done","up","to","the","model","'s","max","length",".","truncation_strategy","(",":","obj",":","`","str","`",",","`","optional","`",",","defaults","to","`","longest_first","`",")",":","String","selected","in","the","following","options",":","-","'longest_first","'","(","default",")","Iteratively","reduce","the","inputs","sequence","until","the","input","is","under","max_seq_len","starting","from","the","longest","one","at","each","token","(","when","there","is","a","pair","of","input","sequences",")","-","'only_first","'",":","Only","truncate","the","first","sequence","-","'only_second","'",":","Only","truncate","the","second","sequence","-","'do_not_truncate","'",":","Does","not","truncate","(","raise","an","error","if","the","input","sequence","is","longer","than","max_seq_len",")","return_position_ids","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","False","`",")",":","Set","to","True","to","return","tokens","position","ids","(","default","True",")",".","return_segment_ids","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","return","token","type","IDs",".","return_input_mask","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","False","`",")",":","Whether","to","return","the","attention","mask",".","return_length","(",":","obj",":","`","int","`",",","defaults","to",":","obj",":","`","True","`",")",":","If","set","the","resulting","dictionary","will","include","the","length","of","each","encoded","inputs","return_overflowing_tokens","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","False","`",")",":","Set","to","True","to","return","overflowing","token","information","(","default","False",")",".","return_special_tokens_mask","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","False","`",")",":","Set","to","True","to","return","special","tokens","mask","information","(","default","False",")",".","Return",":","A","Dictionary","of","shape",":",":","{","input_ids",":","list","[","int","]",",","position_ids",":","list","[","int","]","if","return_position_ids","is","True","(","default",")","segment_ids",":","list","[","int","]","if","return_segment_ids","is","True","(","default",")","input_mask",":","list","[","int","]","if","return_input_mask","is","True","(","default",")","seq_len",":","int","if","return_length","is","True","(","default",")","overflowing_tokens",":","list","[","int","]","if","a","``","max_seq_len","``","is","specified","and","return_overflowing_tokens","is","True","num_truncated_tokens",":","int","if","a","``","max_seq_len","``","is","specified","and","return_overflowing_tokens","is","True","special_tokens_mask",":","list","[","int","]","if","return_special_tokens_mask","is","True","}","With","the","fields",":","-","``","input_ids","``",":","list","of","token","ids","to","be","fed","to","a","model","-","``","position_ids","``",":","list","of","token","position","ids","to","be","fed","to","a","model","-","``","segment_ids","``",":","list","of","token","type","ids","to","be","fed","to","a","model","-","``","input_mask","``",":","list","of","indices","specifying","which","tokens","should","be","attended","to","by","the","model","-","``","length","``",":","the","input_ids","length","-","``","overflowing_tokens","``",":","list","of","overflowing","tokens","if","a","max","length","is","specified",".","-","``","num_truncated_tokens","``",":","number","of","overflowing","tokens","a","``","max_seq_len","``","is","specified","-","``","special_tokens_mask","``",":","if","adding","special","tokens",",","this","is","a","list","of","[","0",",","1","]",",","with","0","specifying","special","added","tokens","and","1","specifying","sequence","tokens","."]
code: def encode(
      self,
      text: Union[str, List[str], List[int]],
      text_pair: Optional[Union[str, List[str], List[int]]] = None,
      max_seq_len: Optional[int] = None,
      pad_to_max_seq_len: bool = True,
      truncation_strategy: str = "longest_first",
      return_position_ids: bool = False,
      return_segment_ids: bool = True,
      return_input_mask: bool = False,
      return_length: bool = True,
      return_overflowing_tokens: bool = False,
      return_special_tokens_mask: bool = False,
  ):
      """
      Returns a dictionary containing the encoded sequence or sequence pair and additional information:
      the mask for sequence classification and the overflowing elements if a ``max_seq_len`` is specified.
      Args:
          text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`):
              The first sequence to be encoded. This can be a string, a list of strings (tokenized string using
              the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
              method)
          text_pair (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`, `optional`, defaults to :obj:`None`):
              Optional second sequence to be encoded. This can be a string, a list of strings (tokenized
              string using the `tokenize` method) or a list of integers (tokenized string ids using the
              `convert_tokens_to_ids` method)
          max_seq_len (:obj:`int`, `optional`, defaults to :int:`None`):
              If set to a number, will limit the total sequence returned so that it has a maximum length.
              If there are overflowing tokens, those will be added to the returned dictionary
          pad_to_max_seq_len (:obj:`bool`, `optional`, defaults to :obj:`True`):
              If set to True, the returned sequences will be padded according to the model's padding side and
              padding index, up to their max length. If no max length is specified, the padding is done up to the
              model's max length.
          truncation_strategy (:obj:`str`, `optional`, defaults to `longest_first`):
              String selected in the following options:
              - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_seq_len
                starting from the longest one at each token (when there is a pair of input sequences)
              - 'only_first': Only truncate the first sequence
              - 'only_second': Only truncate the second sequence
              - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_seq_len)
          return_position_ids (:obj:`bool`, `optional`, defaults to :obj:`False`):
              Set to True to return tokens position ids (default True).
          return_segment_ids (:obj:`bool`, `optional`, defaults to :obj:`True`):
              Whether to return token type IDs.
          return_input_mask (:obj:`bool`, `optional`, defaults to :obj:`False`):
              Whether to return the attention mask.
          return_length (:obj:`int`, defaults to :obj:`True`):
              If set the resulting dictionary will include the length of each encoded inputs
          return_overflowing_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
              Set to True to return overflowing token information (default False).
          return_special_tokens_mask (:obj:`bool`, `optional`, defaults to :obj:`False`):
              Set to True to return special tokens mask information (default False).
      Return:
          A Dictionary of shape::
              {
                  input_ids: list[int],
                  position_ids: list[int] if return_position_ids is True (default)
                  segment_ids: list[int] if return_segment_ids is True (default)
                  input_mask: list[int] if return_input_mask is True (default)
                  seq_len: int if return_length is True (default)
                  overflowing_tokens: list[int] if a ``max_seq_len`` is specified and return_overflowing_tokens is True
                  num_truncated_tokens: int if a ``max_seq_len`` is specified and return_overflowing_tokens is True
                  special_tokens_mask: list[int] if return_special_tokens_mask is True
              }
          With the fields:
          - ``input_ids``: list of token ids to be fed to a model
          - ``position_ids``: list of token position ids to be fed to a model
          - ``segment_ids``: list of token type ids to be fed to a model
          - ``input_mask``: list of indices specifying which tokens should be attended to by the model
          - ``length``: the input_ids length
          - ``overflowing_tokens``: list of overflowing tokens if a max length is specified.
          - ``num_truncated_tokens``: number of overflowing tokens a ``max_seq_len`` is specified
          - ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added
            tokens and 1 specifying sequence tokens.
      """
  
      def get_input_ids(text):
          if isinstance(text, str):
              tokens = self.tokenize(text)
              return self.convert_tokens_to_ids(tokens)
          elif (
              isinstance(text, (list, tuple))
              and len(text) > 0
              and isinstance(text[0], str)
          ):
              return self.convert_tokens_to_ids(text)
          elif (
              isinstance(text, (list, tuple))
              and len(text) > 0
              and isinstance(text[0], int)
          ):
              return text
          else:
              raise ValueError(
                  "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
              )
  
      ids = get_input_ids(text)
      pair_ids = (
          get_input_ids(text_pair) if text_pair is not None else None
      )
  
      pair = bool(pair_ids is not None)
      len_ids = len(ids)
      len_pair_ids = len(pair_ids) if pair else 0
  
      encoded_inputs = {}
  
      # Truncation: Handle max sequence length
      total_len = (
          len_ids
          + len_pair_ids
          + (self.num_special_tokens_to_add(pair=pair))
      )
      if max_seq_len and total_len > max_seq_len:
          ids, pair_ids, overflowing_tokens = self.truncate_sequences(
              ids,
              pair_ids=pair_ids,
              num_tokens_to_remove=total_len - max_seq_len,
              truncation_strategy=truncation_strategy,
          )
          if return_overflowing_tokens:
              encoded_inputs["overflowing_tokens"] = overflowing_tokens
              encoded_inputs["num_truncated_tokens"] = (
                  total_len - max_seq_len
              )
  
      # Add special tokens
      sequence = self.build_inputs_with_special_tokens(ids, pair_ids)
      segment_ids = self.create_segment_ids_from_sequences(
          ids, pair_ids
      )
  
      # Build output dictionnary
      encoded_inputs["input_ids"] = sequence
      if return_segment_ids:
          encoded_inputs["segment_ids"] = segment_ids
      if return_special_tokens_mask:
          encoded_inputs[
              "special_tokens_mask"
          ] = self.get_special_tokens_mask(ids, pair_ids)
      if return_length:
          encoded_inputs["seq_len"] = len(encoded_inputs["input_ids"])
  
      # Check lengths
      assert (
          max_seq_len is None
          or len(encoded_inputs["input_ids"]) <= max_seq_len
      )
  
      # Padding
      needs_to_be_padded = (
          pad_to_max_seq_len
          and max_seq_len
          and len(encoded_inputs["input_ids"]) < max_seq_len
      )
  
      if needs_to_be_padded:
          difference = max_seq_len - len(encoded_inputs["input_ids"])
          if return_input_mask:
              encoded_inputs["input_mask"] = [1] * len(
                  encoded_inputs["input_ids"]
              ) + [0] * difference
          if return_segment_ids:
              encoded_inputs["segment_ids"] = (
                  encoded_inputs["segment_ids"]
                  + [self.pad_token_type_id] * difference
              )
          if return_special_tokens_mask:
              encoded_inputs["special_tokens_mask"] = (
                  encoded_inputs["special_tokens_mask"]
                  + [1] * difference
              )
          encoded_inputs["input_ids"] = (
              encoded_inputs["input_ids"]
              + [self.pad_token_id] * difference
          )
      else:
          if return_input_mask:
              encoded_inputs["input_mask"] = [1] * len(
                  encoded_inputs["input_ids"]
              )
  
      if return_position_ids:
          encoded_inputs["position_ids"] = list(
              range(len(encoded_inputs["input_ids"]))
          )
  
      return encoded_inputs
  
imports: import collections
  import os
  import pickle
  import unicodedata
  from typing import Dict, List, Optional, Union, Tuple
  from paddle.utils import try_import
  from paddlehub.text.utils import load_vocab, is_whitespace, is_control, is_punctuation, whitespace_tokenize, is_chinese_char
label_desc: Use image node_id 16 for calling the BertTokenizer obj's underlying member method code with example usage: obj.encode(text, text_pair, max_seq_len, pad_to_max_seq_len, truncation_strategy, return_position_ids, return_segment_ids, return_input_mask, return_length, return_overflowing_tokens, return_special_tokens_mask) and returns: encoded_inputs, self, self, text