class_docstr: Customtokenizer which will tokenize the input text as words or phases and convert the words (str) to an index (int) using the vocab.
  If you would like tokens, please use `hub.BertTokenizer`.
docstr: Returns a dictionary containing the encoded sequence or sequence pair and additional information:
  the mask for sequence classification and the overflowing elements if a ``max_seq_len`` is specified.
  Args:
      text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`):
          The first sequence to be encoded. This can be a string, a list of strings (tokenized string using
          the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
          method)
      text_pair (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`, `optional`, defaults to :obj:`None`):
          It's nonsense, just for compatible.
      max_seq_len (:obj:`int`, `optional`, defaults to :int:`None`):
          If set to a number, will limit the total sequence returned so that it has a maximum length.
          If there are overflowing tokens, those will be added to the returned dictionary
      pad_to_max_seq_len (:obj:`bool`, `optional`, defaults to :obj:`True`):
          If set to True, the returned sequences will be padded according to the model's padding side and
          padding index, up to their max length. If no max length is specified, the padding is done up to the
          model's max length.
      truncation_strategy (:obj:`str`, `optional`, defaults to `longest_first`):
          String selected in the following options:
          - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_seq_len
            starting from the longest one at each token (when there is a pair of input sequences)
          - 'only_first': Only truncate the first sequence
          - 'only_second': Only truncate the second sequence
          - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_seq_len)
      return_lengths (:obj:`bool`, `optional`, defaults to :obj:`True`):
          If set the resulting dictionary will include the length of each encoded inputs
      return_overflowing_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
          Set to True to return overflowing token information (default False).
  Return:
      A Dictionary of shape::
          {
              text: list[int],
              seq_len: int if return_length is True (default)
              overflowing_tokens: list[int] if a ``max_seq_len`` is specified and return_overflowing_tokens is True
          }
      With the fields:
      - ``text``: list of token ids to be fed to a model
      - ``length``: the input_ids length
      - ``overflowing_tokens``: list of overflowing tokens if a max length is specified.
docstr_tok: ["Returns","a","dictionary","containing","the","encoded","sequence","or","sequence","pair","and","additional","information",":","the","mask","for","sequence","classification","and","the","overflowing","elements","if","a","``","max_seq_len","``","is","specified",".","Args",":","text","(",":","obj",":","`","str","`",",",":","obj",":","`","List","[","str","]","`","or",":","obj",":","`","List","[","int","]","`",")",":","The","first","sequence","to","be","encoded",".","This","can","be","a","string",",","a","list","of","strings","(","tokenized","string","using","the","`","tokenize","`","method",")","or","a","list","of","integers","(","tokenized","string","ids","using","the","`","convert_tokens_to_ids","`","method",")","text_pair","(",":","obj",":","`","str","`",",",":","obj",":","`","List","[","str","]","`","or",":","obj",":","`","List","[","int","]","`",",","`","optional","`",",","defaults","to",":","obj",":","`","None","`",")",":","It","'s","nonsense",",","just","for","compatible",".","max_seq_len","(",":","obj",":","`","int","`",",","`","optional","`",",","defaults","to",":","int",":","`","None","`",")",":","If","set","to","a","number",",","will","limit","the","total","sequence","returned","so","that","it","has","a","maximum","length",".","If","there","are","overflowing","tokens",",","those","will","be","added","to","the","returned","dictionary","pad_to_max_seq_len","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","If","set","to","True",",","the","returned","sequences","will","be","padded","according","to","the","model","'s","padding","side","and","padding","index",",","up","to","their","max","length",".","If","no","max","length","is","specified",",","the","padding","is","done","up","to","the","model","'s","max","length",".","truncation_strategy","(",":","obj",":","`","str","`",",","`","optional","`",",","defaults","to","`","longest_first","`",")",":","String","selected","in","the","following","options",":","-","'longest_first","'","(","default",")","Iteratively","reduce","the","inputs","sequence","until","the","input","is","under","max_seq_len","starting","from","the","longest","one","at","each","token","(","when","there","is","a","pair","of","input","sequences",")","-","'only_first","'",":","Only","truncate","the","first","sequence","-","'only_second","'",":","Only","truncate","the","second","sequence","-","'do_not_truncate","'",":","Does","not","truncate","(","raise","an","error","if","the","input","sequence","is","longer","than","max_seq_len",")","return_lengths","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","If","set","the","resulting","dictionary","will","include","the","length","of","each","encoded","inputs","return_overflowing_tokens","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","False","`",")",":","Set","to","True","to","return","overflowing","token","information","(","default","False",")",".","Return",":","A","Dictionary","of","shape",":",":","{","text",":","list","[","int","]",",","seq_len",":","int","if","return_length","is","True","(","default",")","overflowing_tokens",":","list","[","int","]","if","a","``","max_seq_len","``","is","specified","and","return_overflowing_tokens","is","True","}","With","the","fields",":","-","``","text","``",":","list","of","token","ids","to","be","fed","to","a","model","-","``","length","``",":","the","input_ids","length","-","``","overflowing_tokens","``",":","list","of","overflowing","tokens","if","a","max","length","is","specified","."]
code: def encode(
      self,
      text: str,
      text_pair: Union[str, List[str], List[int]] = None,
      max_seq_len: int = None,
      pad_to_max_seq_len: bool = True,
      truncation_strategy: str = "longest_first",
      return_length: bool = True,
      return_overflowing_tokens: bool = False,
  ):
      """
      Returns a dictionary containing the encoded sequence or sequence pair and additional information:
      the mask for sequence classification and the overflowing elements if a ``max_seq_len`` is specified.
      Args:
          text (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`):
              The first sequence to be encoded. This can be a string, a list of strings (tokenized string using
              the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
              method)
          text_pair (:obj:`str`, :obj:`List[str]` or :obj:`List[int]`, `optional`, defaults to :obj:`None`):
              It's nonsense, just for compatible.
          max_seq_len (:obj:`int`, `optional`, defaults to :int:`None`):
              If set to a number, will limit the total sequence returned so that it has a maximum length.
              If there are overflowing tokens, those will be added to the returned dictionary
          pad_to_max_seq_len (:obj:`bool`, `optional`, defaults to :obj:`True`):
              If set to True, the returned sequences will be padded according to the model's padding side and
              padding index, up to their max length. If no max length is specified, the padding is done up to the
              model's max length.
          truncation_strategy (:obj:`str`, `optional`, defaults to `longest_first`):
              String selected in the following options:
              - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_seq_len
                starting from the longest one at each token (when there is a pair of input sequences)
              - 'only_first': Only truncate the first sequence
              - 'only_second': Only truncate the second sequence
              - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_seq_len)
          return_lengths (:obj:`bool`, `optional`, defaults to :obj:`True`):
              If set the resulting dictionary will include the length of each encoded inputs
          return_overflowing_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
              Set to True to return overflowing token information (default False).
      Return:
          A Dictionary of shape::
              {
                  text: list[int],
                  seq_len: int if return_length is True (default)
                  overflowing_tokens: list[int] if a ``max_seq_len`` is specified and return_overflowing_tokens is True
              }
          With the fields:
          - ``text``: list of token ids to be fed to a model
          - ``length``: the input_ids length
          - ``overflowing_tokens``: list of overflowing tokens if a max length is specified.
      """
  
      def get_input_ids(text: str):
          if isinstance(text, str):
              tokens = self.tokenize(text)
              ids = self.convert_tokens_to_ids(tokens)
              return ids
          elif (
              isinstance(text, (list, tuple))
              and len(text) > 0
              and isinstance(text[0], str)
          ):
              return self.convert_tokens_to_ids(text)
          elif (
              isinstance(text, (list, tuple))
              and len(text) > 0
              and isinstance(text[0], int)
          ):
              return text
          else:
              raise ValueError(
                  "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
              )
  
      ids = get_input_ids(text)
      len_ids = len(ids)
  
      encoded_inputs = {}
      # When all words are not found in the vocab, it will return {}.
      if not len_ids:
          return encoded_inputs
  
      # Truncation: Handle max sequence length
      if max_seq_len and len_ids > max_seq_len:
          ids, pair_ids, overflowing_tokens = self.truncate_sequences(
              ids,
              num_tokens_to_remove=len_ids - max_seq_len,
              truncation_strategy=truncation_strategy,
          )
          if return_overflowing_tokens:
              encoded_inputs["overflowing_tokens"] = overflowing_tokens
              encoded_inputs["num_truncated_tokens"] = (
                  len_ids - max_seq_len
              )
  
      ## Check length and Pad
      if pad_to_max_seq_len and len(ids) < max_seq_len:
          encoded_inputs["text"] = ids + [self.pad_token_id] * (
              max_seq_len - len(ids)
          )
      else:
          encoded_inputs["text"] = ids
  
      if return_length:
          encoded_inputs["seq_len"] = len(ids)
  
      return encoded_inputs
  
imports: import collections
  import inspect
  import os
  from typing import Callable, List, Union
  import paddlehub
  from paddlehub.utils.log import logger
  from paddlehub.text.bert_tokenizer import BasicTokenizer
  from paddlehub.text.utils import load_vocab, whitespace_tokenize
label_desc: Use image node_id 10 for calling the CustomTokenizer obj's underlying member method code with example usage: obj.encode(text, text_pair, max_seq_len, pad_to_max_seq_len, truncation_strategy, return_length, return_overflowing_tokens) and returns: encoded_inputs, encoded_inputs, ids, self, text