class_docstr: Constructs a BERT tokenizer. Based on WordPiece.
  Args:
      vocab_file (:obj:`string`):
          File containing the vocabulary.
      do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`):
          Whether to lowercase the input when tokenizing.
      do_basic_tokenize (:obj:`bool`, `optional`, defaults to :obj:`True`):
          Whether to do basic tokenization before WordPiece.
      never_split (:obj:`bool`, `optional`, defaults to :obj:`True`):
          List of tokens which will never be split during tokenization. Only has an effect when
          :obj:`do_basic_tokenize=True`
      unk_token (:obj:`string`, `optional`, defaults to '[UNK]'):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      sep_token (:obj:`string`, `optional`, defaults to '[SEP]'):
          The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
          for sequence classification or for a text and a question for question answering.
          It is also used as the last token of a sequence built with special tokens.
      pad_token (:obj:`string`, `optional`, defaults to '[PAD]'):
          The token used for padding, for example when batching sequences of different lengths.
      cls_token (:obj:`string`, `optional`, defaults to '[CLS]'):
          The classifier token which is used when doing sequence classification (classification of the whole
          sequence instead of per-token classification). It is the first token of the sequence when built with
          special tokens.
      mask_token (:obj:`string`, `optional`, defaults to '[MASK]'):
          The token used for masking values. This is the token used when training this model with masked language
          modeling. This is the token which the model will try to predict.
      tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
          Whether to tokenize Chinese characters.
          This should likely be deactivated for Japanese:
          see: https://github.com/huggingface/transformers/issues/328
class_docstr_tok: ["Constructs","a","BERT","tokenizer",".","Based","on","WordPiece",".","Args",":","vocab_file","(",":","obj",":","`","string","`",")",":","File","containing","the","vocabulary",".","do_lower_case","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","lowercase","the","input","when","tokenizing",".","do_basic_tokenize","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","do","basic","tokenization","before","WordPiece",".","never_split","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","List","of","tokens","which","will","never","be","split","during","tokenization",".","Only","has","an","effect","when",":","obj",":","`","do_basic_tokenize=True","`","unk_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","UNK","]","'",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","sep_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","SEP","]","'",")",":","The","separator","token",",","which","is","used","when","building","a","sequence","from","multiple","sequences",",","e.g",".","two","sequences","for","sequence","classification","or","for","a","text","and","a","question","for","question","answering",".","It","is","also","used","as","the","last","token","of","a","sequence","built","with","special","tokens",".","pad_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","PAD","]","'",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","cls_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","CLS","]","'",")",":","The","classifier","token","which","is","used","when","doing","sequence","classification","(","classification","of","the","whole","sequence","instead","of","per-token","classification",")",".","It","is","the","first","token","of","the","sequence","when","built","with","special","tokens",".","mask_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","MASK","]","'",")",":","The","token","used","for","masking","values",".","This","is","the","token","used","when","training","this","model","with","masked","language","modeling",".","This","is","the","token","which","the","model","will","try","to","predict",".","tokenize_chinese_chars","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese",":","see",":","https",":","\/\/github.com\/huggingface\/transformers\/issues\/328"]
docstr: Truncates a sequence pair in place to the maximum length.
  Args:
      ids: list of tokenized input ids. Can be obtained from a string by chaining the
          `tokenize` and `convert_tokens_to_ids` methods.
      pair_ids: Optional second list of input ids. Can be obtained from a string by chaining the
          `tokenize` and `convert_tokens_to_ids` methods.
      num_tokens_to_remove (:obj:`int`, `optional`, defaults to ``0``):
          number of tokens to remove using the truncation strategy
      truncation_strategy: string selected in the following options:
          - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_seq_len
              starting from the longest one at each token (when there is a pair of input sequences).
              Overflowing tokens only contains overflow from the first sequence.
          - 'only_first': Only truncate the first sequence. raise an error if the first sequence is shorter or equal to than num_tokens_to_remove.
          - 'only_second': Only truncate the second sequence
          - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_seq_len)
      stride (:obj:`int`, `optional`, defaults to ``0``):
          If set to a number along with max_seq_len, the overflowing tokens returned will contain some tokens
          from the main sequence returned. The value of this argument defines the number of additional tokens.
docstr_tok: ["Truncates","a","sequence","pair","in","place","to","the","maximum","length",".","Args",":","ids",":","list","of","tokenized","input","ids",".","Can","be","obtained","from","a","string","by","chaining","the","`","tokenize","`","and","`","convert_tokens_to_ids","`","methods",".","pair_ids",":","Optional","second","list","of","input","ids",".","Can","be","obtained","from","a","string","by","chaining","the","`","tokenize","`","and","`","convert_tokens_to_ids","`","methods",".","num_tokens_to_remove","(",":","obj",":","`","int","`",",","`","optional","`",",","defaults","to","``","0","``",")",":","number","of","tokens","to","remove","using","the","truncation","strategy","truncation_strategy",":","string","selected","in","the","following","options",":","-","'longest_first","'","(","default",")","Iteratively","reduce","the","inputs","sequence","until","the","input","is","under","max_seq_len","starting","from","the","longest","one","at","each","token","(","when","there","is","a","pair","of","input","sequences",")",".","Overflowing","tokens","only","contains","overflow","from","the","first","sequence",".","-","'only_first","'",":","Only","truncate","the","first","sequence",".","raise","an","error","if","the","first","sequence","is","shorter","or","equal","to","than","num_tokens_to_remove",".","-","'only_second","'",":","Only","truncate","the","second","sequence","-","'do_not_truncate","'",":","Does","not","truncate","(","raise","an","error","if","the","input","sequence","is","longer","than","max_seq_len",")","stride","(",":","obj",":","`","int","`",",","`","optional","`",",","defaults","to","``","0","``",")",":","If","set","to","a","number","along","with","max_seq_len",",","the","overflowing","tokens","returned","will","contain","some","tokens","from","the","main","sequence","returned",".","The","value","of","this","argument","defines","the","number","of","additional","tokens","."]
code: def truncate_sequences(
      self,
      ids: List[int],
      pair_ids: Optional[List[int]] = None,
      num_tokens_to_remove: int = 0,
      truncation_strategy: str = "longest_first",
      stride: int = 0,
  ) -> Tuple[List[int], List[int], List[int]]:
      """Truncates a sequence pair in place to the maximum length.
      Args:
          ids: list of tokenized input ids. Can be obtained from a string by chaining the
              `tokenize` and `convert_tokens_to_ids` methods.
          pair_ids: Optional second list of input ids. Can be obtained from a string by chaining the
              `tokenize` and `convert_tokens_to_ids` methods.
          num_tokens_to_remove (:obj:`int`, `optional`, defaults to ``0``):
              number of tokens to remove using the truncation strategy
          truncation_strategy: string selected in the following options:
              - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_seq_len
                  starting from the longest one at each token (when there is a pair of input sequences).
                  Overflowing tokens only contains overflow from the first sequence.
              - 'only_first': Only truncate the first sequence. raise an error if the first sequence is shorter or equal to than num_tokens_to_remove.
              - 'only_second': Only truncate the second sequence
              - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_seq_len)
          stride (:obj:`int`, `optional`, defaults to ``0``):
              If set to a number along with max_seq_len, the overflowing tokens returned will contain some tokens
              from the main sequence returned. The value of this argument defines the number of additional tokens.
      """
      if num_tokens_to_remove <= 0:
          return ids, pair_ids, []
  
      if truncation_strategy == "longest_first":
          overflowing_tokens = []
          for _ in range(num_tokens_to_remove):
              if pair_ids is None or len(ids) > len(pair_ids):
                  overflowing_tokens = [ids[-1]] + overflowing_tokens
                  ids = ids[:-1]
              else:
                  pair_ids = pair_ids[:-1]
          window_len = min(len(ids), stride)
          if window_len > 0:
              overflowing_tokens = (
                  ids[-window_len:] + overflowing_tokens
              )
      elif truncation_strategy == "only_first":
          assert len(ids) > num_tokens_to_remove
          window_len = min(len(ids), stride + num_tokens_to_remove)
          overflowing_tokens = ids[-window_len:]
          ids = ids[:-num_tokens_to_remove]
      elif truncation_strategy == "only_second":
          assert (
              pair_ids is not None
              and len(pair_ids) > num_tokens_to_remove
          )
          window_len = min(len(pair_ids), stride + num_tokens_to_remove)
          overflowing_tokens = pair_ids[-window_len:]
          pair_ids = pair_ids[:-num_tokens_to_remove]
      elif truncation_strategy == "do_not_truncate":
          raise ValueError(
              "Input sequence are too long for max_seq_len. Please select a truncation strategy."
          )
      else:
          raise ValueError(
              "Truncation_strategy should be selected in ['longest_first', 'only_first', 'only_second', 'do_not_truncate']"
          )
      return (ids, pair_ids, overflowing_tokens)
  
imports: import collections
  import os
  import pickle
  import unicodedata
  from typing import Dict, List, Optional, Union, Tuple
  from paddle.utils import try_import
  from paddlehub.text.utils import load_vocab, is_whitespace, is_control, is_punctuation, whitespace_tokenize, is_chinese_char
label_desc: Use image node_id 15 for calling the BertTokenizer obj's underlying member method code with example usage: obj.truncate_sequences(ids, pair_ids, num_tokens_to_remove, truncation_strategy, stride) and returns: ids,  pair_ids,  overflowing_tokens, ids,  pair_ids,  list