class_docstr: Customtokenizer which will tokenize the input text as words or phases and convert the words (str) to an index (int) using the vocab.
  If you would like tokens, please use `hub.BertTokenizer`.
docstr: Constructs a CustomTokenizer.
  Args:
      vocab_file (:obj:`string`): File containing the vocabulary.
      do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): Whether to lower case the input if the input is in English
      pad_token (:obj:`string`, `optional`, defaults to '[PAD]'): The token used for padding, for example when batching sequences of different lengths.
      tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): Whether to tokenize Chinese characters.
      cut_function(:obj:`function`): It is a function that aims to segment a chinese text and get the word segmentation result (list).
docstr_tok: ["Constructs","a","CustomTokenizer",".","Args",":","vocab_file","(",":","obj",":","`","string","`",")",":","File","containing","the","vocabulary",".","do_lower_case","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","lower","case","the","input","if","the","input","is","in","English","pad_token","(",":","obj",":","`","string","`",",","`","optional","`",",","defaults","to","'","[","PAD","]","'",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","tokenize_chinese_chars","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","True","`",")",":","Whether","to","tokenize","Chinese","characters",".","cut_function","(",":","obj",":","`","function","`",")",":","It","is","a","function","that","aims","to","segment","a","chinese","text","and","get","the","word","segmentation","result","(","list",")","."]
code: def __init__(
      self,
      vocab_file: str,
      do_lower_case: bool = True,
      pad_token: str = "[PAD]",
      tokenize_chinese_chars: bool = True,
      cut_function: Callable = None,
  ):
      """Constructs a CustomTokenizer.
      Args:
          vocab_file (:obj:`string`): File containing the vocabulary.
          do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): Whether to lower case the input if the input is in English
          pad_token (:obj:`string`, `optional`, defaults to '[PAD]'): The token used for padding, for example when batching sequences of different lengths.
          tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): Whether to tokenize Chinese characters.
          cut_function(:obj:`function`): It is a function that aims to segment a chinese text and get the word segmentation result (list).
      """
  
      if not os.path.isfile(vocab_file):
          raise ValueError(
              "Can't find a vocabulary file at path '{}'.".format(
                  vocab_file
              )
          )
      self.vocab = load_vocab(vocab_file)
      self.ids_to_tokens = collections.OrderedDict(
          [(ids, tok) for tok, ids in self.vocab.items()]
      )
      self.pad_token = pad_token
      self.pad_token_id = self.convert_tokens_to_ids(self.pad_token)
  
      self.tokenize_chinese_chars = tokenize_chinese_chars
      self.basic_tokenizer = BasicTokenizer(
          do_lower_case=do_lower_case,
          tokenize_chinese_chars=tokenize_chinese_chars,
      )
  
      self.cut_function = cut_function
      if not self.cut_function:
          lac = hub.Module(name="lac")
          self.cut_function = lac.cut
      elif inspect.isfunction(self.cut_function):
          self.cut_function = cut_function
      else:
          raise RuntimeError(
              "The cut_function (%s) is not a true function."
          )
  
imports: import collections
  import inspect
  import os
  from typing import Callable, List, Union
  import paddlehub
  from paddlehub.utils.log import logger
  from paddlehub.text.bert_tokenizer import BasicTokenizer
  from paddlehub.text.utils import load_vocab, whitespace_tokenize
label_desc: Use image node_id 1 to create a new CustomTokenizer object from inherited base classes: object with example: obj = CustomTokenizer(vocab_file, do_lower_case, pad_token, tokenize_chinese_chars, cut_function)