class_docstr: Bert Like Tokenizer, ref: https://github.com/CyberZHG/keras-bert/blob/master/keras_bert/tokenizer.py
class_docstr_tok: ["Bert","Like","Tokenizer",",","ref",":","https",":","\/\/github.com\/CyberZHG\/keras-bert\/blob\/master\/keras_bert\/tokenizer.py"]
docstr: Initialize tokenizer.
  Args:
      token_dict: A dict maps tokens to indices.
      token_cls: The token represents classification.
      token_sep: The token represents separator.
      token_unk: The token represents unknown token.
      pad_index: The index to pad.
      cased: Whether to keep the case.
docstr_tok: ["Initialize","tokenizer",".","Args",":","token_dict",":","A","dict","maps","tokens","to","indices",".","token_cls",":","The","token","represents","classification",".","token_sep",":","The","token","represents","separator",".","token_unk",":","The","token","represents","unknown","token",".","pad_index",":","The","index","to","pad",".","cased",":","Whether","to","keep","the","case","."]
code: def __init__(
      self,
      *,
      token_dict: Optional[Dict[str, int]] = None,
      token_cls: str = TOKEN_CLS,
      token_sep: str = TOKEN_SEP,
      token_unk: str = TOKEN_UNK,
      pad_index: int = 0,
      cased: bool = False,
  ) -> None:
      """
      Initialize tokenizer.
      Args:
          token_dict: A dict maps tokens to indices.
          token_cls: The token represents classification.
          token_sep: The token represents separator.
          token_unk: The token represents unknown token.
          pad_index: The index to pad.
          cased: Whether to keep the case.
      """
      self._token_dict: Dict[str, int]
  
      if token_dict:
          self._token_dict = token_dict
      else:
          self._token_dict = {}
  
      self._token_dict_inv: Dict[int, str] = {
          v: k for k, v in self._token_dict.items()
      }
      self._token_cls: str = token_cls
      self._token_sep: str = token_sep
      self._token_unk: str = token_unk
      self._pad_index: int = pad_index
      self._cased: bool = cased
  
file_path: Kashgari/kashgari/tokenizers/bert_tokenizer.py
imports: import codecs
  import unicodedata
  from typing import List, Optional, Dict
  from kashgari.tokenizers.base_tokenizer import Tokenizer
label_desc: Use image node_id 1 to create a new BertTokenizer object from inherited base classes: Tokenizer with example: obj = BertTokenizer()