class_docstr: Construct a BERT tokenizer for Japanese text.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer
  to: this superclass for more information regarding those methods.
  
  Args:
      vocab_file (`str`):
          Path to a one-wordpiece-per-line vocabulary file.
      spm_file (`str`, *optional*):
          Path to [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm or .model
          extension) that contains the vocabulary.
      do_lower_case (`bool`, *optional*, defaults to `True`):
          Whether to lower case the input. Only has an effect when do_basic_tokenize=True.
      do_word_tokenize (`bool`, *optional*, defaults to `True`):
          Whether to do word tokenization.
      do_subword_tokenize (`bool`, *optional*, defaults to `True`):
          Whether to do subword tokenization.
      word_tokenizer_type (`str`, *optional*, defaults to `"basic"`):
          Type of word tokenizer. Choose from ["basic", "mecab", "sudachi", "jumanpp"].
      subword_tokenizer_type (`str`, *optional*, defaults to `"wordpiece"`):
          Type of subword tokenizer. Choose from ["wordpiece", "character", "sentencepiece",].
      mecab_kwargs (`dict`, *optional*):
          Dictionary passed to the `MecabTokenizer` constructor.
      sudachi_kwargs (`dict`, *optional*):
          Dictionary passed to the `SudachiTokenizer` constructor.
      jumanpp_kwargs (`dict`, *optional*):
          Dictionary passed to the `JumanppTokenizer` constructor.
class_docstr_tok: ["Construct","a","BERT","tokenizer","for","Japanese","text",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to",":","this","superclass","for","more","information","regarding","those","methods",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","a","one-wordpiece-per-line","vocabulary","file",".","spm_file","(","`","str","`",",","*","optional","*",")",":","Path","to","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")","file","(","generally","has","a",".spm","or",".model","extension",")","that","contains","the","vocabulary",".","do_lower_case","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","to","lower","case","the","input",".","Only","has","an","effect","when","do_basic_tokenize=True",".","do_word_tokenize","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","to","do","word","tokenization",".","do_subword_tokenize","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","to","do","subword","tokenization",".","word_tokenizer_type","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","basic","''","`",")",":","Type","of","word","tokenizer",".","Choose","from","[","``","basic","''",",","``","mecab","''",",","``","sudachi","''",",","``","jumanpp","''","]",".","subword_tokenizer_type","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","wordpiece","''","`",")",":","Type","of","subword","tokenizer",".","Choose","from","[","``","wordpiece","''",",","``","character","''",",","``","sentencepiece","''",",","]",".","mecab_kwargs","(","`","dict","`",",","*","optional","*",")",":","Dictionary","passed","to","the","`","MecabTokenizer","`","constructor",".","sudachi_kwargs","(","`","dict","`",",","*","optional","*",")",":","Dictionary","passed","to","the","`","SudachiTokenizer","`","constructor",".","jumanpp_kwargs","(","`","dict","`",",","*","optional","*",")",":","Dictionary","passed","to","the","`","JumanppTokenizer","`","constructor","."]
code: def __setstate__(self, state):
      self.__dict__ = state
      if self.word_tokenizer_type == "mecab":
          self.word_tokenizer = MecabTokenizer(
              do_lower_case=self.do_lower_case,
              never_split=self.never_split,
              **(self.mecab_kwargs or {}),
          )
      elif self.word_tokenizer_type == "sudachi":
          self.word_tokenizer = SudachiTokenizer(
              do_lower_case=self.do_lower_case,
              never_split=self.never_split,
              **(self.sudachi_kwargs or {}),
          )
      elif self.word_tokenizer_type == "jumanpp":
          self.word_tokenizer = JumanppTokenizer(
              do_lower_case=self.do_lower_case,
              never_split=self.never_split,
              **(self.jumanpp_kwargs or {}),
          )
  
imports: import collections
  import copy
  import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
label_desc: Use image node_id 4 for calling the BertJapaneseTokenizer obj's underlying member method code with example usage: obj.__setstate__(state) without return types