code: def __init__(
      self,
      vocab_file: str,
      mask_token: Optional[Union[str, bool]] = None,
      bos_token: Optional[Union[str, bool]] = None,
      eos_token: Optional[Union[str, bool]] = None,
      pad_token: Optional[Union[str, bool]] = None,
      sep_token: Optional[Union[str, bool]] = None,
      cls_token: Optional[Union[str, bool]] = None,
      unk_token: Optional[Union[str, bool]] = None,
      special_token_to_prepend: Optional[SpecialTokenStringType] = None,
      special_token_to_append: Optional[SpecialTokenStringType] = None,
      special_tokens_to_remove_while_decoding: Union[
          List[SpecialTokenStringType], str
      ] = "all",
  ):
      vocab_file = Path(vocab_file).expanduser()
      with vocab_file.open(encoding="utf-8") as f:
          first_line = f.readline()
          if first_line[0] == "{":
              special_tokens_dict = json.loads(first_line)
              self.check_special_tokens_dict_from_file(
                  special_tokens_dict, vocab_file
              )
              vocab_list = f.readlines()
          else:
              special_tokens_dict = {}
              vocab_list = [first_line] + f.readlines()
      special_tokens_dict = self.update_special_tokens_dict(
          special_tokens_dict,
          mask_token,
          bos_token,
          eos_token,
          pad_token,
          sep_token,
          cls_token,
          unk_token,
      )
      for e in SpecialTokenString:
          name = e.value + "_token"
          setattr(
              self,
              name,
              special_tokens_dict[name]
              if name in special_tokens_dict
              else None,
          )
      for k, v in special_tokens_dict.items():
          setattr(self, k, v)
      for value, name in [
          (special_token_to_prepend, "special_token_to_prepend"),
          (special_token_to_append, "special_token_to_append"),
      ]:
          self.check_special_token_name(
              name, value, special_tokens_dict
          )
          setattr(
              self,
              name,
              value + "_token" if isinstance(value, str) else value,
          )
      self.vocab = {}
      count = 0
      for v in special_tokens_dict.values():
          self.vocab[v] = count
          count += 1
      for i, token in enumerate(vocab_list):
          token = eval(token.strip())
          self.check_token_from_file(token, vocab_file, i)
          if token not in self.vocab:
              self.vocab[token] = count
              count += 1
      self.inv_vocab = {v: k for k, v in self.vocab.items()}
      self.vocab_size = len(self.vocab)
      self.check_special_tokens_to_remove_while_decoding(
          special_tokens_to_remove_while_decoding, special_tokens_dict
      )
      self.special_token_ids_to_remove_while_decoding = (
          self.tokens_to_ids([v for v in special_tokens_dict.values()])
          if special_tokens_to_remove_while_decoding == "all"
          else [
              getattr(self, e + "_id")
              for e in special_tokens_to_remove_while_decoding
          ]
      )
  
imports: import json
  import os
  import warnings
  from collections import Counter
  from enum import Enum
  from pathlib import Path
  from typing import Dict, List, NewType, Optional, Union
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
label_desc: Use image node_id 1 to create a new CharTokenizer object from inherited base classes: TokenizerSpec with example: obj = CharTokenizer(vocab_file, mask_token, bos_token, eos_token, pad_token, sep_token, cls_token, unk_token, special_token_to_prepend, special_token_to_append, special_tokens_to_remove_while_decoding)