class_docstr: This tokenizer inherits from [`PreTrainedTokenizer`] and is based on Japanese special Sub-Word-Encoding that is
  used in this repository (https://github.com/tanreinama/Japanese-BPEEncoder_V2). Check the repository for details.
  Japanese has a relatively large vocabulary and there is no separation between words. Furthermore, the language is a
  combination of hiragana, katakana, and kanji, and variants such as "1" and "①" are often used. In order to cope
  with these, this tokenizer has the following features
  - Subword-by-subword segmentation, which is intermediate between byte strings and morphological analysis.
  - BPEs are created for each Kanji, Hiragana, and Katakana character, and there are no BPEs that cross character
      types, such as Kanji + Hiragana or Hiragana + Katakana.
  - All-byte encoding that does not require <unk>.
  - Independent of UTF codes such as 2-byte and 3-byte characters
  - Conversion of heterographs to the same token_id
  - Emoji and Emoticon are grouped into 12 types as special tags.
  
  Example:
  
  ```python
  >>> from transformers import GPTNeoXJapaneseTokenizer
  
  >>> tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained("abeja/gpt-neox-japanese-2.7b")
  >>> # You can confirm both 慶応 and 慶應 are encoded to 17749
  >>> tokenizer("吾輩は猫である🐯。実は慶応(慶應)大学出身")["input_ids"]
  [30014, 26883, 26638, 27228, 25, 26650, 31732, 31679, 27809, 26638, 17749, 31592, 17749, 31593, 321, 1281]
  
  >>> # Both 慶応 and 慶應 are decoded to 慶応
  >>> tokenizer.decode(tokenizer("吾輩は猫である🐯。実は慶応(慶應)大学出身")["input_ids"])
  '吾輩は猫である🐯。実は慶応(慶応)大学出身'
  ```
  
  Args:
      vocab_file (`str`):
          File containing the vocabulary.
      emoji_file (`str`):
          File containing the emoji.
      unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
          The token used for padding
      bos_token (`str`, *optional*, defaults to `"<|startoftext|>"`):
          The beginning of sequence token.
      eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
          The end of sequence token.
      do_clean_text (`bool`, *optional*, defaults to `False`):
          Whether or not to clean text for URL, EMAIL, TEL, Japanese DATE and Japanese PRICE.
class_docstr_tok: ["This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","and","is","based","on","Japanese","special","Sub-Word-Encoding","that","is","used","in","this","repository","(","https",":","\/\/github.com\/tanreinama\/Japanese-BPEEncoder_V2",")",".","Check","the","repository","for","details",".","Japanese","has","a","relatively","large","vocabulary","and","there","is","no","separation","between","words",".","Furthermore",",","the","language","is","a","combination","of","hiragana",",","katakana",",","and","kanji",",","and","variants","such","as","``","1","''","and","``","\u2460","''","are","often","used",".","In","order","to","cope","with","these",",","this","tokenizer","has","the","following","features","-","Subword-by-subword","segmentation",",","which","is","intermediate","between","byte","strings","and","morphological","analysis",".","-","BPEs","are","created","for","each","Kanji",",","Hiragana",",","and","Katakana","character",",","and","there","are","no","BPEs","that","cross","character","types",",","such","as","Kanji","+","Hiragana","or","Hiragana","+","Katakana",".","-","All-byte","encoding","that","does","not","require","<","unk",">",".","-","Independent","of","UTF","codes","such","as","2-byte","and","3-byte","characters","-","Conversion","of","heterographs","to","the","same","token_id","-","Emoji","and","Emoticon","are","grouped","into","12","types","as","special","tags",".","Example",":","``","`","python",">",">",">","from","transformers","import","GPTNeoXJapaneseTokenizer",">",">",">","tokenizer","=","GPTNeoXJapaneseTokenizer.from_pretrained","(","``","abeja\/gpt-neox-japanese-2.7b","''",")",">",">",">","#","You","can","confirm","both","\u6176\u5fdc","and","\u6176\u61c9","are","encoded","to","17749",">",">",">","tokenizer","(","``","\u543e\u8f29\u306f\u732b\u3067\u3042\u308b\ud83d\udc2f\u3002\u5b9f\u306f\u6176\u5fdc","(","\u6176\u61c9",")","\u5927\u5b66\u51fa\u8eab","''",")","[","``","input_ids","''","]","[","30014",",","26883",",","26638",",","27228",",","25",",","26650",",","31732",",","31679",",","27809",",","26638",",","17749",",","31592",",","17749",",","31593",",","321",",","1281","]",">",">",">","#","Both","\u6176\u5fdc","and","\u6176\u61c9","are","decoded","to","\u6176\u5fdc",">",">",">","tokenizer.decode","(","tokenizer","(","``","\u543e\u8f29\u306f\u732b\u3067\u3042\u308b\ud83d\udc2f\u3002\u5b9f\u306f\u6176\u5fdc","(","\u6176\u61c9",")","\u5927\u5b66\u51fa\u8eab","''",")","[","``","input_ids","''","]",")","'\u543e\u8f29\u306f\u732b\u3067\u3042\u308b\ud83d\udc2f\u3002\u5b9f\u306f\u6176\u5fdc","(","\u6176\u5fdc",")","\u5927\u5b66\u51fa\u8eab'","``","`","Args",":","vocab_file","(","`","str","`",")",":","File","containing","the","vocabulary",".","emoji_file","(","`","str","`",")",":","File","containing","the","emoji",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","|endoftext|",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","|endoftext|",">","''","`",")",":","The","token","used","for","padding","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","|startoftext|",">","''","`",")",":","The","beginning","of","sequence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","|endoftext|",">","''","`",")",":","The","end","of","sequence","token",".","do_clean_text","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","clean","text","for","URL",",","EMAIL",",","TEL",",","Japanese","DATE","and","Japanese","PRICE","."]
code: def __init__(
      self,
      vocab_file,
      emoji_file,
      unk_token="<|endoftext|>",
      pad_token="<|endoftext|>",
      bos_token="<|startoftext|>",
      eos_token="<|endoftext|>",
      do_clean_text=False,
      **kwargs,
  ):
      if not os.path.isfile(vocab_file):
          raise ValueError(
              f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
              " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
          )
      if not os.path.isfile(emoji_file):
          raise ValueError(
              f"Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"
              " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
          )
      self.do_clean_text = do_clean_text
      (
          self.vocab,
          self.raw_vocab,
          self.ids_to_tokens,
          self.emoji,
      ) = load_vocab_and_emoji(vocab_file, emoji_file)
      self.subword_tokenizer = SubWordJapaneseTokenizer(
          vocab=self.vocab,
          ids_to_tokens=self.ids_to_tokens,
          emoji=self.emoji,
      )
      super().__init__(
          unk_token=unk_token,
          pad_token=pad_token,
          bos_token=bos_token,
          eos_token=eos_token,
          do_clean_text=do_clean_text,
          **kwargs,
      )
  
imports: import collections
  import json
  import os
  import re
  from typing import Optional, Tuple
  import numpy
  from ...tokenization_utils_fast import PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 1 to create a new GPTNeoXJapaneseTokenizer object from inherited base classes: PreTrainedTokenizer with example: obj = GPTNeoXJapaneseTokenizer(vocab_file, emoji_file, unk_token, pad_token, bos_token, eos_token, do_clean_text)