





'''simple docstring'''

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class __a			( _snake_case, unittest.TestCase      ):
   __UpperCamelCase :			Optional[Any]						   =						BertJapaneseTokenizer
   __UpperCamelCase :			Optional[Any]						   =						False
   __UpperCamelCase :			str						   =						True





   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      super().setUp()

      __SCREAMING_SNAKE_CASE   				=							[
          """[UNK]""",
          """[CLS]""",
          """[SEP]""",
          """こんにちは""",
          """こん""",
          """にちは""",
          """ばんは""",
          """##こん""",
          """##にちは""",
          """##ばんは""",
          """世界""",
          """##世界""",
          """、""",
          """##、""",
          """。""",
          """##。""",
      ]

      __SCREAMING_SNAKE_CASE   				=							os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""vocab_file"""]     )
      with open(self.vocab_file					,"""w"""					,encoding="""utf-8"""     ) as vocab_writer:
         vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )





   def   UpperCAmelCase__  (       self					:      Union[str, Any]					,lowerCamelCase					:      List[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""こんにちは、世界。 \nこんばんは、世界。"""
      __SCREAMING_SNAKE_CASE   				=							"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
      return input_text, output_text





   def   UpperCAmelCase__  (       self					:      Optional[Any]					,lowerCamelCase					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							self.get_input_output_texts(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							tokenizer.encode(lowerCamelCase					,add_special_tokens=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							tokenizer.decode(lowerCamelCase					,clean_up_tokenization_spaces=lowerCamelCase     )
      return text, ids





   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      pass  # TODO add if relevant





   def   UpperCAmelCase__  (       self					:      Union[str, Any]     ):

      '''simple docstring'''

      pass  # TODO add if relevant





   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      pass  # TODO add if relevant





   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.tokenizer_class(self.vocab_file     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""     )
      self.assertListEqual(lowerCamelCase					,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase     )					,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )





   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.tokenizer_class(self.vocab_file					,word_tokenizer_type="""mecab"""     )
      self.assertIsNotNone(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							"""こんにちは、世界。\nこんばんは、世界。"""
      __SCREAMING_SNAKE_CASE   				=							tokenizer.tokenize(lowerCamelCase     )
      self.assertListEqual(lowerCamelCase					,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase     )					,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )

      __SCREAMING_SNAKE_CASE   				=							os.path.join(self.tmpdirname					,"""tokenizer.bin"""     )
      with open(lowerCamelCase					,"""wb"""     ) as handle:
         pickle.dump(lowerCamelCase					,lowerCamelCase     )

      with open(lowerCamelCase					,"""rb"""     ) as handle:
         __SCREAMING_SNAKE_CASE   				=							pickle.load(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer_new.tokenize(lowerCamelCase     )

      self.assertListEqual(lowerCamelCase					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							MecabTokenizer(mecab_dic="""ipadic"""     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,)





   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      try:
         __SCREAMING_SNAKE_CASE   				=							MecabTokenizer(mecab_dic="""unidic_lite"""     )
      except ModuleNotFoundError:
         return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,)





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      try:
         __SCREAMING_SNAKE_CASE   				=							MecabTokenizer(mecab_dic="""unidic"""     )
      except ModuleNotFoundError:
         return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,)





   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							MecabTokenizer(do_lower_case=lowerCamelCase					,mecab_dic="""ipadic"""     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,)





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      try:
         __SCREAMING_SNAKE_CASE   				=							MecabTokenizer(
             do_lower_case=lowerCamelCase					,normalize_text=lowerCamelCase					,mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""     )
      except RuntimeError:
         # if dict doesn't exist in the system, previous code raises this error.
         return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,)





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							MecabTokenizer(normalize_text=lowerCamelCase					,mecab_dic="""ipadic"""     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]					,)





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.tokenizer_class(self.vocab_file					,word_tokenizer_type="""sudachi"""     )
      self.assertIsNotNone(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							"""こんにちは、世界。\nこんばんは、世界。"""
      __SCREAMING_SNAKE_CASE   				=							tokenizer.tokenize(lowerCamelCase     )
      self.assertListEqual(lowerCamelCase					,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase     )					,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )

      __SCREAMING_SNAKE_CASE   				=							os.path.join(self.tmpdirname					,"""tokenizer.bin"""     )
      with open(lowerCamelCase					,"""wb"""     ) as handle:
         pickle.dump(lowerCamelCase					,lowerCamelCase     )

      with open(lowerCamelCase					,"""rb"""     ) as handle:
         __SCREAMING_SNAKE_CASE   				=							pickle.load(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer_new.tokenize(lowerCamelCase     )

      self.assertListEqual(lowerCamelCase					,lowerCamelCase     )





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SudachiTokenizer(sudachi_dict_type="""core"""     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]					,)





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SudachiTokenizer(sudachi_dict_type="""core"""					,sudachi_split_mode="""A"""     )

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""     )					,["""外国""", """人""", """参政""", """権"""]     )





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SudachiTokenizer(sudachi_dict_type="""core"""					,sudachi_split_mode="""B"""     )

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""     )					,["""外国人""", """参政権"""]     )





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SudachiTokenizer(sudachi_dict_type="""core"""					,sudachi_split_mode="""C"""     )

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""     )					,["""外国人参政権"""]     )





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SudachiTokenizer(do_lower_case=lowerCamelCase					,sudachi_dict_type="""core"""     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]					,)





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SudachiTokenizer(normalize_text=lowerCamelCase					,sudachi_dict_type="""core"""     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]					,)





   @require_sudachi
   def   UpperCAmelCase__  (       self					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							SudachiTokenizer(trim_whitespace=lowerCamelCase					,sudachi_dict_type="""core"""     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,)





   @require_jumanpp
   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.tokenizer_class(self.vocab_file					,word_tokenizer_type="""jumanpp"""     )
      self.assertIsNotNone(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							"""こんにちは、世界。\nこんばんは、世界。"""
      __SCREAMING_SNAKE_CASE   				=							tokenizer.tokenize(lowerCamelCase     )
      self.assertListEqual(lowerCamelCase					,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase     )					,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )

      __SCREAMING_SNAKE_CASE   				=							os.path.join(self.tmpdirname					,"""tokenizer.bin"""     )
      with open(lowerCamelCase					,"""wb"""     ) as handle:
         pickle.dump(lowerCamelCase					,lowerCamelCase     )

      with open(lowerCamelCase					,"""rb"""     ) as handle:
         __SCREAMING_SNAKE_CASE   				=							pickle.load(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer_new.tokenize(lowerCamelCase     )

      self.assertListEqual(lowerCamelCase					,lowerCamelCase     )





   @require_jumanpp
   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,)





   @require_jumanpp
   def   UpperCAmelCase__  (       self					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							JumanppTokenizer(do_lower_case=lowerCamelCase     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,)





   @require_jumanpp
   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							JumanppTokenizer(normalize_text=lowerCamelCase     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,)





   @require_jumanpp
   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							JumanppTokenizer(trim_whitespace=lowerCamelCase     )

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )					,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]					,)





   @require_jumanpp
   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""     )					,["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]					,)





   def   UpperCAmelCase__  (       self					:      Union[str, Any]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

      __SCREAMING_SNAKE_CASE   				=							{}
      for i, token in enumerate(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							i
      __SCREAMING_SNAKE_CASE   				=							WordpieceTokenizer(vocab=lowerCamelCase					,unk_token="""[UNK]"""     )

      self.assertListEqual(tokenizer.tokenize(""""""     )					,[]     )

      self.assertListEqual(tokenizer.tokenize("""こんにちは"""     )					,["""こんにちは"""]     )

      self.assertListEqual(tokenizer.tokenize("""こんばんは"""     )					,["""こん""", """##ばんは"""]     )

      self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""     )					,["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]     )





   def   UpperCAmelCase__  (       self					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""     )
      __SCREAMING_SNAKE_CASE   				=							tokenizer.subword_tokenizer

      __SCREAMING_SNAKE_CASE   				=							subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""     )
      self.assertListEqual(lowerCamelCase					,["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]     )

      __SCREAMING_SNAKE_CASE   				=							subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""     )
      self.assertListEqual(lowerCamelCase					,["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]     )





   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer.encode("""ありがとう。"""					,add_special_tokens=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							tokenizer.encode("""どういたしまして。"""					,add_special_tokens=lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase					,lowerCamelCase     )

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class __a			( _snake_case, unittest.TestCase      ):
   __UpperCamelCase :			Dict						   =						BertJapaneseTokenizer
   __UpperCamelCase :			Optional[int]						   =						False





   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      super().setUp()

      __SCREAMING_SNAKE_CASE   				=							["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

      __SCREAMING_SNAKE_CASE   				=							os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""vocab_file"""]     )
      with open(self.vocab_file					,"""w"""					,encoding="""utf-8"""     ) as vocab_writer:
         vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )





   def   UpperCAmelCase__  (       self					:      Optional[int]					,**lowerCamelCase					:      int     ):

      '''simple docstring'''

      return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,subword_tokenizer_type="""character"""					,**lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Tuple					,lowerCamelCase					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""こんにちは、世界。 \nこんばんは、世界。"""
      __SCREAMING_SNAKE_CASE   				=							"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
      return input_text, output_text





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      pass  # TODO add if relevant





   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      pass  # TODO add if relevant





   def   UpperCAmelCase__  (       self					:      Tuple     ):

      '''simple docstring'''

      pass  # TODO add if relevant





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.tokenizer_class(self.vocab_file					,subword_tokenizer_type="""character"""     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""     )
      self.assertListEqual(
          lowerCamelCase					,["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]     )
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(lowerCamelCase     )					,[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]     )





   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

      __SCREAMING_SNAKE_CASE   				=							{}
      for i, token in enumerate(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							i
      __SCREAMING_SNAKE_CASE   				=							CharacterTokenizer(vocab=lowerCamelCase					,unk_token="""[UNK]"""     )

      self.assertListEqual(tokenizer.tokenize(""""""     )					,[]     )

      self.assertListEqual(tokenizer.tokenize("""こんにちは"""     )					,["""こ""", """ん""", """に""", """ち""", """は"""]     )

      self.assertListEqual(tokenizer.tokenize("""こんにちほ"""     )					,["""こ""", """ん""", """に""", """ち""", """[UNK]"""]     )





   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer.encode("""ありがとう。"""					,add_special_tokens=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							tokenizer.encode("""どういたしまして。"""					,add_special_tokens=lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase					,lowerCamelCase     )

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class __a			( unittest.TestCase      ):





   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""cl-tohoku/bert-base-japanese"""
      __SCREAMING_SNAKE_CASE   				=							AutoTokenizer.from_pretrained(lowerCamelCase     )
      self.assertIsInstance(lowerCamelCase					,lowerCamelCase     )







class __a			( unittest.TestCase      ):





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""cl-tohoku/bert-base-japanese"""
      with self.assertLogs("""transformers"""					,level="""WARNING"""     ) as cm:
         BertTokenizer.from_pretrained(lowerCamelCase     )
         self.assertTrue(
             cm.records[0].message.startswith(
                 """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                 """ is called from."""     )     )
      __SCREAMING_SNAKE_CASE   				=							"""bert-base-cased"""
      with self.assertLogs("""transformers"""					,level="""WARNING"""     ) as cm:
         BertJapaneseTokenizer.from_pretrained(lowerCamelCase     )
         self.assertTrue(
             cm.records[0].message.startswith(
                 """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                 """ is called from."""     )     )












'''simple docstring'''

import numpy as np







class __a			:





   def __init__(       self					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							(0, 0)
      __SCREAMING_SNAKE_CASE   				=							None
      __SCREAMING_SNAKE_CASE   				=							0
      __SCREAMING_SNAKE_CASE   				=							0
      __SCREAMING_SNAKE_CASE   				=							0





   def __eq__(       self					:      int					,lowerCamelCase					:      List[str]     ):

      '''simple docstring'''

      return self.position == cell.position





   def   UpperCAmelCase__  (       self					:      Union[str, Any]     ):

      '''simple docstring'''

      print(self.position     )







class __a			:





   def __init__(       self					:      int					,lowerCamelCase					:      Optional[int]=(5, 5)     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							np.zeros(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							world_size[0]
      __SCREAMING_SNAKE_CASE   				=							world_size[1]





   def   UpperCAmelCase__  (       self					:      Optional[int]     ):

      '''simple docstring'''

      print(self.w     )





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      int     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[
          (-1, -1),
          (-1, 0),
          (-1, 1),
          (0, -1),
          (0, 1),
          (1, -1),
          (1, 0),
          (1, 1),
      ]
      __SCREAMING_SNAKE_CASE   				=							cell.position[0]
      __SCREAMING_SNAKE_CASE   				=							cell.position[1]
      __SCREAMING_SNAKE_CASE   				=							[]
      for n in neughbour_cord:
         __SCREAMING_SNAKE_CASE   				=							current_x + n[0]
         __SCREAMING_SNAKE_CASE   				=							current_y + n[1]
         if 0 <= x < self.world_x_limit and 0 <= y < self.world_y_limit:
            __SCREAMING_SNAKE_CASE   				=							Cell()
            __SCREAMING_SNAKE_CASE   				=							(x, y)
            __SCREAMING_SNAKE_CASE   				=							cell
            neighbours.append(lowerCamelCase     )
      return neighbours





def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase				,						__UpperCAmelCase   )    ->       Any:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE   				=							[]
   __SCREAMING_SNAKE_CASE   				=							[]
   _open.append(__UpperCAmelCase   )

   while _open:
      __SCREAMING_SNAKE_CASE   				=							np.argmin([n.f for n in _open]   )
      __SCREAMING_SNAKE_CASE   				=							_open[min_f]
      _closed.append(_open.pop(__UpperCAmelCase   )   )
      if current == goal:
         break
      for n in world.get_neigbours(__UpperCAmelCase   ):
         for c in _closed:
            if c == n:
               continue
         __SCREAMING_SNAKE_CASE   				=							current.g + 1
         __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							n.position
         __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							goal.position
         __SCREAMING_SNAKE_CASE   				=							(ya - ya) ** 2 + (xa - xa) ** 2
         __SCREAMING_SNAKE_CASE   				=							n.h + n.g

         for c in _open:
            if c == n and c.f < n.f:
               continue
         _open.append(__UpperCAmelCase   )
   __SCREAMING_SNAKE_CASE   				=							[]
   while current.parent is not None:
      path.append(current.position   )
      __SCREAMING_SNAKE_CASE   				=							current.parent
   path.append(current.position   )
   return path[::-1]


if __name__ == "__main__":
    a   	=    Gridworld()
    # Start position and goal
    a   	=    Cell()
    a   	=    (0, 0)
    a   	=    Cell()
    a   	=    (4, 4)
    print(F'''path from {start.position} to {goal.position}''')
    a   	=    astar(world, start, goal)
    #   Just for visual reasons.
    for i in s:
        a   	=    1
    print(world.w)





