
"""simple docstring"""






import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
 from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE							=     get_tests_dir('fixtures/test_sentencepiece.model')

__SCREAMING_SNAKE_CASE							=     {'target_lang': 'fi', 'source_lang': 'en'}
__SCREAMING_SNAKE_CASE							=     '>>zh<<'
__SCREAMING_SNAKE_CASE							=     'Helsinki-NLP/'

if is_torch_available():
 __SCREAMING_SNAKE_CASE							=     'pt'
elif is_tf_available():
 __SCREAMING_SNAKE_CASE							=     'tf'
else:
 __SCREAMING_SNAKE_CASE							=     'jax'





@require_sentencepiece
class 				a__					(						snake_case_   ,							unittest.TestCase       ):
     UpperCAmelCase__         =      MarianTokenizer
     UpperCAmelCase__         =      False
     UpperCAmelCase__         =      True



     def  lowerCamelCase_						(							self :Any   ):







      '''simple docstring'''



      super().setUp()
      UpperCamelCase_     :					Optional[Any]   	=["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
      UpperCamelCase_     :					Optional[int]   	=dict(zip(lowercase__      ,  range(len(lowercase__   )   )   )   )
      UpperCamelCase_     :					List[str]   	=Path(self.tmpdirname   )
      save_json(lowercase__      ,  save_dir / VOCAB_FILES_NAMES['vocab']   )
      save_json(lowercase__      ,  save_dir / VOCAB_FILES_NAMES['tokenizer_config_file']   )
      if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
       copyfile(lowercase__      ,  save_dir / VOCAB_FILES_NAMES['source_spm']   )
       copyfile(lowercase__      ,  save_dir / VOCAB_FILES_NAMES['target_spm']   )

      UpperCamelCase_     :					Dict   	=MarianTokenizer.from_pretrained(self.tmpdirname   )
      tokenizer.save_pretrained(self.tmpdirname   )



     def  lowerCamelCase_						(							self :Tuple      ,  **_lowerCamelCase :int   ):







      '''simple docstring'''



      return MarianTokenizer.from_pretrained(self.tmpdirname      ,  **lowercase__   )



     def  lowerCamelCase_						(							self :int      ,  _lowerCamelCase :int   ):







      '''simple docstring'''



      return (
          "This is a test",
          "This is a test",
      )



     def  lowerCamelCase_						(							self :List[str]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	="""</s>"""
      UpperCamelCase_     :					Optional[int]   	=0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__   )      ,  lowercase__   )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__   )      ,  lowercase__   )



     def  lowerCamelCase_						(							self :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=list(self.get_tokenizer().get_vocab().keys()   )

      self.assertEqual(vocab_keys[0]      ,  '</s>'   )
      self.assertEqual(vocab_keys[1]      ,  '<unk>'   )
      self.assertEqual(vocab_keys[-1]      ,  '<pad>'   )
      self.assertEqual(len(lowercase__   )      ,  9   )



     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      self.assertEqual(self.get_tokenizer().vocab_size      ,  9   )



     def  lowerCamelCase_						(							self :str   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=MarianTokenizer.from_pretrained(f'''{ORG_NAME}opus-mt-en-de'''   )
      UpperCamelCase_     :					Any   	=en_de_tokenizer(['I am a small frog']      ,  return_tensors=lowercase__   )
      self.assertIsInstance(lowercase__      ,  lowercase__   )
      UpperCamelCase_     :					str   	=[38, 121, 14, 697, 38_848, 0]
      self.assertListEqual(lowercase__      ,  batch.input_ids[0]   )

      UpperCamelCase_     :					Union[str, Any]   	=tempfile.mkdtemp()
      en_de_tokenizer.save_pretrained(lowercase__   )
      UpperCamelCase_     :					Union[str, Any]   	=[x.name for x in Path(lowercase__   ).glob('*'   )]
      self.assertIn('source.spm'      ,  lowercase__   )
      MarianTokenizer.from_pretrained(lowercase__   )



     def  lowerCamelCase_						(							self :str   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self.get_tokenizer()

      UpperCamelCase_     :					Dict   	=tok(
          ['I am a small frog' * 1_000, 'I am a small frog']      ,  padding=lowercase__      ,  truncation=lowercase__      ,  return_tensors=lowercase__   )
      self.assertIsInstance(lowercase__      ,  lowercase__   )
      self.assertEqual(batch.input_ids.shape      ,  (2, 512)   )



     def  lowerCamelCase_						(							self :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=self.get_tokenizer()
      UpperCamelCase_     :					Dict   	=tok(['I am a tiny frog', 'I am a small frog']      ,  padding=lowercase__      ,  return_tensors=lowercase__   )
      self.assertIsInstance(lowercase__      ,  lowercase__   )
      self.assertEqual(batch_smaller.input_ids.shape      ,  (2, 10)   )



     @slow
     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	={"""input_ids""": [[43_495, 462, 20, 42_164, 1_369, 52, 464, 132, 1_703, 492, 13, 7_491, 38_999, 6, 8, 464, 132, 1_703, 492, 13, 4_669, 37_867, 13, 7_525, 27, 1_593, 988, 13, 33_972, 7_029, 6, 20, 8_251, 383, 2, 270, 5_866, 3_788, 2, 2_353, 8_251, 12_338, 2, 13_958, 387, 2, 3_629, 6_953, 188, 2_900, 2, 13_958, 8_011, 11_501, 23, 8_460, 4_073, 34_009, 20, 435, 11_439, 27, 8, 8_460, 4_073, 6_004, 20, 9_988, 375, 27, 33, 266, 1_945, 1_076, 1_350, 37_867, 3_288, 5, 577, 1_076, 4_374, 8, 5_082, 5, 26_453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 10_767, 6, 316, 304, 4_239, 3, 0], [148, 15_722, 19, 1_839, 12, 1_350, 13, 22_327, 5_082, 5_418, 47_567, 35_938, 59, 318, 19_552, 108, 2_183, 54, 14_976, 4_835, 32, 547, 1_114, 8, 315, 2_417, 5, 92, 19_088, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100], [36, 6_395, 12_570, 39_147, 11_597, 6, 266, 4, 45_405, 7_296, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=lowercase__      ,  model_name='Helsinki-NLP/opus-mt-en-de'      ,  revision='1a8c2263da11e68e50938f97e10cd57820bd504c'      ,  decode_kwargs={'use_source_tokenizer': True}      ,  )




     def  lowerCamelCase_						(							self :int   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=MarianTokenizer.from_pretrained('hf-internal-testing/test-marian-two-vocabs'   )

      UpperCamelCase_     :					Tuple   	="""Tämä on testi"""
      UpperCamelCase_     :					Union[str, Any]   	="""This is a test"""

      UpperCamelCase_     :					Union[str, Any]   	=[76, 7, 2_047, 2]
      UpperCamelCase_     :					Optional[int]   	=[69, 12, 11, 940, 2]

      UpperCamelCase_     :					Optional[Any]   	=tokenizer(lowercase__   ).input_ids
      self.assertListEqual(lowercase__      ,  lowercase__   )

      UpperCamelCase_     :					Optional[int]   	=tokenizer(text_target=lowercase__   ).input_ids
      self.assertListEqual(lowercase__      ,  lowercase__   )

      UpperCamelCase_     :					str   	=tokenizer.decode(lowercase__      ,  skip_special_tokens=lowercase__   )
      self.assertEqual(lowercase__      ,  lowercase__   )






'''simple docstring'''


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class      SCREAMING_SNAKE_CASE    (						snake_case_							,      unittest.TestCase				):
      __magic_name__  :       Optional[Any]   							=					BertJapaneseTokenizer
      __magic_name__  :       int   							=					False
      __magic_name__  :       Tuple   							=					True






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           super().setUp()

           a_     :							List[Any]     						=  [
               """[UNK]""",
               """[CLS]""",
               """[SEP]""",
               """こんにちは""",
               """こん""",
               """にちは""",
               """ばんは""",
               """##こん""",
               """##にちは""",
               """##ばんは""",
               """世界""",
               """##世界""",
               """、""",
               """##、""",
               """。""",
               """##。""",
           ]

           a_     :							Optional[int]     						=  os.path.join(self.tmpdirname						,						VOCAB_FILES_NAMES["""vocab_file"""]       )
           with open(self.vocab_file						,						"""w"""						,						encoding="""utf-8"""       ) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






      def 							lowercase_						( self   :  List[Any]						,						lowercase__   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Dict     						=  """こんにちは、世界。 \nこんばんは、世界。"""
           a_     :							Tuple     						=  """こんにちは 、 世界 。 こんばんは 、 世界 。"""
           return input_text, output_text






      def 							lowercase_						( self   :  Optional[int]						,						lowercase__   :  str       ):


           '''simple docstring'''

           a_						,     a_     :							int     						=  self.get_input_output_texts(lowercase__       )
           a_     :							Tuple     						=  tokenizer.encode(lowercase__						,						add_special_tokens=lowercase__       )
           a_     :							Union[str, Any]     						=  tokenizer.decode(lowercase__						,						clean_up_tokenization_spaces=lowercase__       )
           return text, ids






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Tuple     						=  self.tokenizer_class(self.vocab_file       )

           a_     :							Union[str, Any]     						=  tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )






      def 							lowercase_						( self   :  Union[str, Any]       ):


           '''simple docstring'''

           a_     :							Tuple     						=  self.tokenizer_class(self.vocab_file						,						word_tokenizer_type="""mecab"""       )
           self.assertIsNotNone(lowercase__       )

           a_     :							Optional[Any]     						=  """こんにちは、世界。\nこんばんは、世界。"""
           a_     :							Any     						=  tokenizer.tokenize(lowercase__       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

           a_     :							Dict     						=  os.path.join(self.tmpdirname						,						"""tokenizer.bin"""       )
           with open(lowercase__						,						"""wb"""       ) as handle:
                pickle.dump(lowercase__						,						lowercase__       )

           with open(lowercase__						,						"""rb"""       ) as handle:
                a_     :							Optional[int]     						=  pickle.load(lowercase__       )

           a_     :							Tuple     						=  tokenizer_new.tokenize(lowercase__       )

           self.assertListEqual(lowercase__						,						lowercase__       )






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							List[str]     						=  MecabTokenizer(mecab_dic="""ipadic"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           try:
                a_     :							Tuple     						=  MecabTokenizer(mecab_dic="""unidic_lite"""       )
           except ModuleNotFoundError:
                return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           try:
                a_     :							Union[str, Any]     						=  MecabTokenizer(mecab_dic="""unidic"""       )
           except ModuleNotFoundError:
                return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  MecabTokenizer(do_lower_case=lowercase__						,						mecab_dic="""ipadic"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           try:
                a_     :							Any     						=  MecabTokenizer(
                    do_lower_case=lowercase__						,						normalize_text=lowercase__						,						mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""       )
           except RuntimeError:
                # if dict doesn't exist in the system, previous code raises this error.
                return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							str     						=  MecabTokenizer(normalize_text=lowercase__						,						mecab_dic="""ipadic"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           a_     :							int     						=  self.tokenizer_class(self.vocab_file						,						word_tokenizer_type="""sudachi"""       )
           self.assertIsNotNone(lowercase__       )

           a_     :							Union[str, Any]     						=  """こんにちは、世界。\nこんばんは、世界。"""
           a_     :							Optional[int]     						=  tokenizer.tokenize(lowercase__       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

           a_     :							str     						=  os.path.join(self.tmpdirname						,						"""tokenizer.bin"""       )
           with open(lowercase__						,						"""wb"""       ) as handle:
                pickle.dump(lowercase__						,						lowercase__       )

           with open(lowercase__						,						"""rb"""       ) as handle:
                a_     :							List[Any]     						=  pickle.load(lowercase__       )

           a_     :							Optional[int]     						=  tokenizer_new.tokenize(lowercase__       )

           self.assertListEqual(lowercase__						,						lowercase__       )






      @require_sudachi
      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  SudachiTokenizer(sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  SudachiTokenizer(sudachi_dict_type="""core"""						,						sudachi_split_mode="""A"""       )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )						,						["""外国""", """人""", """参政""", """権"""]       )






      @require_sudachi
      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Dict     						=  SudachiTokenizer(sudachi_dict_type="""core"""						,						sudachi_split_mode="""B"""       )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )						,						["""外国人""", """参政権"""]       )






      @require_sudachi
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							Tuple     						=  SudachiTokenizer(sudachi_dict_type="""core"""						,						sudachi_split_mode="""C"""       )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )						,						["""外国人参政権"""]       )






      @require_sudachi
      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							List[Any]     						=  SudachiTokenizer(do_lower_case=lowercase__						,						sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							Tuple     						=  SudachiTokenizer(normalize_text=lowercase__						,						sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]						,						)






      @require_sudachi
      def 							lowercase_						( self   :  Optional[int]       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  SudachiTokenizer(trim_whitespace=lowercase__						,						sudachi_dict_type="""core"""       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  self.tokenizer_class(self.vocab_file						,						word_tokenizer_type="""jumanpp"""       )
           self.assertIsNotNone(lowercase__       )

           a_     :							Union[str, Any]     						=  """こんにちは、世界。\nこんばんは、世界。"""
           a_     :							Union[str, Any]     						=  tokenizer.tokenize(lowercase__       )
           self.assertListEqual(lowercase__						,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 12, 10, 14, 4, 9, 12, 10, 14]       )

           a_     :							Optional[Any]     						=  os.path.join(self.tmpdirname						,						"""tokenizer.bin"""       )
           with open(lowercase__						,						"""wb"""       ) as handle:
                pickle.dump(lowercase__						,						lowercase__       )

           with open(lowercase__						,						"""rb"""       ) as handle:
                a_     :							Any     						=  pickle.load(lowercase__       )

           a_     :							List[Any]     						=  tokenizer_new.tokenize(lowercase__       )

           self.assertListEqual(lowercase__						,						lowercase__       )






      @require_jumanpp
      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							int     						=  JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							str     						=  JumanppTokenizer(do_lower_case=lowercase__       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Tuple       ):


           '''simple docstring'''

           a_     :							List[Any]     						=  JumanppTokenizer(normalize_text=lowercase__       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  JumanppTokenizer(trim_whitespace=lowercase__       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )						,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]						,						)






      @require_jumanpp
      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""       )						,						["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]						,						)






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							int     						=  ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

           a_     :							int     						=  {}
           for i, token in enumerate(lowercase__       ):
                a_     :							str     						=  i
           a_     :							int     						=  WordpieceTokenizer(vocab=lowercase__						,						unk_token="""[UNK]"""       )

           self.assertListEqual(tokenizer.tokenize(""""""       )						,						[]       )

           self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )						,						["""こんにちは"""]       )

           self.assertListEqual(tokenizer.tokenize("""こんばんは"""       )						,						["""こん""", """##ばんは"""]       )

           self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""       )						,						["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]       )






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""       )
           a_     :							List[str]     						=  tokenizer.subword_tokenizer

           a_     :							Any     						=  subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""       )
           self.assertListEqual(lowercase__						,						["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]       )

           a_     :							Optional[Any]     						=  subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""       )
           self.assertListEqual(lowercase__						,						["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]       )






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           a_     :							Union[str, Any]     						=  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""       )

           a_     :							Union[str, Any]     						=  tokenizer.encode("""ありがとう。"""						,						add_special_tokens=lowercase__       )
           a_     :							Any     						=  tokenizer.encode("""どういたしまして。"""						,						add_special_tokens=lowercase__       )

           a_     :							str     						=  tokenizer.build_inputs_with_special_tokens(lowercase__       )
           a_     :							Optional[int]     						=  tokenizer.build_inputs_with_special_tokens(lowercase__						,						lowercase__       )

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      SCREAMING_SNAKE_CASE    (						snake_case_							,      unittest.TestCase				):
      __magic_name__  :       Optional[Any]   							=					BertJapaneseTokenizer
      __magic_name__  :       int   							=					False






      def 							lowercase_						( self   :  List[Any]       ):


           '''simple docstring'''

           super().setUp()

           a_     :							int     						=  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

           a_     :							Tuple     						=  os.path.join(self.tmpdirname						,						VOCAB_FILES_NAMES["""vocab_file"""]       )
           with open(self.vocab_file						,						"""w"""						,						encoding="""utf-8"""       ) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






      def 							lowercase_						( self   :  str						,						**lowercase__   :  Tuple       ):


           '''simple docstring'''

           return BertJapaneseTokenizer.from_pretrained(self.tmpdirname						,						subword_tokenizer_type="""character"""						,						**lowercase__       )






      def 							lowercase_						( self   :  str						,						lowercase__   :  Union[str, Any]       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  """こんにちは、世界。 \nこんばんは、世界。"""
           a_     :							List[Any]     						=  """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
           return input_text, output_text






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  Any       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           pass  # TODO add if relevant






      def 							lowercase_						( self   :  str       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  self.tokenizer_class(self.vocab_file						,						subword_tokenizer_type="""character"""       )

           a_     :							Tuple     						=  tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""       )
           self.assertListEqual(
               lowercase__						,						["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]       )
           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(lowercase__       )						,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]       )






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							str     						=  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

           a_     :							Dict     						=  {}
           for i, token in enumerate(lowercase__       ):
                a_     :							List[Any]     						=  i
           a_     :							Dict     						=  CharacterTokenizer(vocab=lowercase__						,						unk_token="""[UNK]"""       )

           self.assertListEqual(tokenizer.tokenize(""""""       )						,						[]       )

           self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )						,						["""こ""", """ん""", """に""", """ち""", """は"""]       )

           self.assertListEqual(tokenizer.tokenize("""こんにちほ"""       )						,						["""こ""", """ん""", """に""", """ち""", """[UNK]"""]       )






      def 							lowercase_						( self   :  Dict       ):


           '''simple docstring'''

           a_     :							Dict     						=  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""       )

           a_     :							List[Any]     						=  tokenizer.encode("""ありがとう。"""						,						add_special_tokens=lowercase__       )
           a_     :							List[Any]     						=  tokenizer.encode("""どういたしまして。"""						,						add_special_tokens=lowercase__       )

           a_     :							str     						=  tokenizer.build_inputs_with_special_tokens(lowercase__       )
           a_     :							Optional[Any]     						=  tokenizer.build_inputs_with_special_tokens(lowercase__						,						lowercase__       )

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      SCREAMING_SNAKE_CASE    (						unittest.TestCase				):






      def 							lowercase_						( self   :  List[str]       ):


           '''simple docstring'''

           a_     :							Optional[Any]     						=  """cl-tohoku/bert-base-japanese"""
           a_     :							Union[str, Any]     						=  AutoTokenizer.from_pretrained(lowercase__       )
           self.assertIsInstance(lowercase__						,						lowercase__       )







class      SCREAMING_SNAKE_CASE    (						unittest.TestCase				):






      def 							lowercase_						( self   :  int       ):


           '''simple docstring'''

           a_     :							Any     						=  """cl-tohoku/bert-base-japanese"""
           with self.assertLogs("""transformers"""						,						level="""WARNING"""       ) as cm:
                BertTokenizer.from_pretrained(lowercase__       )
                self.assertTrue(
                    cm.records[0].message.startswith(
                        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                        """ is called from."""       )       )
           a_     :							Optional[Any]     						=  """bert-base-cased"""
           with self.assertLogs("""transformers"""						,						level="""WARNING"""       ) as cm:
                BertJapaneseTokenizer.from_pretrained(lowercase__       )
                self.assertTrue(
                    cm.records[0].message.startswith(
                        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                        """ is called from."""       )       )
