






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 				lowerCAmelCase_	(		SCREAMING_SNAKE_CASE_   ,		unittest.TestCase			):
				_UpperCamelCase					:       Tuple							       = BertJapaneseTokenizer
				_UpperCamelCase					:       Optional[int]							       = False
				_UpperCamelCase					:       Dict							       = True


				def 						__a					(					self      ):
											super().setUp()

											_lowercase							:   Optional[int] 							=   [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    'こんにちは',
											    'こん',
											    'にちは',
											    'ばんは',
											    '##こん',
											    '##にちは',
											    '##ばんは',
											    '世界',
											    '##世界',
											    '、',
											    '##、',
											    '。',
											    '##。',
											]

											_lowercase							:   Union[str, Any] 							=   os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['vocab_file']      )
											with open(self.vocab_file ,						'w' ,						encoding='utf-8'      ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )


				def 						__a					(					self ,						_lowerCAmelCase      ):
											_lowercase							:   Dict 							=   'こんにちは、世界。 \nこんばんは、世界。'
											_lowercase							:   Dict 							=   'こんにちは 、 世界 。 こんばんは 、 世界 。'
											return input_text, output_text


				def 						__a					(					self ,						_lowerCAmelCase      ):
											_lowercase							:   str 							=   self.get_input_output_texts(_a      )
											_lowercase							:   Dict 							=   tokenizer.encode(_a ,						add_special_tokens=_a      )
											_lowercase							:   List[Any] 							=   tokenizer.decode(_a ,						clean_up_tokenization_spaces=_a      )
											return text, ids


				def 						__a					(					self      ):
											pass  # TODO add if relevant


				def 						__a					(					self      ):
											pass  # TODO add if relevant


				def 						__a					(					self      ):
											pass  # TODO add if relevant


				def 						__a					(					self      ):
											_lowercase							:   Tuple 							=   self.tokenizer_class(self.vocab_file      )

											_lowercase							:   Optional[int] 							=   tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'      )
											self.assertListEqual(_a ,						['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_a      ) ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )


				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   self.tokenizer_class(self.vocab_file ,						word_tokenizer_type='mecab'      )
											self.assertIsNotNone(_a      )

											_lowercase							:   int 							=   'こんにちは、世界。\nこんばんは、世界。'
											_lowercase							:   List[str] 							=   tokenizer.tokenize(_a      )
											self.assertListEqual(_a ,						['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_a      ) ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

											_lowercase							:   Optional[Any] 							=   os.path.join(self.tmpdirname ,						'tokenizer.bin'      )
											with open(_a ,						'wb'      ) as handle:
																		pickle.dump(_a ,						_a      )

											with open(_a ,						'rb'      ) as handle:
																		_lowercase							:   List[Any] 							=   pickle.load(_a      )

											_lowercase							:   Dict 							=   tokenizer_new.tokenize(_a      )

											self.assertListEqual(_a ,						_a      )


				def 						__a					(					self      ):
											_lowercase							:   List[str] 							=   MecabTokenizer(mecab_dic='ipadic'      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。'] ,						)


				def 						__a					(					self      ):
											try:
																		_lowercase							:   List[str] 							=   MecabTokenizer(mecab_dic='unidic_lite'      )
											except ModuleNotFoundError:
																		return

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。'] ,						)


				def 						__a					(					self      ):
											try:
																		_lowercase							:   List[str] 							=   MecabTokenizer(mecab_dic='unidic'      )
											except ModuleNotFoundError:
																		return

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。'] ,						)


				def 						__a					(					self      ):
											_lowercase							:   List[Any] 							=   MecabTokenizer(do_lower_case=_a ,						mecab_dic='ipadic'      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。'] ,						)


				def 						__a					(					self      ):
											try:
																		_lowercase							:   int 							=   MecabTokenizer(
																		    do_lower_case=_a ,						normalize_text=_a ,						mecab_option='-d /usr/local/lib/mecab/dic/jumandic'      )
											except RuntimeError:
																		# if dict doesn't exist in the system, previous code raises this error.
																		return

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。'] ,						)


				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   MecabTokenizer(normalize_text=_a ,						mecab_dic='ipadic'      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。'] ,						)


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   List[str] 							=   self.tokenizer_class(self.vocab_file ,						word_tokenizer_type='sudachi'      )
											self.assertIsNotNone(_a      )

											_lowercase							:   Dict 							=   'こんにちは、世界。\nこんばんは、世界。'
											_lowercase							:   List[Any] 							=   tokenizer.tokenize(_a      )
											self.assertListEqual(_a ,						['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_a      ) ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

											_lowercase							:   List[str] 							=   os.path.join(self.tmpdirname ,						'tokenizer.bin'      )
											with open(_a ,						'wb'      ) as handle:
																		pickle.dump(_a ,						_a      )

											with open(_a ,						'rb'      ) as handle:
																		_lowercase							:   Union[str, Any] 							=   pickle.load(_a      )

											_lowercase							:   Tuple 							=   tokenizer_new.tokenize(_a      )

											self.assertListEqual(_a ,						_a      )


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   Optional[Any] 							=   SudachiTokenizer(sudachi_dict_type='core'      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						[' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' '] ,						)


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   int 							=   SudachiTokenizer(sudachi_dict_type='core' ,						sudachi_split_mode='A'      )

											self.assertListEqual(tokenizer.tokenize('外国人参政権'      ) ,						['外国', '人', '参政', '権']      )


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   SudachiTokenizer(sudachi_dict_type='core' ,						sudachi_split_mode='B'      )

											self.assertListEqual(tokenizer.tokenize('外国人参政権'      ) ,						['外国人', '参政権']      )


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   Optional[Any] 							=   SudachiTokenizer(sudachi_dict_type='core' ,						sudachi_split_mode='C'      )

											self.assertListEqual(tokenizer.tokenize('外国人参政権'      ) ,						['外国人参政権']      )


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   Optional[int] 							=   SudachiTokenizer(do_lower_case=_a ,						sudachi_dict_type='core'      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						[' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' '] ,						)


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   str 							=   SudachiTokenizer(normalize_text=_a ,						sudachi_dict_type='core'      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						[' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' '] ,						)


				@require_sudachi
				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   SudachiTokenizer(trim_whitespace=_a ,						sudachi_dict_type='core'      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。'] ,						)


				@require_jumanpp
				def 						__a					(					self      ):
											_lowercase							:   Any 							=   self.tokenizer_class(self.vocab_file ,						word_tokenizer_type='jumanpp'      )
											self.assertIsNotNone(_a      )

											_lowercase							:   Any 							=   'こんにちは、世界。\nこんばんは、世界。'
											_lowercase							:   Optional[Any] 							=   tokenizer.tokenize(_a      )
											self.assertListEqual(_a ,						['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_a      ) ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

											_lowercase							:   Union[str, Any] 							=   os.path.join(self.tmpdirname ,						'tokenizer.bin'      )
											with open(_a ,						'wb'      ) as handle:
																		pickle.dump(_a ,						_a      )

											with open(_a ,						'rb'      ) as handle:
																		_lowercase							:   str 							=   pickle.load(_a      )

											_lowercase							:   int 							=   tokenizer_new.tokenize(_a      )

											self.assertListEqual(_a ,						_a      )


				@require_jumanpp
				def 						__a					(					self      ):
											_lowercase							:   List[Any] 							=   JumanppTokenizer()

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。'] ,						)


				@require_jumanpp
				def 						__a					(					self      ):
											_lowercase							:   Any 							=   JumanppTokenizer(do_lower_case=_a      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。'] ,						)


				@require_jumanpp
				def 						__a					(					self      ):
											_lowercase							:   int 							=   JumanppTokenizer(normalize_text=_a      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。'] ,						)


				@require_jumanpp
				def 						__a					(					self      ):
											_lowercase							:   Any 							=   JumanppTokenizer(trim_whitespace=_a      )

											self.assertListEqual(
											    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      ) ,						['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。'] ,						)


				@require_jumanpp
				def 						__a					(					self      ):
											_lowercase							:   str 							=   JumanppTokenizer()

											self.assertListEqual(
											    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'      ) ,						['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。'] ,						)


				def 						__a					(					self      ):
											_lowercase							:   List[Any] 							=   ['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

											_lowercase							:   List[Any] 							=   {}
											for i, token in enumerate(_a      ):
																		_lowercase							:   Tuple 							=   i
											_lowercase							:   Dict 							=   WordpieceTokenizer(vocab=_a ,						unk_token='[UNK]'      )

											self.assertListEqual(tokenizer.tokenize(''      ) ,						[]      )

											self.assertListEqual(tokenizer.tokenize('こんにちは'      ) ,						['こんにちは']      )

											self.assertListEqual(tokenizer.tokenize('こんばんは'      ) ,						['こん', '##ばんは']      )

											self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'      ) ,						['こん', '##ばんは', '[UNK]', 'こんにちは']      )


				def 						__a					(					self      ):
											_lowercase							:   str 							=   BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'      )
											_lowercase							:   List[str] 							=   tokenizer.subword_tokenizer

											_lowercase							:   int 							=   subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'      )
											self.assertListEqual(_a ,						['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']      )

											_lowercase							:   List[Any] 							=   subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'      )
											self.assertListEqual(_a ,						['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']      )


				def 						__a					(					self      ):
											_lowercase							:   Optional[int] 							=   self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'      )

											_lowercase							:   List[Any] 							=   tokenizer.encode('ありがとう。' ,						add_special_tokens=_a      )
											_lowercase							:   List[Any] 							=   tokenizer.encode('どういたしまして。' ,						add_special_tokens=_a      )

											_lowercase							:   Union[str, Any] 							=   tokenizer.build_inputs_with_special_tokens(_a      )
											_lowercase							:   Tuple 							=   tokenizer.build_inputs_with_special_tokens(_a ,						_a      )

											# 2 is for "[CLS]", 3 is for "[SEP]"
											assert encoded_sentence == [2] + text + [3]
											assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 				lowerCAmelCase_	(		SCREAMING_SNAKE_CASE_   ,		unittest.TestCase			):
				_UpperCamelCase					:       Union[str, Any]							       = BertJapaneseTokenizer
				_UpperCamelCase					:       List[str]							       = False


				def 						__a					(					self      ):
											super().setUp()

											_lowercase							:   Optional[Any] 							=   ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

											_lowercase							:   Optional[int] 							=   os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['vocab_file']      )
											with open(self.vocab_file ,						'w' ,						encoding='utf-8'      ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )


				def 						__a					(					self ,						**_lowerCAmelCase      ):
											return BertJapaneseTokenizer.from_pretrained(self.tmpdirname ,						subword_tokenizer_type='character' ,						**_a      )


				def 						__a					(					self ,						_lowerCAmelCase      ):
											_lowercase							:   str 							=   'こんにちは、世界。 \nこんばんは、世界。'
											_lowercase							:   Dict 							=   'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
											return input_text, output_text


				def 						__a					(					self      ):
											pass  # TODO add if relevant


				def 						__a					(					self      ):
											pass  # TODO add if relevant


				def 						__a					(					self      ):
											pass  # TODO add if relevant


				def 						__a					(					self      ):
											_lowercase							:   str 							=   self.tokenizer_class(self.vocab_file ,						subword_tokenizer_type='character'      )

											_lowercase							:   Optional[int] 							=   tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'      )
											self.assertListEqual(
											    _a ,						['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']      )
											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(_a      ) ,						[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]      )


				def 						__a					(					self      ):
											_lowercase							:   Tuple 							=   ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

											_lowercase							:   List[Any] 							=   {}
											for i, token in enumerate(_a      ):
																		_lowercase							:   Dict 							=   i
											_lowercase							:   Union[str, Any] 							=   CharacterTokenizer(vocab=_a ,						unk_token='[UNK]'      )

											self.assertListEqual(tokenizer.tokenize(''      ) ,						[]      )

											self.assertListEqual(tokenizer.tokenize('こんにちは'      ) ,						['こ', 'ん', 'に', 'ち', 'は']      )

											self.assertListEqual(tokenizer.tokenize('こんにちほ'      ) ,						['こ', 'ん', 'に', 'ち', '[UNK]']      )


				def 						__a					(					self      ):
											_lowercase							:   Any 							=   self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'      )

											_lowercase							:   Any 							=   tokenizer.encode('ありがとう。' ,						add_special_tokens=_a      )
											_lowercase							:   Dict 							=   tokenizer.encode('どういたしまして。' ,						add_special_tokens=_a      )

											_lowercase							:   Tuple 							=   tokenizer.build_inputs_with_special_tokens(_a      )
											_lowercase							:   Union[str, Any] 							=   tokenizer.build_inputs_with_special_tokens(_a ,						_a      )

											# 2 is for "[CLS]", 3 is for "[SEP]"
											assert encoded_sentence == [2] + text + [3]
											assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 				lowerCAmelCase_	(		unittest.TestCase			):


				def 						__a					(					self      ):
											_lowercase							:   Optional[int] 							=   'cl-tohoku/bert-base-japanese'
											_lowercase							:   List[Any] 							=   AutoTokenizer.from_pretrained(_a      )
											self.assertIsInstance(_a ,						_a      )




class 				lowerCAmelCase_	(		unittest.TestCase			):


				def 						__a					(					self      ):
											_lowercase							:   Tuple 							=   'cl-tohoku/bert-base-japanese'
											with self.assertLogs('transformers' ,						level='WARNING'      ) as cm:
																		BertTokenizer.from_pretrained(_a      )
																		self.assertTrue(
																		    cm.records[0].message.startswith(
																		        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																		        ' is called from.'      )      )
											_lowercase							:   List[str] 							=   'bert-base-cased'
											with self.assertLogs('transformers' ,						level='WARNING'      ) as cm:
																		BertJapaneseTokenizer.from_pretrained(_a      )
																		self.assertTrue(
																		    cm.records[0].message.startswith(
																		        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
																		        ' is called from.'      )      )









'''simple docstring'''





from __future__ import annotations

_UpperCamelCase											=		10

def 			a_	(				_lowerCAmelCase			) -> list[int]:
  __lowerCamelCase			:			str       			=					1
  __lowerCamelCase			:			Union[str, Any]       			=					max(_lowerCAmelCase			)
  while placement <= max_digit:
    # declare and initialize empty buckets
    __lowerCamelCase			:			list[list]       			=					[[] for _ in range(_lowerCAmelCase			)]
    # split list_of_ints between the buckets
    for i in list_of_ints:
      __lowerCamelCase			:			List[str]       			=					int((i / placement) % RADIX			)
      buckets[tmp].append(_lowerCAmelCase			)
    # put each buckets' contents into list_of_ints
    __lowerCamelCase			:			Dict       			=					0
    for b in range(_lowerCAmelCase			):
      for i in buckets[b]:
        __lowerCamelCase			:			List[str]       			=					i
        a += 1
        # move to next
    placement *= RADIX
  return list_of_ints


if __name__ == "__main__":
   import doctest

   doctest.testmod()






