






'''simple docstring'''





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class  snake_case__      (					UpperCamelCase     ,				unittest.TestCase):
					a_        =					BertJapaneseTokenizer
					a_        =					False
					a_        =					True




					def 		A      (  self  :		Any      )    ->							Any:
								super().setUp()

								UpperCAmelCase_      :       Any          =       [
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''こんにちは''',
								    '''こん''',
								    '''にちは''',
								    '''ばんは''',
								    '''##こん''',
								    '''##にちは''',
								    '''##ばんは''',
								    '''世界''',
								    '''##世界''',
								    '''、''',
								    '''##、''',
								    '''。''',
								    '''##。''',
								]

								UpperCAmelCase_      :       Optional[Any]          =       os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES['''vocab_file''']      )
								with open(self.vocab_file				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )




					def 		A      (  self  :		Any				, _A  :		Tuple      )    ->							List[str]:
								UpperCAmelCase_      :       int          =       '''こんにちは、世界。 \nこんばんは、世界。'''
								UpperCAmelCase_      :       Tuple          =       '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
								return input_text, output_text




					def 		A      (  self  :		Tuple				, _A  :		Dict      )    ->							Dict:
								UpperCAmelCase_    ,      UpperCAmelCase_      :       List[str]          =       self.get_input_output_texts(_A      )
								UpperCAmelCase_      :       Dict          =       tokenizer.encode(_A				, add_special_tokens=_A      )
								UpperCAmelCase_      :       int          =       tokenizer.decode(_A				, clean_up_tokenization_spaces=_A      )
								return text, ids




					def 		A      (  self  :		int      )    ->							int:
								pass  # TODO add if relevant




					def 		A      (  self  :		List[str]      )    ->							Tuple:
								pass  # TODO add if relevant




					def 		A      (  self  :		str      )    ->							Dict:
								pass  # TODO add if relevant




					def 		A      (  self  :		int      )    ->							Dict:
								UpperCAmelCase_      :       int          =       self.tokenizer_class(self.vocab_file      )

								UpperCAmelCase_      :       int          =       tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''      )
								self.assertListEqual(_A				, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )				, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )




					def 		A      (  self  :		Any      )    ->							Dict:
								UpperCAmelCase_      :       int          =       self.tokenizer_class(self.vocab_file				, word_tokenizer_type='''mecab'''      )
								self.assertIsNotNone(_A      )

								UpperCAmelCase_      :       List[str]          =       '''こんにちは、世界。\nこんばんは、世界。'''
								UpperCAmelCase_      :       int          =       tokenizer.tokenize(_A      )
								self.assertListEqual(_A				, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )				, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

								UpperCAmelCase_      :       Union[str, Any]          =       os.path.join(self.tmpdirname				, '''tokenizer.bin'''      )
								with open(_A				, '''wb'''      ) as handle:
											pickle.dump(_A				, _A      )

								with open(_A				, '''rb'''      ) as handle:
											UpperCAmelCase_      :       Any          =       pickle.load(_A      )

								UpperCAmelCase_      :       str          =       tokenizer_new.tokenize(_A      )

								self.assertListEqual(_A				, _A      )




					def 		A      (  self  :		int      )    ->							List[str]:
								UpperCAmelCase_      :       int          =       MecabTokenizer(mecab_dic='''ipadic'''      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				, )




					def 		A      (  self  :		Optional[int]      )    ->							str:
								try:
											UpperCAmelCase_      :       str          =       MecabTokenizer(mecab_dic='''unidic_lite'''      )
								except ModuleNotFoundError:
											return

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				, )




					def 		A      (  self  :		Dict      )    ->							Dict:
								try:
											UpperCAmelCase_      :       str          =       MecabTokenizer(mecab_dic='''unidic'''      )
								except ModuleNotFoundError:
											return

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				, )




					def 		A      (  self  :		Any      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Any          =       MecabTokenizer(do_lower_case=_A				, mecab_dic='''ipadic'''      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				, )




					def 		A      (  self  :		Optional[Any]      )    ->							Any:
								try:
											UpperCAmelCase_      :       Optional[int]          =       MecabTokenizer(
											    do_lower_case=_A				, normalize_text=_A				, mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''      )
								except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				, )




					def 		A      (  self  :		Union[str, Any]      )    ->							int:
								UpperCAmelCase_      :       Union[str, Any]          =       MecabTokenizer(normalize_text=_A				, mecab_dic='''ipadic'''      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']				, )




					@require_sudachi
					def 		A      (  self  :		Union[str, Any]      )    ->							Dict:
								UpperCAmelCase_      :       Union[str, Any]          =       self.tokenizer_class(self.vocab_file				, word_tokenizer_type='''sudachi'''      )
								self.assertIsNotNone(_A      )

								UpperCAmelCase_      :       int          =       '''こんにちは、世界。\nこんばんは、世界。'''
								UpperCAmelCase_      :       Dict          =       tokenizer.tokenize(_A      )
								self.assertListEqual(_A				, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )				, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

								UpperCAmelCase_      :       List[str]          =       os.path.join(self.tmpdirname				, '''tokenizer.bin'''      )
								with open(_A				, '''wb'''      ) as handle:
											pickle.dump(_A				, _A      )

								with open(_A				, '''rb'''      ) as handle:
											UpperCAmelCase_      :       int          =       pickle.load(_A      )

								UpperCAmelCase_      :       Any          =       tokenizer_new.tokenize(_A      )

								self.assertListEqual(_A				, _A      )




					@require_sudachi
					def 		A      (  self  :		List[str]      )    ->							Optional[int]:
								UpperCAmelCase_      :       Tuple          =       SudachiTokenizer(sudachi_dict_type='''core'''      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']				, )




					@require_sudachi
					def 		A      (  self  :		Tuple      )    ->							Dict:
								UpperCAmelCase_      :       List[Any]          =       SudachiTokenizer(sudachi_dict_type='''core'''				, sudachi_split_mode='''A'''      )

								self.assertListEqual(tokenizer.tokenize('''外国人参政権'''      )				, ['''外国''', '''人''', '''参政''', '''権''']      )




					@require_sudachi
					def 		A      (  self  :		Optional[int]      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Any          =       SudachiTokenizer(sudachi_dict_type='''core'''				, sudachi_split_mode='''B'''      )

								self.assertListEqual(tokenizer.tokenize('''外国人参政権'''      )				, ['''外国人''', '''参政権''']      )




					@require_sudachi
					def 		A      (  self  :		List[str]      )    ->							List[str]:
								UpperCAmelCase_      :       List[str]          =       SudachiTokenizer(sudachi_dict_type='''core'''				, sudachi_split_mode='''C'''      )

								self.assertListEqual(tokenizer.tokenize('''外国人参政権'''      )				, ['''外国人参政権''']      )




					@require_sudachi
					def 		A      (  self  :		Dict      )    ->							List[str]:
								UpperCAmelCase_      :       Optional[int]          =       SudachiTokenizer(do_lower_case=_A				, sudachi_dict_type='''core'''      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, [''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']				, )




					@require_sudachi
					def 		A      (  self  :		Dict      )    ->							List[Any]:
								UpperCAmelCase_      :       Dict          =       SudachiTokenizer(normalize_text=_A				, sudachi_dict_type='''core'''      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, [''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']				, )




					@require_sudachi
					def 		A      (  self  :		Dict      )    ->							int:
								UpperCAmelCase_      :       Optional[int]          =       SudachiTokenizer(trim_whitespace=_A				, sudachi_dict_type='''core'''      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']				, )




					@require_jumanpp
					def 		A      (  self  :		Any      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Union[str, Any]          =       self.tokenizer_class(self.vocab_file				, word_tokenizer_type='''jumanpp'''      )
								self.assertIsNotNone(_A      )

								UpperCAmelCase_      :       Optional[Any]          =       '''こんにちは、世界。\nこんばんは、世界。'''
								UpperCAmelCase_      :       Any          =       tokenizer.tokenize(_A      )
								self.assertListEqual(_A				, ['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )				, [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

								UpperCAmelCase_      :       str          =       os.path.join(self.tmpdirname				, '''tokenizer.bin'''      )
								with open(_A				, '''wb'''      ) as handle:
											pickle.dump(_A				, _A      )

								with open(_A				, '''rb'''      ) as handle:
											UpperCAmelCase_      :       List[Any]          =       pickle.load(_A      )

								UpperCAmelCase_      :       List[Any]          =       tokenizer_new.tokenize(_A      )

								self.assertListEqual(_A				, _A      )




					@require_jumanpp
					def 		A      (  self  :		List[Any]      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Optional[int]          =       JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				, )




					@require_jumanpp
					def 		A      (  self  :		Optional[Any]      )    ->							Tuple:
								UpperCAmelCase_      :       int          =       JumanppTokenizer(do_lower_case=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				, )




					@require_jumanpp
					def 		A      (  self  :		Optional[int]      )    ->							Tuple:
								UpperCAmelCase_      :       int          =       JumanppTokenizer(normalize_text=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']				, )




					@require_jumanpp
					def 		A      (  self  :		Dict      )    ->							int:
								UpperCAmelCase_      :       int          =       JumanppTokenizer(trim_whitespace=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''      )				, ['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']				, )




					@require_jumanpp
					def 		A      (  self  :		Union[str, Any]      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Tuple          =       JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''      )				, ['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']				, )




					def 		A      (  self  :		List[Any]      )    ->							Any:
								UpperCAmelCase_      :       Union[str, Any]          =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

								UpperCAmelCase_      :       Optional[Any]          =       {}
								for i, token in enumerate(_A      ):
											UpperCAmelCase_      :       Optional[int]          =       i
								UpperCAmelCase_      :       Optional[int]          =       WordpieceTokenizer(vocab=_A				, unk_token='''[UNK]'''      )

								self.assertListEqual(tokenizer.tokenize(''''''      )				, []      )

								self.assertListEqual(tokenizer.tokenize('''こんにちは'''      )				, ['''こんにちは''']      )

								self.assertListEqual(tokenizer.tokenize('''こんばんは'''      )				, ['''こん''', '''##ばんは''']      )

								self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''      )				, ['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']      )




					def 		A      (  self  :		str      )    ->							int:
								UpperCAmelCase_      :       List[Any]          =       BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''      )
								UpperCAmelCase_      :       List[str]          =       tokenizer.subword_tokenizer

								UpperCAmelCase_      :       List[str]          =       subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''      )
								self.assertListEqual(_A				, ['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']      )

								UpperCAmelCase_      :       List[Any]          =       subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''      )
								self.assertListEqual(_A				, ['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']      )





					def 		A      (  self  :		List[Any]      )    ->							int:
								UpperCAmelCase_      :       Dict          =       self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''      )

								UpperCAmelCase_      :       int          =       tokenizer.encode('''ありがとう。'''				, add_special_tokens=_A      )
								UpperCAmelCase_      :       Dict          =       tokenizer.encode('''どういたしまして。'''				, add_special_tokens=_A      )

								UpperCAmelCase_      :       Any          =       tokenizer.build_inputs_with_special_tokens(_A      )
								UpperCAmelCase_      :       List[str]          =       tokenizer.build_inputs_with_special_tokens(_A				, _A      )

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class  snake_case__      (					UpperCamelCase     ,				unittest.TestCase):
					a_        =					BertJapaneseTokenizer
					a_        =					False




					def 		A      (  self  :		Optional[Any]      )    ->							List[str]:
								super().setUp()

								UpperCAmelCase_      :       Dict          =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

								UpperCAmelCase_      :       str          =       os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES['''vocab_file''']      )
								with open(self.vocab_file				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )




					def 		A      (  self  :		int				, **_A  :		Union[str, Any]      )    ->							Optional[Any]:
								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname				, subword_tokenizer_type='''character'''				, **_A      )




					def 		A      (  self  :		int				, _A  :		Optional[int]      )    ->							List[str]:
								UpperCAmelCase_      :       Optional[int]          =       '''こんにちは、世界。 \nこんばんは、世界。'''
								UpperCAmelCase_      :       Any          =       '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
								return input_text, output_text




					def 		A      (  self  :		str      )    ->							Optional[Any]:
								pass  # TODO add if relevant




					def 		A      (  self  :		Union[str, Any]      )    ->							Optional[Any]:
								pass  # TODO add if relevant




					def 		A      (  self  :		Optional[int]      )    ->							Any:
								pass  # TODO add if relevant




					def 		A      (  self  :		Union[str, Any]      )    ->							str:
								UpperCAmelCase_      :       Tuple          =       self.tokenizer_class(self.vocab_file				, subword_tokenizer_type='''character'''      )

								UpperCAmelCase_      :       Union[str, Any]          =       tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''      )
								self.assertListEqual(
								    _A				, ['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']      )
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_A      )				, [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]      )




					def 		A      (  self  :		List[str]      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Tuple          =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

								UpperCAmelCase_      :       Any          =       {}
								for i, token in enumerate(_A      ):
											UpperCAmelCase_      :       Tuple          =       i
								UpperCAmelCase_      :       int          =       CharacterTokenizer(vocab=_A				, unk_token='''[UNK]'''      )

								self.assertListEqual(tokenizer.tokenize(''''''      )				, []      )

								self.assertListEqual(tokenizer.tokenize('''こんにちは'''      )				, ['''こ''', '''ん''', '''に''', '''ち''', '''は''']      )

								self.assertListEqual(tokenizer.tokenize('''こんにちほ'''      )				, ['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']      )





					def 		A      (  self  :		str      )    ->							List[Any]:
								UpperCAmelCase_      :       Dict          =       self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''      )

								UpperCAmelCase_      :       Tuple          =       tokenizer.encode('''ありがとう。'''				, add_special_tokens=_A      )
								UpperCAmelCase_      :       Optional[Any]          =       tokenizer.encode('''どういたしまして。'''				, add_special_tokens=_A      )

								UpperCAmelCase_      :       Tuple          =       tokenizer.build_inputs_with_special_tokens(_A      )
								UpperCAmelCase_      :       Dict          =       tokenizer.build_inputs_with_special_tokens(_A				, _A      )

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class  snake_case__      (					unittest.TestCase):




					def 		A      (  self  :		Optional[int]      )    ->							Optional[int]:
								UpperCAmelCase_      :       Optional[Any]          =       '''cl-tohoku/bert-base-japanese'''
								UpperCAmelCase_      :       int          =       AutoTokenizer.from_pretrained(_A      )
								self.assertIsInstance(_A				, _A      )






class  snake_case__      (					unittest.TestCase):




					def 		A      (  self  :		Union[str, Any]      )    ->							Optional[int]:
								UpperCAmelCase_      :       int          =       '''cl-tohoku/bert-base-japanese'''
								with self.assertLogs('''transformers'''				, level='''WARNING'''      ) as cm:
											BertTokenizer.from_pretrained(_A      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
											        ''' is called from.'''      )      )
								UpperCAmelCase_      :       List[str]          =       '''bert-base-cased'''
								with self.assertLogs('''transformers'''				, level='''WARNING'''      ) as cm:
											BertJapaneseTokenizer.from_pretrained(_A      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
											        ''' is called from.'''      )      )










'''simple docstring'''





import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


_UpperCamelCase						:						List[str]  =					logging.getLogger(__name__)

_UpperCamelCase						:						int  =					'pytorch_model.bin'


@dataclasses.dataclass
class  snake_case__      :
					a_        =					dataclasses.field(
					    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models."})
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co."}     ,				)






@dataclasses.dataclass
class  snake_case__      :
					a_        =					dataclasses.field(metadata={"help": "A csv or a json file containing the training data."})
					a_        =					dataclasses.field(metadata={"help": "A csv or a json file containing the data to predict on."})
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "A csv or a json file containing the validation data."})
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "The name of the task to train on."}     ,				)
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "The list of labels for the task."})






@dataclasses.dataclass
class  snake_case__      :
					a_        =					dataclasses.field(
					    metadata={"help": "The output directory where the model predictions and checkpoints will be written."})
					a_        =					dataclasses.field(
					    default="accuracy"     ,				metadata={"help": "The evaluation metric used for the task."})
					a_        =					dataclasses.field(
					    default="no"     ,				metadata={
					        "help": "The evaluation strategy to adopt during training. Possible values are: [\"no\", \"step\", \"epoch]"
					    }     ,				)
					a_        =					dataclasses.field(
					    default=10     ,				metadata={"help": "Number of evaluation calls with no improvement after which training will be stopped."}     ,				)
					a_        =					dataclasses.field(
					    default=0.0     ,				metadata={
					        "help": "How much the specified evaluation metric must improve to satisfy early stopping conditions."
					    }     ,				)
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "Whether to filter the pseudo-labeled data based on the confidence score."}     ,				)
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "Whether to filter the pseudo-labeled data based on the validation performance."}     ,				)
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "Whether to fine-tune on labeled data after pseudo training."}     ,				)
					a_        =					dataclasses.field(
					    default=0.0     ,				metadata={"help": "Confidence threshold for pseudo-labeled data filtering."}     ,				)
					a_        =					dataclasses.field(
					    default=100     ,				metadata={"help": "Number of evaluation calls with no improvement after which training will be stopped."}     ,				)
					a_        =					dataclasses.field(
					    default=UpperCamelCase     ,				metadata={"help": "Random seed for initialization."}     ,				)






def __UpperCAmelCase   (					A					:			str			,       A					:			Optional[Any]			,       A					:			List[Any]			,       A					:			Any			,       A					:			Union[str, Any]			,       A					:			Dict		)    ->   int:
			UpperCAmelCase_      :       Tuple          =       datasets.concatenate_datasets([infer_input, infer_output]			,       axis=1		)

			if args.do_filter_by_confidence:
						UpperCAmelCase_      :       Tuple          =       dataset.filter(lambda A		: example["probability"] > args.confidence_threshold		)

			if args.do_filter_by_val_performance:
						assert eval_result >= 0.0 and eval_result <= 1.0
						UpperCAmelCase_      :       Dict          =       int(eval_result * len(A		)		)
						print(A		)
						UpperCAmelCase_      :       Optional[int]          =       dataset.sort('''probability'''			,       reverse=A		)
						UpperCAmelCase_      :       Optional[Any]          =       dataset.select(range(A		)		)

			UpperCAmelCase_      :       Dict          =       dataset.remove_columns(['''label''', '''probability''']		)
			UpperCAmelCase_      :       List[Any]          =       dataset.rename_column('''prediction'''			,       '''label'''		)
			UpperCAmelCase_      :       List[str]          =       dataset.map(lambda A		: {"label": idalabel[example["label"]]}		)
			UpperCAmelCase_      :       Union[str, Any]          =       dataset.shuffle(seed=args.seed		)

			UpperCAmelCase_      :       str          =       os.path.join(A			,       F"train_pseudo.{args.data_file_extension}"		)
			if args.data_file_extension == "csv":
						dataset.to_csv(A			,       index=A		)
			else:
						dataset.to_json(A		)







def __UpperCAmelCase   (					A					:			Any			,       A					:			int			,       A					:			Union[str, Any]			,       A					:			Dict			,       **A					:			Any		)    ->   Dict:
			UpperCAmelCase_      :       List[str]          =       Accelerator()
			# Make one log on every process with the configuration for debugging.
			logging.basicConfig(
			    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''			,       datefmt='''%m/%d/%Y %H:%M:%S'''			,       level=logging.INFO			,       )
			logger.info(accelerator.state		)

			# Setup logging, we only want one process per machine to log things on the
			# screen. accelerator.is_local_main_process is only True for one process per
			# machine.
			logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR		)

			if accelerator.is_local_main_process:
						datasets.utils.logging.set_verbosity_warning()
						transformers.utils.logging.set_verbosity_info()
			else:
						datasets.utils.logging.set_verbosity_error()
						transformers.utils.logging.set_verbosity_error()

			UpperCAmelCase_      :       Tuple          =       STModelArguments(model_name_or_path=A		)
			UpperCAmelCase_      :       int          =       STDataArguments(train_file=A			,       infer_file=A		)
			UpperCAmelCase_      :       Optional[Any]          =       STTrainingArguments(output_dir=A		)
			UpperCAmelCase_      :       Optional[int]          =       argparse.Namespace()

			for arg_class in (model_args, data_args, training_args):
						for key, value in vars(A		).items():
									setattr(A			,       A			,       A		)

			for key, value in kwargs.items():
						if hasattr(A			,       A		):
									setattr(A			,       A			,       A		)

    # Sanity checks
			UpperCAmelCase_      :       List[str]          =       {}
			UpperCAmelCase_      :       Any          =       None

			# You need to provide the training data and the data to predict on
			assert args.train_file is not None
			assert args.infer_file is not None
			UpperCAmelCase_      :       List[Any]          =       args.train_file
			UpperCAmelCase_      :       Optional[int]          =       args.infer_file

			if args.evaluation_strategy != IntervalStrategy.NO.value:
						assert args.eval_file is not None
						UpperCAmelCase_      :       List[str]          =       args.eval_file

			for key in data_files:
						UpperCAmelCase_      :       Dict          =       data_files[key].split('''.'''		)[-1]
						assert extension in ["csv", "json"], F"`{key}_file` should be a csv or a json file."
						if args.data_file_extension is None:
									UpperCAmelCase_      :       Any          =       extension
						else:
									assert extension == args.data_file_extension, F"`{key}_file` should be a {args.data_file_extension} file`."

			assert (
			    args.eval_metric in datasets.list_metrics()
			), F"{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}."

			# If passed along, set the training seed now.
			if args.seed is not None:
						set_seed(args.seed		)

			logger.info('''Creating the initial data directory for self-training...'''		)
			UpperCAmelCase_      :       Any          =       F"{args.output_dir}/self-train_iter-{{}}".format
			UpperCAmelCase_      :       Any          =       data_dir_format(0		)

			if accelerator.is_main_process:
						if args.output_dir is not None:
									os.makedirs(args.output_dir			,       exist_ok=A		)
									os.makedirs(A			,       exist_ok=A		)
			accelerator.wait_for_everyone()

			UpperCAmelCase_      :       Union[str, Any]          =       None
			UpperCAmelCase_      :       Tuple          =       None
			UpperCAmelCase_      :       Optional[Any]          =       0
			UpperCAmelCase_      :       int          =       False
			# Show the progress bar
			UpperCAmelCase_      :       int          =       tqdm(range(args.max_selftrain_iterations		)			,       disable=not accelerator.is_local_main_process		)

			# Self-train
			for iteration in range(0			,       int(args.max_selftrain_iterations		)		):
						UpperCAmelCase_      :       str          =       data_dir_format(A		)
						assert os.path.exists(A		)

						# Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
						# iteration > 0
						UpperCAmelCase_      :       List[Any]          =       os.path.join(A			,       '''stage-1'''		)
						UpperCAmelCase_      :       Dict          =       {
						    '''accelerator''': accelerator,
						    '''model_name_or_path''': args.model_name_or_path,
						    '''cache_dir''': args.cache_dir,
						    '''do_train''': True,
						    '''train_file''': data_files['''train'''] if iteration == 0 else data_files['''train_pseudo'''],
						    '''do_eval''': True if args.eval_file is not None else False,
						    '''eval_file''': data_files['''eval'''],
						    '''do_predict''': True,
						    '''infer_file''': data_files['''infer'''],
						    '''task_name''': args.task_name,
						    '''label_list''': args.label_list,
						    '''output_dir''': current_output_dir,
						    '''eval_metric''': args.eval_metric,
						    '''evaluation_strategy''': args.evaluation_strategy,
						    '''early_stopping_patience''': args.early_stopping_patience,
						    '''early_stopping_threshold''': args.early_stopping_threshold,
						    '''seed''': args.seed,
						}
						# Add additional training arguments
						for key, value in kwargs.items():
									if key not in arguments_dict and not hasattr(A			,       A		):
												arguments_dict.update({key: value}		)

						UpperCAmelCase_      :       int          =       os.path.join(A			,       '''best-checkpoint'''			,       A		)
						if os.path.exists(A		):
									logger.info(
									    '''Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1.'''			,       A			,       A			,       )
						else:
									logger.info('''***** Running self-training: iteration: %d, stage: 1 *****'''			,       A		)
									finetune(**A		)
									accelerator.wait_for_everyone()
									assert os.path.exists(A		)
									logger.info('''Self-training job completed: iteration: %d, stage: 1.'''			,       A		)

						if iteration > 0 and args.finetune_on_labeled_data:
									# Stage 2 (optional): fine-tuning on the original labeled data
									UpperCAmelCase_      :       Any          =       os.path.join(A			,       '''best-checkpoint'''		)
									UpperCAmelCase_      :       Optional[int]          =       os.path.join(A			,       '''stage-2'''		)
									# Update arguments_dict
									UpperCAmelCase_      :       List[Any]          =       model_path
									UpperCAmelCase_      :       Optional[int]          =       data_files['''train''']
									UpperCAmelCase_      :       Union[str, Any]          =       current_output_dir

									UpperCAmelCase_      :       Optional[int]          =       os.path.join(A			,       '''best-checkpoint'''			,       A		)
									if os.path.exists(A		):
												logger.info(
												    '''Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2.'''			,       A			,       A			,       )
									else:
												logger.info('''***** Running self-training: iteration: %d, stage: 2 *****'''			,       A		)
												finetune(**A		)
												accelerator.wait_for_everyone()
												assert os.path.exists(A		)
												logger.info('''Self-training job completed: iteration: %d, stage: 2.'''			,       A		)

						UpperCAmelCase_      :       Dict          =       iteration
						UpperCAmelCase_      :       Optional[Any]          =       data_dir_format(iteration + 1		)

						UpperCAmelCase_      :       Any          =       AutoConfig.from_pretrained(os.path.join(A			,       '''best-checkpoint'''		)		)
						UpperCAmelCase_      :       Optional[Any]          =       config.idalabel
						UpperCAmelCase_      :       List[Any]          =       os.path.join(A			,       '''eval_results_best-checkpoint.json'''		)
						UpperCAmelCase_      :       Union[str, Any]          =       os.path.join(A			,       '''test_results_best-checkpoint.json'''		)
						assert os.path.exists(A		)

						with open(A			,       '''r'''		) as f:
									UpperCAmelCase_      :       Optional[int]          =       float(json.load(A		)[args.eval_metric]		)
						UpperCAmelCase_      :       List[Any]          =       os.path.join(A			,       '''infer_output_best-checkpoint.csv'''		)
						assert os.path.exists(A		)
						# Loading the dataset from local csv or json files.
						UpperCAmelCase_      :       str          =       load_dataset(args.data_file_extension			,       data_files={'''data''': data_files['''infer''']}		)['''data''']
						UpperCAmelCase_      :       Union[str, Any]          =       load_dataset('''csv'''			,       data_files={'''data''': infer_output_file}		)['''data''']

						if accelerator.is_main_process:
									os.makedirs(A			,       exist_ok=A		)
									shutil.copy(A			,       os.path.join(A			,       F"eval_results_iter-{iteration}.json"		)		)
									if os.path.exists(A		):
												shutil.copy(A			,       os.path.join(A			,       F"test_results_iter-{iteration}.json"		)		)
									create_pseudo_labeled_data(A			,       A			,       A			,       A			,       A			,       A		)
						accelerator.wait_for_everyone()

						UpperCAmelCase_      :       int          =       os.path.join(A			,       F"train_pseudo.{args.data_file_extension}"		)

						if args.evaluation_strategy != IntervalStrategy.NO.value:
									UpperCAmelCase_      :       List[str]          =       eval_result

									if best_iteration is None:
												UpperCAmelCase_      :       Any          =       new_iteration
												UpperCAmelCase_      :       List[Any]          =       new_eval_result
									else:
												if new_eval_result - best_eval_result > args.early_stopping_threshold:
															UpperCAmelCase_      :       str          =       new_iteration
															UpperCAmelCase_      :       Optional[int]          =       new_eval_result
															UpperCAmelCase_      :       Any          =       0
												else:
															if new_eval_result == best_eval_result:
																		UpperCAmelCase_      :       int          =       new_iteration
																		UpperCAmelCase_      :       List[Any]          =       new_eval_result
															early_stopping_patience_counter += 1

												if early_stopping_patience_counter >= args.early_stopping_patience:
															UpperCAmelCase_      :       List[Any]          =       True

						progress_bar.update(1		)

						if should_training_stop:
									break

			if best_iteration is not None:
						# Save the best iteration
						logger.info('''Best iteration: %d'''			,       A		)
						logger.info('''Best evaluation result: %s = %f'''			,       args.eval_metric			,       A		)
						accelerator.wait_for_everyone()
						if accelerator.is_main_process:
									shutil.copy(
									    os.path.join(A			,       F"eval_results_iter-{iteration}.json"		)			,       os.path.join(A			,       '''eval_results_best-iteration.json'''		)			,       )
			else:
						# Assume that the last iteration is the best
						logger.info('''Best iteration: %d'''			,       args.max_selftrain_iterations - 1		)
						logger.info('''Best evaluation result: %s = %f'''			,       args.eval_metric			,       A		)
						accelerator.wait_for_everyone()
						if accelerator.is_main_process:
									shutil.copy(
									    os.path.join(A			,       F"eval_results_iter-{args.max_selftrain_iterations - 1}.json"		)			,       os.path.join(A			,       '''eval_results_best-iteration.json'''		)			,       )


