






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_			,       unittest.TestCase						):





					"""simple docstring"""






					a_     =		BertJapaneseTokenizer
					a_     =		False
					a_     =		True



					def 	_lowercase					( self					:  Optional[Any]	):
						super().setUp()

						snake_case__			:				int       =  [
						    "[UNK]",
						    "[CLS]",
						    "[SEP]",
						    "こんにちは",
						    "こん",
						    "にちは",
						    "ばんは",
						    "##こん",
						    "##にちは",
						    "##ばんは",
						    "世界",
						    "##世界",
						    "、",
						    "##、",
						    "。",
						    "##。",
						]

						snake_case__			:				Optional[int]       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["vocab_file"]	)
						with open(self.vocab_file							,		"w"							,		encoding="utf-8"	) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)



					def 	_lowercase					( self					:  Dict							,		__A					:  int	):
						snake_case__			:				Optional[Any]       =  "こんにちは、世界。 \nこんばんは、世界。"
						snake_case__			:				Any       =  "こんにちは 、 世界 。 こんばんは 、 世界 。"
						return input_text, output_text



					def 	_lowercase					( self					:  Optional[int]							,		__A					:  Optional[int]	):
						snake_case__,						snake_case__			:				List[str]       =  self.get_input_output_texts(__A	)
						snake_case__			:				List[str]       =  tokenizer.encode(__A							,		add_special_tokens=__A	)
						snake_case__			:				Dict       =  tokenizer.decode(__A							,		clean_up_tokenization_spaces=__A	)
						return text, ids



					def 	_lowercase					( self					:  Optional[Any]	):
						pass  # TODO add if relevant



					def 	_lowercase					( self					:  Optional[int]	):
						pass  # TODO add if relevant



					def 	_lowercase					( self					:  Tuple	):
						pass  # TODO add if relevant



					def 	_lowercase					( self					:  Tuple	):
						snake_case__			:				List[Any]       =  self.tokenizer_class(self.vocab_file	)

						snake_case__			:				List[str]       =  tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"	)
						self.assertListEqual(__A							,		["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]	)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)							,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]	)



					def 	_lowercase					( self					:  int	):
						snake_case__			:				List[Any]       =  self.tokenizer_class(self.vocab_file							,		word_tokenizer_type="mecab"	)
						self.assertIsNotNone(__A	)

						snake_case__			:				str       =  "こんにちは、世界。\nこんばんは、世界。"
						snake_case__			:				Optional[int]       =  tokenizer.tokenize(__A	)
						self.assertListEqual(__A							,		["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]	)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)							,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]	)

						snake_case__			:				str       =  os.path.join(self.tmpdirname							,		"tokenizer.bin"	)
						with open(__A							,		"wb"	) as handle:
							pickle.dump(__A							,		__A	)

						with open(__A							,		"rb"	) as handle:
							snake_case__			:				Tuple       =  pickle.load(__A	)

						snake_case__			:				int       =  tokenizer_new.tokenize(__A	)

						self.assertListEqual(__A							,		__A	)



					def 	_lowercase					( self					:  List[Any]	):
						snake_case__			:				Optional[int]       =  MecabTokenizer(mecab_dic="ipadic"	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,		)



					def 	_lowercase					( self					:  Tuple	):
						try:
							snake_case__			:				List[Any]       =  MecabTokenizer(mecab_dic="unidic_lite"	)
						except ModuleNotFoundError:
							return

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,		)



					def 	_lowercase					( self					:  List[Any]	):
						try:
							snake_case__			:				Union[str, Any]       =  MecabTokenizer(mecab_dic="unidic"	)
						except ModuleNotFoundError:
							return

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,		)



					def 	_lowercase					( self					:  int	):
						snake_case__			:				int       =  MecabTokenizer(do_lower_case=__A							,		mecab_dic="ipadic"	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]							,		)



					def 	_lowercase					( self					:  Optional[int]	):
						try:
							snake_case__			:				List[Any]       =  MecabTokenizer(
							    do_lower_case=__A							,		normalize_text=__A							,		mecab_option="-d /usr/local/lib/mecab/dic/jumandic"	)
						except RuntimeError:
							# if dict doesn't exist in the system, previous code raises this error.
							return

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]							,		)



					def 	_lowercase					( self					:  List[Any]	):
						snake_case__			:				Dict       =  MecabTokenizer(normalize_text=__A							,		mecab_dic="ipadic"	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]							,		)



					@require_sudachi
					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				int       =  self.tokenizer_class(self.vocab_file							,		word_tokenizer_type="sudachi"	)
						self.assertIsNotNone(__A	)

						snake_case__			:				str       =  "こんにちは、世界。\nこんばんは、世界。"
						snake_case__			:				Union[str, Any]       =  tokenizer.tokenize(__A	)
						self.assertListEqual(__A							,		["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]	)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)							,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]	)

						snake_case__			:				int       =  os.path.join(self.tmpdirname							,		"tokenizer.bin"	)
						with open(__A							,		"wb"	) as handle:
							pickle.dump(__A							,		__A	)

						with open(__A							,		"rb"	) as handle:
							snake_case__			:				Optional[Any]       =  pickle.load(__A	)

						snake_case__			:				Any       =  tokenizer_new.tokenize(__A	)

						self.assertListEqual(__A							,		__A	)



					@require_sudachi
					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				Any       =  SudachiTokenizer(sudachi_dict_type="core"	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]							,		)



					@require_sudachi
					def 	_lowercase					( self					:  Any	):
						snake_case__			:				List[Any]       =  SudachiTokenizer(sudachi_dict_type="core"							,		sudachi_split_mode="A"	)

						self.assertListEqual(tokenizer.tokenize("外国人参政権"	)							,		["外国", "人", "参政", "権"]	)



					@require_sudachi
					def 	_lowercase					( self					:  List[Any]	):
						snake_case__			:				Any       =  SudachiTokenizer(sudachi_dict_type="core"							,		sudachi_split_mode="B"	)

						self.assertListEqual(tokenizer.tokenize("外国人参政権"	)							,		["外国人", "参政権"]	)



					@require_sudachi
					def 	_lowercase					( self					:  Dict	):
						snake_case__			:				Union[str, Any]       =  SudachiTokenizer(sudachi_dict_type="core"							,		sudachi_split_mode="C"	)

						self.assertListEqual(tokenizer.tokenize("外国人参政権"	)							,		["外国人参政権"]	)



					@require_sudachi
					def 	_lowercase					( self					:  Dict	):
						snake_case__			:				Optional[Any]       =  SudachiTokenizer(do_lower_case=__A							,		sudachi_dict_type="core"	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]							,		)



					@require_sudachi
					def 	_lowercase					( self					:  List[Any]	):
						snake_case__			:				Tuple       =  SudachiTokenizer(normalize_text=__A							,		sudachi_dict_type="core"	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]							,		)



					@require_sudachi
					def 	_lowercase					( self					:  str	):
						snake_case__			:				Optional[int]       =  SudachiTokenizer(trim_whitespace=__A							,		sudachi_dict_type="core"	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]							,		)



					@require_jumanpp
					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				Any       =  self.tokenizer_class(self.vocab_file							,		word_tokenizer_type="jumanpp"	)
						self.assertIsNotNone(__A	)

						snake_case__			:				List[str]       =  "こんにちは、世界。\nこんばんは、世界。"
						snake_case__			:				Optional[Any]       =  tokenizer.tokenize(__A	)
						self.assertListEqual(__A							,		["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]	)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)							,		[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]	)

						snake_case__			:				Union[str, Any]       =  os.path.join(self.tmpdirname							,		"tokenizer.bin"	)
						with open(__A							,		"wb"	) as handle:
							pickle.dump(__A							,		__A	)

						with open(__A							,		"rb"	) as handle:
							snake_case__			:				Union[str, Any]       =  pickle.load(__A	)

						snake_case__			:				Union[str, Any]       =  tokenizer_new.tokenize(__A	)

						self.assertListEqual(__A							,		__A	)



					@require_jumanpp
					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				Dict       =  JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,		)



					@require_jumanpp
					def 	_lowercase					( self					:  Dict	):
						snake_case__			:				Dict       =  JumanppTokenizer(do_lower_case=__A	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,		)



					@require_jumanpp
					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				int       =  JumanppTokenizer(normalize_text=__A	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]							,		)



					@require_jumanpp
					def 	_lowercase					( self					:  str	):
						snake_case__			:				List[str]       =  JumanppTokenizer(trim_whitespace=__A	)

						self.assertListEqual(
						    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "	)							,		["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]							,		)



					@require_jumanpp
					def 	_lowercase					( self					:  List[str]	):
						snake_case__			:				Any       =  JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"	)							,		["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]							,		)



					def 	_lowercase					( self					:  Any	):
						snake_case__			:				Optional[Any]       =  ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

						snake_case__			:				List[Any]       =  {}
						for i, token in enumerate(__A	):
							snake_case__			:				str       =  i
						snake_case__			:				List[Any]       =  WordpieceTokenizer(vocab=__A							,		unk_token="[UNK]"	)

						self.assertListEqual(tokenizer.tokenize(""	)							,		[]	)

						self.assertListEqual(tokenizer.tokenize("こんにちは"	)							,		["こんにちは"]	)

						self.assertListEqual(tokenizer.tokenize("こんばんは"	)							,		["こん", "##ばんは"]	)

						self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"	)							,		["こん", "##ばんは", "[UNK]", "こんにちは"]	)



					def 	_lowercase					( self					:  Optional[Any]	):
						snake_case__			:				Optional[int]       =  BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"	)
						snake_case__			:				Tuple       =  tokenizer.subword_tokenizer

						snake_case__			:				Union[str, Any]       =  subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"	)
						self.assertListEqual(__A							,		["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]	)

						snake_case__			:				Optional[int]       =  subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"	)
						self.assertListEqual(__A							,		["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]	)






					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				List[str]       =  self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"	)

						snake_case__			:				Any       =  tokenizer.encode("ありがとう。"							,		add_special_tokens=__A	)
						snake_case__			:				Optional[Any]       =  tokenizer.encode("どういたしまして。"							,		add_special_tokens=__A	)

						snake_case__			:				List[str]       =  tokenizer.build_inputs_with_special_tokens(__A	)
						snake_case__			:				Tuple       =  tokenizer.build_inputs_with_special_tokens(__A							,		__A	)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_			,       unittest.TestCase						):





					"""simple docstring"""






					a_     =		BertJapaneseTokenizer
					a_     =		False



					def 	_lowercase					( self					:  Any	):
						super().setUp()

						snake_case__			:				List[str]       =  ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

						snake_case__			:				Dict       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["vocab_file"]	)
						with open(self.vocab_file							,		"w"							,		encoding="utf-8"	) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)



					def 	_lowercase					( self					:  List[str]							,		**__A					:  Tuple	):
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,		subword_tokenizer_type="character"							,		**__A	)



					def 	_lowercase					( self					:  Tuple							,		__A					:  List[Any]	):
						snake_case__			:				str       =  "こんにちは、世界。 \nこんばんは、世界。"
						snake_case__			:				Optional[int]       =  "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
						return input_text, output_text



					def 	_lowercase					( self					:  Union[str, Any]	):
						pass  # TODO add if relevant



					def 	_lowercase					( self					:  Optional[Any]	):
						pass  # TODO add if relevant



					def 	_lowercase					( self					:  Union[str, Any]	):
						pass  # TODO add if relevant



					def 	_lowercase					( self					:  int	):
						snake_case__			:				Dict       =  self.tokenizer_class(self.vocab_file							,		subword_tokenizer_type="character"	)

						snake_case__			:				Optional[int]       =  tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"	)
						self.assertListEqual(
						    __A							,		["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]	)
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__A	)							,		[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]	)



					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				Any       =  ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

						snake_case__			:				Optional[Any]       =  {}
						for i, token in enumerate(__A	):
							snake_case__			:				str       =  i
						snake_case__			:				List[str]       =  CharacterTokenizer(vocab=__A							,		unk_token="[UNK]"	)

						self.assertListEqual(tokenizer.tokenize(""	)							,		[]	)

						self.assertListEqual(tokenizer.tokenize("こんにちは"	)							,		["こ", "ん", "に", "ち", "は"]	)

						self.assertListEqual(tokenizer.tokenize("こんにちほ"	)							,		["こ", "ん", "に", "ち", "[UNK]"]	)






					def 	_lowercase					( self					:  List[Any]	):
						snake_case__			:				Tuple       =  self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"	)

						snake_case__			:				List[Any]       =  tokenizer.encode("ありがとう。"							,		add_special_tokens=__A	)
						snake_case__			:				List[Any]       =  tokenizer.encode("どういたしまして。"							,		add_special_tokens=__A	)

						snake_case__			:				Any       =  tokenizer.build_inputs_with_special_tokens(__A	)
						snake_case__			:				Optional[int]       =  tokenizer.build_inputs_with_special_tokens(__A							,		__A	)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]



@custom_tokenizers
class 							SCREAMING_SNAKE_CASE__						(    unittest.TestCase						):





					"""simple docstring"""






					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				Any       =  "cl-tohoku/bert-base-japanese"
						snake_case__			:				Dict       =  AutoTokenizer.from_pretrained(__A	)
						self.assertIsInstance(__A							,		__A	)



class 							SCREAMING_SNAKE_CASE__						(    unittest.TestCase						):





					"""simple docstring"""






					def 	_lowercase					( self					:  int	):
						snake_case__			:				Union[str, Any]       =  "cl-tohoku/bert-base-japanese"
						with self.assertLogs("transformers"							,		level="WARNING"	) as cm:
							BertTokenizer.from_pretrained(__A	)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
							        " is called from."	)	)
						snake_case__			:				Union[str, Any]       =  "bert-base-cased"
						with self.assertLogs("transformers"							,		level="WARNING"	) as cm:
							BertJapaneseTokenizer.from_pretrained(__A	)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
							        " is called from."	)	)











import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_			,       unittest.TestCase						):





					"""simple docstring"""






					a_     =		PhobertTokenizer
					a_     =		False



					def 	_lowercase					( self					:  List[str]	):
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						snake_case__			:				Optional[int]       =  ["T@@", "i", "I", "R@@", "r", "e@@"]
						snake_case__			:				int       =  dict(zip(__A							,		range(len(__A	)	)	)	)
						snake_case__			:				Dict       =  ["#version: 0.2", "l à</w>"]
						snake_case__			:				Optional[Any]       =  {"unk_token": "<unk>"}

						snake_case__			:				Tuple       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["vocab_file"]	)
						snake_case__			:				Union[str, Any]       =  os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["merges_file"]	)

						with open(self.vocab_file							,		"w"							,		encoding="utf-8"	) as fp:
							for token in vocab_tokens:
								fp.write(f'''{token} {vocab_tokens[token]}\n'''	)
						with open(self.merges_file							,		"w"							,		encoding="utf-8"	) as fp:
							fp.write("\n".join(__A	)	)



					def 	_lowercase					( self					:  List[str]							,		**__A					:  Any	):
						kwargs.update(self.special_tokens_map	)
						return PhobertTokenizer.from_pretrained(self.tmpdirname							,		**__A	)



					def 	_lowercase					( self					:  Tuple							,		__A					:  List[Any]	):
						snake_case__			:				str       =  "Tôi là VinAI Research"
						snake_case__			:				int       =  "T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
						return input_text, output_text






					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				int       =  PhobertTokenizer(self.vocab_file							,		self.merges_file							,		**self.special_tokens_map	)
						snake_case__			:				Tuple       =  "Tôi là VinAI Research"
						snake_case__			:				List[Any]       =  "T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
						snake_case__			:				int       =  tokenizer.tokenize(__A	)
						print(__A	)
						self.assertListEqual(__A							,		__A	)

						snake_case__			:				Any       =  tokens + [tokenizer.unk_token]

						snake_case__			:				Any       =  [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A	)							,		__A	)



