

"""simple docstring"""






import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class    snake_case__	(			lowerCAmelCase_       ,	unittest.TestCase  ):
			SCREAMING_SNAKE_CASE__  					=    GPTSanJapaneseTokenizer
			SCREAMING_SNAKE_CASE__  					=    False
			SCREAMING_SNAKE_CASE__  					=    {'''do_clean_text''': False, '''add_prefix_space''': False}







			def       __lowerCAmelCase					(      self      :			List[str]	):





								'''simple docstring'''



								super().setUp()

								# fmt: off
								UpperCAmelCase      : str   =					["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
								# fmt: on
								UpperCAmelCase      : Optional[int]   =					{"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
								UpperCAmelCase      : str   =					{"unk_token": "<unk>"}

								UpperCAmelCase      : Any   =					os.path.join(self.tmpdirname      ,	VOCAB_FILES_NAMES["vocab_file"]	)
								UpperCAmelCase      : Optional[Any]   =					os.path.join(self.tmpdirname      ,	VOCAB_FILES_NAMES["emoji_file"]	)
								with open(self.vocab_file      ,	"w"      ,	encoding="utf-8"	) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
								with open(self.emoji_file      ,	"w"	) as emoji_writer:
													emoji_writer.write(json.dumps(__UpperCamelCase	)	)







			def       __lowerCAmelCase					(      self      :			Dict      ,	**lowercase      :			int	):





								'''simple docstring'''



								kwargs.update(self.special_tokens_map	)
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname      ,	**__UpperCamelCase	)







			def       __lowerCAmelCase					(      self      :			List[str]      ,	lowercase      :			str	):





								'''simple docstring'''



								UpperCAmelCase      : Any   =					"こんにちは、世界。 \nこんばんは、㔺界。😀"
								UpperCAmelCase      : Optional[int]   =					"こんにちは、世界。 \nこんばんは、世界。😀"
								return input_text, output_text







			def       __lowerCAmelCase					(      self      :			str      ,	lowercase      :			Optional[int]	):





								'''simple docstring'''



								UpperCAmelCase   ,				UpperCAmelCase      : Optional[Any]   =					self.get_input_output_texts(__UpperCamelCase	)
								UpperCAmelCase      : str   =					tokenizer.encode(__UpperCamelCase      ,	add_special_tokens=__UpperCamelCase	)
								UpperCAmelCase      : Tuple   =					tokenizer.decode(__UpperCamelCase      ,	clean_up_tokenization_spaces=__UpperCamelCase	)
								return text, ids







			def       __lowerCAmelCase					(      self      :			Dict	):





								'''simple docstring'''



								pass  # TODO add if relevant







			def       __lowerCAmelCase					(      self      :			Union[str, Any]	):





								'''simple docstring'''



								pass  # TODO add if relevant







			def       __lowerCAmelCase					(      self      :			Union[str, Any]	):





								'''simple docstring'''



								pass  # TODO add if relevant







			def       __lowerCAmelCase					(      self      :			Any	):





								'''simple docstring'''



								UpperCAmelCase      : List[Any]   =					self.get_tokenizer()

								# Testing tokenization
								UpperCAmelCase      : Tuple   =					"こんにちは、世界。　こんばんは、㔺界。"
								UpperCAmelCase      : str   =					["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
								UpperCAmelCase      : Tuple   =					tokenizer.tokenize(__UpperCamelCase	)
								self.assertListEqual(__UpperCamelCase      ,	__UpperCamelCase	)

								# Testing conversion to ids without special tokens
								UpperCAmelCase      : List[str]   =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								UpperCAmelCase      : str   =					tokenizer.convert_tokens_to_ids(__UpperCamelCase	)
								self.assertListEqual(__UpperCamelCase      ,	__UpperCamelCase	)

								# Testing conversion to ids with special tokens
								UpperCAmelCase      : Optional[int]   =					tokens + [tokenizer.unk_token]
								UpperCAmelCase      : Tuple   =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
								UpperCAmelCase      : Union[str, Any]   =					tokenizer.convert_tokens_to_ids(__UpperCamelCase	)
								self.assertListEqual(__UpperCamelCase      ,	__UpperCamelCase	)







			def       __lowerCAmelCase					(      self      :			int	):





								'''simple docstring'''



								UpperCAmelCase      : Any   =					self.get_tokenizer()

								# Testing tokenization
								UpperCAmelCase      : List[Any]   =					"こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
								UpperCAmelCase      : str   =					"こんにちは、、、、世界。こんばんは、、、、世界。"
								UpperCAmelCase      : List[Any]   =					tokenizer.encode(__UpperCamelCase	)
								UpperCAmelCase      : List[Any]   =					tokenizer.decode(__UpperCamelCase	)
								self.assertEqual(__UpperCamelCase      ,	__UpperCamelCase	)







			@slow
			def       __lowerCAmelCase					(      self      :			Any	):





								'''simple docstring'''



								UpperCAmelCase      : List[Any]   =					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								# Testing tokenization
								UpperCAmelCase      : Optional[int]   =					"こんにちは、世界。"
								UpperCAmelCase      : Optional[Any]   =					"こんばんは、㔺界。😀"
								UpperCAmelCase      : List[str]   =					"こんにちは、世界。こんばんは、世界。😀"
								UpperCAmelCase      : int   =					tokenizer.encode(prefix_text + input_text	)
								UpperCAmelCase      : Dict   =					tokenizer.encode(""      ,	prefix_text=prefix_text + input_text	)
								UpperCAmelCase      : Dict   =					tokenizer.encode(__UpperCamelCase      ,	prefix_text=__UpperCamelCase	)
								UpperCAmelCase      : List[str]   =					tokenizer.decode(__UpperCamelCase	)
								UpperCAmelCase      : Optional[int]   =					tokenizer.decode(__UpperCamelCase	)
								UpperCAmelCase      : Union[str, Any]   =					tokenizer.decode(__UpperCamelCase	)
								self.assertEqual(__UpperCamelCase      ,	__UpperCamelCase	)
								self.assertEqual(__UpperCamelCase      ,	__UpperCamelCase	)
								self.assertEqual(__UpperCamelCase      ,	__UpperCamelCase	)







			@slow
			def       __lowerCAmelCase					(      self      :			List[Any]	):





								'''simple docstring'''



								UpperCAmelCase      : Union[str, Any]   =					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								# Testing tokenization
								UpperCAmelCase      : List[str]   =					"こんにちは、世界。"
								UpperCAmelCase      : Dict   =					"こんばんは、㔺界。😀"

								UpperCAmelCase      : Union[str, Any]   =					len(tokenizer.encode(__UpperCamelCase	)	) - 2
								UpperCAmelCase      : Optional[Any]   =					len(tokenizer.encode(__UpperCamelCase	)	) - 2

								UpperCAmelCase      : int   =					[1] + [0] * (len_prefix + len_text + 1)
								UpperCAmelCase      : Tuple   =					[1] * (len_prefix + len_text + 1) + [0]
								UpperCAmelCase      : List[Any]   =					[1] + [1] * (len_prefix) + [0] * (len_text + 1)

								UpperCAmelCase      : List[str]   =					tokenizer(prefix_text + input_text	).token_type_ids
								UpperCAmelCase      : Dict   =					tokenizer(""      ,	prefix_text=prefix_text + input_text	).token_type_ids
								UpperCAmelCase      : Union[str, Any]   =					tokenizer(__UpperCamelCase      ,	prefix_text=__UpperCamelCase	).token_type_ids
								self.assertListEqual(__UpperCamelCase      ,	__UpperCamelCase	)
								self.assertListEqual(__UpperCamelCase      ,	__UpperCamelCase	)
								self.assertListEqual(__UpperCamelCase      ,	__UpperCamelCase	)







			@slow
			def       __lowerCAmelCase					(      self      :			Any	):





								'''simple docstring'''



								UpperCAmelCase      : str   =					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								UpperCAmelCase      : Tuple   =					tokenizer.encode("あンいワ"	)
								UpperCAmelCase      : int   =					tokenizer.encode(""      ,	prefix_text="あンいワ"	)
								UpperCAmelCase      : Optional[int]   =					tokenizer.encode("いワ"      ,	prefix_text="あン"	)

								self.assertEqual(tokenizer.decode(__UpperCamelCase	)      ,	tokenizer.decode(__UpperCamelCase	)	)
								self.assertEqual(tokenizer.decode(__UpperCamelCase	)      ,	tokenizer.decode(__UpperCamelCase	)	)
								self.assertNotEqual(__UpperCamelCase      ,	__UpperCamelCase	)
								self.assertNotEqual(__UpperCamelCase      ,	__UpperCamelCase	)
								self.assertEqual(x_token_a[1]      ,	x_token_a[-1]	)  # SEG token
								self.assertEqual(x_token_a[1]      ,	x_token_a[3]	)  # SEG token







			@slow
			def       __lowerCAmelCase					(      self      :			Tuple	):





								'''simple docstring'''



								UpperCAmelCase      : str   =					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

								UpperCAmelCase      : Union[str, Any]   =					[["武田信玄", "は、"], ["織田信長", "の配下の、"]]
								UpperCAmelCase      : int   =					tokenizer(__UpperCamelCase      ,	padding=__UpperCamelCase	)
								UpperCAmelCase      : str   =					tokenizer.batch_encode_plus(__UpperCamelCase      ,	padding=__UpperCamelCase	)

								# fmt: off
								UpperCAmelCase      : Optional[int]   =					[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
								UpperCAmelCase      : Dict   =					[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								UpperCAmelCase      : List[Any]   =					[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids      ,	__UpperCamelCase	)
								self.assertListEqual(x_token.token_type_ids      ,	__UpperCamelCase	)
								self.assertListEqual(x_token.attention_mask      ,	__UpperCamelCase	)
								self.assertListEqual(x_token_a.input_ids      ,	__UpperCamelCase	)
								self.assertListEqual(x_token_a.token_type_ids      ,	__UpperCamelCase	)
								self.assertListEqual(x_token_a.attention_mask      ,	__UpperCamelCase	)







			def       __lowerCAmelCase					(      self      :			int	):





								'''simple docstring'''



								pass







			def       __lowerCAmelCase					(      self      :			List[Any]	):





								'''simple docstring'''



								pass




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


lowercase   =	{}


try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase   =	['''MLukeTokenizer''']

if TYPE_CHECKING:
		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_mluke import MLukeTokenizer


else:
		import sys

		lowercase   =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





