

'''simple docstring'''


import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class      a      (					SCREAMING_SNAKE_CASE_      ,      unittest.TestCase							):






						"""simple docstring"""




						__UpperCAmelCase        =     GPTSanJapaneseTokenizer
						__UpperCAmelCase        =     False
						__UpperCAmelCase        =     {"""do_clean_text""": False, """add_prefix_space""": False}







						def    __magic_name__				(				self       :   Optional[Any]	):



								'''simple docstring'''



								super().setUp()

								# fmt: off
								snake_case__    :							Tuple							    =	['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
								# fmt: on
								snake_case__    :							Union[str, Any]							    =	{'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
								snake_case__    :							int							    =	{'unk_token': '<unk>'}

								snake_case__    :							Union[str, Any]							    =	os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['''vocab_file''']	)
								snake_case__    :							Tuple							    =	os.path.join(self.tmpdirname      ,      VOCAB_FILES_NAMES['''emoji_file''']	)
								with open(self.vocab_file      ,      '''w'''      ,      encoding='''utf-8'''	) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
								with open(self.emoji_file      ,      '''w'''	) as emoji_writer:
										emoji_writer.write(json.dumps(snake_case_	)	)







						def    __magic_name__				(				self       :   List[str]      ,      **snake_case_       :   str	):



								'''simple docstring'''



								kwargs.update(self.special_tokens_map	)
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname      ,      **snake_case_	)







						def    __magic_name__				(				self       :   Any      ,      snake_case_       :   Dict	):



								'''simple docstring'''



								snake_case__    :							Any							    =	'こんにちは、世界。 \nこんばんは、㔺界。😀'
								snake_case__    :							str							    =	'こんにちは、世界。 \nこんばんは、世界。😀'
								return input_text, output_text







						def    __magic_name__				(				self       :   str      ,      snake_case_       :   Tuple	):



								'''simple docstring'''



								snake_case__    :							Optional[int]							    =	self.get_input_output_texts(snake_case_	)
								snake_case__    :							str							    =	tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)
								snake_case__    :							Tuple							    =	tokenizer.decode(snake_case_      ,      clean_up_tokenization_spaces=snake_case_	)
								return text, ids







						def    __magic_name__				(				self       :   Any	):



								'''simple docstring'''



								pass  # TODO add if relevant







						def    __magic_name__				(				self       :   Optional[Any]	):



								'''simple docstring'''



								pass  # TODO add if relevant







						def    __magic_name__				(				self       :   Dict	):



								'''simple docstring'''



								pass  # TODO add if relevant







						def    __magic_name__				(				self       :   Optional[Any]	):



								'''simple docstring'''



								snake_case__    :							List[str]							    =	self.get_tokenizer()

								# Testing tokenization
								snake_case__    :							Optional[int]							    =	'こんにちは、世界。　こんばんは、㔺界。'
								snake_case__    :							Union[str, Any]							    =	['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
								snake_case__    :							int							    =	tokenizer.tokenize(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								# Testing conversion to ids without special tokens
								snake_case__    :							Optional[Any]							    =	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								snake_case__    :							Optional[int]							    =	tokenizer.convert_tokens_to_ids(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								# Testing conversion to ids with special tokens
								snake_case__    :							Optional[int]							    =	tokens + [tokenizer.unk_token]
								snake_case__    :							str							    =	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
								snake_case__    :							Union[str, Any]							    =	tokenizer.convert_tokens_to_ids(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   int	):



								'''simple docstring'''



								snake_case__    :							List[str]							    =	self.get_tokenizer()

								# Testing tokenization
								snake_case__    :							str							    =	'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
								snake_case__    :							Any							    =	'こんにちは、、、、世界。こんばんは、、、、世界。'
								snake_case__    :							Any							    =	tokenizer.encode(snake_case_	)
								snake_case__    :							List[str]							    =	tokenizer.decode(snake_case_	)
								self.assertEqual(snake_case_      ,      snake_case_	)







						@slow
						def    __magic_name__				(				self       :   List[str]	):



								'''simple docstring'''



								snake_case__    :							Dict							    =	self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''	)

								# Testing tokenization
								snake_case__    :							Dict							    =	'こんにちは、世界。'
								snake_case__    :							Optional[Any]							    =	'こんばんは、㔺界。😀'
								snake_case__    :							Dict							    =	'こんにちは、世界。こんばんは、世界。😀'
								snake_case__    :							int							    =	tokenizer.encode(prefix_text + input_text	)
								snake_case__    :							Tuple							    =	tokenizer.encode(''''''      ,      prefix_text=prefix_text + input_text	)
								snake_case__    :							Tuple							    =	tokenizer.encode(snake_case_      ,      prefix_text=snake_case_	)
								snake_case__    :							List[str]							    =	tokenizer.decode(snake_case_	)
								snake_case__    :							int							    =	tokenizer.decode(snake_case_	)
								snake_case__    :							List[Any]							    =	tokenizer.decode(snake_case_	)
								self.assertEqual(snake_case_      ,      snake_case_	)
								self.assertEqual(snake_case_      ,      snake_case_	)
								self.assertEqual(snake_case_      ,      snake_case_	)







						@slow
						def    __magic_name__				(				self       :   Tuple	):



								'''simple docstring'''



								snake_case__    :							Dict							    =	self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''	)

								# Testing tokenization
								snake_case__    :							List[Any]							    =	'こんにちは、世界。'
								snake_case__    :							Tuple							    =	'こんばんは、㔺界。😀'

								snake_case__    :							List[str]							    =	len(tokenizer.encode(snake_case_	)	) - 2
								snake_case__    :							List[Any]							    =	len(tokenizer.encode(snake_case_	)	) - 2

								snake_case__    :							Union[str, Any]							    =	[1] + [0] * (len_prefix + len_text + 1)
								snake_case__    :							int							    =	[1] * (len_prefix + len_text + 1) + [0]
								snake_case__    :							Optional[int]							    =	[1] + [1] * (len_prefix) + [0] * (len_text + 1)

								snake_case__    :							Any							    =	tokenizer(prefix_text + input_text	).token_type_ids
								snake_case__    :							Tuple							    =	tokenizer(''''''      ,      prefix_text=prefix_text + input_text	).token_type_ids
								snake_case__    :							Dict							    =	tokenizer(snake_case_      ,      prefix_text=snake_case_	).token_type_ids
								self.assertListEqual(snake_case_      ,      snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)







						@slow
						def    __magic_name__				(				self       :   Optional[Any]	):



								'''simple docstring'''



								snake_case__    :							List[Any]							    =	self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''	)

								snake_case__    :							int							    =	tokenizer.encode('''あンいワ'''	)
								snake_case__    :							Optional[int]							    =	tokenizer.encode(''''''      ,      prefix_text='''あンいワ'''	)
								snake_case__    :							int							    =	tokenizer.encode('''いワ'''      ,      prefix_text='''あン'''	)

								self.assertEqual(tokenizer.decode(snake_case_	)      ,      tokenizer.decode(snake_case_	)	)
								self.assertEqual(tokenizer.decode(snake_case_	)      ,      tokenizer.decode(snake_case_	)	)
								self.assertNotEqual(snake_case_      ,      snake_case_	)
								self.assertNotEqual(snake_case_      ,      snake_case_	)
								self.assertEqual(x_token_a[1]      ,      x_token_a[-1]	)  # SEG token
								self.assertEqual(x_token_a[1]      ,      x_token_a[3]	)  # SEG token







						@slow
						def    __magic_name__				(				self       :   List[Any]	):



								'''simple docstring'''



								snake_case__    :							Tuple							    =	self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''	)

								snake_case__    :							Dict							    =	[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
								snake_case__    :							List[str]							    =	tokenizer(snake_case_      ,      padding=snake_case_	)
								snake_case__    :							Optional[int]							    =	tokenizer.batch_encode_plus(snake_case_      ,      padding=snake_case_	)

								# fmt: off
								snake_case__    :							str							    =	[[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
								snake_case__    :							Union[str, Any]							    =	[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								snake_case__    :							List[Any]							    =	[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids      ,      snake_case_	)
								self.assertListEqual(x_token.token_type_ids      ,      snake_case_	)
								self.assertListEqual(x_token.attention_mask      ,      snake_case_	)
								self.assertListEqual(x_token_a.input_ids      ,      snake_case_	)
								self.assertListEqual(x_token_a.token_type_ids      ,      snake_case_	)
								self.assertListEqual(x_token_a.attention_mask      ,      snake_case_	)







						def    __magic_name__				(				self       :   Tuple	):



								'''simple docstring'''



								pass







						def    __magic_name__				(				self       :   Union[str, Any]	):



								'''simple docstring'''



								pass








"""simple docstring"""







# Lint as: python3
import itertools
import os
import re


a_							=    re.compile(r"""([A-Z]+)([A-Z][a-z])""")
a_							=    re.compile(r"""([a-z\d])([A-Z])""")

a_							=    re.compile(r"""(?<!_)_(?!_)""")
a_							=    re.compile(r"""(_{2,})""")

a_							=    r"""^\w+(\.\w+)*$"""

a_							=    r"""<>:/\|?*"""

def     UpperCAmelCase_					(				__a   :			Optional[int]						):




			'''simple docstring'''



			_lowerCamelCase					:					str	       =    _uppercase_uppercase_re.sub(r'\1_\2'      ,							__a						)
			_lowerCamelCase					:					Tuple	       =    _lowercase_uppercase_re.sub(r'\1_\2'      ,							__a						)
			return name.lower()

def     UpperCAmelCase_					(				__a   :			Optional[int]						):




			'''simple docstring'''



			_lowerCamelCase					:					Dict	       =    _single_underscore_re.split(__a						)
			_lowerCamelCase					:					Tuple	       =    [_multiple_underscores_re.split(__a						) for n in name]
			return "".join(n.capitalize() for n in itertools.chain.from_iterable(__a						) if n != ''						)

def     UpperCAmelCase_					(				__a   :			List[Any]						):




			'''simple docstring'''



			if os.path.basename(__a						) != name:
						raise ValueError(f"Should be a dataset name, not a path: {name}"						)
			return camelcase_to_snakecase(__a						)

def     UpperCAmelCase_					(				__a   :			Union[str, Any]      ,							__a   :			Optional[int]						):




			'''simple docstring'''



			if os.path.basename(__a						) != name:
						raise ValueError(f"Should be a dataset name, not a path: {name}"						)
			if not re.match(_split_re      ,							__a						):
						raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'."						)
			return f"{filename_prefix_for_name(__a						)}-{split}"

def     UpperCAmelCase_					(				__a   :			Any      ,							__a   :			Union[str, Any]      ,							__a   :			List[Any]      ,							__a   :			List[str]=None						):




			'''simple docstring'''



			_lowerCamelCase					:					List[Any]	       =    filename_prefix_for_split(__a      ,							__a						)
			if filetype_suffix:
						prefix += f".{filetype_suffix}"
			_lowerCamelCase					:					List[str]	       =    os.path.join(__a      ,							__a						)
			return f"{filepath}*"



def     UpperCAmelCase_					(				__a   :			str      ,							__a   :			List[Any]      ,							__a   :			List[str]      ,							__a   :			Tuple=None      ,							__a   :			Tuple=None						):




			'''simple docstring'''



			_lowerCamelCase					:					Tuple	       =    filename_prefix_for_split(__a      ,							__a						)
			_lowerCamelCase					:					List[str]	       =    os.path.join(__a      ,							__a						)

			if shard_lengths:
						_lowerCamelCase					:					Union[str, Any]	       =    len(__a						)
						_lowerCamelCase					:					str	       =    [f"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(__a						)]
						if filetype_suffix:
									_lowerCamelCase					:					int	       =    [filename + f".{filetype_suffix}" for filename in filenames]
						return filenames
			else:
						_lowerCamelCase					:					int	       =    prefix
						if filetype_suffix:
									filename += f".{filetype_suffix}"
						return [filename]





