





"""simple docstring"""





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class        _lowerCamelCase      (       UpperCAmelCase_  ,			unittest.TestCase ):
						UpperCAmelCase_ =			GPTSanJapaneseTokenizer
						UpperCAmelCase_ =			False
						UpperCAmelCase_ =			{'do_clean_text': False, 'add_prefix_space': False}


						def     snake_case_   (self     :   Any		) ->   List[Any]:
								super().setUp()

								# fmt: off
								UpperCamelCase   =	['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
								# fmt: on
								UpperCamelCase   =	{'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
								UpperCamelCase   =	{'unk_token': '<unk>'}

								UpperCamelCase   =	os.path.join(self.tmpdirname   ,							VOCAB_FILES_NAMES["vocab_file"]		)
								UpperCamelCase   =	os.path.join(self.tmpdirname   ,							VOCAB_FILES_NAMES["emoji_file"]		)
								with open(self.vocab_file   ,							"w"   ,							encoding="utf-8"		) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
								with open(self.emoji_file   ,							"w"		) as emoji_writer:
										emoji_writer.write(json.dumps(_lowercase		)		)


						def     snake_case_   (self     :   Optional[Any]   ,							**__a     :   List[str]		) ->   List[str]:
								kwargs.update(self.special_tokens_map		)
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname   ,							**_lowercase		)


						def     snake_case_   (self     :   Dict   ,							__a     :   Optional[Any]		) ->   str:
								UpperCamelCase   =	'こんにちは、世界。 \nこんばんは、㔺界。😀'
								UpperCamelCase   =	'こんにちは、世界。 \nこんばんは、世界。😀'
								return input_text, output_text


						def     snake_case_   (self     :   Any   ,							__a     :   List[str]		) ->   Union[str, Any]:
								UpperCamelCase   =	self.get_input_output_texts(_lowercase		)
								UpperCamelCase   =	tokenizer.encode(_lowercase   ,							add_special_tokens=_lowercase		)
								UpperCamelCase   =	tokenizer.decode(_lowercase   ,							clean_up_tokenization_spaces=_lowercase		)
								return text, ids


						def     snake_case_   (self     :   Union[str, Any]		) ->   Tuple:
								pass  # TODO add if relevant


						def     snake_case_   (self     :   Optional[int]		) ->   int:
								pass  # TODO add if relevant


						def     snake_case_   (self     :   Tuple		) ->   int:
								pass  # TODO add if relevant


						def     snake_case_   (self     :   int		) ->   List[str]:
								UpperCamelCase   =	self.get_tokenizer()

								# Testing tokenization
								UpperCamelCase   =	'こんにちは、世界。　こんばんは、㔺界。'
								UpperCamelCase   =	['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
								UpperCamelCase   =	tokenizer.tokenize(_lowercase		)
								self.assertListEqual(_lowercase   ,							_lowercase		)

								# Testing conversion to ids without special tokens
								UpperCamelCase   =	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								UpperCamelCase   =	tokenizer.convert_tokens_to_ids(_lowercase		)
								self.assertListEqual(_lowercase   ,							_lowercase		)

								# Testing conversion to ids with special tokens
								UpperCamelCase   =	tokens + [tokenizer.unk_token]
								UpperCamelCase   =	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
								UpperCamelCase   =	tokenizer.convert_tokens_to_ids(_lowercase		)
								self.assertListEqual(_lowercase   ,							_lowercase		)


						def     snake_case_   (self     :   Optional[int]		) ->   List[str]:
								UpperCamelCase   =	self.get_tokenizer()

								# Testing tokenization
								UpperCamelCase   =	'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
								UpperCamelCase   =	'こんにちは、、、、世界。こんばんは、、、、世界。'
								UpperCamelCase   =	tokenizer.encode(_lowercase		)
								UpperCamelCase   =	tokenizer.decode(_lowercase		)
								self.assertEqual(_lowercase   ,							_lowercase		)


						@slow
						def     snake_case_   (self     :   Tuple		) ->   Optional[Any]:
								UpperCamelCase   =	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

								# Testing tokenization
								UpperCamelCase   =	'こんにちは、世界。'
								UpperCamelCase   =	'こんばんは、㔺界。😀'
								UpperCamelCase   =	'こんにちは、世界。こんばんは、世界。😀'
								UpperCamelCase   =	tokenizer.encode(prefix_text + input_text		)
								UpperCamelCase   =	tokenizer.encode(""   ,							prefix_text=prefix_text + input_text		)
								UpperCamelCase   =	tokenizer.encode(_lowercase   ,							prefix_text=_lowercase		)
								UpperCamelCase   =	tokenizer.decode(_lowercase		)
								UpperCamelCase   =	tokenizer.decode(_lowercase		)
								UpperCamelCase   =	tokenizer.decode(_lowercase		)
								self.assertEqual(_lowercase   ,							_lowercase		)
								self.assertEqual(_lowercase   ,							_lowercase		)
								self.assertEqual(_lowercase   ,							_lowercase		)


						@slow
						def     snake_case_   (self     :   str		) ->   Union[str, Any]:
								UpperCamelCase   =	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

								# Testing tokenization
								UpperCamelCase   =	'こんにちは、世界。'
								UpperCamelCase   =	'こんばんは、㔺界。😀'

								UpperCamelCase   =	len(tokenizer.encode(_lowercase		)		) - 2
								UpperCamelCase   =	len(tokenizer.encode(_lowercase		)		) - 2

								UpperCamelCase   =	[1] + [0] * (len_prefix + len_text + 1)
								UpperCamelCase   =	[1] * (len_prefix + len_text + 1) + [0]
								UpperCamelCase   =	[1] + [1] * (len_prefix) + [0] * (len_text + 1)

								UpperCamelCase   =	tokenizer(prefix_text + input_text		).token_type_ids
								UpperCamelCase   =	tokenizer(""   ,							prefix_text=prefix_text + input_text		).token_type_ids
								UpperCamelCase   =	tokenizer(_lowercase   ,							prefix_text=_lowercase		).token_type_ids
								self.assertListEqual(_lowercase   ,							_lowercase		)
								self.assertListEqual(_lowercase   ,							_lowercase		)
								self.assertListEqual(_lowercase   ,							_lowercase		)


						@slow
						def     snake_case_   (self     :   List[str]		) ->   List[str]:
								UpperCamelCase   =	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

								UpperCamelCase   =	tokenizer.encode("あンいワ"		)
								UpperCamelCase   =	tokenizer.encode(""   ,							prefix_text="あンいワ"		)
								UpperCamelCase   =	tokenizer.encode("いワ"   ,							prefix_text="あン"		)

								self.assertEqual(tokenizer.decode(_lowercase		)   ,							tokenizer.decode(_lowercase		)		)
								self.assertEqual(tokenizer.decode(_lowercase		)   ,							tokenizer.decode(_lowercase		)		)
								self.assertNotEqual(_lowercase   ,							_lowercase		)
								self.assertNotEqual(_lowercase   ,							_lowercase		)
								self.assertEqual(x_token_a[1]   ,							x_token_a[-1]		)  # SEG token
								self.assertEqual(x_token_a[1]   ,							x_token_a[3]		)  # SEG token


						@slow
						def     snake_case_   (self     :   str		) ->   str:
								UpperCamelCase   =	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

								UpperCamelCase   =	[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
								UpperCamelCase   =	tokenizer(_lowercase   ,							padding=_lowercase		)
								UpperCamelCase   =	tokenizer.batch_encode_plus(_lowercase   ,							padding=_lowercase		)

								# fmt: off
								UpperCamelCase   =	[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
								UpperCamelCase   =	[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								UpperCamelCase   =	[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids   ,							_lowercase		)
								self.assertListEqual(x_token.token_type_ids   ,							_lowercase		)
								self.assertListEqual(x_token.attention_mask   ,							_lowercase		)
								self.assertListEqual(x_token_a.input_ids   ,							_lowercase		)
								self.assertListEqual(x_token_a.token_type_ids   ,							_lowercase		)
								self.assertListEqual(x_token_a.attention_mask   ,							_lowercase		)


						def     snake_case_   (self     :   Optional[int]		) ->   int:
								# Intentionally convert some words to accommodate character fluctuations unique to Japanese
								pass




						def     snake_case_   (self     :   Union[str, Any]		) ->   Any:
								# tokenizer has no padding token
								pass













"""simple docstring"""





import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows


def       a__					(					_SCREAMING_SNAKE_CASE    ):  # picklable for multiprocessing




		"""simple docstring"""


		return i + 1


@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def       a__					(					):




		"""simple docstring"""


		with parallel_backend("spark"    ):
				assert ParallelBackendConfig.backend_name == "spark"

		UpperCamelCase   =	[1, 2, 3]
		with pytest.raises(_SCREAMING_SNAKE_CASE    ):
				with parallel_backend("unsupported backend"    ):
						map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=2    )

		with pytest.raises(_SCREAMING_SNAKE_CASE    ):
				with parallel_backend("unsupported backend"    ):
						map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=-1    )






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("num_proc"  ,	[2, -1]    )
def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	[1, 2]
		UpperCamelCase   =	{"a": 1, "b": 2}
		UpperCamelCase   =	{"a": [1, 2], "b": [3, 4]}
		UpperCamelCase   =	{"a": {"1": 1}, "b": 2}
		UpperCamelCase   =	{"a": 1, "b": 2, "c": 3, "d": 4}
		UpperCamelCase   =	[2, 3]
		UpperCamelCase   =	{"a": 2, "b": 3}
		UpperCamelCase   =	{"a": [2, 3], "b": [4, 5]}
		UpperCamelCase   =	{"a": {"1": 2}, "b": 3}
		UpperCamelCase   =	{"a": 2, "b": 3, "c": 4, "d": 5}

		with parallel_backend("spark"    ):
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa






