






import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 							__UpperCamelCase     (  _lowerCAmelCase ,      unittest.TestCase ):
	__snake_case						:List[Any] =							GPTSanJapaneseTokenizer
	__snake_case						:List[Any] =							False
	__snake_case						:Optional[Any] =							{'do_clean_text': False, 'add_prefix_space': False}







	def 			_a (	self   :				Optional[Any]      )		->			List[str]:
		"""simple docstring"""







		super().setUp()

		# fmt: off
		__lowercase   							=	["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
		# fmt: on
		__lowercase   							=	{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
		__lowercase   							=	{"""unk_token""": """<unk>"""}

		__lowercase   							=	os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES["""vocab_file"""]      )
		__lowercase   							=	os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES["""emoji_file"""]      )
		with open(self.vocab_file							,					"""w"""							,					encoding="""utf-8"""      ) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )
		with open(self.emoji_file							,					"""w"""      ) as emoji_writer:
			emoji_writer.write(json.dumps(_lowerCAmelCase      )      )







	def 			_a (	self   :				Any							,					**_lowerCAmelCase   :				Any      )		->			Union[str, Any]:
		"""simple docstring"""







		kwargs.update(self.special_tokens_map      )
		return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,					**_lowerCAmelCase      )







	def 			_a (	self   :				Optional[Any]							,					_lowerCAmelCase   :				int      )		->			Union[str, Any]:
		"""simple docstring"""







		__lowercase   							=	"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
		__lowercase   							=	"""こんにちは、世界。 \nこんばんは、世界。😀"""
		return input_text, output_text







	def 			_a (	self   :				List[Any]							,					_lowerCAmelCase   :				str      )		->			int:
		"""simple docstring"""







		__lowercase							,    __lowercase   							=	self.get_input_output_texts(_lowerCAmelCase      )
		__lowercase   							=	tokenizer.encode(_lowerCAmelCase							,					add_special_tokens=_lowerCAmelCase      )
		__lowercase   							=	tokenizer.decode(_lowerCAmelCase							,					clean_up_tokenization_spaces=_lowerCAmelCase      )
		return text, ids







	def 			_a (	self   :				Tuple      )		->			Optional[Any]:
		"""simple docstring"""







		pass  # TODO add if relevant







	def 			_a (	self   :				Tuple      )		->			List[Any]:
		"""simple docstring"""







		pass  # TODO add if relevant







	def 			_a (	self   :				Tuple      )		->			List[Any]:
		"""simple docstring"""







		pass  # TODO add if relevant







	def 			_a (	self   :				Tuple      )		->			int:
		"""simple docstring"""







		__lowercase   							=	self.get_tokenizer()

		# Testing tokenization
		__lowercase   							=	"""こんにちは、世界。　こんばんは、㔺界。"""
		__lowercase   							=	["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
		__lowercase   							=	tokenizer.tokenize(_lowerCAmelCase      )
		self.assertListEqual(_lowerCAmelCase							,					_lowerCAmelCase      )

		# Testing conversion to ids without special tokens
		__lowercase   							=	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
		__lowercase   							=	tokenizer.convert_tokens_to_ids(_lowerCAmelCase      )
		self.assertListEqual(_lowerCAmelCase							,					_lowerCAmelCase      )

		# Testing conversion to ids with special tokens
		__lowercase   							=	tokens + [tokenizer.unk_token]
		__lowercase   							=	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
		__lowercase   							=	tokenizer.convert_tokens_to_ids(_lowerCAmelCase      )
		self.assertListEqual(_lowerCAmelCase							,					_lowerCAmelCase      )







	def 			_a (	self   :				int      )		->			Union[str, Any]:
		"""simple docstring"""







		__lowercase   							=	self.get_tokenizer()

		# Testing tokenization
		__lowercase   							=	"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
		__lowercase   							=	"""こんにちは、、、、世界。こんばんは、、、、世界。"""
		__lowercase   							=	tokenizer.encode(_lowerCAmelCase      )
		__lowercase   							=	tokenizer.decode(_lowerCAmelCase      )
		self.assertEqual(_lowerCAmelCase							,					_lowerCAmelCase      )







	@slow
	def 			_a (	self   :				Optional[int]      )		->			Any:
		"""simple docstring"""







		__lowercase   							=	self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""      )

		# Testing tokenization
		__lowercase   							=	"""こんにちは、世界。"""
		__lowercase   							=	"""こんばんは、㔺界。😀"""
		__lowercase   							=	"""こんにちは、世界。こんばんは、世界。😀"""
		__lowercase   							=	tokenizer.encode(prefix_text + input_text      )
		__lowercase   							=	tokenizer.encode(""""""							,					prefix_text=prefix_text + input_text      )
		__lowercase   							=	tokenizer.encode(_lowerCAmelCase							,					prefix_text=_lowerCAmelCase      )
		__lowercase   							=	tokenizer.decode(_lowerCAmelCase      )
		__lowercase   							=	tokenizer.decode(_lowerCAmelCase      )
		__lowercase   							=	tokenizer.decode(_lowerCAmelCase      )
		self.assertEqual(_lowerCAmelCase							,					_lowerCAmelCase      )
		self.assertEqual(_lowerCAmelCase							,					_lowerCAmelCase      )
		self.assertEqual(_lowerCAmelCase							,					_lowerCAmelCase      )







	@slow
	def 			_a (	self   :				Any      )		->			List[Any]:
		"""simple docstring"""







		__lowercase   							=	self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""      )

		# Testing tokenization
		__lowercase   							=	"""こんにちは、世界。"""
		__lowercase   							=	"""こんばんは、㔺界。😀"""

		__lowercase   							=	len(tokenizer.encode(_lowerCAmelCase      )      ) - 2
		__lowercase   							=	len(tokenizer.encode(_lowerCAmelCase      )      ) - 2

		__lowercase   							=	[1] + [0] * (len_prefix + len_text + 1)
		__lowercase   							=	[1] * (len_prefix + len_text + 1) + [0]
		__lowercase   							=	[1] + [1] * (len_prefix) + [0] * (len_text + 1)

		__lowercase   							=	tokenizer(prefix_text + input_text      ).token_type_ids
		__lowercase   							=	tokenizer(""""""							,					prefix_text=prefix_text + input_text      ).token_type_ids
		__lowercase   							=	tokenizer(_lowerCAmelCase							,					prefix_text=_lowerCAmelCase      ).token_type_ids
		self.assertListEqual(_lowerCAmelCase							,					_lowerCAmelCase      )
		self.assertListEqual(_lowerCAmelCase							,					_lowerCAmelCase      )
		self.assertListEqual(_lowerCAmelCase							,					_lowerCAmelCase      )







	@slow
	def 			_a (	self   :				Optional[int]      )		->			List[str]:
		"""simple docstring"""







		__lowercase   							=	self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""      )

		__lowercase   							=	tokenizer.encode("""あンいワ"""      )
		__lowercase   							=	tokenizer.encode(""""""							,					prefix_text="""あンいワ"""      )
		__lowercase   							=	tokenizer.encode("""いワ"""							,					prefix_text="""あン"""      )

		self.assertEqual(tokenizer.decode(_lowerCAmelCase      )							,					tokenizer.decode(_lowerCAmelCase      )      )
		self.assertEqual(tokenizer.decode(_lowerCAmelCase      )							,					tokenizer.decode(_lowerCAmelCase      )      )
		self.assertNotEqual(_lowerCAmelCase							,					_lowerCAmelCase      )
		self.assertNotEqual(_lowerCAmelCase							,					_lowerCAmelCase      )
		self.assertEqual(x_token_a[1]							,					x_token_a[-1]      )  # SEG token
		self.assertEqual(x_token_a[1]							,					x_token_a[3]      )  # SEG token







	@slow
	def 			_a (	self   :				Optional[int]      )		->			Optional[int]:
		"""simple docstring"""







		__lowercase   							=	self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""      )

		__lowercase   							=	[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
		__lowercase   							=	tokenizer(_lowerCAmelCase							,					padding=_lowerCAmelCase      )
		__lowercase   							=	tokenizer.batch_encode_plus(_lowerCAmelCase							,					padding=_lowerCAmelCase      )

		# fmt: off
		__lowercase   							=	[[3_5993, 8640, 2_5948, 3_5998, 3_0647, 3_5675, 3_5999, 3_5999], [3_5993, 1_0382, 9868, 3_5998, 3_0646, 9459, 3_0646, 3_5675]]
		__lowercase   							=	[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
		__lowercase   							=	[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
		# fmt: on
		self.assertListEqual(x_token.input_ids							,					_lowerCAmelCase      )
		self.assertListEqual(x_token.token_type_ids							,					_lowerCAmelCase      )
		self.assertListEqual(x_token.attention_mask							,					_lowerCAmelCase      )
		self.assertListEqual(x_token_a.input_ids							,					_lowerCAmelCase      )
		self.assertListEqual(x_token_a.token_type_ids							,					_lowerCAmelCase      )
		self.assertListEqual(x_token_a.attention_mask							,					_lowerCAmelCase      )







	def 			_a (	self   :				Optional[Any]      )		->			Any:
		"""simple docstring"""







		pass







	def 			_a (	self   :				Tuple      )		->			Any:
		"""simple docstring"""







		pass




import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


lowercase_   							=  5_0_0_0_0_0

lowercase_							,		lowercase_   							=  os.path.split(__file__)
lowercase_   							=  os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))
@get_duration
def 	a (     A__      :  datasets.Dataset ,  **A__      :  List[str]	)       -> Optional[Any]:





      """simple docstring"""



      _lowercase      			=dataset.map(**A__	)
@get_duration
def 	a (     A__      :  datasets.Dataset ,  **A__      :  List[Any]	)       -> List[str]:





      """simple docstring"""



      _lowercase      			=dataset.filter(**A__	)
def 	a (     )       -> Union[str, Any]:





      """simple docstring"""



      _lowercase      			={'num examples': SPEED_TEST_N_EXAMPLES}
      with tempfile.TemporaryDirectory() as tmp_dir:
            _lowercase      			=datasets.Features({'text': datasets.Value('string'	), 'numbers': datasets.Value('float32'	)}	)
            _lowercase      			=generate_example_dataset(
                os.path.join(A__ ,  'dataset.arrow'	) ,  A__ ,  num_examples=A__	)

            _lowercase      			=transformers.AutoTokenizer.from_pretrained('bert-base-cased' ,  use_fast=A__	)

            def tokenize(A__      :  int	):
                  return tokenizer(examples['text']	)

            _lowercase      			=map(A__	)

            _lowercase      			=map(A__ ,  batched=A__	)

            _lowercase      			=map(A__ ,  function=lambda A__	: None ,  batched=A__	)

            with dataset.formatted_as(type='numpy'	):
                  _lowercase      			=map(A__ ,  function=lambda A__	: None ,  batched=A__	)

            with dataset.formatted_as(type='pandas'	):
                  _lowercase      			=map(A__ ,  function=lambda A__	: None ,  batched=A__	)

            with dataset.formatted_as(type='torch' ,  columns='numbers'	):
                  _lowercase      			=map(A__ ,  function=lambda A__	: None ,  batched=A__	)

            with dataset.formatted_as(type='tensorflow' ,  columns='numbers'	):
                  _lowercase      			=map(A__ ,  function=lambda A__	: None ,  batched=A__	)

            _lowercase      			=map(A__ ,  function=A__ ,  batched=A__	)

            _lowercase      			=filter(A__	)

            # Activate later when tokenizer support batched inputs
            # with dataset.formatted_as(type='numpy'):
            #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

      with open(A__ ,  'wb'	) as f:
            f.write(json.dumps(A__	).encode('utf-8'	)	)


if __name__ == "__main__":  # useful to run the profiler
  benchmark_map_filter()


