





import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin






class 						UpperCamelCase_  (  snake_case__  ,     unittest.TestCase							):
					_a      :						Tuple						    =					ProphetNetTokenizer
					_a      :						Optional[Any]						    =					False



					def   __a     (		self     :     Union[str, Any]						):
							super().setUp()

							lowerCamelCase_	:     Any 						=      [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							lowerCamelCase_	:     List[str] 						=      os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES['vocab_file']						)
							with open(self.vocab_file       ,      'w'       ,      encoding='utf-8'						) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)



					def   __a     (		self     :     Tuple       ,      lowerCamelCase     :     List[Any]						):
							lowerCamelCase_	:     List[Any] 						=      'UNwant\u00E9d,running'
							lowerCamelCase_	:     List[Any] 						=      'unwanted, running'
							return input_text, output_text



					def   __a     (		self     :     Optional[int]						):
							lowerCamelCase_	:     str 						=      self.tokenizer_class(self.vocab_file						)

							lowerCamelCase_	:     Any 						=      tokenizer.tokenize('UNwant\u00E9d,running'						)
							self.assertListEqual(lowerCamelCase       ,      ['un', '##want', '##ed', ',', 'runn', '##ing']						)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,      [9, 6, 7, 12, 10, 11]						)



					def   __a     (		self     :     str						):
							lowerCamelCase_	:     List[Any] 						=      BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'						)       ,      ['ah', '\u535A', '\u63A8', 'zz']						)



					def   __a     (		self     :     int						):
							lowerCamelCase_	:     Optional[Any] 						=      BasicTokenizer(do_lower_case=lowerCamelCase						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)       ,      ['hello', '!', 'how', 'are', 'you', '?']						)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)       ,      ['hello']						)



					def   __a     (		self     :     List[str]						):
							lowerCamelCase_	:     Tuple 						=      BasicTokenizer(do_lower_case=lowerCamelCase       ,      strip_accents=lowerCamelCase						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)       ,      ['hällo', '!', 'how', 'are', 'you', '?']						)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)       ,      ['h\u00E9llo']						)



					def   __a     (		self     :     int						):
							lowerCamelCase_	:     Optional[Any] 						=      BasicTokenizer(do_lower_case=lowerCamelCase       ,      strip_accents=lowerCamelCase						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)       ,      ['hallo', '!', 'how', 'are', 'you', '?']						)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)       ,      ['hello']						)



					def   __a     (		self     :     Optional[Any]						):
							lowerCamelCase_	:     Optional[Any] 						=      BasicTokenizer(do_lower_case=lowerCamelCase						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)       ,      ['hallo', '!', 'how', 'are', 'you', '?']						)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)       ,      ['hello']						)



					def   __a     (		self     :     Dict						):
							lowerCamelCase_	:     str 						=      BasicTokenizer(do_lower_case=lowerCamelCase						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)       ,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?']						)



					def   __a     (		self     :     Optional[int]						):
							lowerCamelCase_	:     Optional[Any] 						=      BasicTokenizer(do_lower_case=lowerCamelCase       ,      strip_accents=lowerCamelCase						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)       ,      ['HäLLo', '!', 'how', 'Are', 'yoU', '?']						)



					def   __a     (		self     :     Tuple						):
							lowerCamelCase_	:     Optional[int] 						=      BasicTokenizer(do_lower_case=lowerCamelCase       ,      strip_accents=lowerCamelCase						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)       ,      ['HaLLo', '!', 'how', 'Are', 'yoU', '?']						)



					def   __a     (		self     :     Union[str, Any]						):
							lowerCamelCase_	:     Union[str, Any] 						=      BasicTokenizer(do_lower_case=lowerCamelCase       ,      never_split=['[UNK]']						)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'						)       ,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']						)



					def   __a     (		self     :     List[Any]						):
							lowerCamelCase_	:     str 						=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

							lowerCamelCase_	:     int 						=      {}
							for i, token in enumerate(lowerCamelCase						):
									lowerCamelCase_	:     int 						=      i
							lowerCamelCase_	:     Optional[int] 						=      WordpieceTokenizer(vocab=lowerCamelCase       ,      unk_token='[UNK]'						)

							self.assertListEqual(tokenizer.tokenize(''						)       ,      []						)

							self.assertListEqual(tokenizer.tokenize('unwanted running'						)       ,      ['un', '##want', '##ed', 'runn', '##ing']						)

							self.assertListEqual(tokenizer.tokenize('unwantedX running'						)       ,      ['[UNK]', 'runn', '##ing']						)



					@require_torch
					def   __a     (		self     :     Optional[Any]						):
							lowerCamelCase_	:     Optional[Any] 						=      self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'						)

							lowerCamelCase_	:     Dict 						=      ['A long paragraph for summarization.', 'Another paragraph for summarization.']
							lowerCamelCase_	:     Optional[Any] 						=      [10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
							lowerCamelCase_	:     int 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      return_tensors='pt'						)
							self.assertIsInstance(lowerCamelCase       ,      lowerCamelCase						)
							lowerCamelCase_	:     List[Any] 						=      list(batch.input_ids.numpy()[0]						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							self.assertEqual((2, 9)       ,      batch.input_ids.shape						)
							self.assertEqual((2, 9)       ,      batch.attention_mask.shape						)



					def   __a     (		self     :     Union[str, Any]						):
							self.assertTrue(_is_whitespace(' '						)						)
							self.assertTrue(_is_whitespace('\t'						)						)
							self.assertTrue(_is_whitespace('\r'						)						)
							self.assertTrue(_is_whitespace('\n'						)						)
							self.assertTrue(_is_whitespace('\u00A0'						)						)

							self.assertFalse(_is_whitespace('A'						)						)
							self.assertFalse(_is_whitespace('-'						)						)



					def   __a     (		self     :     int						):
							self.assertTrue(_is_control('\u0005'						)						)

							self.assertFalse(_is_control('A'						)						)
							self.assertFalse(_is_control(' '						)						)
							self.assertFalse(_is_control('\t'						)						)
							self.assertFalse(_is_control('\r'						)						)



					def   __a     (		self     :     Optional[Any]						):
							self.assertTrue(_is_punctuation('-'						)						)
							self.assertTrue(_is_punctuation('$'						)						)
							self.assertTrue(_is_punctuation('`'						)						)
							self.assertTrue(_is_punctuation('.'						)						)

							self.assertFalse(_is_punctuation('A'						)						)
							self.assertFalse(_is_punctuation(' '						)						)





					@slow
					def   __a     (		self     :     Optional[int]						):
							lowerCamelCase_	:     Tuple 						=      self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'						)

							lowerCamelCase_	:     str 						=      tokenizer.encode('sequence builders'       ,      add_special_tokens=lowerCamelCase						)
							lowerCamelCase_	:     Dict 						=      tokenizer.encode('multi-sequence build'       ,      add_special_tokens=lowerCamelCase						)

							lowerCamelCase_	:     Optional[Any] 						=      tokenizer.build_inputs_with_special_tokens(lowerCamelCase						)
							lowerCamelCase_	:     List[str] 						=      tokenizer.build_inputs_with_special_tokens(lowerCamelCase       ,      lowerCamelCase						)

							assert encoded_sentence == text + [1_02]
							assert encoded_pair == text + [1_02] + text_a + [1_02]







import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


_lowercase		:  List[str]			       =5_0000
_lowercase		:  str			       =5000

_lowercase       , _lowercase		:  List[str]			       =os.path.split(__file__)
_lowercase		:  Union[str, Any]			       =os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))


@get_duration
def 		_SCREAMING_SNAKE_CASE     ( lowerCAmelCase__							,lowerCAmelCase__						):
		for i in range(lowerCAmelCase__						):
				lowerCamelCase_	:     Tuple 						=      dataset[i]


@get_duration
def 		_SCREAMING_SNAKE_CASE     ( lowerCAmelCase__							,lowerCAmelCase__							,lowerCAmelCase__						):
		for i in range(0							,len(lowerCAmelCase__						)							,lowerCAmelCase__						):
				lowerCamelCase_	:     Tuple 						=      dataset[i : i + batch_size]


@get_duration
def 		_SCREAMING_SNAKE_CASE     ( lowerCAmelCase__							,lowerCAmelCase__							,lowerCAmelCase__						):
		with dataset.formatted_as(type=lowerCAmelCase__						):
				for i in range(lowerCAmelCase__						):
						lowerCamelCase_	:     Tuple 						=      dataset[i]


@get_duration
def 		_SCREAMING_SNAKE_CASE     ( lowerCAmelCase__							,lowerCAmelCase__							,lowerCAmelCase__							,lowerCAmelCase__						):
		with dataset.formatted_as(type=lowerCAmelCase__						):
				for i in range(0							,lowerCAmelCase__							,lowerCAmelCase__						):
						lowerCamelCase_	:     Optional[int] 						=      dataset[i : i + batch_size]


def 		_SCREAMING_SNAKE_CASE     ( ):
		lowerCamelCase_	:     Any 						=      {'num examples': SPEED_TEST_N_EXAMPLES}
		lowerCamelCase_	:     Optional[Any] 						=      [
		    (read, {'length': SMALL_TEST}),
		    (read, {'length': SPEED_TEST_N_EXAMPLES}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_00}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10_00}),
		    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
		    (read_formatted, {'type': 'pandas', 'length': SMALL_TEST}),
		    (read_formatted, {'type': 'torch', 'length': SMALL_TEST}),
		    (read_formatted, {'type': 'tensorflow', 'length': SMALL_TEST}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10_00}),
		]

		lowerCamelCase_	:     Optional[Any] 						=      [
		    (read, {'length': SMALL_TEST}),
		    (read, {'length': SPEED_TEST_N_EXAMPLES}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_00}),
		    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10_00}),
		    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
		    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10_00}),
		]
		with tempfile.TemporaryDirectory() as tmp_dir:
				print('generating dataset'						)
				lowerCamelCase_	:     List[Any] 						=      datasets.Features(
				    {'list': datasets.Sequence(datasets.Value('float32'						)						), 'numbers': datasets.Value('float32'						)}						)
				lowerCamelCase_	:     Union[str, Any] 						=      generate_example_dataset(
				    os.path.join(lowerCAmelCase__							,'dataset.arrow'						)							,lowerCAmelCase__							,num_examples=lowerCAmelCase__							,seq_shapes={'list': (1_00,)}							,)
				print('first set of iterations'						)
				for func, kwargs in functions:
						print(func.__name__							,str(lowerCAmelCase__						)						)
						lowerCamelCase_	:     Optional[int] 						=      func(lowerCAmelCase__							,**lowerCAmelCase__						)

				print('shuffling dataset'						)
				lowerCamelCase_	:     int 						=      dataset.shuffle()
				print('Second set of iterations (after shuffling'						)
				for func, kwargs in functions_shuffled:
						print('shuffled '							,func.__name__							,str(lowerCAmelCase__						)						)
						lowerCamelCase_	:     Optional[Any] 						=      func(
						    lowerCAmelCase__							,**lowerCAmelCase__						)

		with open(lowerCAmelCase__							,'wb'						) as f:
				f.write(json.dumps(lowerCAmelCase__						).encode('utf-8'						)						)


if __name__ == "__main__":  # useful to run the profiler
		benchmark_iterating()
