

import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class 						__UpperCamelCase    (					UpperCamelCase_     ,     unittest.TestCase						):
	SCREAMING_SNAKE_CASE__    				=     DebertaTokenizer
	SCREAMING_SNAKE_CASE__    				=     True
	SCREAMING_SNAKE_CASE__    				=     DebertaTokenizerFast




	def       __A					(		self	:						List[Any]							):


		'''simple docstring'''







		super().setUp()

		# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
		UpperCAmelCase_       				=			[
		    '''l''',
		    '''o''',
		    '''w''',
		    '''e''',
		    '''r''',
		    '''s''',
		    '''t''',
		    '''i''',
		    '''d''',
		    '''n''',
		    '''\u0120''',
		    '''\u0120l''',
		    '''\u0120n''',
		    '''\u0120lo''',
		    '''\u0120low''',
		    '''er''',
		    '''\u0120lowest''',
		    '''\u0120newer''',
		    '''\u0120wider''',
		    '''[UNK]''',
		]
		UpperCAmelCase_       				=			dict(zip(UpperCamelCase__		,	range(len(UpperCamelCase__							)							)							)							)
		UpperCAmelCase_       				=			['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
		UpperCAmelCase_       				=			{'''unk_token''': '''[UNK]'''}

		UpperCAmelCase_       				=			os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES["vocab_file"]							)
		UpperCAmelCase_       				=			os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES["merges_file"]							)
		with open(self.vocab_file		,	"w"		,	encoding="utf-8"							) as fp:
			fp.write(json.dumps(UpperCamelCase__							) + "\n"							)
		with open(self.merges_file		,	"w"		,	encoding="utf-8"							) as fp:
			fp.write("\n".join(UpperCamelCase__							)							)




	def       __A					(		self	:						Any		,	**lowerCAmelCase	:						str							):


		'''simple docstring'''







		kwargs.update(self.special_tokens_map							)
		return self.tokenizer_class.from_pretrained(self.tmpdirname		,	**UpperCamelCase__							)




	def       __A					(		self	:						Tuple		,	lowerCAmelCase	:						Any							):


		'''simple docstring'''







		UpperCAmelCase_       				=			'''lower newer'''
		UpperCAmelCase_       				=			'''lower newer'''
		return input_text, output_text




	def       __A					(		self	:						Tuple							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.get_tokenizer()
		UpperCAmelCase_       				=			'''lower newer'''
		UpperCAmelCase_       				=			['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
		UpperCAmelCase_       				=			tokenizer.tokenize(UpperCamelCase__							)
		self.assertListEqual(UpperCamelCase__		,	UpperCamelCase__							)

		UpperCAmelCase_       				=			tokens + [tokenizer.unk_token]
		UpperCAmelCase_       				=			[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
		self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__							)		,	UpperCamelCase__							)




	def       __A					(		self	:						Union[str, Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.get_tokenizer()
		UpperCAmelCase_       				=			tokenizer("Hello"		,	"World"							)
		UpperCAmelCase_       				=			[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
		self.assertListEqual(tokd["token_type_ids"]		,	UpperCamelCase__							)




	@slow
	def       __A					(		self	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.tokenizer_class.from_pretrained("microsoft/deberta-base"							)

		UpperCAmelCase_       				=			tokenizer.encode("sequence builders"		,	add_special_tokens=UpperCamelCase__							)
		UpperCAmelCase_       				=			tokenizer.encode("multi-sequence build"		,	add_special_tokens=UpperCamelCase__							)

		UpperCAmelCase_       				=			tokenizer.encode(
		    "sequence builders"		,	add_special_tokens=UpperCamelCase__		,	add_prefix_space=UpperCamelCase__							)
		UpperCAmelCase_       				=			tokenizer.encode(
		    "sequence builders"		,	"multi-sequence build"		,	add_special_tokens=UpperCamelCase__		,	add_prefix_space=UpperCamelCase__							)

		UpperCAmelCase_       				=			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__							)
		UpperCAmelCase_       				=			tokenizer.build_inputs_with_special_tokens(UpperCamelCase__		,	UpperCamelCase__							)

		assert encoded_sentence == encoded_text_from_decode
		assert encoded_pair == encoded_pair_from_decode




	@slow
	def       __A					(		self	:						Optional[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			[self.tokenizer_class]
		if self.test_rust_tokenizer:
			tokenizer_classes.append(self.rust_tokenizer_class							)

		for tokenizer_class in tokenizer_classes:
			UpperCAmelCase_       				=			tokenizer_class.from_pretrained("microsoft/deberta-base"							)

			UpperCAmelCase_       				=			[
			    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
			    '''ALBERT incorporates two parameter reduction techniques''',
			    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
			    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
			    ''' vocabulary embedding.''',
			]

			UpperCAmelCase_       				=			tokenizer(UpperCamelCase__		,	padding=UpperCamelCase__							)
			UpperCAmelCase_       				=			[tokenizer.decode(UpperCamelCase__		,	skip_special_tokens=UpperCamelCase__							) for seq in encoding['''input_ids''']]

			# fmt: off
			UpperCAmelCase_       				=			{
			    '''input_ids''': [
			        [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
			        [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
			        [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
			    ],
			    '''token_type_ids''': [
			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
			        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
			    ],
			    '''attention_mask''': [
			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
			        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
			    ]
			}
			# fmt: on

			UpperCAmelCase_       				=			[
			    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
			    '''ALBERT incorporates two parameter reduction techniques''',
			    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
			    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
			    ''' vocabulary embedding.''',
			]

			self.assertDictEqual(encoding.data		,	UpperCamelCase__							)

			for expected, decoded in zip(UpperCamelCase__		,	UpperCamelCase__							):
				self.assertEqual(UpperCamelCase__		,	UpperCamelCase__							)




import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_a:    str 							=			get_tests_dir("""fixtures/test_sentencepiece.model""")

@require_sentencepiece
@require_tokenizers
class 						__UpperCamelCase    (					lowercase     ,     unittest.TestCase						):
	SCREAMING_SNAKE_CASE__    				=     ReformerTokenizer
	SCREAMING_SNAKE_CASE__    				=     ReformerTokenizerFast
	SCREAMING_SNAKE_CASE__    				=     True
	SCREAMING_SNAKE_CASE__    				=     False
	SCREAMING_SNAKE_CASE__    				=     True




	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		super().setUp()

		UpperCAmelCase_       				=			ReformerTokenizer(lowerCAmelCase		,	keep_accents=lowerCAmelCase							)
		tokenizer.save_pretrained(self.tmpdirname							)




	def       __A					(		self	:						List[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			"<s>"
		UpperCAmelCase_       				=			1

		self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase							)		,	lowerCAmelCase							)
		self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase							)		,	lowerCAmelCase							)




	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			list(self.get_tokenizer().get_vocab().keys()							)

		self.assertEqual(vocab_keys[0]		,	"<unk>"							)
		self.assertEqual(vocab_keys[1]		,	"<s>"							)
		self.assertEqual(vocab_keys[-1]		,	"j"							)
		self.assertEqual(len(lowerCAmelCase							)		,	1_000							)




	def       __A					(		self	:						List[Any]							):


		'''simple docstring'''







		self.assertEqual(self.get_tokenizer().vocab_size		,	1_000							)




	def       __A					(		self	:						str							):


		'''simple docstring'''







		if not self.test_rust_tokenizer:
			return

		UpperCAmelCase_       				=			self.get_tokenizer()
		UpperCAmelCase_       				=			self.get_rust_tokenizer()

		UpperCAmelCase_       				=			"I was born in 92000, and this is falsé."

		UpperCAmelCase_       				=			tokenizer.tokenize(lowerCAmelCase							)
		UpperCAmelCase_       				=			rust_tokenizer.tokenize(lowerCAmelCase							)
		self.assertListEqual(lowerCAmelCase		,	lowerCAmelCase							)

		UpperCAmelCase_       				=			tokenizer.encode(lowerCAmelCase		,	add_special_tokens=lowerCAmelCase							)
		UpperCAmelCase_       				=			rust_tokenizer.encode(lowerCAmelCase		,	add_special_tokens=lowerCAmelCase							)
		self.assertListEqual(lowerCAmelCase		,	lowerCAmelCase							)

		UpperCAmelCase_       				=			self.get_rust_tokenizer()
		UpperCAmelCase_       				=			tokenizer.encode(lowerCAmelCase							)
		UpperCAmelCase_       				=			rust_tokenizer.encode(lowerCAmelCase							)
		self.assertListEqual(lowerCAmelCase		,	lowerCAmelCase							)




	def       __A					(		self	:						List[Any]		,	lowerCAmelCase	:						Optional[int]=15							):


		'''simple docstring'''







		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
				UpperCAmelCase_       				=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase		,	**lowerCAmelCase							)

				# Simple input
				UpperCAmelCase_       				=			"This is a simple input"
				UpperCAmelCase_       				=			["This is a simple input 1", "This is a simple input 2"]
				UpperCAmelCase_       				=			("This is a simple input", "This is a pair")
				UpperCAmelCase_       				=			[
				    ("This is a simple input 1", "This is a simple input 2"),
				    ("This is a simple pair 1", "This is a simple pair 2"),
				]

				# Simple input tests
				self.assertRaises(lowerCAmelCase		,	tokenizer_r.encode		,	lowerCAmelCase		,	max_length=lowerCAmelCase		,	padding="max_length"							)

				# Simple input
				self.assertRaises(lowerCAmelCase		,	tokenizer_r.encode_plus		,	lowerCAmelCase		,	max_length=lowerCAmelCase		,	padding="max_length"							)

				# Simple input
				self.assertRaises(
				    lowerCAmelCase		,	tokenizer_r.batch_encode_plus		,	lowerCAmelCase		,	max_length=lowerCAmelCase		,	padding="max_length"		,	)

				# Pair input
				self.assertRaises(lowerCAmelCase		,	tokenizer_r.encode		,	lowerCAmelCase		,	max_length=lowerCAmelCase		,	padding="max_length"							)

				# Pair input
				self.assertRaises(lowerCAmelCase		,	tokenizer_r.encode_plus		,	lowerCAmelCase		,	max_length=lowerCAmelCase		,	padding="max_length"							)

				# Pair input
				self.assertRaises(
				    lowerCAmelCase		,	tokenizer_r.batch_encode_plus		,	lowerCAmelCase		,	max_length=lowerCAmelCase		,	padding="max_length"		,	)




	def       __A					(		self	:						Union[str, Any]							):


		'''simple docstring'''







		pass




	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			ReformerTokenizer(lowerCAmelCase		,	keep_accents=lowerCAmelCase							)

		UpperCAmelCase_       				=			tokenizer.tokenize("This is a test"							)
		self.assertListEqual(lowerCAmelCase		,	["▁This", "▁is", "▁a", "▁t", "est"]							)

		self.assertListEqual(
		    tokenizer.convert_tokens_to_ids(lowerCAmelCase							)		,	[285, 46, 10, 170, 382]		,	)

		UpperCAmelCase_       				=			tokenizer.tokenize("I was born in 92000, and this is falsé."							)
		self.assertListEqual(
		    lowerCAmelCase		,	[
		        SPIECE_UNDERLINE + "I",
		        SPIECE_UNDERLINE + "was",
		        SPIECE_UNDERLINE + "b",
		        "or",
		        "n",
		        SPIECE_UNDERLINE + "in",
		        SPIECE_UNDERLINE + "",
		        "9",
		        "2",
		        "0",
		        "0",
		        "0",
		        ",",
		        SPIECE_UNDERLINE + "and",
		        SPIECE_UNDERLINE + "this",
		        SPIECE_UNDERLINE + "is",
		        SPIECE_UNDERLINE + "f",
		        "al",
		        "s",
		        "é",
		        ".",
		    ]		,	)
		UpperCAmelCase_       				=			tokenizer.convert_tokens_to_ids(lowerCAmelCase							)
		self.assertListEqual(
		    lowerCAmelCase		,	[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		,	)

		UpperCAmelCase_       				=			tokenizer.convert_ids_to_tokens(lowerCAmelCase							)
		self.assertListEqual(
		    lowerCAmelCase		,	[
		        SPIECE_UNDERLINE + "I",
		        SPIECE_UNDERLINE + "was",
		        SPIECE_UNDERLINE + "b",
		        "or",
		        "n",
		        SPIECE_UNDERLINE + "in",
		        SPIECE_UNDERLINE + "",
		        "<unk>",
		        "2",
		        "0",
		        "0",
		        "0",
		        ",",
		        SPIECE_UNDERLINE + "and",
		        SPIECE_UNDERLINE + "this",
		        SPIECE_UNDERLINE + "is",
		        SPIECE_UNDERLINE + "f",
		        "al",
		        "s",
		        "<unk>",
		        ".",
		    ]		,	)




	@cached_property
	def       __A					(		self	:						int							):


		'''simple docstring'''







		return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"							)




	@slow
	def       __A					(		self	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			"Hello World!"
		UpperCAmelCase_       				=			[126, 32, 262, 152, 38, 72, 287]

		self.assertListEqual(lowerCAmelCase		,	self.big_tokenizer.encode(lowerCAmelCase							)							)




	@slow
	def       __A					(		self	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			(
		    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
		    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
		)
		UpperCAmelCase_       				=			[
		    108,
		    265,
		    24,
		    111,
		    4,
		    258,
		    156,
		    35,
		    28,
		    275,
		    3,
		    259,
		    297,
		    260,
		    84,
		    4,
		    35,
		    110,
		    44,
		    8,
		    259,
		    91,
		    268,
		    21,
		    11,
		    209,
		    274,
		    109,
		    266,
		    277,
		    117,
		    86,
		    93,
		    315,
		    258,
		    278,
		    258,
		    277,
		    258,
		    0,
		    258,
		    288,
		    258,
		    319,
		    258,
		    0,
		    258,
		    0,
		    258,
		    0,
		    258,
		    0,
		    258,
		    287,
		    258,
		    315,
		    258,
		    289,
		    258,
		    278,
		    99,
		    269,
		    266,
		    262,
		    8,
		    259,
		    241,
		    4,
		    217,
		    230,
		    268,
		    266,
		    55,
		    168,
		    106,
		    75,
		    193,
		    266,
		    223,
		    27,
		    49,
		    26,
		    282,
		    25,
		    264,
		    299,
		    19,
		    26,
		    0,
		    258,
		    277,
		    117,
		    86,
		    93,
		    176,
		    183,
		    270,
		    11,
		    262,
		    42,
		    61,
		    265,
		]

		self.assertListEqual(lowerCAmelCase		,	self.big_tokenizer.encode(lowerCAmelCase							)							)




	@require_torch
	@slow
	def       __A					(		self	:						Tuple							):


		'''simple docstring'''







		import torch

		from transformers import ReformerConfig, ReformerModel

		# Build sequence
		UpperCAmelCase_       				=			list(self.big_tokenizer.get_vocab().keys()							)[:10]
		UpperCAmelCase_       				=			" ".join(lowerCAmelCase							)
		UpperCAmelCase_       				=			self.big_tokenizer.encode_plus(lowerCAmelCase		,	return_tensors="pt"							)
		UpperCAmelCase_       				=			self.big_tokenizer.batch_encode_plus([sequence, sequence]		,	return_tensors="pt"							)

		UpperCAmelCase_       				=			ReformerConfig()
		# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
		UpperCAmelCase_       				=			encoded_sequence["input_ids"].shape
		UpperCAmelCase_       				=			ReformerModel(lowerCAmelCase							)

		# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
		assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

		with torch.no_grad():
			model(**lowerCAmelCase							)
			model(**lowerCAmelCase							)




	@slow
	def       __A					(		self	:						Any							):


		'''simple docstring'''







		UpperCAmelCase_       				=			{"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
		# fmt: on

		# This tokenizer does not know some characters like ")".
		# That is the reason why we use very simple texts here.
		# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
		UpperCAmelCase_       				=			[
		    "This is a very simple sentence.",
		    "The quick brown fox jumps over the lazy dog.",
		]

		self.tokenizer_integration_test_util(
		    expected_encoding=lowerCAmelCase		,	model_name="google/reformer-crime-and-punishment"		,	revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"		,	padding=lowerCAmelCase		,	sequences=lowerCAmelCase		,	)