


"""simple docstring"""




import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase		= get_tests_dir('''fixtures/test_sentencepiece.model''')
@require_sentencepiece
@require_tokenizers
class    lowerCAmelCase      (   A_      ,       unittest.TestCase       ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_		:	List[str]														=	ReformerTokenizer
		SCREAMING_SNAKE_CASE_		:	List[str]														=	ReformerTokenizerFast
		SCREAMING_SNAKE_CASE_		:	Tuple														=	True
		SCREAMING_SNAKE_CASE_		:	List[Any]														=	False
		SCREAMING_SNAKE_CASE_		:	List[Any]														=	True







		def 				__A		(      self							)	->				List[Any]:
								super().setUp()

								SCREAMING_SNAKE_CASE						=   ReformerTokenizer(snake_case__		,			keep_accents=snake_case__							)
								tokenizer.save_pretrained(self.tmpdirname							)







		def 				__A		(      self							)	->				List[str]:
								SCREAMING_SNAKE_CASE						=   "<s>"
								SCREAMING_SNAKE_CASE						=   1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__							)		,			snake_case__							)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__							)		,			snake_case__							)







		def 				__A		(      self							)	->				Dict:
								SCREAMING_SNAKE_CASE						=   list(self.get_tokenizer().get_vocab().keys()							)

								self.assertEqual(vocab_keys[0]		,			'<unk>'							)
								self.assertEqual(vocab_keys[1]		,			'<s>'							)
								self.assertEqual(vocab_keys[-1]		,			'j'							)
								self.assertEqual(len(snake_case__							)		,			1_000							)







		def 				__A		(      self							)	->				List[Any]:
								self.assertEqual(self.get_tokenizer().vocab_size		,			1_000							)







		def 				__A		(      self							)	->				Optional[Any]:


								if not self.test_rust_tokenizer:
														return

								SCREAMING_SNAKE_CASE						=   self.get_tokenizer()
								SCREAMING_SNAKE_CASE						=   self.get_rust_tokenizer()

								SCREAMING_SNAKE_CASE						=   "I was born in 92000, and this is falsé."

								SCREAMING_SNAKE_CASE						=   tokenizer.tokenize(snake_case__							)
								SCREAMING_SNAKE_CASE						=   rust_tokenizer.tokenize(snake_case__							)
								self.assertListEqual(snake_case__		,			snake_case__							)

								SCREAMING_SNAKE_CASE						=   tokenizer.encode(snake_case__		,			add_special_tokens=snake_case__							)
								SCREAMING_SNAKE_CASE						=   rust_tokenizer.encode(snake_case__		,			add_special_tokens=snake_case__							)
								self.assertListEqual(snake_case__		,			snake_case__							)

								SCREAMING_SNAKE_CASE						=   self.get_rust_tokenizer()
								SCREAMING_SNAKE_CASE						=   tokenizer.encode(snake_case__							)
								SCREAMING_SNAKE_CASE						=   rust_tokenizer.encode(snake_case__							)
								self.assertListEqual(snake_case__		,			snake_case__							)







		def 				__A		(      self		,			lowerCAmelCase__=15							)	->				Optional[int]:


								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
																				SCREAMING_SNAKE_CASE						=   self.rust_tokenizer_class.from_pretrained(snake_case__		,			**snake_case__							)

																				# Simple input
																				SCREAMING_SNAKE_CASE						=   "This is a simple input"
																				SCREAMING_SNAKE_CASE						=   ["This is a simple input 1", "This is a simple input 2"]
																				SCREAMING_SNAKE_CASE						=   ("This is a simple input", "This is a pair")
																				SCREAMING_SNAKE_CASE						=   [
																				    ("This is a simple input 1", "This is a simple input 2"),
																				    ("This is a simple pair 1", "This is a simple pair 2"),
																				]

																				# Simple input tests
																				self.assertRaises(snake_case__		,			tokenizer_r.encode		,			snake_case__		,			max_length=snake_case__		,			padding='max_length'							)

																				# Simple input
																				self.assertRaises(snake_case__		,			tokenizer_r.encode_plus		,			snake_case__		,			max_length=snake_case__		,			padding='max_length'							)

																				# Simple input
																				self.assertRaises(
																				    snake_case__		,			tokenizer_r.batch_encode_plus		,			snake_case__		,			max_length=snake_case__		,			padding='max_length'		,			)

																				# Pair input
																				self.assertRaises(snake_case__		,			tokenizer_r.encode		,			snake_case__		,			max_length=snake_case__		,			padding='max_length'							)

																				# Pair input
																				self.assertRaises(snake_case__		,			tokenizer_r.encode_plus		,			snake_case__		,			max_length=snake_case__		,			padding='max_length'							)

																				# Pair input
																				self.assertRaises(
																				    snake_case__		,			tokenizer_r.batch_encode_plus		,			snake_case__		,			max_length=snake_case__		,			padding='max_length'		,			)







		def 				__A		(      self							)	->				str:
								pass







		def 				__A		(      self							)	->				Optional[Any]:
								SCREAMING_SNAKE_CASE						=   ReformerTokenizer(snake_case__		,			keep_accents=snake_case__							)

								SCREAMING_SNAKE_CASE						=   tokenizer.tokenize('This is a test'							)
								self.assertListEqual(snake_case__		,			['▁This', '▁is', '▁a', '▁t', 'est']							)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(snake_case__							)		,			[285, 46, 10, 170, 382]		,			)

								SCREAMING_SNAKE_CASE						=   tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
								self.assertListEqual(
								    snake_case__		,			[
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]		,			)
								SCREAMING_SNAKE_CASE						=   tokenizer.convert_tokens_to_ids(snake_case__							)
								self.assertListEqual(
								    snake_case__		,			[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		,			)

								SCREAMING_SNAKE_CASE						=   tokenizer.convert_ids_to_tokens(snake_case__							)
								self.assertListEqual(
								    snake_case__		,			[
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '<unk>',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '<unk>',
								        '.',
								    ]		,			)







		@cached_property
		def 				__A		(      self							)	->				Dict:
								return ReformerTokenizer.from_pretrained('google/reformer-crime-and-punishment'							)







		@slow
		def 				__A		(      self							)	->				Union[str, Any]:
								SCREAMING_SNAKE_CASE						=   "Hello World!"
								SCREAMING_SNAKE_CASE						=   [126, 32, 262, 152, 38, 72, 287]

								self.assertListEqual(snake_case__		,			self.big_tokenizer.encode(snake_case__							)							)







		@slow
		def 				__A		(      self							)	->				Optional[int]:
								SCREAMING_SNAKE_CASE						=   (
								    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
								    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
								)
								SCREAMING_SNAKE_CASE						=   [
								    108,
								    265,
								    24,
								    111,
								    4,
								    258,
								    156,
								    35,
								    28,
								    275,
								    3,
								    259,
								    297,
								    260,
								    84,
								    4,
								    35,
								    110,
								    44,
								    8,
								    259,
								    91,
								    268,
								    21,
								    11,
								    209,
								    274,
								    109,
								    266,
								    277,
								    117,
								    86,
								    93,
								    315,
								    258,
								    278,
								    258,
								    277,
								    258,
								    0,
								    258,
								    288,
								    258,
								    319,
								    258,
								    0,
								    258,
								    0,
								    258,
								    0,
								    258,
								    0,
								    258,
								    287,
								    258,
								    315,
								    258,
								    289,
								    258,
								    278,
								    99,
								    269,
								    266,
								    262,
								    8,
								    259,
								    241,
								    4,
								    217,
								    230,
								    268,
								    266,
								    55,
								    168,
								    106,
								    75,
								    193,
								    266,
								    223,
								    27,
								    49,
								    26,
								    282,
								    25,
								    264,
								    299,
								    19,
								    26,
								    0,
								    258,
								    277,
								    117,
								    86,
								    93,
								    176,
								    183,
								    270,
								    11,
								    262,
								    42,
								    61,
								    265,
								]

								self.assertListEqual(snake_case__		,			self.big_tokenizer.encode(snake_case__							)							)







		@require_torch
		@slow
		def 				__A		(      self							)	->				List[Any]:
								import torch

								from transformers import ReformerConfig, ReformerModel

								# Build sequence
								SCREAMING_SNAKE_CASE						=   list(self.big_tokenizer.get_vocab().keys()							)[:10]
								SCREAMING_SNAKE_CASE						=   " ".join(snake_case__							)
								SCREAMING_SNAKE_CASE						=   self.big_tokenizer.encode_plus(snake_case__		,			return_tensors='pt'							)
								SCREAMING_SNAKE_CASE						=   self.big_tokenizer.batch_encode_plus([sequence, sequence]		,			return_tensors='pt'							)

								SCREAMING_SNAKE_CASE						=   ReformerConfig()
								# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
								SCREAMING_SNAKE_CASE						=   encoded_sequence["input_ids"].shape
								SCREAMING_SNAKE_CASE						=   ReformerModel(snake_case__							)

								# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
								assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

								with torch.no_grad():
														model(**snake_case__							)
														model(**snake_case__							)







		@slow
		def 				__A		(      self							)	->				Optional[int]:
								SCREAMING_SNAKE_CASE						=   {"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								# This tokenizer does not know some characters like ")".
								# That is the reason why we use very simple texts here.
								# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
								SCREAMING_SNAKE_CASE						=   [
								    "This is a very simple sentence.",
								    "The quick brown fox jumps over the lazy dog.",
								]

								self.tokenizer_integration_test_util(
								    expected_encoding=snake_case__		,			model_name='google/reformer-crime-and-punishment'		,			revision='0e6c3decb8211d49bf881013425dc8b0448b3f5a'		,			padding=snake_case__		,			sequences=snake_case__		,			)




__lowerCamelCase				=			{
    0: """0""",
    1: """1""",
    2: """2""",
    3: """3""",
    4: """4""",
    5: """5""",
    6: """6""",
    7: """7""",
    8: """8""",
    9: """9""",
    10: """a""",
    11: """b""",
    12: """c""",
    13: """d""",
    14: """e""",
    15: """f""",
}


def      UpperCamelCase   (       __lowerCamelCase	:					float       ):
       assert type(__lowerCamelCase       ) in (int, float) and decimal == int(__lowerCamelCase       )
       snake_case			:  str     							= int(__lowerCamelCase       )
       snake_case			:  str     							= ""
       snake_case			:  str     							= False
       if decimal < 0:
              snake_case			:  Optional[Any]     							= True
              decimal *= -1
       while decimal > 0:
              snake_case							,   snake_case			:  Union[str, Any]     							= divmod(__lowerCamelCase							,   16       )
              snake_case			:  Tuple     							= values[remainder] + hexadecimal
       snake_case			:  Tuple     							= "0x" + hexadecimal
       if negative:
              snake_case			:  Any     							= "-" + hexadecimal
       return hexadecimal


if __name__ == "__main__":
   import doctest

   doctest.testmod()





