



"""simple docstring"""







import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_  		=  get_tests_dir('fixtures/test_sentencepiece.model')
@require_sentencepiece
@require_tokenizers
class 			__A			(      lowerCamelCase__	,unittest.TestCase       ):







		'''simple docstring'''





		lowerCAmelCase     :   Tuple        =							ReformerTokenizer
		lowerCAmelCase     :   str        =							ReformerTokenizerFast
		lowerCAmelCase     :   Optional[Any]        =							True
		lowerCAmelCase     :   str        =							False
		lowerCAmelCase     :   Dict        =							True







		def    UpperCAmelCase       (      self		:    Optional[Any]		)   ->						str:






				"""simple docstring"""



				super().setUp()

				lowercase__  :       Union[str, Any]												=     ReformerTokenizer(snake_case__     ,keep_accents=snake_case__		)
				tokenizer.save_pretrained(self.tmpdirname		)







		def    UpperCAmelCase       (      self		:    str		)   ->						Any:






				"""simple docstring"""



				lowercase__  :       str												=     """<s>"""
				lowercase__  :       List[str]												=     1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__		)     ,snake_case__		)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__		)     ,snake_case__		)







		def    UpperCAmelCase       (      self		:    List[str]		)   ->						str:






				"""simple docstring"""



				lowercase__  :       Any												=     list(self.get_tokenizer().get_vocab().keys()		)

				self.assertEqual(vocab_keys[0]     ,'''<unk>'''		)
				self.assertEqual(vocab_keys[1]     ,'''<s>'''		)
				self.assertEqual(vocab_keys[-1]     ,'''j'''		)
				self.assertEqual(len(snake_case__		)     ,1_000		)







		def    UpperCAmelCase       (      self		:    Tuple		)   ->						Optional[Any]:






				"""simple docstring"""



				self.assertEqual(self.get_tokenizer().vocab_size     ,1_000		)







		def    UpperCAmelCase       (      self		:    List[Any]		)   ->						Any:






				"""simple docstring"""



				if not self.test_rust_tokenizer:
						return

				lowercase__  :       List[Any]												=     self.get_tokenizer()
				lowercase__  :       List[str]												=     self.get_rust_tokenizer()

				lowercase__  :       int												=     """I was born in 92000, and this is falsé."""

				lowercase__  :       int												=     tokenizer.tokenize(snake_case__		)
				lowercase__  :       str												=     rust_tokenizer.tokenize(snake_case__		)
				self.assertListEqual(snake_case__     ,snake_case__		)

				lowercase__  :       int												=     tokenizer.encode(snake_case__     ,add_special_tokens=snake_case__		)
				lowercase__  :       Union[str, Any]												=     rust_tokenizer.encode(snake_case__     ,add_special_tokens=snake_case__		)
				self.assertListEqual(snake_case__     ,snake_case__		)

				lowercase__  :       str												=     self.get_rust_tokenizer()
				lowercase__  :       Dict												=     tokenizer.encode(snake_case__		)
				lowercase__  :       Optional[Any]												=     rust_tokenizer.encode(snake_case__		)
				self.assertListEqual(snake_case__     ,snake_case__		)







		def    UpperCAmelCase       (      self		:    Dict     ,_snake_case		:    List[Any]=15		)   ->						int:






				"""simple docstring"""



				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
								lowercase__  :       Union[str, Any]												=     self.rust_tokenizer_class.from_pretrained(snake_case__     ,**snake_case__		)

								# Simple input
								lowercase__  :       Union[str, Any]												=     """This is a simple input"""
								lowercase__  :       Optional[int]												=     ["""This is a simple input 1""", """This is a simple input 2"""]
								lowercase__  :       Optional[Any]												=     ("""This is a simple input""", """This is a pair""")
								lowercase__  :       Union[str, Any]												=     [
								    ("""This is a simple input 1""", """This is a simple input 2"""),
								    ("""This is a simple pair 1""", """This is a simple pair 2"""),
								]

								# Simple input tests
								self.assertRaises(snake_case__     ,tokenizer_r.encode     ,snake_case__     ,max_length=snake_case__     ,padding='''max_length'''		)

								# Simple input
								self.assertRaises(snake_case__     ,tokenizer_r.encode_plus     ,snake_case__     ,max_length=snake_case__     ,padding='''max_length'''		)

								# Simple input
								self.assertRaises(
								    snake_case__     ,tokenizer_r.batch_encode_plus     ,snake_case__     ,max_length=snake_case__     ,padding='''max_length'''     ,)

								# Pair input
								self.assertRaises(snake_case__     ,tokenizer_r.encode     ,snake_case__     ,max_length=snake_case__     ,padding='''max_length'''		)

								# Pair input
								self.assertRaises(snake_case__     ,tokenizer_r.encode_plus     ,snake_case__     ,max_length=snake_case__     ,padding='''max_length'''		)

								# Pair input
								self.assertRaises(
								    snake_case__     ,tokenizer_r.batch_encode_plus     ,snake_case__     ,max_length=snake_case__     ,padding='''max_length'''     ,)







		def    UpperCAmelCase       (      self		:    Dict		)   ->						str:






				"""simple docstring"""



				pass







		def    UpperCAmelCase       (      self		:    str		)   ->						List[str]:






				"""simple docstring"""



				lowercase__  :       Tuple												=     ReformerTokenizer(snake_case__     ,keep_accents=snake_case__		)

				lowercase__  :       List[Any]												=     tokenizer.tokenize('''This is a test'''		)
				self.assertListEqual(snake_case__     ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(snake_case__		)     ,[285, 46, 10, 170, 382]     ,)

				lowercase__  :       Tuple												=     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
				self.assertListEqual(
				    snake_case__     ,[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''9''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''é''',
				        '''.''',
				    ]     ,)
				lowercase__  :       List[str]												=     tokenizer.convert_tokens_to_ids(snake_case__		)
				self.assertListEqual(
				    snake_case__     ,[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]     ,)

				lowercase__  :       List[Any]												=     tokenizer.convert_ids_to_tokens(snake_case__		)
				self.assertListEqual(
				    snake_case__     ,[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''<unk>''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''<unk>''',
				        '''.''',
				    ]     ,)







		@cached_property
		def    UpperCAmelCase       (      self		:    Tuple		)   ->						Optional[int]:






				"""simple docstring"""



				return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''		)







		@slow
		def    UpperCAmelCase       (      self		:    List[str]		)   ->						List[str]:






				"""simple docstring"""



				lowercase__  :       str												=     """Hello World!"""
				lowercase__  :       List[str]												=     [126, 32, 262, 152, 38, 72, 287]

				self.assertListEqual(snake_case__     ,self.big_tokenizer.encode(snake_case__		)		)







		@slow
		def    UpperCAmelCase       (      self		:    Optional[int]		)   ->						str:






				"""simple docstring"""



				lowercase__  :       str												=     (
				    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
				    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
				)
				lowercase__  :       str												=     [
				    108,
				    265,
				    24,
				    111,
				    4,
				    258,
				    156,
				    35,
				    28,
				    275,
				    3,
				    259,
				    297,
				    260,
				    84,
				    4,
				    35,
				    110,
				    44,
				    8,
				    259,
				    91,
				    268,
				    21,
				    11,
				    209,
				    274,
				    109,
				    266,
				    277,
				    117,
				    86,
				    93,
				    315,
				    258,
				    278,
				    258,
				    277,
				    258,
				    0,
				    258,
				    288,
				    258,
				    319,
				    258,
				    0,
				    258,
				    0,
				    258,
				    0,
				    258,
				    0,
				    258,
				    287,
				    258,
				    315,
				    258,
				    289,
				    258,
				    278,
				    99,
				    269,
				    266,
				    262,
				    8,
				    259,
				    241,
				    4,
				    217,
				    230,
				    268,
				    266,
				    55,
				    168,
				    106,
				    75,
				    193,
				    266,
				    223,
				    27,
				    49,
				    26,
				    282,
				    25,
				    264,
				    299,
				    19,
				    26,
				    0,
				    258,
				    277,
				    117,
				    86,
				    93,
				    176,
				    183,
				    270,
				    11,
				    262,
				    42,
				    61,
				    265,
				]

				self.assertListEqual(snake_case__     ,self.big_tokenizer.encode(snake_case__		)		)







		@require_torch
		@slow
		def    UpperCAmelCase       (      self		:    Optional[Any]		)   ->						Optional[Any]:






				"""simple docstring"""



				import torch

				from transformers import ReformerConfig, ReformerModel

				# Build sequence
				lowercase__  :       List[str]												=     list(self.big_tokenizer.get_vocab().keys()		)[:10]
				lowercase__  :       Optional[Any]												=     """ """.join(snake_case__		)
				lowercase__  :       Union[str, Any]												=     self.big_tokenizer.encode_plus(snake_case__     ,return_tensors='''pt'''		)
				lowercase__  :       Optional[int]												=     self.big_tokenizer.batch_encode_plus([sequence, sequence]     ,return_tensors='''pt'''		)

				lowercase__  :       Optional[int]												=     ReformerConfig()
				# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
				lowercase__  :       Optional[int]												=     encoded_sequence["""input_ids"""].shape
				lowercase__  :       List[Any]												=     ReformerModel(snake_case__		)

				# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
				assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

				with torch.no_grad():
						model(**snake_case__		)
						model(**snake_case__		)







		@slow
		def    UpperCAmelCase       (      self		:    Union[str, Any]		)   ->						Optional[int]:






				"""simple docstring"""



				lowercase__  :       Optional[int]												=     {"""input_ids""": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
				# fmt: on

				# This tokenizer does not know some characters like ")".
				# That is the reason why we use very simple texts here.
				# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
				lowercase__  :       Optional[Any]												=     [
				    """This is a very simple sentence.""",
				    """The quick brown fox jumps over the lazy dog.""",
				]

				self.tokenizer_integration_test_util(
				    expected_encoding=snake_case__     ,model_name='''google/reformer-crime-and-punishment'''     ,revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''     ,padding=snake_case__     ,sequences=snake_case__     ,)







UpperCAmelCase_							=				8.31_44_62  # Unit - J mol-1 K-1





def  __magic_name__  (				lowercase				,       lowercase				,       lowercase				)				-> float:

							"""simple docstring"""



							if moles < 0 or kelvin < 0 or volume < 0:
														raise ValueError("""Invalid inputs. Enter positive value."""				)
							return moles * kelvin * UNIVERSAL_GAS_CONSTANT / volume





def  __magic_name__  (				lowercase				,       lowercase				,       lowercase				)				-> float:

							"""simple docstring"""



							if moles < 0 or kelvin < 0 or pressure < 0:
														raise ValueError("""Invalid inputs. Enter positive value."""				)
							return moles * kelvin * UNIVERSAL_GAS_CONSTANT / pressure


if __name__ == "__main__":
							from doctest import testmod

							testmod()