




"""simple docstring"""

import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase  :			Union[str, Any]        =       get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class        lowercase      (    __UpperCAmelCase     ,	unittest.TestCase):

				'''simple docstring'''





				__lowerCAmelCase    :    Tuple         =   ReformerTokenizer
				__lowerCAmelCase    :    List[Any]         =   ReformerTokenizerFast
				__lowerCAmelCase    :    Any         =   True
				__lowerCAmelCase    :    str         =   False
				__lowerCAmelCase    :    Union[str, Any]         =   True







				def 					a_  (							self   :					Optional[int]       ):



					"""simple docstring"""

					super().setUp()

					A_  :				Tuple	   =		ReformerTokenizer(_lowerCamelCase      ,     keep_accents=_lowerCamelCase       )
					tokenizer.save_pretrained(self.tmpdirname       )







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				Tuple	   =		'''<s>'''
					A_  :				Optional[int]	   =		1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase       )      ,     _lowerCamelCase       )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase       )      ,     _lowerCamelCase       )







				def 					a_  (							self   :					Union[str, Any]       ):



					"""simple docstring"""

					A_  :				int	   =		list(self.get_tokenizer().get_vocab().keys()       )

					self.assertEqual(vocab_keys[0]      ,     '''<unk>'''       )
					self.assertEqual(vocab_keys[1]      ,     '''<s>'''       )
					self.assertEqual(vocab_keys[-1]      ,     '''j'''       )
					self.assertEqual(len(_lowerCamelCase       )      ,     10_00       )







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					self.assertEqual(self.get_tokenizer().vocab_size      ,     10_00       )







				def 					a_  (							self   :					Any       ):



					"""simple docstring"""

					if not self.test_rust_tokenizer:
						return

					A_  :				Union[str, Any]	   =		self.get_tokenizer()
					A_  :				Tuple	   =		self.get_rust_tokenizer()

					A_  :				str	   =		'''I was born in 92000, and this is falsé.'''

					A_  :				List[Any]	   =		tokenizer.tokenize(_lowerCamelCase       )
					A_  :				Tuple	   =		rust_tokenizer.tokenize(_lowerCamelCase       )
					self.assertListEqual(_lowerCamelCase      ,     _lowerCamelCase       )

					A_  :				Union[str, Any]	   =		tokenizer.encode(_lowerCamelCase      ,     add_special_tokens=_lowerCamelCase       )
					A_  :				Dict	   =		rust_tokenizer.encode(_lowerCamelCase      ,     add_special_tokens=_lowerCamelCase       )
					self.assertListEqual(_lowerCamelCase      ,     _lowerCamelCase       )

					A_  :				List[str]	   =		self.get_rust_tokenizer()
					A_  :				Any	   =		tokenizer.encode(_lowerCamelCase       )
					A_  :				Any	   =		rust_tokenizer.encode(_lowerCamelCase       )
					self.assertListEqual(_lowerCamelCase      ,     _lowerCamelCase       )







				def 					a_  (							self   :					Dict      ,     _lowerCamelCase   :					int=15       ):



					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
							A_  :				List[str]	   =		self.rust_tokenizer_class.from_pretrained(_lowerCamelCase      ,     **_lowerCamelCase       )

							# Simple input
							A_  :				Dict	   =		'''This is a simple input'''
							A_  :				List[Any]	   =		['''This is a simple input 1''', '''This is a simple input 2''']
							A_  :				int	   =		('''This is a simple input''', '''This is a pair''')
							A_  :				Union[str, Any]	   =		[
							    ('''This is a simple input 1''', '''This is a simple input 2'''),
							    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
							]

							# Simple input tests
							self.assertRaises(_lowerCamelCase      ,     tokenizer_r.encode      ,     _lowerCamelCase      ,     max_length=_lowerCamelCase      ,     padding='''max_length'''       )

							# Simple input
							self.assertRaises(_lowerCamelCase      ,     tokenizer_r.encode_plus      ,     _lowerCamelCase      ,     max_length=_lowerCamelCase      ,     padding='''max_length'''       )

							# Simple input
							self.assertRaises(
							    _lowerCamelCase      ,     tokenizer_r.batch_encode_plus      ,     _lowerCamelCase      ,     max_length=_lowerCamelCase      ,     padding='''max_length'''      ,     )

							# Pair input
							self.assertRaises(_lowerCamelCase      ,     tokenizer_r.encode      ,     _lowerCamelCase      ,     max_length=_lowerCamelCase      ,     padding='''max_length'''       )

							# Pair input
							self.assertRaises(_lowerCamelCase      ,     tokenizer_r.encode_plus      ,     _lowerCamelCase      ,     max_length=_lowerCamelCase      ,     padding='''max_length'''       )

							# Pair input
							self.assertRaises(
							    _lowerCamelCase      ,     tokenizer_r.batch_encode_plus      ,     _lowerCamelCase      ,     max_length=_lowerCamelCase      ,     padding='''max_length'''      ,     )







				def 					a_  (							self   :					Optional[int]       ):



					"""simple docstring"""

					pass







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				int	   =		ReformerTokenizer(_lowerCamelCase      ,     keep_accents=_lowerCamelCase       )

					A_  :				Optional[int]	   =		tokenizer.tokenize('''This is a test'''       )
					self.assertListEqual(_lowerCamelCase      ,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_lowerCamelCase       )      ,     [2_85, 46, 10, 1_70, 3_82]      ,     )

					A_  :				Union[str, Any]	   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''é''',
					        '''.''',
					    ]      ,     )
					A_  :				Tuple	   =		tokenizer.convert_tokens_to_ids(_lowerCamelCase       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,     )

					A_  :				List[Any]	   =		tokenizer.convert_ids_to_tokens(_lowerCamelCase       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''<unk>''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''<unk>''',
					        '''.''',
					    ]      ,     )







				@cached_property
				def 					a_  (							self   :					Any       ):



					"""simple docstring"""

					return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''       )







				@slow
				def 					a_  (							self   :					Any       ):



					"""simple docstring"""

					A_  :				Union[str, Any]	   =		'''Hello World!'''
					A_  :				Any	   =		[1_26, 32, 2_62, 1_52, 38, 72, 2_87]

					self.assertListEqual(_lowerCamelCase      ,     self.big_tokenizer.encode(_lowerCamelCase       )       )







				@slow
				def 					a_  (							self   :					int       ):



					"""simple docstring"""

					A_  :				Union[str, Any]	   =		(
					    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
					    ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
					)
					A_  :				List[Any]	   =		[
					    1_08,
					    2_65,
					    24,
					    1_11,
					    4,
					    2_58,
					    1_56,
					    35,
					    28,
					    2_75,
					    3,
					    2_59,
					    2_97,
					    2_60,
					    84,
					    4,
					    35,
					    1_10,
					    44,
					    8,
					    2_59,
					    91,
					    2_68,
					    21,
					    11,
					    2_09,
					    2_74,
					    1_09,
					    2_66,
					    2_77,
					    1_17,
					    86,
					    93,
					    3_15,
					    2_58,
					    2_78,
					    2_58,
					    2_77,
					    2_58,
					    0,
					    2_58,
					    2_88,
					    2_58,
					    3_19,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    0,
					    2_58,
					    2_87,
					    2_58,
					    3_15,
					    2_58,
					    2_89,
					    2_58,
					    2_78,
					    99,
					    2_69,
					    2_66,
					    2_62,
					    8,
					    2_59,
					    2_41,
					    4,
					    2_17,
					    2_30,
					    2_68,
					    2_66,
					    55,
					    1_68,
					    1_06,
					    75,
					    1_93,
					    2_66,
					    2_23,
					    27,
					    49,
					    26,
					    2_82,
					    25,
					    2_64,
					    2_99,
					    19,
					    26,
					    0,
					    2_58,
					    2_77,
					    1_17,
					    86,
					    93,
					    1_76,
					    1_83,
					    2_70,
					    11,
					    2_62,
					    42,
					    61,
					    2_65,
					]

					self.assertListEqual(_lowerCamelCase      ,     self.big_tokenizer.encode(_lowerCamelCase       )       )







				@require_torch
				@slow
				def 					a_  (							self   :					Dict       ):



					"""simple docstring"""

					import torch

					from transformers import ReformerConfig, ReformerModel

					# Build sequence
					A_  :				Tuple	   =		list(self.big_tokenizer.get_vocab().keys()       )[:10]
					A_  :				Dict	   =		''' '''.join(_lowerCamelCase       )
					A_  :				Any	   =		self.big_tokenizer.encode_plus(_lowerCamelCase      ,     return_tensors='''pt'''       )
					A_  :				Any	   =		self.big_tokenizer.batch_encode_plus([sequence, sequence]      ,     return_tensors='''pt'''       )

					A_  :				Optional[Any]	   =		ReformerConfig()
					# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
					A_  :				Optional[Any]	   =		encoded_sequence['''input_ids'''].shape
					A_  :				List[Any]	   =		ReformerModel(_lowerCamelCase       )

					# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
					assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

					with torch.no_grad():
						model(**_lowerCamelCase       )
						model(**_lowerCamelCase       )







				@slow
				def 					a_  (							self   :					Any       ):



					"""simple docstring"""

					A_  :				Optional[Any]	   =		{'''input_ids''': [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
					# fmt: on

					# This tokenizer does not know some characters like ")".
					# That is the reason why we use very simple texts here.
					# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
					A_  :				Dict	   =		[
					    '''This is a very simple sentence.''',
					    '''The quick brown fox jumps over the lazy dog.''',
					]

					self.tokenizer_integration_test_util(
					    expected_encoding=_lowerCamelCase      ,     model_name='''google/reformer-crime-and-punishment'''      ,     revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''      ,     padding=_lowerCamelCase      ,     sequences=_lowerCamelCase      ,     )











"""simple docstring"""

from typing import Any

import numpy as np

def 		lowercase_      (		_UpperCAmelCase	):




	"""simple docstring"""






	return np.array_equal(_UpperCAmelCase  ,      matrix.conjugate().T	)

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase	):




	"""simple docstring"""






	A_  :				Tuple	   =		v.conjugate().T
	A_  :				Tuple	   =		v_star.dot(_UpperCAmelCase	)
	assert isinstance(_UpperCAmelCase  ,      np.ndarray	)
	return (v_star_dot.dot(_UpperCAmelCase	)) / (v_star.dot(_UpperCAmelCase	))

def 		lowercase_      (		):




	"""simple docstring"""






	A_  :				Optional[Any]	   =		np.array([[2, 2 + 1J, 4], [2 - 1J, 3, 1J], [4, -1J, 1]]	)
	A_  :				str	   =		np.array([[1], [2], [3]]	)
	assert is_hermitian(_UpperCAmelCase	), f"""{a} is not hermitian."""
	print(rayleigh_quotient(_UpperCAmelCase  ,      _UpperCAmelCase	)	)

	A_  :				int	   =		np.array([[1, 2, 4], [2, 3, -1], [4, -1, 1]]	)
	assert is_hermitian(_UpperCAmelCase	), f"""{a} is not hermitian."""
	assert rayleigh_quotient(_UpperCAmelCase  ,      _UpperCAmelCase	) == float(3	)


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		tests()





