




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast







class 				lowerCAmelCase_       (					unittest.TestCase							):






				def 		snake_case	(			self ):
							# A mock response for an HTTP head request to emulate server down
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      mock.Mock()
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      500
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      {}
							SCREAMING_SNAKE_CASE_   : Dict					  =      HTTPError
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      {}

							# Download this model to make sure it's in the cache.
							SCREAMING_SNAKE_CASE_   : Tuple					  =      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'      ,return_value=snake_case__ ) as mock_head:
										SCREAMING_SNAKE_CASE_   : Dict					  =      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )
										# This check we did call the fake head request
										mock_head.assert_called()






				@require_tokenizers
				def 		snake_case	(			self ):
							# A mock response for an HTTP head request to emulate server down
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      mock.Mock()
							SCREAMING_SNAKE_CASE_   : List[str]					  =      500
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      {}
							SCREAMING_SNAKE_CASE_   : str					  =      HTTPError
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      {}

							# Download this model to make sure it's in the cache.
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      GPTaTokenizerFast.from_pretrained('gpt2' )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'      ,return_value=snake_case__ ) as mock_head:
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      GPTaTokenizerFast.from_pretrained('gpt2' )
										# This check we did call the fake head request
										mock_head.assert_called()






				def 		snake_case	(			self ):
							# This test is for deprecated behavior and can be removed in v5
							try:
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tempfile.mktemp()
										with open(snake_case__      ,'wb' ) as f:
													http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'      ,snake_case__ )

										SCREAMING_SNAKE_CASE_   : int					  =      AlbertTokenizer.from_pretrained(snake_case__ )
							finally:
										os.remove(snake_case__ )

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile('tokenizer.json' ):
										# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
										return
							try:
										with open('tokenizer.json'      ,'wb' ) as f:
													http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'      ,snake_case__ )
										SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2' )
										# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
										self.assertEqual(tokenizer.vocab_size      ,1000 )
										# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
										os.remove('tokenizer.json' )






				def 		snake_case	(			self ):
							# This test is for deprecated behavior and can be removed in v5
							SCREAMING_SNAKE_CASE_   : List[str]					  =      AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model' )







@is_staging_test
class 				lowerCAmelCase_       (					unittest.TestCase							):
				__a     :       int						=     ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]






				@classmethod
				def 		snake_case	(			cls ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      TOKEN
							HfFolder.save_token(snake_case__ )






				@classmethod
				def 		snake_case	(			cls ):
							try:
										delete_repo(token=cls._token      ,repo_id='test-tokenizer' )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token      ,repo_id='valid_org/test-tokenizer-org' )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token      ,repo_id='test-dynamic-tokenizer' )
							except HTTPError:
										pass






				def 		snake_case	(			self ):
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : Any					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      BertTokenizer(snake_case__ )

							tokenizer.push_to_hub('test-tokenizer'      ,use_auth_token=self._token )
							SCREAMING_SNAKE_CASE_   : str					  =      BertTokenizer.from_pretrained(F'{USER}/test-tokenizer' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )

							# Reset repo
							delete_repo(token=self._token      ,repo_id='test-tokenizer' )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(snake_case__      ,repo_id='test-tokenizer'      ,push_to_hub=snake_case__      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      BertTokenizer.from_pretrained(F'{USER}/test-tokenizer' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )






				def 		snake_case	(			self ):
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : str					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
										SCREAMING_SNAKE_CASE_   : List[str]					  =      BertTokenizer(snake_case__ )

							tokenizer.push_to_hub('valid_org/test-tokenizer-org'      ,use_auth_token=self._token )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )

							# Reset repo
							delete_repo(token=self._token      ,repo_id='valid_org/test-tokenizer-org' )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(
										    snake_case__      ,repo_id='valid_org/test-tokenizer-org'      ,push_to_hub=snake_case__      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : Tuple					  =      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )






				@require_tokenizers
				def 		snake_case	(			self ):
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      CustomTokenizer(snake_case__ )

							# No fast custom tokenizer
							tokenizer.push_to_hub('test-dynamic-tokenizer'      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'      ,trust_remote_code=snake_case__ )
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizer' )

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : List[Any]					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )

										SCREAMING_SNAKE_CASE_   : List[str]					  =      BertTokenizerFast.from_pretrained(snake_case__ )
										bert_tokenizer.save_pretrained(snake_case__ )
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      CustomTokenizerFast.from_pretrained(snake_case__ )

							tokenizer.push_to_hub('test-dynamic-tokenizer'      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : int					  =      AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'      ,trust_remote_code=snake_case__ )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizerFast' )
							SCREAMING_SNAKE_CASE_   : Any					  =      AutoTokenizer.from_pretrained(
							    F'{USER}/test-dynamic-tokenizer'      ,use_fast=snake_case__      ,trust_remote_code=snake_case__ )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizer' )







class 				lowerCAmelCase_       (					unittest.TestCase							):






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      Trie()
							trie.add('Hello 友達' )
							self.assertEqual(trie.data      ,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}} )
							trie.add('Hello' )
							trie.data
							self.assertEqual(trie.data      ,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}} )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      Trie()
							self.assertEqual(trie.split('[CLS] This is a extra_id_100' )      ,['[CLS] This is a extra_id_100'] )
							trie.add('[CLS]' )
							trie.add('extra_id_1' )
							trie.add('extra_id_100' )
							self.assertEqual(trie.split('[CLS] This is a extra_id_100' )      ,['[CLS]', ' This is a ', 'extra_id_100'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      Trie()
							trie.add('A' )
							self.assertEqual(trie.split('ABC' )      ,['A', 'BC'] )
							self.assertEqual(trie.split('BCA' )      ,['BC', 'A'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      Trie()
							trie.add('TOKEN]' )
							trie.add('[SPECIAL_TOKEN]' )
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]' )      ,['This is something ', '[SPECIAL_TOKEN]'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : int					  =      Trie()
							trie.add('A' )
							trie.add('P' )
							trie.add('[SPECIAL_TOKEN]' )
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]' )      ,['This is something ', '[SPECIAL_TOKEN]'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      Trie()
							trie.add('AB' )
							trie.add('B' )
							trie.add('C' )
							self.assertEqual(trie.split('ABC' )      ,['AB', 'C'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      Trie()
							trie.add('ABC' )
							trie.add('B' )
							trie.add('CD' )
							self.assertEqual(trie.split('ABCD' )      ,['ABC', 'D'] )






				def 		snake_case	(			self ):
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							SCREAMING_SNAKE_CASE_   : Tuple					  =      Trie()
							SCREAMING_SNAKE_CASE_   : str					  =      trie.cut_text('ABC'      ,[0, 0, 2, 1, 2, 3] )
							self.assertEqual(snake_case__      ,['AB', 'C'] )








import numpy as np







def     UpperCAmelCase_		(					snake_case__    ,			snake_case__    ,			snake_case__ = 1E-12    ,			snake_case__ = 100    ,			)	->     tuple[float, np.ndarray]:



				"""simple docstring"""





				assert np.shape(snake_case__					)[0] == np.shape(snake_case__					)[1]
				# Ensure proper dimensionality.
				assert np.shape(snake_case__					)[0] == np.shape(snake_case__					)[0]
				# Ensure inputs are either both complex or both real
				assert np.iscomplexobj(snake_case__					) == np.iscomplexobj(snake_case__					)
				lowerCAmelCase__		  =  np.iscomplexobj(snake_case__					)
				if is_complex:
								# Ensure complex input_matrix is Hermitian
								assert np.array_equal(snake_case__    ,			input_matrix.conj().T					)

				# Set convergence to False. Will define convergence when we exceed max_iterations
				# or when we have small changes from one iteration to next.

				lowerCAmelCase__		  =  False
				lowerCAmelCase__		  =  0
				lowerCAmelCase__		  =  0
				lowerCAmelCase__		  =  1E12

				while not convergence:
								# Multiple matrix by the vector.
								lowerCAmelCase__		  =  np.dot(snake_case__    ,			snake_case__					)
								# Normalize the resulting output vector.
								lowerCAmelCase__		  =  w / np.linalg.norm(snake_case__					)
								# Find rayleigh quotient
								# (faster than usual b/c we know vector is normalized already)
								lowerCAmelCase__		  =  vector.conj().T if is_complex else vector.T
								lowerCAmelCase__		  =  np.dot(snake_case__    ,			np.dot(snake_case__    ,			snake_case__					)					)

								# Check convergence.
								lowerCAmelCase__		  =  np.abs(lambda_ - lambda_previous					) / lambda_
								iterations += 1

								if error <= error_tol or iterations >= max_iterations:
												lowerCAmelCase__		  =  True

								lowerCAmelCase__		  =  lambda_

				if is_complex:
								lowerCAmelCase__		  =  np.real(lambda_					)

				return lambda_, vector







def     UpperCAmelCase_		(					)	->     None:



				"""simple docstring"""





				lowerCAmelCase__		  =  np.array([[41, 4, 20], [4, 26, 30], [20, 30, 50]]					)
				lowerCAmelCase__		  =  np.array([41, 4, 20]					)
				lowerCAmelCase__		  =  real_input_matrix.astype(np.complexaaa					)
				lowerCAmelCase__		  =  np.triu(1j * complex_input_matrix    ,			1					)
				complex_input_matrix += imag_matrix
				complex_input_matrix += -1 * imag_matrix.T
				lowerCAmelCase__		  =  np.array([41, 4, 20]					).astype(np.complexaaa					)

				for problem_type in ["real", "complex"]:
								if problem_type == "real":
												lowerCAmelCase__		  =  real_input_matrix
												lowerCAmelCase__		  =  real_vector
								elif problem_type == "complex":
												lowerCAmelCase__		  =  complex_input_matrix
												lowerCAmelCase__		  =  complex_vector

								# Our implementation.
								lowerCAmelCase__  ,   lowerCAmelCase__		  =  power_iteration(snake_case__    ,			snake_case__					)

								# Numpy implementation.

								# Get eigenvalues and eigenvectors using built-in numpy
								# eigh (eigh used for symmetric or hermetian matrices).
								lowerCAmelCase__  ,   lowerCAmelCase__		  =  np.linalg.eigh(snake_case__					)
								# Last eigenvalue is the maximum one.
								lowerCAmelCase__		  =  eigen_values[-1]
								# Last column in this matrix is eigenvector corresponding to largest eigenvalue.
								lowerCAmelCase__		  =  eigen_vectors[:, -1]

								# Check our implementation and numpy gives close answers.
								assert np.abs(eigen_value - eigen_value_max					) <= 1E-6
								# Take absolute values element wise of each eigenvector.
								# as they are only unique to a minus sign.
								assert np.linalg.norm(np.abs(snake_case__					) - np.abs(snake_case__					)					) <= 1E-6


if __name__ == "__main__":
	import doctest

	doctest.testmod()
	test_power_iteration()
