




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast







class 				lowerCAmelCase_       (					unittest.TestCase							):






				def 		snake_case	(			self ):
							# A mock response for an HTTP head request to emulate server down
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      mock.Mock()
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      500
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      {}
							SCREAMING_SNAKE_CASE_   : Dict					  =      HTTPError
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      {}

							# Download this model to make sure it's in the cache.
							SCREAMING_SNAKE_CASE_   : Tuple					  =      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'      ,return_value=snake_case__ ) as mock_head:
										SCREAMING_SNAKE_CASE_   : Dict					  =      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert' )
										# This check we did call the fake head request
										mock_head.assert_called()






				@require_tokenizers
				def 		snake_case	(			self ):
							# A mock response for an HTTP head request to emulate server down
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      mock.Mock()
							SCREAMING_SNAKE_CASE_   : List[str]					  =      500
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      {}
							SCREAMING_SNAKE_CASE_   : str					  =      HTTPError
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      {}

							# Download this model to make sure it's in the cache.
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      GPTaTokenizerFast.from_pretrained('gpt2' )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('requests.Session.request'      ,return_value=snake_case__ ) as mock_head:
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      GPTaTokenizerFast.from_pretrained('gpt2' )
										# This check we did call the fake head request
										mock_head.assert_called()






				def 		snake_case	(			self ):
							# This test is for deprecated behavior and can be removed in v5
							try:
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tempfile.mktemp()
										with open(snake_case__      ,'wb' ) as f:
													http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'      ,snake_case__ )

										SCREAMING_SNAKE_CASE_   : int					  =      AlbertTokenizer.from_pretrained(snake_case__ )
							finally:
										os.remove(snake_case__ )

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile('tokenizer.json' ):
										# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
										return
							try:
										with open('tokenizer.json'      ,'wb' ) as f:
													http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'      ,snake_case__ )
										SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2' )
										# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
										self.assertEqual(tokenizer.vocab_size      ,1000 )
										# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
										os.remove('tokenizer.json' )






				def 		snake_case	(			self ):
							# This test is for deprecated behavior and can be removed in v5
							SCREAMING_SNAKE_CASE_   : List[str]					  =      AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model' )







@is_staging_test
class 				lowerCAmelCase_       (					unittest.TestCase							):
				__a     :       int						=     ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]






				@classmethod
				def 		snake_case	(			cls ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      TOKEN
							HfFolder.save_token(snake_case__ )






				@classmethod
				def 		snake_case	(			cls ):
							try:
										delete_repo(token=cls._token      ,repo_id='test-tokenizer' )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token      ,repo_id='valid_org/test-tokenizer-org' )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token      ,repo_id='test-dynamic-tokenizer' )
							except HTTPError:
										pass






				def 		snake_case	(			self ):
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : Any					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      BertTokenizer(snake_case__ )

							tokenizer.push_to_hub('test-tokenizer'      ,use_auth_token=self._token )
							SCREAMING_SNAKE_CASE_   : str					  =      BertTokenizer.from_pretrained(F'{USER}/test-tokenizer' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )

							# Reset repo
							delete_repo(token=self._token      ,repo_id='test-tokenizer' )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(snake_case__      ,repo_id='test-tokenizer'      ,push_to_hub=snake_case__      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      BertTokenizer.from_pretrained(F'{USER}/test-tokenizer' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )






				def 		snake_case	(			self ):
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : str					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
										SCREAMING_SNAKE_CASE_   : List[str]					  =      BertTokenizer(snake_case__ )

							tokenizer.push_to_hub('valid_org/test-tokenizer-org'      ,use_auth_token=self._token )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )

							# Reset repo
							delete_repo(token=self._token      ,repo_id='valid_org/test-tokenizer-org' )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(
										    snake_case__      ,repo_id='valid_org/test-tokenizer-org'      ,push_to_hub=snake_case__      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : Tuple					  =      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org' )
							self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab )






				@require_tokenizers
				def 		snake_case	(			self ):
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      CustomTokenizer(snake_case__ )

							# No fast custom tokenizer
							tokenizer.push_to_hub('test-dynamic-tokenizer'      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'      ,trust_remote_code=snake_case__ )
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizer' )

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										SCREAMING_SNAKE_CASE_   : List[Any]					  =      os.path.join(snake_case__      ,'vocab.txt' )
										with open(snake_case__      ,'w'      ,encoding='utf-8' ) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens] ) )

										SCREAMING_SNAKE_CASE_   : List[str]					  =      BertTokenizerFast.from_pretrained(snake_case__ )
										bert_tokenizer.save_pretrained(snake_case__ )
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      CustomTokenizerFast.from_pretrained(snake_case__ )

							tokenizer.push_to_hub('test-dynamic-tokenizer'      ,use_auth_token=self._token )

							SCREAMING_SNAKE_CASE_   : int					  =      AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'      ,trust_remote_code=snake_case__ )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizerFast' )
							SCREAMING_SNAKE_CASE_   : Any					  =      AutoTokenizer.from_pretrained(
							    F'{USER}/test-dynamic-tokenizer'      ,use_fast=snake_case__      ,trust_remote_code=snake_case__ )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizer' )







class 				lowerCAmelCase_       (					unittest.TestCase							):






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      Trie()
							trie.add('Hello 友達' )
							self.assertEqual(trie.data      ,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}} )
							trie.add('Hello' )
							trie.data
							self.assertEqual(trie.data      ,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}} )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      Trie()
							self.assertEqual(trie.split('[CLS] This is a extra_id_100' )      ,['[CLS] This is a extra_id_100'] )
							trie.add('[CLS]' )
							trie.add('extra_id_1' )
							trie.add('extra_id_100' )
							self.assertEqual(trie.split('[CLS] This is a extra_id_100' )      ,['[CLS]', ' This is a ', 'extra_id_100'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      Trie()
							trie.add('A' )
							self.assertEqual(trie.split('ABC' )      ,['A', 'BC'] )
							self.assertEqual(trie.split('BCA' )      ,['BC', 'A'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      Trie()
							trie.add('TOKEN]' )
							trie.add('[SPECIAL_TOKEN]' )
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]' )      ,['This is something ', '[SPECIAL_TOKEN]'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : int					  =      Trie()
							trie.add('A' )
							trie.add('P' )
							trie.add('[SPECIAL_TOKEN]' )
							self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]' )      ,['This is something ', '[SPECIAL_TOKEN]'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      Trie()
							trie.add('AB' )
							trie.add('B' )
							trie.add('C' )
							self.assertEqual(trie.split('ABC' )      ,['AB', 'C'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      Trie()
							trie.add('ABC' )
							trie.add('B' )
							trie.add('CD' )
							self.assertEqual(trie.split('ABCD' )      ,['ABC', 'D'] )






				def 		snake_case	(			self ):
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							SCREAMING_SNAKE_CASE_   : Tuple					  =      Trie()
							SCREAMING_SNAKE_CASE_   : str					  =      trie.cut_text('ABC'      ,[0, 0, 2, 1, 2, 3] )
							self.assertEqual(snake_case__      ,['AB', 'C'] )









from __future__ import annotations

import math

import numpy as np
from numpy.linalg import norm





def        __UpperCAmelCase						(							lowerCamelCase_ :     np.ndarray  ,							lowerCamelCase_ :     np.ndarray  )       ->  float:







			"""simple docstring"""





			return math.sqrt(sum(pow(a - b  ,							2  ) for a, b in zip(lowerCamelCase_  ,							lowerCamelCase_  )  )  )





def        __UpperCAmelCase						(							lowerCamelCase_ :     np.ndarray  ,							lowerCamelCase_ :     np.ndarray  )       ->  list[list[list[float] | float]]:







			"""simple docstring"""





			if dataset.ndim != value_array.ndim:
						SCREAMING_SNAKE_CASE_   : Optional[int]					  =      (
						    'Wrong input data\'s dimensions... '
						    F'dataset : {dataset.ndim}, value_array : {value_array.ndim}'
						)
						raise ValueError(lowerCamelCase_  )

			try:
						if dataset.shape[1] != value_array.shape[1]:
									SCREAMING_SNAKE_CASE_   : List[Any]					  =      (
									    'Wrong input data\'s shape... '
									    F'dataset : {dataset.shape[1]}, value_array : {value_array.shape[1]}'
									)
									raise ValueError(lowerCamelCase_  )
			except IndexError:
						if dataset.ndim != value_array.ndim:
									raise TypeError('Wrong shape'  )

			if dataset.dtype != value_array.dtype:
						SCREAMING_SNAKE_CASE_   : Tuple					  =      (
						    'Input data have different datatype... '
						    F'dataset : {dataset.dtype}, value_array : {value_array.dtype}'
						)
						raise TypeError(lowerCamelCase_  )

			SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      []

			for value in value_array:
						SCREAMING_SNAKE_CASE_   : str					  =      euclidean(lowerCamelCase_  ,							dataset[0]  )
						SCREAMING_SNAKE_CASE_   : List[Any]					  =      dataset[0].tolist()

						for dataset_value in dataset[1:]:
									SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      euclidean(lowerCamelCase_  ,							lowerCamelCase_  )

									if dist > temp_dist:
												SCREAMING_SNAKE_CASE_   : Optional[int]					  =      temp_dist
												SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      dataset_value.tolist()

						answer.append([vector, dist]  )

			return answer





def        __UpperCAmelCase						(							lowerCamelCase_ :     np.ndarray  ,							lowerCamelCase_ :     np.ndarray  )       ->  float:







			"""simple docstring"""





			return np.dot(lowerCamelCase_  ,							lowerCamelCase_  ) / (norm(lowerCamelCase_  ) * norm(lowerCamelCase_  ))


if __name__ == "__main__":
						import doctest

						doctest.testmod()



