

import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast


class SCREAMING_SNAKE_CASE	(		unittest.TestCase				):






			"""simple docstring"""




			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			mock.Mock()
				snake_case_   		=			5_00
				snake_case_   		=			{}
				snake_case_   		=			HTTPError
				snake_case_   		=			{}

				# Download this model to make sure it's in the cache.
				snake_case_   		=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'				)

				# Under the mock environment we get a 500 error when trying to reach the tokenizer.
				with mock.patch('requests.Session.request'			,					return_value=__UpperCamelCase				) as mock_head:
					snake_case_   		=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'				)
					# This check we did call the fake head request
					mock_head.assert_called()


			@require_tokenizers
			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			mock.Mock()
				snake_case_   		=			5_00
				snake_case_   		=			{}
				snake_case_   		=			HTTPError
				snake_case_   		=			{}

				# Download this model to make sure it's in the cache.
				snake_case_   		=			GPTaTokenizerFast.from_pretrained('gpt2'				)

				# Under the mock environment we get a 500 error when trying to reach the tokenizer.
				with mock.patch('requests.Session.request'			,					return_value=__UpperCamelCase				) as mock_head:
					snake_case_   		=			GPTaTokenizerFast.from_pretrained('gpt2'				)
					# This check we did call the fake head request
					mock_head.assert_called()


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				try:
					snake_case_   		=			tempfile.mktemp()
					with open(__UpperCamelCase			,					'wb'				) as f:
						http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'			,					__UpperCamelCase				)

					snake_case_   		=			AlbertTokenizer.from_pretrained(__UpperCamelCase				)
				finally:
					os.remove(__UpperCamelCase				)

				# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
				# the current folder and have the right name.
				if os.path.isfile('tokenizer.json'				):
					# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
					return
				try:
					with open('tokenizer.json'			,					'wb'				) as f:
						http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'			,					__UpperCamelCase				)
					snake_case_   		=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'				)
					# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
					self.assertEqual(tokenizer.vocab_size			,					10_00				)
					# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

				finally:
					os.remove('tokenizer.json'				)







			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'				)







@is_staging_test
class SCREAMING_SNAKE_CASE	(		unittest.TestCase				):






			"""simple docstring"""




			__A    	=	["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]


			@classmethod
			def       __lowerCAmelCase			(     cls				):

				"""simple docstring"""


				snake_case_   		=			TOKEN
				HfFolder.save_token(__UpperCamelCase				)


			@classmethod
			def       __lowerCAmelCase			(     cls				):

				"""simple docstring"""


				try:
					delete_repo(token=cls._token			,					repo_id='test-tokenizer'				)
				except HTTPError:
					pass

				try:
					delete_repo(token=cls._token			,					repo_id='valid_org/test-tokenizer-org'				)
				except HTTPError:
					pass

				try:
					delete_repo(token=cls._token			,					repo_id='test-dynamic-tokenizer'				)
				except HTTPError:
					pass


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				with tempfile.TemporaryDirectory() as tmp_dir:
					snake_case_   		=			os.path.join(__UpperCamelCase			,					'vocab.txt'				)
					with open(__UpperCamelCase			,					'w'			,					encoding='utf-8'				) as vocab_writer:
						vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
					snake_case_   		=			BertTokenizer(__UpperCamelCase				)

				tokenizer.push_to_hub('test-tokenizer'			,					use_auth_token=self._token				)
				snake_case_   		=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""				)
				self.assertDictEqual(new_tokenizer.vocab			,					tokenizer.vocab				)

				# Reset repo
				delete_repo(token=self._token			,					repo_id='test-tokenizer'				)

				# Push to hub via save_pretrained
				with tempfile.TemporaryDirectory() as tmp_dir:
					tokenizer.save_pretrained(__UpperCamelCase			,					repo_id='test-tokenizer'			,					push_to_hub=__UpperCamelCase			,					use_auth_token=self._token				)

				snake_case_   		=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""				)
				self.assertDictEqual(new_tokenizer.vocab			,					tokenizer.vocab				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				with tempfile.TemporaryDirectory() as tmp_dir:
					snake_case_   		=			os.path.join(__UpperCamelCase			,					'vocab.txt'				)
					with open(__UpperCamelCase			,					'w'			,					encoding='utf-8'				) as vocab_writer:
						vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
					snake_case_   		=			BertTokenizer(__UpperCamelCase				)

				tokenizer.push_to_hub('valid_org/test-tokenizer-org'			,					use_auth_token=self._token				)
				snake_case_   		=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'				)
				self.assertDictEqual(new_tokenizer.vocab			,					tokenizer.vocab				)

				# Reset repo
				delete_repo(token=self._token			,					repo_id='valid_org/test-tokenizer-org'				)

				# Push to hub via save_pretrained
				with tempfile.TemporaryDirectory() as tmp_dir:
					tokenizer.save_pretrained(
					    __UpperCamelCase			,					repo_id='valid_org/test-tokenizer-org'			,					push_to_hub=__UpperCamelCase			,					use_auth_token=self._token				)

				snake_case_   		=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'				)
				self.assertDictEqual(new_tokenizer.vocab			,					tokenizer.vocab				)







			@require_tokenizers
			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				CustomTokenizer.register_for_auto_class()
				with tempfile.TemporaryDirectory() as tmp_dir:
					snake_case_   		=			os.path.join(__UpperCamelCase			,					'vocab.txt'				)
					with open(__UpperCamelCase			,					'w'			,					encoding='utf-8'				) as vocab_writer:
						vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)
					snake_case_   		=			CustomTokenizer(__UpperCamelCase				)

				# No fast custom tokenizer
				tokenizer.push_to_hub('test-dynamic-tokenizer'			,					use_auth_token=self._token				)

				snake_case_   		=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""			,					trust_remote_code=__UpperCamelCase				)
				# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__			,					'CustomTokenizer'				)

				# Fast and slow custom tokenizer
				CustomTokenizerFast.register_for_auto_class()
				with tempfile.TemporaryDirectory() as tmp_dir:
					snake_case_   		=			os.path.join(__UpperCamelCase			,					'vocab.txt'				)
					with open(__UpperCamelCase			,					'w'			,					encoding='utf-8'				) as vocab_writer:
						vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]				)				)

					snake_case_   		=			BertTokenizerFast.from_pretrained(__UpperCamelCase				)
					bert_tokenizer.save_pretrained(__UpperCamelCase				)
					snake_case_   		=			CustomTokenizerFast.from_pretrained(__UpperCamelCase				)

				tokenizer.push_to_hub('test-dynamic-tokenizer'			,					use_auth_token=self._token				)

				snake_case_   		=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""			,					trust_remote_code=__UpperCamelCase				)
				# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__			,					'CustomTokenizerFast'				)
				snake_case_   		=			AutoTokenizer.from_pretrained(
				    f"""{USER}/test-dynamic-tokenizer"""			,					use_fast=__UpperCamelCase			,					trust_remote_code=__UpperCamelCase				)
				# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
				self.assertEqual(tokenizer.__class__.__name__			,					'CustomTokenizer'				)







class SCREAMING_SNAKE_CASE	(		unittest.TestCase				):






			"""simple docstring"""




			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				trie.add('Hello 友達'				)
				self.assertEqual(trie.data			,					{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}				)
				trie.add('Hello'				)
				trie.data
				self.assertEqual(trie.data			,					{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				self.assertEqual(trie.split('[CLS] This is a extra_id_100'				)			,					['[CLS] This is a extra_id_100']				)
				trie.add('[CLS]'				)
				trie.add('extra_id_1'				)
				trie.add('extra_id_100'				)
				self.assertEqual(trie.split('[CLS] This is a extra_id_100'				)			,					['[CLS]', ' This is a ', 'extra_id_100']				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				trie.add('A'				)
				self.assertEqual(trie.split('ABC'				)			,					['A', 'BC']				)
				self.assertEqual(trie.split('BCA'				)			,					['BC', 'A']				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				trie.add('TOKEN]'				)
				trie.add('[SPECIAL_TOKEN]'				)
				self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'				)			,					['This is something ', '[SPECIAL_TOKEN]']				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				trie.add('A'				)
				trie.add('P'				)
				trie.add('[SPECIAL_TOKEN]'				)
				self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'				)			,					['This is something ', '[SPECIAL_TOKEN]']				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				trie.add('AB'				)
				trie.add('B'				)
				trie.add('C'				)
				self.assertEqual(trie.split('ABC'				)			,					['AB', 'C']				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				trie.add('ABC'				)
				trie.add('B'				)
				trie.add('CD'				)
				self.assertEqual(trie.split('ABCD'				)			,					['ABC', 'D']				)







			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			Trie()
				snake_case_   		=			trie.cut_text('ABC'			,					[0, 0, 2, 1, 2, 3]				)
				self.assertEqual(__UpperCamelCase			,					['AB', 'C']				)



from ..utils import DummyObject, requires_backends


class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)
