



'''simple docstring'''

import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
							from test_module.custom_tokenization_fast import CustomTokenizerFast

class       __magic_name__		(       unittest.TestCase  ):







							def 				__lowercase     (				self   :			str						):
									# A mock response for an HTTP head request to emulate server down
									_a	: Optional[int]								=				mock.Mock()
									_a	: List[Any]								=				500
									_a	: Optional[int]								=				{}
									_a	: int								=				HTTPError
									_a	: Optional[Any]								=				{}

									# Download this model to make sure it's in the cache.
									_a	: Optional[int]								=				BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'						)

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('requests.Session.request'      ,return_value=_UpperCAmelCase						) as mock_head:
											_a	: Optional[Any]								=				BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'						)
											# This check we did call the fake head request
											mock_head.assert_called()







							@require_tokenizers
							def 				__lowercase     (				self   :			Dict						):
									# A mock response for an HTTP head request to emulate server down
									_a	: List[str]								=				mock.Mock()
									_a	: List[Any]								=				500
									_a	: Dict								=				{}
									_a	: List[str]								=				HTTPError
									_a	: List[str]								=				{}

									# Download this model to make sure it's in the cache.
									_a	: int								=				GPTaTokenizerFast.from_pretrained('gpt2'						)

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('requests.Session.request'      ,return_value=_UpperCAmelCase						) as mock_head:
											_a	: List[str]								=				GPTaTokenizerFast.from_pretrained('gpt2'						)
											# This check we did call the fake head request
											mock_head.assert_called()







							def 				__lowercase     (				self   :			int						):
									# This test is for deprecated behavior and can be removed in v5
									try:
											_a	: List[str]								=				tempfile.mktemp()
											with open(_UpperCAmelCase      ,'wb'						) as f:
													http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'      ,_UpperCAmelCase						)

											_a	: Tuple								=				AlbertTokenizer.from_pretrained(_UpperCAmelCase						)
									finally:
											os.remove(_UpperCAmelCase						)

									# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
									# the current folder and have the right name.
									if os.path.isfile('tokenizer.json'						):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
									try:
											with open('tokenizer.json'      ,'wb'						) as f:
													http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'      ,_UpperCAmelCase						)
											_a	: Any								=				AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'						)
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size      ,1000						)
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

									finally:
											os.remove('tokenizer.json'						)







							def 				__lowercase     (				self   :			Optional[int]						):
									# This test is for deprecated behavior and can be removed in v5
									_a	: Dict								=				AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						)

@is_staging_test
class       __magic_name__		(       unittest.TestCase  ):
							lowerCAmelCase			:       Any  							=   ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']







							@classmethod
							def 				__lowercase     (				cls   :			Optional[Any]						):
									_a	: Tuple								=				TOKEN
									HfFolder.save_token(_UpperCAmelCase						)







							@classmethod
							def 				__lowercase     (				cls   :			List[Any]						):
									try:
											delete_repo(token=cls._token      ,repo_id='test-tokenizer'						)
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token      ,repo_id='valid_org/test-tokenizer-org'						)
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token      ,repo_id='test-dynamic-tokenizer'						)
									except HTTPError:
											pass







							def 				__lowercase     (				self   :			Optional[Any]						):
									with tempfile.TemporaryDirectory() as tmp_dir:
											_a	: int								=				os.path.join(_UpperCAmelCase      ,'vocab.txt'						)
											with open(_UpperCAmelCase      ,'w'      ,encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
											_a	: Optional[int]								=				BertTokenizer(_UpperCAmelCase						)

									tokenizer.push_to_hub('test-tokenizer'      ,use_auth_token=self._token						)
									_a	: Dict								=				BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""						)
									self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab						)

									# Reset repo
									delete_repo(token=self._token      ,repo_id='test-tokenizer'						)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(_UpperCAmelCase      ,repo_id='test-tokenizer'      ,push_to_hub=_UpperCAmelCase      ,use_auth_token=self._token						)

									_a	: List[Any]								=				BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""						)
									self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab						)







							def 				__lowercase     (				self   :			Optional[int]						):
									with tempfile.TemporaryDirectory() as tmp_dir:
											_a	: Any								=				os.path.join(_UpperCAmelCase      ,'vocab.txt'						)
											with open(_UpperCAmelCase      ,'w'      ,encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
											_a	: Optional[Any]								=				BertTokenizer(_UpperCAmelCase						)

									tokenizer.push_to_hub('valid_org/test-tokenizer-org'      ,use_auth_token=self._token						)
									_a	: List[str]								=				BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'						)
									self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab						)

									# Reset repo
									delete_repo(token=self._token      ,repo_id='valid_org/test-tokenizer-org'						)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    _UpperCAmelCase      ,repo_id='valid_org/test-tokenizer-org'      ,push_to_hub=_UpperCAmelCase      ,use_auth_token=self._token						)

									_a	: Any								=				BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'						)
									self.assertDictEqual(new_tokenizer.vocab      ,tokenizer.vocab						)







							@require_tokenizers
							def 				__lowercase     (				self   :			Optional[Any]						):
									CustomTokenizer.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											_a	: Any								=				os.path.join(_UpperCAmelCase      ,'vocab.txt'						)
											with open(_UpperCAmelCase      ,'w'      ,encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)
											_a	: List[str]								=				CustomTokenizer(_UpperCAmelCase						)

									# No fast custom tokenizer
									tokenizer.push_to_hub('test-dynamic-tokenizer'      ,use_auth_token=self._token						)

									_a	: Tuple								=				AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""      ,trust_remote_code=_UpperCAmelCase						)
									# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizer'						)

									# Fast and slow custom tokenizer
									CustomTokenizerFast.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											_a	: str								=				os.path.join(_UpperCAmelCase      ,'vocab.txt'						)
											with open(_UpperCAmelCase      ,'w'      ,encoding='utf-8'						) as vocab_writer:
													vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]						)						)

											_a	: Any								=				BertTokenizerFast.from_pretrained(_UpperCAmelCase						)
											bert_tokenizer.save_pretrained(_UpperCAmelCase						)
											_a	: Optional[Any]								=				CustomTokenizerFast.from_pretrained(_UpperCAmelCase						)

									tokenizer.push_to_hub('test-dynamic-tokenizer'      ,use_auth_token=self._token						)

									_a	: List[str]								=				AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""      ,trust_remote_code=_UpperCAmelCase						)
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizerFast'						)
									_a	: Tuple								=				AutoTokenizer.from_pretrained(
									    F"""{USER}/test-dynamic-tokenizer"""      ,use_fast=_UpperCAmelCase      ,trust_remote_code=_UpperCAmelCase						)
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__      ,'CustomTokenizer'						)

class       __magic_name__		(       unittest.TestCase  ):







							def 				__lowercase     (				self   :			Tuple						):
									_a	: Tuple								=				Trie()
									trie.add('Hello 友達'						)
									self.assertEqual(trie.data      ,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}						)
									trie.add('Hello'						)
									trie.data
									self.assertEqual(trie.data      ,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}						)







							def 				__lowercase     (				self   :			Tuple						):
									_a	: Optional[Any]								=				Trie()
									self.assertEqual(trie.split('[CLS] This is a extra_id_100'						)      ,['[CLS] This is a extra_id_100']						)
									trie.add('[CLS]'						)
									trie.add('extra_id_1'						)
									trie.add('extra_id_100'						)
									self.assertEqual(trie.split('[CLS] This is a extra_id_100'						)      ,['[CLS]', ' This is a ', 'extra_id_100']						)







							def 				__lowercase     (				self   :			int						):
									_a	: Optional[Any]								=				Trie()
									trie.add('A'						)
									self.assertEqual(trie.split('ABC'						)      ,['A', 'BC']						)
									self.assertEqual(trie.split('BCA'						)      ,['BC', 'A']						)







							def 				__lowercase     (				self   :			Optional[int]						):
									_a	: str								=				Trie()
									trie.add('TOKEN]'						)
									trie.add('[SPECIAL_TOKEN]'						)
									self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'						)      ,['This is something ', '[SPECIAL_TOKEN]']						)







							def 				__lowercase     (				self   :			Optional[Any]						):
									_a	: Any								=				Trie()
									trie.add('A'						)
									trie.add('P'						)
									trie.add('[SPECIAL_TOKEN]'						)
									self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'						)      ,['This is something ', '[SPECIAL_TOKEN]']						)







							def 				__lowercase     (				self   :			str						):
									_a	: str								=				Trie()
									trie.add('AB'						)
									trie.add('B'						)
									trie.add('C'						)
									self.assertEqual(trie.split('ABC'						)      ,['AB', 'C']						)







							def 				__lowercase     (				self   :			Optional[Any]						):
									_a	: Optional[Any]								=				Trie()
									trie.add('ABC'						)
									trie.add('B'						)
									trie.add('CD'						)
									self.assertEqual(trie.split('ABCD'						)      ,['ABC', 'D']						)







							def 				__lowercase     (				self   :			Union[str, Any]						):
									# Even if the offsets are wrong, we necessarily output correct string
									# parts.
									_a	: Dict								=				Trie()
									_a	: Any								=				trie.cut_text('ABC'      ,[0, 0, 2, 1, 2, 3]						)
									self.assertEqual(_UpperCAmelCase      ,['AB', 'C']						)









'''simple docstring'''

import warnings

from ...utils import logging
from .image_processing_deit import DeiTImageProcessor


__lowerCAmelCase		     =  logging.get_logger(__name__)

class       __magic_name__		(       _UpperCamelCase  ):







							def __init__(				self   :			Tuple      ,*_UpperCAmelCase   :			str      ,**_UpperCAmelCase   :			int						):
									warnings.warn(
									    'The class DeiTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'
									    ' use DeiTImageProcessor instead.'      ,_UpperCAmelCase      ,)
									super().__init__(*_UpperCAmelCase      ,**_UpperCAmelCase						)




