



"""simple docstring"""





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
					from test_module.custom_tokenization_fast import CustomTokenizerFast



class  _lowerCAmelCase     (					unittest.TestCase		):





		"""simple docstring"""






		def  UpperCAmelCase__				(       self )			->    Dict:







								'''simple docstring'''

								snake_case_ :			Union[str, Any]            =      mock.Mock()
								snake_case_ :			Union[str, Any]            =      5_0_0
								snake_case_ :			str            =      {}
								snake_case_ :			Optional[Any]            =      HTTPError
								snake_case_ :			Any            =      {}

								# Download this model to make sure it's in the cache.
								snake_case_ :			Optional[Any]            =      BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""" )

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch("""requests.Session.request"""				,     return_value=__lowerCamelCase ) as mock_head:
														snake_case_ :			Optional[Any]            =      BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert""" )
														# This check we did call the fake head request
														mock_head.assert_called()





		@require_tokenizers
		def  UpperCAmelCase__				(       self )			->    Optional[Any]:







								'''simple docstring'''

								snake_case_ :			Optional[Any]            =      mock.Mock()
								snake_case_ :			Any            =      5_0_0
								snake_case_ :			Dict            =      {}
								snake_case_ :			int            =      HTTPError
								snake_case_ :			Union[str, Any]            =      {}

								# Download this model to make sure it's in the cache.
								snake_case_ :			Any            =      GPTaTokenizerFast.from_pretrained("""gpt2""" )

								# Under the mock environment we get a 500 error when trying to reach the tokenizer.
								with mock.patch("""requests.Session.request"""				,     return_value=__lowerCamelCase ) as mock_head:
														snake_case_ :			List[Any]            =      GPTaTokenizerFast.from_pretrained("""gpt2""" )
														# This check we did call the fake head request
														mock_head.assert_called()





		def  UpperCAmelCase__				(       self )			->    Any:







								'''simple docstring'''

								try:
														snake_case_ :			List[Any]            =      tempfile.mktemp()
														with open(__lowerCamelCase				,     """wb""" ) as f:
																				http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""				,     __lowerCamelCase )

														snake_case_ :			Optional[int]            =      AlbertTokenizer.from_pretrained(__lowerCamelCase )
								finally:
														os.remove(__lowerCamelCase )

								# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
								# the current folder and have the right name.
								if os.path.isfile("""tokenizer.json""" ):
														# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
														return
								try:
														with open("""tokenizer.json"""				,     """wb""" ) as f:
																				http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""				,     __lowerCamelCase )
														snake_case_ :			List[Any]            =      AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2""" )
														# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
														self.assertEqual(tokenizer.vocab_size				,     1_0_0_0 )
														# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

								finally:
														os.remove("""tokenizer.json""" )





		def  UpperCAmelCase__				(       self )			->    Union[str, Any]:







								'''simple docstring'''

								snake_case_ :			Tuple            =      AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model""" )




@is_staging_test
class  _lowerCAmelCase     (					unittest.TestCase		):





		"""simple docstring"""






		_lowerCamelCase      	=							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





		@classmethod
		def  UpperCAmelCase__				(       cls )			->    Any:







								'''simple docstring'''

								snake_case_ :			Dict            =      TOKEN
								HfFolder.save_token(__lowerCamelCase )





		@classmethod
		def  UpperCAmelCase__				(       cls )			->    int:







								'''simple docstring'''

								try:
														delete_repo(token=cls._token				,     repo_id="""test-tokenizer""" )
								except HTTPError:
														pass

								try:
														delete_repo(token=cls._token				,     repo_id="""valid_org/test-tokenizer-org""" )
								except HTTPError:
														pass

								try:
														delete_repo(token=cls._token				,     repo_id="""test-dynamic-tokenizer""" )
								except HTTPError:
														pass





		def  UpperCAmelCase__				(       self )			->    Optional[int]:







								'''simple docstring'''

								with tempfile.TemporaryDirectory() as tmp_dir:
														snake_case_ :			List[Any]            =      os.path.join(__lowerCamelCase				,     """vocab.txt""" )
														with open(__lowerCamelCase				,     """w"""				,     encoding="""utf-8""" ) as vocab_writer:
																				vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )
														snake_case_ :			Optional[Any]            =      BertTokenizer(__lowerCamelCase )

								tokenizer.push_to_hub("""test-tokenizer"""				,     use_auth_token=self._token )
								snake_case_ :			List[Any]            =      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer' )
								self.assertDictEqual(new_tokenizer.vocab				,     tokenizer.vocab )

								# Reset repo
								delete_repo(token=self._token				,     repo_id="""test-tokenizer""" )

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(__lowerCamelCase				,     repo_id="""test-tokenizer"""				,     push_to_hub=__lowerCamelCase				,     use_auth_token=self._token )

								snake_case_ :			str            =      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer' )
								self.assertDictEqual(new_tokenizer.vocab				,     tokenizer.vocab )





		def  UpperCAmelCase__				(       self )			->    int:







								'''simple docstring'''

								with tempfile.TemporaryDirectory() as tmp_dir:
														snake_case_ :			List[str]            =      os.path.join(__lowerCamelCase				,     """vocab.txt""" )
														with open(__lowerCamelCase				,     """w"""				,     encoding="""utf-8""" ) as vocab_writer:
																				vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )
														snake_case_ :			List[str]            =      BertTokenizer(__lowerCamelCase )

								tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""				,     use_auth_token=self._token )
								snake_case_ :			Tuple            =      BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""" )
								self.assertDictEqual(new_tokenizer.vocab				,     tokenizer.vocab )

								# Reset repo
								delete_repo(token=self._token				,     repo_id="""valid_org/test-tokenizer-org""" )

								# Push to hub via save_pretrained
								with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(
														    __lowerCamelCase				,     repo_id="""valid_org/test-tokenizer-org"""				,     push_to_hub=__lowerCamelCase				,     use_auth_token=self._token )

								snake_case_ :			Union[str, Any]            =      BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org""" )
								self.assertDictEqual(new_tokenizer.vocab				,     tokenizer.vocab )





		@require_tokenizers
		def  UpperCAmelCase__				(       self )			->    int:







								'''simple docstring'''

								CustomTokenizer.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
														snake_case_ :			Any            =      os.path.join(__lowerCamelCase				,     """vocab.txt""" )
														with open(__lowerCamelCase				,     """w"""				,     encoding="""utf-8""" ) as vocab_writer:
																				vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )
														snake_case_ :			Optional[Any]            =      CustomTokenizer(__lowerCamelCase )

								# No fast custom tokenizer
								tokenizer.push_to_hub("""test-dynamic-tokenizer"""				,     use_auth_token=self._token )

								snake_case_ :			str            =      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'				,     trust_remote_code=__lowerCamelCase )
								# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,     """CustomTokenizer""" )

								# Fast and slow custom tokenizer
								CustomTokenizerFast.register_for_auto_class()
								with tempfile.TemporaryDirectory() as tmp_dir:
														snake_case_ :			List[Any]            =      os.path.join(__lowerCamelCase				,     """vocab.txt""" )
														with open(__lowerCamelCase				,     """w"""				,     encoding="""utf-8""" ) as vocab_writer:
																				vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens] ) )

														snake_case_ :			List[Any]            =      BertTokenizerFast.from_pretrained(__lowerCamelCase )
														bert_tokenizer.save_pretrained(__lowerCamelCase )
														snake_case_ :			Optional[Any]            =      CustomTokenizerFast.from_pretrained(__lowerCamelCase )

								tokenizer.push_to_hub("""test-dynamic-tokenizer"""				,     use_auth_token=self._token )

								snake_case_ :			List[str]            =      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'				,     trust_remote_code=__lowerCamelCase )
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,     """CustomTokenizerFast""" )
								snake_case_ :			Tuple            =      AutoTokenizer.from_pretrained(
								    f'{USER}/test-dynamic-tokenizer'				,     use_fast=__lowerCamelCase				,     trust_remote_code=__lowerCamelCase )
								# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
								self.assertEqual(tokenizer.__class__.__name__				,     """CustomTokenizer""" )




class  _lowerCAmelCase     (					unittest.TestCase		):





		"""simple docstring"""






		def  UpperCAmelCase__				(       self )			->    Tuple:







								'''simple docstring'''

								snake_case_ :			Any            =      Trie()
								trie.add("""Hello 友達""" )
								self.assertEqual(trie.data				,     {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}} )
								trie.add("""Hello""" )
								trie.data
								self.assertEqual(trie.data				,     {"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}} )





		def  UpperCAmelCase__				(       self )			->    Dict:







								'''simple docstring'''

								snake_case_ :			int            =      Trie()
								self.assertEqual(trie.split("""[CLS] This is a extra_id_100""" )				,     ["""[CLS] This is a extra_id_100"""] )
								trie.add("""[CLS]""" )
								trie.add("""extra_id_1""" )
								trie.add("""extra_id_100""" )
								self.assertEqual(trie.split("""[CLS] This is a extra_id_100""" )				,     ["""[CLS]""", """ This is a """, """extra_id_100"""] )





		def  UpperCAmelCase__				(       self )			->    Union[str, Any]:







								'''simple docstring'''

								snake_case_ :			List[str]            =      Trie()
								trie.add("""A""" )
								self.assertEqual(trie.split("""ABC""" )				,     ["""A""", """BC"""] )
								self.assertEqual(trie.split("""BCA""" )				,     ["""BC""", """A"""] )





		def  UpperCAmelCase__				(       self )			->    str:







								'''simple docstring'''

								snake_case_ :			Any            =      Trie()
								trie.add("""TOKEN]""" )
								trie.add("""[SPECIAL_TOKEN]""" )
								self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""" )				,     ["""This is something """, """[SPECIAL_TOKEN]"""] )





		def  UpperCAmelCase__				(       self )			->    Any:







								'''simple docstring'''

								snake_case_ :			str            =      Trie()
								trie.add("""A""" )
								trie.add("""P""" )
								trie.add("""[SPECIAL_TOKEN]""" )
								self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]""" )				,     ["""This is something """, """[SPECIAL_TOKEN]"""] )





		def  UpperCAmelCase__				(       self )			->    Optional[Any]:







								'''simple docstring'''

								snake_case_ :			Tuple            =      Trie()
								trie.add("""AB""" )
								trie.add("""B""" )
								trie.add("""C""" )
								self.assertEqual(trie.split("""ABC""" )				,     ["""AB""", """C"""] )





		def  UpperCAmelCase__				(       self )			->    Tuple:







								'''simple docstring'''

								snake_case_ :			int            =      Trie()
								trie.add("""ABC""" )
								trie.add("""B""" )
								trie.add("""CD""" )
								self.assertEqual(trie.split("""ABCD""" )				,     ["""ABC""", """D"""] )





		def  UpperCAmelCase__				(       self )			->    int:







								'''simple docstring'''

								snake_case_ :			Union[str, Any]            =      Trie()
								snake_case_ :			Dict            =      trie.cut_text("""ABC"""				,     [0, 0, 2, 1, 2, 3] )
								self.assertEqual(__lowerCamelCase				,     ["""AB""", """C"""] )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a__: List[Any]							   =						{
    'configuration_x_clip': [
        'XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XCLIPConfig',
        'XCLIPTextConfig',
        'XCLIPVisionConfig',
    ],
    'processing_x_clip': ['XCLIPProcessor'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a__: Any							   =						[
           'XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
           'XCLIPModel',
           'XCLIPPreTrainedModel',
           'XCLIPTextModel',
           'XCLIPVisionModel',
       ]

if TYPE_CHECKING:
       from .configuration_x_clip import (
           XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           XCLIPConfig,
           XCLIPTextConfig,
           XCLIPVisionConfig,
       )
       from .processing_x_clip import XCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_x_clip import (
                  XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XCLIPModel,
                  XCLIPPreTrainedModel,
                  XCLIPTextModel,
                  XCLIPVisionModel,
              )

else:
       import sys

       a__: Optional[Any]							   =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





