



'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_albert import AlbertTokenizer
else:
  lowercase_									=    None

lowercase_									=    logging.get_logger(__name__)
lowercase_									=    {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

lowercase_									=    {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json",
    },
}

lowercase_									=    {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

lowercase_									=    "▁"






class __A     (    A		):



 '''simple docstring'''







 __lowerCamelCase       :			Optional[int]		=			VOCAB_FILES_NAMES
 __lowerCamelCase       :			Tuple		=			PRETRAINED_VOCAB_FILES_MAP
 __lowerCamelCase       :			Optional[int]		=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCamelCase       :			List[str]		=			AlbertTokenizer



 def __init__(self					,  A=None					,  A=None					,  A=True					,  A=True					,  A=False					,  A="[CLS]"					,  A="[SEP]"					,  A="<unk>"					,  A="[SEP]"					,  A="<pad>"					,  A="[CLS]"					,  A="[MASK]"					,  **A					,  )       ->	Tuple:


      """simple docstring"""



      _a           =    (
          AddedToken(A					,  lstrip=A					,  rstrip=A					,  normalized=A			)
          if isinstance(A					,  A			)
          else mask_token
      )

      super().__init__(
          A					,  tokenizer_file=A					,  do_lower_case=A					,  remove_space=A					,  keep_accents=A					,  bos_token=A					,  eos_token=A					,  unk_token=A					,  sep_token=A					,  pad_token=A					,  cls_token=A					,  mask_token=A					,  **A					,  )

      _a           =    do_lower_case
      _a           =    remove_space
      _a           =    keep_accents
      _a           =    vocab_file
      _a           =    False if not self.vocab_file else True



 def    a__    (self					,  A					,  A = None			)       ->	List[int]:


      """simple docstring"""



      _a           =    [self.sep_token_id]
      _a           =    [self.cls_token_id]
      if token_ids_a is None:
           return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep



 def    a__    (self					,  A					,  A = None			)       ->	List[int]:


      """simple docstring"""



      _a           =    [self.sep_token_id]
      _a           =    [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
      return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]



 def    a__    (self					,  A					,  A = None			)       ->	Tuple[str]:


      """simple docstring"""



      if not self.can_save_slow_tokenizer:
           raise ValueError(
               '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
               '''tokenizer.'''			)

      if not os.path.isdir(A			):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
           return
      _a           =    os.path.join(
          A					,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

      if os.path.abspath(self.vocab_file			) != os.path.abspath(A			):
           copyfile(self.vocab_file					,  A			)

      return (out_vocab_file,)










import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402





class 				a			(      unittest.TestCase			):






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[int]           =   0






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   AutoImageProcessor.from_pretrained('''openai/clip-vit-base-patch32'''					)
										self.assertIsInstance(_lowerCAmelCase			, _lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmpdirname:
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   Path(_lowerCAmelCase					) / '''preprocessor_config.json'''
																__SCREAMING_SNAKE_CASE:				List[str]           =   Path(_lowerCAmelCase					) / '''config.json'''
																json.dump(
																    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}			, open(_lowerCAmelCase			, '''w'''					)			, )
																json.dump({'''model_type''': '''clip'''}			, open(_lowerCAmelCase			, '''w'''					)					)

																__SCREAMING_SNAKE_CASE:				Dict           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase					)
																self.assertIsInstance(_lowerCAmelCase			, _lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmpdirname:
																__SCREAMING_SNAKE_CASE:				int           =   Path(_lowerCAmelCase					) / '''preprocessor_config.json'''
																__SCREAMING_SNAKE_CASE:				Tuple           =   Path(_lowerCAmelCase					) / '''config.json'''
																json.dump(
																    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}			, open(_lowerCAmelCase			, '''w'''					)			, )
																json.dump({'''model_type''': '''clip'''}			, open(_lowerCAmelCase			, '''w'''					)					)

																__SCREAMING_SNAKE_CASE:				str           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase					)
																self.assertIsInstance(_lowerCAmelCase			, _lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmpdirname:
																__SCREAMING_SNAKE_CASE:				int           =   CLIPConfig()

																# Create a dummy config file with image_proceesor_type
																__SCREAMING_SNAKE_CASE:				Any           =   Path(_lowerCAmelCase					) / '''preprocessor_config.json'''
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   Path(_lowerCAmelCase					) / '''config.json'''
																json.dump(
																    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}			, open(_lowerCAmelCase			, '''w'''					)			, )
																json.dump({'''model_type''': '''clip'''}			, open(_lowerCAmelCase			, '''w'''					)					)

																# remove image_processor_type to make sure config.json alone is enough to load image processor locally
																__SCREAMING_SNAKE_CASE:				Any           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase					).to_dict()

																config_dict.pop('''image_processor_type'''					)
																__SCREAMING_SNAKE_CASE:				List[str]           =   CLIPImageProcessor(**_lowerCAmelCase					)

																# save in new folder
																model_config.save_pretrained(_lowerCAmelCase					)
																config.save_pretrained(_lowerCAmelCase					)

																__SCREAMING_SNAKE_CASE:				Dict           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase					)

																# make sure private variable is not incorrectly saved
																__SCREAMING_SNAKE_CASE:				int           =   json.loads(config.to_json_string()					)
																self.assertTrue('''_processor_class''' not in dict_as_saved					)

										self.assertIsInstance(_lowerCAmelCase			, _lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with tempfile.TemporaryDirectory() as tmpdirname:
																__SCREAMING_SNAKE_CASE:				Optional[int]           =   Path(_lowerCAmelCase					) / '''preprocessor_config.json'''
																json.dump(
																    {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}			, open(_lowerCAmelCase			, '''w'''					)			, )

																__SCREAMING_SNAKE_CASE:				int           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase					)
																self.assertIsInstance(_lowerCAmelCase			, _lowerCAmelCase					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with self.assertRaisesRegex(
										    _lowerCAmelCase			, '''clip-base is not a local folder and is not a valid model identifier'''					):
																__SCREAMING_SNAKE_CASE:				Tuple           =   AutoImageProcessor.from_pretrained('''clip-base'''					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with self.assertRaisesRegex(
										    _lowerCAmelCase			, R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''					):
																__SCREAMING_SNAKE_CASE:				Dict           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase			, revision='''aaaaaa'''					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with self.assertRaisesRegex(
										    _lowerCAmelCase			, '''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''			, ):
																__SCREAMING_SNAKE_CASE:				int           =   AutoImageProcessor.from_pretrained('''hf-internal-testing/config-no-model'''					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										with self.assertRaises(_lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''					)
										# If remote code is disabled, we can't load this config.
										with self.assertRaises(_lowerCAmelCase					):
																__SCREAMING_SNAKE_CASE:				Any           =   AutoImageProcessor.from_pretrained(
																    '''hf-internal-testing/test_dynamic_image_processor'''			, trust_remote_code=_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   AutoImageProcessor.from_pretrained(
										    '''hf-internal-testing/test_dynamic_image_processor'''			, trust_remote_code=_lowerCAmelCase					)
										self.assertEqual(image_processor.__class__.__name__			, '''NewImageProcessor'''					)

										# Test image processor can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
																image_processor.save_pretrained(_lowerCAmelCase					)
																__SCREAMING_SNAKE_CASE:				List[str]           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase			, trust_remote_code=_lowerCAmelCase					)
										self.assertEqual(reloaded_image_processor.__class__.__name__			, '''NewImageProcessor'''					)






				def 			snake_case_    (  self					):

										"""simple docstring"""




										try:
																AutoConfig.register('''custom'''			, _lowerCAmelCase					)
																AutoImageProcessor.register(_lowerCAmelCase			, _lowerCAmelCase					)
																# Trying to register something existing in the Transformers library will raise an error
																with self.assertRaises(_lowerCAmelCase					):
																						AutoImageProcessor.register(_lowerCAmelCase			, _lowerCAmelCase					)

																with tempfile.TemporaryDirectory() as tmpdirname:
																						__SCREAMING_SNAKE_CASE:				Optional[int]           =   Path(_lowerCAmelCase					) / '''preprocessor_config.json'''
																						__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   Path(_lowerCAmelCase					) / '''config.json'''
																						json.dump(
																						    {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}			, open(_lowerCAmelCase			, '''w'''					)			, )
																						json.dump({'''model_type''': '''clip'''}			, open(_lowerCAmelCase			, '''w'''					)					)

																						__SCREAMING_SNAKE_CASE:				List[str]           =   CustomImageProcessor.from_pretrained(_lowerCAmelCase					)

																# Now that the config is registered, it can be used as any other config with the auto-API
																with tempfile.TemporaryDirectory() as tmp_dir:
																						image_processor.save_pretrained(_lowerCAmelCase					)
																						__SCREAMING_SNAKE_CASE:				int           =   AutoImageProcessor.from_pretrained(_lowerCAmelCase					)
																						self.assertIsInstance(_lowerCAmelCase			, _lowerCAmelCase					)

										finally:
																if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
																if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																						del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]






				def 			snake_case_    (  self					):

										"""simple docstring"""




										class 				a			(      __lowercase			):
														SCREAMING_SNAKE_CASE__     :   Tuple		      =     True

										try:
																AutoConfig.register('''custom'''			, _lowerCAmelCase					)
																AutoImageProcessor.register(_lowerCAmelCase			, _lowerCAmelCase					)
																# If remote code is not set, the default is to use local
																__SCREAMING_SNAKE_CASE:				str           =   AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''					)
																self.assertEqual(image_processor.__class__.__name__			, '''NewImageProcessor'''					)
																self.assertTrue(image_processor.is_local					)

																# If remote code is disabled, we load the local one.
																__SCREAMING_SNAKE_CASE:				Tuple           =   AutoImageProcessor.from_pretrained(
																    '''hf-internal-testing/test_dynamic_image_processor'''			, trust_remote_code=_lowerCAmelCase					)
																self.assertEqual(image_processor.__class__.__name__			, '''NewImageProcessor'''					)
																self.assertTrue(image_processor.is_local					)

																# If remote is enabled, we load from the Hub
																__SCREAMING_SNAKE_CASE:				Any           =   AutoImageProcessor.from_pretrained(
																    '''hf-internal-testing/test_dynamic_image_processor'''			, trust_remote_code=_lowerCAmelCase					)
																self.assertEqual(image_processor.__class__.__name__			, '''NewImageProcessor'''					)
																self.assertTrue(not hasattr(_lowerCAmelCase			, '''is_local'''					)					)

										finally:
																if "custom" in CONFIG_MAPPING._extra_content:
																						del CONFIG_MAPPING._extra_content["custom"]
																if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																						del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]



