



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase						=       logging.get_logger(__name__)
__lowerCamelCase						=       {"""vocab_file""": """spiece.model"""}

__lowerCamelCase						=       {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

__lowerCamelCase						=       {
    """albert-base-v1""": 5_1_2,
    """albert-large-v1""": 5_1_2,
    """albert-xlarge-v1""": 5_1_2,
    """albert-xxlarge-v1""": 5_1_2,
    """albert-base-v2""": 5_1_2,
    """albert-large-v2""": 5_1_2,
    """albert-xlarge-v2""": 5_1_2,
    """albert-xxlarge-v2""": 5_1_2,
}

__lowerCamelCase						=       """▁"""



class 		_snake_case   (      _snake_case					):





 """simple docstring"""



 lowerCamelCase_        =      VOCAB_FILES_NAMES
 lowerCamelCase_        =      PRETRAINED_VOCAB_FILES_MAP
 lowerCamelCase_        =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 def __init__(    self						,     a						,     a=True						,     a=True						,     a=False						,     a="[CLS]"						,     a="[SEP]"						,     a="<unk>"						,     a="[SEP]"						,     a="<pad>"						,     a="[CLS]"						,     a="[MASK]"						,     a = None						,     **a						,     )						->				Optional[int]:




   """simple docstring"""







   _A							     =     (
       AddedToken(a						,     lstrip=a						,     rstrip=a						,     normalized=a       )
       if isinstance(a						,     a       )
       else mask_token
   )

   _A							     =     {} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=a						,     remove_space=a						,     keep_accents=a						,     bos_token=a						,     eos_token=a						,     unk_token=a						,     sep_token=a						,     pad_token=a						,     cls_token=a						,     mask_token=a						,     sp_model_kwargs=self.sp_model_kwargs						,     **a						,     )

   _A							     =     do_lower_case
   _A							     =     remove_space
   _A							     =     keep_accents
   _A							     =     vocab_file

   _A							     =     spm.SentencePieceProcessor(**self.sp_model_kwargs       )
   self.sp_model.Load(a       )
 @property
 def lowercase_		(    self       )						->				int:




   """simple docstring"""







   return len(self.sp_model       )
 def lowercase_		(    self       )						->				int:




   """simple docstring"""







   _A							     =     {self.convert_ids_to_tokens(a       ): i for i in range(self.vocab_size       )}
   vocab.update(self.added_tokens_encoder       )
   return vocab
 def __getstate__(    self       )						->				str:




   """simple docstring"""







   _A							     =     self.__dict__.copy()
   _A							     =     None
   return state
 def __setstate__(    self						,     a       )						->				List[str]:




   """simple docstring"""







   _A							     =     d

   # for backward compatibility
   if not hasattr(self						,     '''sp_model_kwargs'''       ):
     _A							     =     {}

   _A							     =     spm.SentencePieceProcessor(**self.sp_model_kwargs       )
   self.sp_model.Load(self.vocab_file       )
 def lowercase_		(    self						,     a       )						->				Optional[int]:




   """simple docstring"""







   if self.remove_space:
     _A							     =     ''' '''.join(inputs.strip().split()       )
   else:
     _A							     =     inputs
   _A							     =     outputs.replace('''``'''						,     '''\"'''       ).replace('''\'\''''						,     '''\"'''       )

   if not self.keep_accents:
     _A							     =     unicodedata.normalize('''NFKD'''						,     a       )
     _A							     =     ''''''.join([c for c in outputs if not unicodedata.combining(a       )]       )
   if self.do_lower_case:
     _A							     =     outputs.lower()

   return outputs
 def lowercase_		(    self						,     a       )						->				List[Any]:




   """simple docstring"""







   _A							     =     self.preprocess_text(a       )
   _A							     =     self.sp_model.encode(a						,     out_type=a       )
   _A							     =     []
   for piece in pieces:
     if len(a       ) > 1 and piece[-1] == str(''','''       ) and piece[-2].isdigit():
       _A							     =     self.sp_model.EncodeAsPieces(piece[:-1].replace(a						,     ''''''       )       )
       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
         if len(cur_pieces[0]       ) == 1:
           _A							     =     cur_pieces[1:]
         else:
           _A							     =     cur_pieces[0][1:]
       cur_pieces.append(piece[-1]       )
       new_pieces.extend(a       )
     else:
       new_pieces.append(a       )

   return new_pieces
 def lowercase_		(    self						,     a       )						->				Optional[Any]:




   """simple docstring"""







   return self.sp_model.PieceToId(a       )
 def lowercase_		(    self						,     a       )						->				List[Any]:




   """simple docstring"""







   return self.sp_model.IdToPiece(a       )
 def lowercase_		(    self						,     a       )						->				str:




   """simple docstring"""







   _A							     =     []
   _A							     =     ''''''
   _A							     =     False
   for token in tokens:
     # make sure that special tokens are not decoded using sentencepiece model
     if token in self.all_special_tokens:
       if not prev_is_special:
         out_string += " "
       out_string += self.sp_model.decode(a       ) + token
       _A							     =     True
       _A							     =     []
     else:
       current_sub_tokens.append(a       )
       _A							     =     False
   out_string += self.sp_model.decode(a       )
   return out_string.strip()
 def lowercase_		(    self						,     a						,     a = None       )						->				str:




   """simple docstring"""







   _A							     =     [self.sep_token_id]
   _A							     =     [self.cls_token_id]
   if token_ids_a is None:
     return cls + token_ids_a + sep
   return cls + token_ids_a + sep + token_ids_a + sep
 def lowercase_		(    self						,     a						,     a = None						,     a = False       )						->				Optional[Any]:




   """simple docstring"""







   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=a						,     token_ids_a=a						,     already_has_special_tokens=a       )

   if token_ids_a is not None:
     return [1] + ([0] * len(a       )) + [1] + ([0] * len(a       )) + [1]
   return [1] + ([0] * len(a       )) + [1]
 def lowercase_		(    self						,     a						,     a = None       )						->				Union[str, Any]:




   """simple docstring"""







   _A							     =     [self.sep_token_id]
   _A							     =     [self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep       ) * [0]
   return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]



 def lowercase_		(    self						,     a						,     a = None       )						->				Optional[int]:




   """simple docstring"""







   if not os.path.isdir(a       ):
     logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
     return
   _A							     =     os.path.join(
       a						,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

   if os.path.abspath(self.vocab_file       ) != os.path.abspath(a       ) and os.path.isfile(self.vocab_file       ):
     copyfile(self.vocab_file						,     a       )
   elif not os.path.isfile(self.vocab_file       ):
     with open(a						,     '''wb'''       ) as fi:
       _A							     =     self.sp_model.serialized_model_proto()
       fi.write(a       )

   return (out_vocab_file,)






'''simple docstring'''

import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402







class __a			( unittest.TestCase      ):





   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							0





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained("""openai/clip-vit-base-patch32"""     )
      self.assertIsInstance(lowerCamelCase					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmpdirname:
         __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """preprocessor_config.json"""
         __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """config.json"""
         json.dump(
             {"""image_processor_type""": """CLIPImageProcessor""", """processor_class""": """CLIPProcessor"""}					,open(lowerCamelCase					,"""w"""     )					,)
         json.dump({"""model_type""": """clip"""}					,open(lowerCamelCase					,"""w"""     )     )

         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase     )
         self.assertIsInstance(lowerCamelCase					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmpdirname:
         __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """preprocessor_config.json"""
         __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """config.json"""
         json.dump(
             {"""feature_extractor_type""": """CLIPFeatureExtractor""", """processor_class""": """CLIPProcessor"""}					,open(lowerCamelCase					,"""w"""     )					,)
         json.dump({"""model_type""": """clip"""}					,open(lowerCamelCase					,"""w"""     )     )

         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase     )
         self.assertIsInstance(lowerCamelCase					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmpdirname:
         __SCREAMING_SNAKE_CASE   				=							CLIPConfig()

         # Create a dummy config file with image_proceesor_type
         __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """preprocessor_config.json"""
         __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """config.json"""
         json.dump(
             {"""image_processor_type""": """CLIPImageProcessor""", """processor_class""": """CLIPProcessor"""}					,open(lowerCamelCase					,"""w"""     )					,)
         json.dump({"""model_type""": """clip"""}					,open(lowerCamelCase					,"""w"""     )     )

         # remove image_processor_type to make sure config.json alone is enough to load image processor locally
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase     ).to_dict()

         config_dict.pop("""image_processor_type"""     )
         __SCREAMING_SNAKE_CASE   				=							CLIPImageProcessor(**lowerCamelCase     )

         # save in new folder
         model_config.save_pretrained(lowerCamelCase     )
         config.save_pretrained(lowerCamelCase     )

         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase     )

         # make sure private variable is not incorrectly saved
         __SCREAMING_SNAKE_CASE   				=							json.loads(config.to_json_string()     )
         self.assertTrue("""_processor_class""" not in dict_as_saved     )

      self.assertIsInstance(lowerCamelCase					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Union[str, Any]     ):

      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmpdirname:
         __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """preprocessor_config.json"""
         json.dump(
             {"""image_processor_type""": """CLIPImageProcessor""", """processor_class""": """CLIPProcessor"""}					,open(lowerCamelCase					,"""w"""     )					,)

         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase     )
         self.assertIsInstance(lowerCamelCase					,lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Dict     ):

      '''simple docstring'''

      with self.assertRaisesRegex(
          lowerCamelCase					,"""clip-base is not a local folder and is not a valid model identifier"""     ):
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained("""clip-base"""     )





   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      with self.assertRaisesRegex(
          lowerCamelCase					,r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""     ):
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase					,revision="""aaaaaa"""     )





   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      with self.assertRaisesRegex(
          lowerCamelCase					,"""hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."""					,):
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained("""hf-internal-testing/config-no-model"""     )





   def   UpperCAmelCase__  (       self					:      Any     ):

      '''simple docstring'''

      with self.assertRaises(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained("""hf-internal-testing/test_dynamic_image_processor"""     )
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(
             """hf-internal-testing/test_dynamic_image_processor"""					,trust_remote_code=lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(
          """hf-internal-testing/test_dynamic_image_processor"""					,trust_remote_code=lowerCamelCase     )
      self.assertEqual(image_processor.__class__.__name__					,"""NewImageProcessor"""     )

      # Test image processor can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
         image_processor.save_pretrained(lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase					,trust_remote_code=lowerCamelCase     )
      self.assertEqual(reloaded_image_processor.__class__.__name__					,"""NewImageProcessor"""     )





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      try:
         AutoConfig.register("""custom"""					,lowerCamelCase     )
         AutoImageProcessor.register(lowerCamelCase					,lowerCamelCase     )
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(lowerCamelCase     ):
            AutoImageProcessor.register(lowerCamelCase					,lowerCamelCase     )

         with tempfile.TemporaryDirectory() as tmpdirname:
            __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """preprocessor_config.json"""
            __SCREAMING_SNAKE_CASE   				=							Path(lowerCamelCase     ) / """config.json"""
            json.dump(
                {"""feature_extractor_type""": """CLIPFeatureExtractor""", """processor_class""": """CLIPProcessor"""}					,open(lowerCamelCase					,"""w"""     )					,)
            json.dump({"""model_type""": """clip"""}					,open(lowerCamelCase					,"""w"""     )     )

            __SCREAMING_SNAKE_CASE   				=							CustomImageProcessor.from_pretrained(lowerCamelCase     )

         # Now that the config is registered, it can be used as any other config with the auto-API
         with tempfile.TemporaryDirectory() as tmp_dir:
            image_processor.save_pretrained(lowerCamelCase     )
            __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(lowerCamelCase     )
            self.assertIsInstance(lowerCamelCase					,lowerCamelCase     )

      finally:
         if "custom" in CONFIG_MAPPING._extra_content:
            del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
            del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      class __a			( _snake_case      ):
         __UpperCamelCase :			int						   =						True

      try:
         AutoConfig.register("""custom"""					,lowerCamelCase     )
         AutoImageProcessor.register(lowerCamelCase					,lowerCamelCase     )
         # If remote code is not set, the default is to use local
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained("""hf-internal-testing/test_dynamic_image_processor"""     )
         self.assertEqual(image_processor.__class__.__name__					,"""NewImageProcessor"""     )
         self.assertTrue(image_processor.is_local     )

         # If remote code is disabled, we load the local one.
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(
             """hf-internal-testing/test_dynamic_image_processor"""					,trust_remote_code=lowerCamelCase     )
         self.assertEqual(image_processor.__class__.__name__					,"""NewImageProcessor"""     )
         self.assertTrue(image_processor.is_local     )

         # If remote is enabled, we load from the Hub
         __SCREAMING_SNAKE_CASE   				=							AutoImageProcessor.from_pretrained(
             """hf-internal-testing/test_dynamic_image_processor"""					,trust_remote_code=lowerCamelCase     )
         self.assertEqual(image_processor.__class__.__name__					,"""NewImageProcessor"""     )
         self.assertTrue(not hasattr(lowerCamelCase					,"""is_local"""     )     )

      finally:
         if "custom" in CONFIG_MAPPING._extra_content:
            del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
            del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]





