from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


lowerCamelCase__   = logging.get_logger(__name__)

lowerCamelCase__   = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase__   = {
    '''vocab_file''': {
        '''openai/clip-vit-base-patch32''': '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''openai/clip-vit-base-patch32''': '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''openai/clip-vit-base-patch32''': (
            '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase__   = {
    '''openai/clip-vit-base-patch32''': 77,
}




class  _UpperCAmelCase       (			lowerCAmelCase   ):



 '''simple docstring'''




 __A           = VOCAB_FILES_NAMES
 __A           = PRETRAINED_VOCAB_FILES_MAP
 __A           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __A           = ['''input_ids''', '''attention_mask''']
 __A           = CLIPTokenizer







 def __init__(						self							:					List[str]				,      lowercase_							:					Optional[int]=None				,      lowercase_							:					Union[str, Any]=None				,      lowercase_							:					Optional[Any]=None				,      lowercase_							:					Optional[Any]="<|endoftext|>"				,      lowercase_							:					List[Any]="<|startoftext|>"				,      lowercase_							:					Any="<|endoftext|>"				,      lowercase_							:					str="<|endoftext|>"				,      **lowercase_							:					Any				,      )		->     List[Any]:


   """simple docstring"""







   super().__init__(
       lowercase_				,      lowercase_				,      tokenizer_file=lowercase_				,      unk_token=lowercase_				,      bos_token=lowercase_				,      eos_token=lowercase_				,      pad_token=lowercase_				,      **lowercase_				,      )

   if not isinstance(self.backend_tokenizer.pre_tokenizer				,      pre_tokenizers.Sequence):
     raise ValueError(
         "The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"
         " heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"
         " to be compatible with this version.The easiest way to do so is"
         " `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want"
         " to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"
         " transformers.")

   self._wrap_decode_method_backend_tokenizer()







 def 						__UpperCAmelCase							(						self							:					Optional[int])		->     Optional[int]:


   """simple docstring"""







   _UpperCamelCase				=  self.backend_tokenizer.decode

   def new_decode_method(*lowercase_							:					str				,      **lowercase_							:					Tuple):
     _UpperCamelCase				=  orig_decode_method(*lowercase_				,      **lowercase_)
     _UpperCamelCase				=  text.replace(self.backend_tokenizer.model.end_of_word_suffix				,      " ").strip()
     return text

   _UpperCamelCase				=  new_decode_method







 def 						__UpperCAmelCase							(						self							:					str				,      lowercase_							:					List[int]				,      lowercase_							:					Optional[List[int]] = None)		->     List[int]:


   """simple docstring"""







   _UpperCamelCase				=  [self.bos_token_id]
   _UpperCamelCase				=  [self.eos_token_id]

   if token_ids_a is None:
     return bos_token + token_ids_a + eos_token
   return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token







 def 						__UpperCAmelCase							(						self							:					str				,      lowercase_							:					List[int]				,      lowercase_							:					Optional[List[int]] = None)		->     List[int]:


   """simple docstring"""







   _UpperCamelCase				=  [self.bos_token_id]
   _UpperCamelCase				=  [self.eos_token_id]

   if token_ids_a is None:
     return len(bos_token + token_ids_a + eos_token) * [0]
   return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token) * [0]







 def 						__UpperCAmelCase							(						self							:					int				,      lowercase_							:					str				,      lowercase_							:					Optional[str] = None)		->     Tuple[str]:


   """simple docstring"""







   _UpperCamelCase				=  self._tokenizer.model.save(lowercase_				,      name=lowercase_)
   return tuple(lowercase_)





lowerCamelCase__   = '''
# Transformers installation
! pip install transformers datasets
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

lowerCamelCase__   = [{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
lowerCamelCase__   = {
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}




