






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


lowercase  :							Dict			  =    logging.get_logger(__name__)


lowercase  :							Dict			  =    {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all BART models at https://huggingface.co/models?filter=bart
lowercase  :							Optional[Any]			  =    {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json''',
    },
}

lowercase  :							Tuple			  =    {
    '''facebook/bart-base''': 10_24,
    '''facebook/bart-large''': 10_24,
    '''facebook/bart-large-mnli''': 10_24,
    '''facebook/bart-large-cnn''': 10_24,
    '''facebook/bart-large-xsum''': 10_24,
    '''yjernite/bart_eli5''': 10_24,
}




class 		UpperCAmelCase_						(	_UpperCAmelCase					):


						'''simple docstring'''




						A     :   Union[str, Any]						=					VOCAB_FILES_NAMES
						A     :   Optional[int]						=					PRETRAINED_VOCAB_FILES_MAP
						A     :   str						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						A     :   Optional[int]						=					['input_ids', 'attention_mask']
						A     :   Any						=					BartTokenizer






						def __init__(	self		,				_SCREAMING_SNAKE_CASE=None		,				_SCREAMING_SNAKE_CASE=None		,				_SCREAMING_SNAKE_CASE=None		,				_SCREAMING_SNAKE_CASE="replace"		,				_SCREAMING_SNAKE_CASE="<s>"		,				_SCREAMING_SNAKE_CASE="</s>"		,				_SCREAMING_SNAKE_CASE="</s>"		,				_SCREAMING_SNAKE_CASE="<s>"		,				_SCREAMING_SNAKE_CASE="<unk>"		,				_SCREAMING_SNAKE_CASE="<pad>"		,				_SCREAMING_SNAKE_CASE="<mask>"		,				_SCREAMING_SNAKE_CASE=False		,				_SCREAMING_SNAKE_CASE=True		,				**_SCREAMING_SNAKE_CASE		,				) ->				Any:
								super().__init__(
								    lowercase__		,				lowercase__		,				tokenizer_file=lowercase__		,				errors=lowercase__		,				bos_token=lowercase__		,				eos_token=lowercase__		,				sep_token=lowercase__		,				cls_token=lowercase__		,				unk_token=lowercase__		,				pad_token=lowercase__		,				mask_token=lowercase__		,				add_prefix_space=lowercase__		,				trim_offsets=lowercase__		,				**lowercase__		,				)

								snake_case_				:							List[Any]						 =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
								if pre_tok_state.get("add_prefix_space"		,				lowercase__				) != add_prefix_space:
										snake_case_				:							Dict						 =       getattr(lowercase__		,				pre_tok_state.pop("type"				)				)
										snake_case_				:							Optional[int]						 =       add_prefix_space
										snake_case_				:							int						 =       pre_tok_class(**lowercase__				)

								snake_case_				:							List[str]						 =       add_prefix_space

								# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
								snake_case_				:							Dict						 =       "post_processor"
								snake_case_				:							Dict						 =       getattr(self.backend_tokenizer		,				lowercase__		,				lowercase__				)
								if tokenizer_component_instance:
										snake_case_				:							Union[str, Any]						 =       json.loads(tokenizer_component_instance.__getstate__()				)

										# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
										if "sep" in state:
												snake_case_				:							List[Any]						 =       tuple(state["sep"]				)
										if "cls" in state:
												snake_case_				:							Optional[int]						 =       tuple(state["cls"]				)

										snake_case_				:							List[str]						 =       False

										if state.get("add_prefix_space"		,				lowercase__				) != add_prefix_space:
												snake_case_				:							Optional[int]						 =       add_prefix_space
												snake_case_				:							List[str]						 =       True

										if state.get("trim_offsets"		,				lowercase__				) != trim_offsets:
												snake_case_				:							List[str]						 =       trim_offsets
												snake_case_				:							Optional[Any]						 =       True

										if changes_to_apply:
												snake_case_				:							str						 =       getattr(lowercase__		,				state.pop("type"				)				)
												snake_case_				:							str						 =       component_class(**lowercase__				)
												setattr(self.backend_tokenizer		,				lowercase__		,				lowercase__				)






						@property
						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:






								if self._mask_token is None:
										if self.verbose:
												logger.error("Using mask_token, but it is not set yet."				)
										return None
								return str(self._mask_token				)






						@mask_token.setter
						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				List[str]:
								snake_case_				:							Optional[Any]						 =       AddedToken(lowercase__		,				lstrip=lowercase__		,				rstrip=lowercase__				) if isinstance(lowercase__		,				lowercase__				) else value
								snake_case_				:							List[str]						 =       value






						def       _lowerCAmelCase       (	self		,				*_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				) ->				int:
								snake_case_				:							Optional[Any]						 =       kwargs.get("is_split_into_words"		,				lowercase__				)

								if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."				)

								return super()._batch_encode_plus(*lowercase__		,				**lowercase__				)






						def       _lowerCAmelCase       (	self		,				*_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				) ->				int:
								snake_case_				:							Union[str, Any]						 =       kwargs.get("is_split_into_words"		,				lowercase__				)

								if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."				)

								return super()._encode_plus(*lowercase__		,				**lowercase__				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None				) ->				Union[str, Any]:
								snake_case_				:							Optional[int]						 =       self._tokenizer.model.save(lowercase__		,				name=lowercase__				)
								return tuple(lowercase__				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE=None				) ->				List[Any]:
								snake_case_				:							Tuple						 =       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
										return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None				) ->				List[Any]:
								snake_case_				:							int						 =       [self.sep_token_id]
								snake_case_				:							str						 =       [self.cls_token_id]

								if token_ids_a is None:
										return len(cls + token_ids_a + sep				) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]








import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase  :							int			  =    logging.get_logger(__name__)

lowercase  :							Any			  =    {
    '''BAAI/AltCLIP''': '''https://huggingface.co/BAAI/AltCLIP/resolve/main/config.json''',
    # See all AltCLIP models at https://huggingface.co/models?filter=altclip
}




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						A     :   int						=					'altclip_text_model'






						def __init__(	self		,				_SCREAMING_SNAKE_CASE=25_0002		,				_SCREAMING_SNAKE_CASE=1024		,				_SCREAMING_SNAKE_CASE=24		,				_SCREAMING_SNAKE_CASE=16		,				_SCREAMING_SNAKE_CASE=4096		,				_SCREAMING_SNAKE_CASE="gelu"		,				_SCREAMING_SNAKE_CASE=0.1		,				_SCREAMING_SNAKE_CASE=0.1		,				_SCREAMING_SNAKE_CASE=514		,				_SCREAMING_SNAKE_CASE=1		,				_SCREAMING_SNAKE_CASE=0.02		,				_SCREAMING_SNAKE_CASE=0.02		,				_SCREAMING_SNAKE_CASE=1e-05		,				_SCREAMING_SNAKE_CASE=1		,				_SCREAMING_SNAKE_CASE=0		,				_SCREAMING_SNAKE_CASE=2		,				_SCREAMING_SNAKE_CASE="absolute"		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=768		,				**_SCREAMING_SNAKE_CASE		,				) ->				List[str]:
								super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE		,				bos_token_id=_SCREAMING_SNAKE_CASE		,				eos_token_id=_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[Any]						 =       vocab_size
								snake_case_				:							List[str]						 =       hidden_size
								snake_case_				:							Dict						 =       num_hidden_layers
								snake_case_				:							List[str]						 =       num_attention_heads
								snake_case_				:							Optional[int]						 =       hidden_act
								snake_case_				:							Optional[Any]						 =       intermediate_size
								snake_case_				:							List[str]						 =       hidden_dropout_prob
								snake_case_				:							Union[str, Any]						 =       attention_probs_dropout_prob
								snake_case_				:							Optional[Any]						 =       max_position_embeddings
								snake_case_				:							Any						 =       type_vocab_size
								snake_case_				:							int						 =       initializer_range
								snake_case_				:							str						 =       initializer_factor
								snake_case_				:							List[str]						 =       layer_norm_eps
								snake_case_				:							Any						 =       position_embedding_type
								snake_case_				:							Optional[int]						 =       use_cache
								snake_case_				:							int						 =       project_dim




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						A     :   List[Any]						=					'altclip_vision_model'






						def __init__(	self		,				_SCREAMING_SNAKE_CASE=768		,				_SCREAMING_SNAKE_CASE=3072		,				_SCREAMING_SNAKE_CASE=512		,				_SCREAMING_SNAKE_CASE=12		,				_SCREAMING_SNAKE_CASE=12		,				_SCREAMING_SNAKE_CASE=3		,				_SCREAMING_SNAKE_CASE=224		,				_SCREAMING_SNAKE_CASE=32		,				_SCREAMING_SNAKE_CASE="quick_gelu"		,				_SCREAMING_SNAKE_CASE=1e-5		,				_SCREAMING_SNAKE_CASE=0.0		,				_SCREAMING_SNAKE_CASE=0.02		,				_SCREAMING_SNAKE_CASE=1.0		,				**_SCREAMING_SNAKE_CASE		,				) ->				Optional[Any]:
								super().__init__(**_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Tuple						 =       hidden_size
								snake_case_				:							Dict						 =       intermediate_size
								snake_case_				:							Optional[int]						 =       projection_dim
								snake_case_				:							int						 =       num_hidden_layers
								snake_case_				:							Union[str, Any]						 =       num_attention_heads
								snake_case_				:							int						 =       num_channels
								snake_case_				:							Optional[int]						 =       patch_size
								snake_case_				:							Dict						 =       image_size
								snake_case_				:							Tuple						 =       initializer_range
								snake_case_				:							Optional[Any]						 =       initializer_factor
								snake_case_				:							Any						 =       attention_dropout
								snake_case_				:							Tuple						 =       layer_norm_eps
								snake_case_				:							int						 =       hidden_act






						@classmethod
						def       _lowerCAmelCase       (	cls		,				_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				) ->				"PretrainedConfig":
								cls._set_token_in_kwargs(_SCREAMING_SNAKE_CASE				)

								snake_case_ ,       snake_case_				:							str						 =       cls.get_config_dict(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

								# get the vision config dict if we are loading from AltCLIPConfig
								if config_dict.get("model_type"				) == "altclip":
										snake_case_				:							List[Any]						 =       config_dict["vision_config"]

								if "model_type" in config_dict and hasattr(cls		,				"model_type"				) and config_dict["model_type"] != cls.model_type:
										logger.warning(
										    f'''You are using a model of type {config_dict['model_type']} to instantiate a model of type '''
										    f'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''				)

								return cls.from_dict(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						A     :   str						=					'altclip'
						A     :   Dict						=					True






						def __init__(	self		,				_SCREAMING_SNAKE_CASE=None		,				_SCREAMING_SNAKE_CASE=None		,				_SCREAMING_SNAKE_CASE=768		,				_SCREAMING_SNAKE_CASE=2.6592		,				**_SCREAMING_SNAKE_CASE				) ->				Optional[int]:
								# If `_config_dict` exist, we use them for the backward compatibility.
								# We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
								# of confusion!).
								snake_case_				:							Optional[Any]						 =       kwargs.pop("text_config_dict"		,				_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Dict						 =       kwargs.pop("vision_config_dict"		,				_SCREAMING_SNAKE_CASE				)

								super().__init__(**_SCREAMING_SNAKE_CASE				)

								# Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
								# `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
								# cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
								if text_config_dict is not None:
										if text_config is None:
												snake_case_				:							Any						 =       {}

										# This is the complete result when using `text_config_dict`.
										snake_case_				:							str						 =       AltCLIPTextConfig(**_SCREAMING_SNAKE_CASE				).to_dict()

										# Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
										for key, value in _text_config_dict.items():
												if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
														# If specified in `text_config_dict`
														if key in text_config_dict:
																snake_case_				:							Optional[Any]						 =       (
																    f'''`{key}` is found in both `text_config_dict` and `text_config` but with different values. '''
																    f'''The value `text_config_dict["{key}"]` will be used instead.'''
																)
														# If inferred from default argument values (just to be super careful)
														else:
																snake_case_				:							Dict						 =       (
																    f'''`text_config_dict` is provided which will be used to initialize `AltCLIPTextConfig`. The '''
																    f'''value `text_config["{key}"]` will be overriden.'''
																)
														logger.warning(_SCREAMING_SNAKE_CASE				)

            # Update all values in `text_config` with the ones in `_text_config_dict`.
										text_config.update(_text_config_dict				)

								if vision_config_dict is not None:
										if vision_config is None:
												snake_case_				:							str						 =       {}

										# This is the complete result when using `vision_config_dict`.
										snake_case_				:							Optional[Any]						 =       AltCLIPVisionConfig(**_SCREAMING_SNAKE_CASE				).to_dict()
										# convert keys to string instead of integer
										if "id2label" in _vision_config_dict:
												snake_case_				:							Any						 =       {
												    str(_SCREAMING_SNAKE_CASE				): value for key, value in _vision_config_dict["id2label"].items()
												}

										# Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
										for key, value in _vision_config_dict.items():
												if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
														# If specified in `vision_config_dict`
														if key in vision_config_dict:
																snake_case_				:							Dict						 =       (
																    f'''`{key}` is found in both `vision_config_dict` and `vision_config` but with different '''
																    f'''values. The value `vision_config_dict["{key}"]` will be used instead.'''
																)
														# If inferred from default argument values (just to be super careful)
														else:
																snake_case_				:							Optional[Any]						 =       (
																    f'''`vision_config_dict` is provided which will be used to initialize `AltCLIPVisionConfig`. '''
																    f'''The value `vision_config["{key}"]` will be overriden.'''
																)
														logger.warning(_SCREAMING_SNAKE_CASE				)

            # Update all values in `vision_config` with the ones in `_vision_config_dict`.
										vision_config.update(_vision_config_dict				)

								if text_config is None:
										snake_case_				:							Tuple						 =       {}
										logger.info("`text_config` is `None`. Initializing the `AltCLIPTextConfig` with default values."				)

								if vision_config is None:
										snake_case_				:							Optional[Any]						 =       {}
										logger.info("`vision_config` is `None`. initializing the `AltCLIPVisionConfig` with default values."				)

								snake_case_				:							List[Any]						 =       AltCLIPTextConfig(**_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Union[str, Any]						 =       AltCLIPVisionConfig(**_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[int]						 =       projection_dim
								snake_case_				:							Tuple						 =       logit_scale_init_value
								snake_case_				:							List[Any]						 =       1.0






						@classmethod
						def       _lowerCAmelCase       (	cls		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				) ->				Optional[Any]:
								return cls(text_config=text_config.to_dict()		,				vision_config=vision_config.to_dict()		,				**_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							Any						 =       copy.deepcopy(self.__dict__				)
								snake_case_				:							Union[str, Any]						 =       self.text_config.to_dict()
								snake_case_				:							List[Any]						 =       self.vision_config.to_dict()
								snake_case_				:							Tuple						 =       self.__class__.model_type
								return output
