



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__A       =						logging.get_logger(__name__)


__A       =						{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all BART models at https://huggingface.co/models?filter=bart
__A       =						{
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json''',
    },
}

__A       =						{
    '''facebook/bart-base''': 1024,
    '''facebook/bart-large''': 1024,
    '''facebook/bart-large-mnli''': 1024,
    '''facebook/bart-large-cnn''': 1024,
    '''facebook/bart-large-xsum''': 1024,
    '''yjernite/bart_eli5''': 1024,
}






class 		lowercase						(			snake_case__):






	"""simple docstring"""


	a__				: Any		       =							VOCAB_FILES_NAMES
	a__				: Dict		       =							PRETRAINED_VOCAB_FILES_MAP
	a__				: str		       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a__				: Tuple		       =							["input_ids", "attention_mask"]
	a__				: Optional[Any]		       =							BartTokenizer




	def __init__(     self     :						Tuple						,  __UpperCAmelCase     :						List[Any]=None						,  __UpperCAmelCase     :						Dict=None						,  __UpperCAmelCase     :						Union[str, Any]=None						,  __UpperCAmelCase     :						Dict="replace"						,  __UpperCAmelCase     :						int="<s>"						,  __UpperCAmelCase     :						str="</s>"						,  __UpperCAmelCase     :						List[str]="</s>"						,  __UpperCAmelCase     :						List[Any]="<s>"						,  __UpperCAmelCase     :						int="<unk>"						,  __UpperCAmelCase     :						str="<pad>"						,  __UpperCAmelCase     :						str="<mask>"						,  __UpperCAmelCase     :						str=False						,  __UpperCAmelCase     :						Dict=True						,  **__UpperCAmelCase     :						List[str]						,  )      ->			Optional[int]:
							super().__init__(
							    __UpperCAmelCase						,  __UpperCAmelCase						,  tokenizer_file=__UpperCAmelCase						,  errors=__UpperCAmelCase						,  bos_token=__UpperCAmelCase						,  eos_token=__UpperCAmelCase						,  sep_token=__UpperCAmelCase						,  cls_token=__UpperCAmelCase						,  unk_token=__UpperCAmelCase						,  pad_token=__UpperCAmelCase						,  mask_token=__UpperCAmelCase						,  add_prefix_space=__UpperCAmelCase						,  trim_offsets=__UpperCAmelCase						,  **__UpperCAmelCase						,  )

							UpperCAmelCase_=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
							if pre_tok_state.get("""add_prefix_space"""						,  __UpperCAmelCase					) != add_prefix_space:
													UpperCAmelCase_=       getattr(__UpperCAmelCase						,  pre_tok_state.pop("""type"""					)					)
													UpperCAmelCase_=       add_prefix_space
													UpperCAmelCase_=       pre_tok_class(**__UpperCAmelCase					)

							UpperCAmelCase_=       add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							UpperCAmelCase_=       """post_processor"""
							UpperCAmelCase_=       getattr(self.backend_tokenizer						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							if tokenizer_component_instance:
													UpperCAmelCase_=       json.loads(tokenizer_component_instance.__getstate__()					)

													# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
													if "sep" in state:
																			UpperCAmelCase_=       tuple(state["""sep"""]					)
													if "cls" in state:
																			UpperCAmelCase_=       tuple(state["""cls"""]					)

													UpperCAmelCase_=       False

													if state.get("""add_prefix_space"""						,  __UpperCAmelCase					) != add_prefix_space:
																			UpperCAmelCase_=       add_prefix_space
																			UpperCAmelCase_=       True

													if state.get("""trim_offsets"""						,  __UpperCAmelCase					) != trim_offsets:
																			UpperCAmelCase_=       trim_offsets
																			UpperCAmelCase_=       True

													if changes_to_apply:
																			UpperCAmelCase_=       getattr(__UpperCAmelCase						,  state.pop("""type"""					)					)
																			UpperCAmelCase_=       component_class(**__UpperCAmelCase					)
																			setattr(self.backend_tokenizer						,  __UpperCAmelCase						,  __UpperCAmelCase					)




	@property
	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			str:
							if self._mask_token is None:
													if self.verbose:
																			logger.error("""Using mask_token, but it is not set yet."""					)
													return None
							return str(self._mask_token					)




	@mask_token.setter
	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]						,  __UpperCAmelCase     :						Optional[Any]					)      ->			Tuple:
							UpperCAmelCase_=       AddedToken(__UpperCAmelCase						,  lstrip=__UpperCAmelCase						,  rstrip=__UpperCAmelCase					) if isinstance(__UpperCAmelCase						,  __UpperCAmelCase					) else value
							UpperCAmelCase_=       value




	def   _SCREAMING_SNAKE_CASE		(     self     :						Tuple						,  *__UpperCAmelCase     :						Any						,  **__UpperCAmelCase     :						Tuple					)      ->			BatchEncoding:
							UpperCAmelCase_=       kwargs.get("""is_split_into_words"""						,  __UpperCAmelCase					)

							if is_split_into_words and not self.add_prefix_space:
													raise ValueError(
													    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    """to use it with pretokenized inputs."""					)

							return super()._batch_encode_plus(*__UpperCAmelCase						,  **__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Any						,  *__UpperCAmelCase     :						Union[str, Any]						,  **__UpperCAmelCase     :						str					)      ->			BatchEncoding:
							UpperCAmelCase_=       kwargs.get("""is_split_into_words"""						,  __UpperCAmelCase					)

							if is_split_into_words and not self.add_prefix_space:
													raise ValueError(
													    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    """to use it with pretokenized inputs."""					)

							return super()._encode_plus(*__UpperCAmelCase						,  **__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]						,  __UpperCAmelCase     :						str						,  __UpperCAmelCase     :						Optional[str] = None					)      ->			Tuple[str]:
							UpperCAmelCase_=       self._tokenizer.model.save(__UpperCAmelCase						,  name=__UpperCAmelCase					)
							return tuple(__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						str						,  __UpperCAmelCase     :						int						,  __UpperCAmelCase     :						List[Any]=None					)      ->			int:
							UpperCAmelCase_=       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
													return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]						,  __UpperCAmelCase     :						List[int]						,  __UpperCAmelCase     :						Optional[List[int]] = None					)      ->			List[int]:
							UpperCAmelCase_=       [self.sep_token_id]
							UpperCAmelCase_=       [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


__A       =						{
    '''configuration_clip''': [
        '''CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''CLIPConfig''',
        '''CLIPOnnxConfig''',
        '''CLIPTextConfig''',
        '''CLIPVisionConfig''',
    ],
    '''processing_clip''': ['''CLIPProcessor'''],
    '''tokenization_clip''': ['''CLIPTokenizer'''],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A       =						['''CLIPTokenizerFast''']

try:
	if not is_vision_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A       =						['''CLIPFeatureExtractor''']
	__A       =						['''CLIPImageProcessor''']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A       =						[
	    '''CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''CLIPModel''',
	    '''CLIPPreTrainedModel''',
	    '''CLIPTextModel''',
	    '''CLIPTextModelWithProjection''',
	    '''CLIPVisionModel''',
	    '''CLIPVisionModelWithProjection''',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A       =						[
	    '''TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''TFCLIPModel''',
	    '''TFCLIPPreTrainedModel''',
	    '''TFCLIPTextModel''',
	    '''TFCLIPVisionModel''',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A       =						[
	    '''FlaxCLIPModel''',
	    '''FlaxCLIPPreTrainedModel''',
	    '''FlaxCLIPTextModel''',
	    '''FlaxCLIPTextPreTrainedModel''',
	    '''FlaxCLIPVisionModel''',
	    '''FlaxCLIPVisionPreTrainedModel''',
	]


if TYPE_CHECKING:
	from .configuration_clip import (
	    CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    CLIPConfig,
	    CLIPOnnxConfig,
	    CLIPTextConfig,
	    CLIPVisionConfig,
	)
	from .processing_clip import CLIPProcessor
	from .tokenization_clip import CLIPTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_clip_fast import CLIPTokenizerFast

	try:
		if not is_vision_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .feature_extraction_clip import CLIPFeatureExtractor
		from .image_processing_clip import CLIPImageProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_clip import (
		    CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    CLIPModel,
		    CLIPPreTrainedModel,
		    CLIPTextModel,
		    CLIPTextModelWithProjection,
		    CLIPVisionModel,
		    CLIPVisionModelWithProjection,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_clip import (
		    TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFCLIPModel,
		    TFCLIPPreTrainedModel,
		    TFCLIPTextModel,
		    TFCLIPVisionModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_clip import (
		    FlaxCLIPModel,
		    FlaxCLIPPreTrainedModel,
		    FlaxCLIPTextModel,
		    FlaxCLIPTextPreTrainedModel,
		    FlaxCLIPVisionModel,
		    FlaxCLIPVisionPreTrainedModel,
		)


else:
	import sys

	__A       =						_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




