






"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


A__			:							Optional[Any]					   =   logging.get_logger(__name__)


A__			:							Optional[int]					   =   {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
A__			:							str					   =   {
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

A__			:							List[Any]					   =   {
    'facebook/bart-base': 1_024,
    'facebook/bart-large': 1_024,
    'facebook/bart-large-mnli': 1_024,
    'facebook/bart-large-cnn': 1_024,
    'facebook/bart-large-xsum': 1_024,
    'yjernite/bart_eli5': 1_024,
}







class   lowercase__     (				__SCREAMING_SNAKE_CASE  ):
			_UpperCAmelCase						:int       =					VOCAB_FILES_NAMES
			_UpperCAmelCase						:int       =					PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase						:Optional[int]       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase						:str       =					["input_ids", "attention_mask"]
			_UpperCAmelCase						:Optional[int]       =					BartTokenizer


			def __init__( self	:    Union[str, Any]  ,			snake_case__	:    List[str]=None  ,			snake_case__	:    Dict=None  ,			snake_case__	:    int=None  ,			snake_case__	:    int="replace"  ,			snake_case__	:    Any="<s>"  ,			snake_case__	:    Dict="</s>"  ,			snake_case__	:    Optional[int]="</s>"  ,			snake_case__	:    Tuple="<s>"  ,			snake_case__	:    int="<unk>"  ,			snake_case__	:    Dict="<pad>"  ,			snake_case__	:    Optional[Any]="<mask>"  ,			snake_case__	:    Any=False  ,			snake_case__	:    int=True  ,			**snake_case__	:    int  ,			):
										super().__init__(
										    _SCREAMING_SNAKE_CASE  ,			_SCREAMING_SNAKE_CASE  ,			tokenizer_file=_SCREAMING_SNAKE_CASE  ,			errors=_SCREAMING_SNAKE_CASE  ,			bos_token=_SCREAMING_SNAKE_CASE  ,			eos_token=_SCREAMING_SNAKE_CASE  ,			sep_token=_SCREAMING_SNAKE_CASE  ,			cls_token=_SCREAMING_SNAKE_CASE  ,			unk_token=_SCREAMING_SNAKE_CASE  ,			pad_token=_SCREAMING_SNAKE_CASE  ,			mask_token=_SCREAMING_SNAKE_CASE  ,			add_prefix_space=_SCREAMING_SNAKE_CASE  ,			trim_offsets=_SCREAMING_SNAKE_CASE  ,			**_SCREAMING_SNAKE_CASE  ,			)

										lowerCamelCase_			:				Optional[int]										=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
										if pre_tok_state.get("add_prefix_space"  ,			_SCREAMING_SNAKE_CASE   ) != add_prefix_space:
																	lowerCamelCase_			:				str										=getattr(_SCREAMING_SNAKE_CASE  ,			pre_tok_state.pop("type"   )   )
																	lowerCamelCase_			:				Optional[Any]										=add_prefix_space
																	lowerCamelCase_			:				List[str]										=pre_tok_class(**_SCREAMING_SNAKE_CASE   )

										lowerCamelCase_			:				str										=add_prefix_space

										# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
										lowerCamelCase_			:				Dict										="post_processor"
										lowerCamelCase_			:				Optional[int]										=getattr(self.backend_tokenizer  ,			_SCREAMING_SNAKE_CASE  ,			_SCREAMING_SNAKE_CASE   )
										if tokenizer_component_instance:
																	lowerCamelCase_			:				Optional[int]										=json.loads(tokenizer_component_instance.__getstate__()   )

																	# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																	if "sep" in state:
																								lowerCamelCase_			:				Tuple										=tuple(state["sep"]   )
																	if "cls" in state:
																								lowerCamelCase_			:				Optional[Any]										=tuple(state["cls"]   )

																	lowerCamelCase_			:				Any										=False

																	if state.get("add_prefix_space"  ,			_SCREAMING_SNAKE_CASE   ) != add_prefix_space:
																								lowerCamelCase_			:				Tuple										=add_prefix_space
																								lowerCamelCase_			:				Union[str, Any]										=True

																	if state.get("trim_offsets"  ,			_SCREAMING_SNAKE_CASE   ) != trim_offsets:
																								lowerCamelCase_			:				int										=trim_offsets
																								lowerCamelCase_			:				Any										=True

																	if changes_to_apply:
																								lowerCamelCase_			:				int										=getattr(_SCREAMING_SNAKE_CASE  ,			state.pop("type"   )   )
																								lowerCamelCase_			:				Optional[int]										=component_class(**_SCREAMING_SNAKE_CASE   )
																								setattr(self.backend_tokenizer  ,			_SCREAMING_SNAKE_CASE  ,			_SCREAMING_SNAKE_CASE   )


			@property
			def       UpperCAmelCase__		( self	:    List[Any]   ):

										if self._mask_token is None:
																	if self.verbose:
																								logger.error("Using mask_token, but it is not set yet."   )
																	return None
										return str(self._mask_token   )


			@mask_token.setter
			def       UpperCAmelCase__		( self	:    Optional[int]  ,			snake_case__	:    Optional[Any]   ):
										lowerCamelCase_			:				Tuple										=AddedToken(_SCREAMING_SNAKE_CASE  ,			lstrip=_SCREAMING_SNAKE_CASE  ,			rstrip=_SCREAMING_SNAKE_CASE   ) if isinstance(_SCREAMING_SNAKE_CASE  ,			_SCREAMING_SNAKE_CASE   ) else value
										lowerCamelCase_			:				str										=value


			def       UpperCAmelCase__		( self	:    List[str]  ,			*snake_case__	:    Dict  ,			**snake_case__	:    str   ):
										lowerCamelCase_			:				int										=kwargs.get("is_split_into_words"  ,			_SCREAMING_SNAKE_CASE   )

										if is_split_into_words and not self.add_prefix_space:
																	raise ValueError(
																	    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																	    "to use it with pretokenized inputs."   )

										return super()._batch_encode_plus(*_SCREAMING_SNAKE_CASE  ,			**_SCREAMING_SNAKE_CASE   )


			def       UpperCAmelCase__		( self	:    Tuple  ,			*snake_case__	:    Tuple  ,			**snake_case__	:    Optional[Any]   ):
										lowerCamelCase_			:				Tuple										=kwargs.get("is_split_into_words"  ,			_SCREAMING_SNAKE_CASE   )

										if is_split_into_words and not self.add_prefix_space:
																	raise ValueError(
																	    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
																	    "to use it with pretokenized inputs."   )

										return super()._encode_plus(*_SCREAMING_SNAKE_CASE  ,			**_SCREAMING_SNAKE_CASE   )


			def       UpperCAmelCase__		( self	:    int  ,			snake_case__	:    str  ,			snake_case__	:    Optional[str] = None   ):
										lowerCamelCase_			:				List[str]										=self._tokenizer.model.save(_SCREAMING_SNAKE_CASE  ,			name=_SCREAMING_SNAKE_CASE   )
										return tuple(_SCREAMING_SNAKE_CASE   )


			def       UpperCAmelCase__		( self	:    Any  ,			snake_case__	:    Union[str, Any]  ,			snake_case__	:    Any=None   ):
										lowerCamelCase_			:				Optional[Any]										=[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
																	return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


			def       UpperCAmelCase__		( self	:    Union[str, Any]  ,			snake_case__	:    List[int]  ,			snake_case__	:    Optional[List[int]] = None   ):
										lowerCamelCase_			:				List[Any]										=[self.sep_token_id]
										lowerCamelCase_			:				Optional[Any]										=[self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]








"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case				:  Dict =       logging.get_logger(__name__)

__snake_case				:  int =       {
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}



class      A__  (    __SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      'visual_bert'

	def __init__(	self:						int			,     _SCREAMING_SNAKE_CASE:						Dict=3_0522			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=768			,     _SCREAMING_SNAKE_CASE:						List[str]=512			,     _SCREAMING_SNAKE_CASE:						Optional[int]=12			,     _SCREAMING_SNAKE_CASE:						List[str]=12			,     _SCREAMING_SNAKE_CASE:						Tuple=3072			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]="gelu"			,     _SCREAMING_SNAKE_CASE:						Tuple=0.1			,     _SCREAMING_SNAKE_CASE:						str=0.1			,     _SCREAMING_SNAKE_CASE:						List[Any]=512			,     _SCREAMING_SNAKE_CASE:						Any=2			,     _SCREAMING_SNAKE_CASE:						Optional[int]=0.02			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=1e-12			,     _SCREAMING_SNAKE_CASE:						List[str]=False			,     _SCREAMING_SNAKE_CASE:						int=True			,     _SCREAMING_SNAKE_CASE:						str=1			,     _SCREAMING_SNAKE_CASE:						Tuple=0			,     _SCREAMING_SNAKE_CASE:						Any=2			,     **_SCREAMING_SNAKE_CASE:						List[str]			,     )							->			str:

				"""simple docstring"""

				super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE			,     bos_token_id=_SCREAMING_SNAKE_CASE			,     eos_token_id=_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase :    str				=			vocab_size
				__lowerCAmelCase :    Tuple				=			max_position_embeddings
				__lowerCAmelCase :    Any				=			hidden_size
				__lowerCAmelCase :    Union[str, Any]				=			visual_embedding_dim
				__lowerCAmelCase :    List[str]				=			num_hidden_layers
				__lowerCAmelCase :    Optional[int]				=			num_attention_heads
				__lowerCAmelCase :    str				=			intermediate_size
				__lowerCAmelCase :    int				=			hidden_act
				__lowerCAmelCase :    Union[str, Any]				=			hidden_dropout_prob
				__lowerCAmelCase :    int				=			attention_probs_dropout_prob
				__lowerCAmelCase :    List[Any]				=			initializer_range
				__lowerCAmelCase :    Dict				=			type_vocab_size
				__lowerCAmelCase :    Optional[int]				=			layer_norm_eps
				__lowerCAmelCase :    Union[str, Any]				=			bypass_transformer
				__lowerCAmelCase :    List[str]				=			special_visual_initialize