



"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


__snake_case				:  Optional[Any] =       logging.get_logger(__name__)


__snake_case				:  Optional[int] =       {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all BART models at https://huggingface.co/models?filter=bart
__snake_case				:  str =       {
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json',
    },
}

__snake_case				:  List[Any] =       {
    'facebook/bart-base': 1_024,
    'facebook/bart-large': 1_024,
    'facebook/bart-large-mnli': 1_024,
    'facebook/bart-large-cnn': 1_024,
    'facebook/bart-large-xsum': 1_024,
    'yjernite/bart_eli5': 1_024,
}



class      A__  (    __SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE        =      PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE        =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE        =      ['input_ids', 'attention_mask']
	SCREAMING_SNAKE_CASE        =      BartTokenizer

	def __init__(	self:						Union[str, Any]			,     _SCREAMING_SNAKE_CASE:						List[str]=None			,     _SCREAMING_SNAKE_CASE:						Dict=None			,     _SCREAMING_SNAKE_CASE:						int=None			,     _SCREAMING_SNAKE_CASE:						int="replace"			,     _SCREAMING_SNAKE_CASE:						Any="<s>"			,     _SCREAMING_SNAKE_CASE:						Dict="</s>"			,     _SCREAMING_SNAKE_CASE:						Optional[int]="</s>"			,     _SCREAMING_SNAKE_CASE:						Tuple="<s>"			,     _SCREAMING_SNAKE_CASE:						int="<unk>"			,     _SCREAMING_SNAKE_CASE:						Dict="<pad>"			,     _SCREAMING_SNAKE_CASE:						Optional[Any]="<mask>"			,     _SCREAMING_SNAKE_CASE:						Any=False			,     _SCREAMING_SNAKE_CASE:						int=True			,     **_SCREAMING_SNAKE_CASE:						int			,     )							->			Optional[int]:

				"""simple docstring"""

				super().__init__(
				    _SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE			,     tokenizer_file=_SCREAMING_SNAKE_CASE			,     errors=_SCREAMING_SNAKE_CASE			,     bos_token=_SCREAMING_SNAKE_CASE			,     eos_token=_SCREAMING_SNAKE_CASE			,     sep_token=_SCREAMING_SNAKE_CASE			,     cls_token=_SCREAMING_SNAKE_CASE			,     unk_token=_SCREAMING_SNAKE_CASE			,     pad_token=_SCREAMING_SNAKE_CASE			,     mask_token=_SCREAMING_SNAKE_CASE			,     add_prefix_space=_SCREAMING_SNAKE_CASE			,     trim_offsets=_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE			,     )

				__lowerCAmelCase :    Optional[int]				=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
				if pre_tok_state.get("add_prefix_space"			,     _SCREAMING_SNAKE_CASE) != add_prefix_space:
							__lowerCAmelCase :    str				=			getattr(_SCREAMING_SNAKE_CASE			,     pre_tok_state.pop("type"))
							__lowerCAmelCase :    Optional[Any]				=			add_prefix_space
							__lowerCAmelCase :    List[str]				=			pre_tok_class(**_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase :    str				=			add_prefix_space

				# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
				__lowerCAmelCase :    Dict				=			"post_processor"
				__lowerCAmelCase :    Optional[int]				=			getattr(self.backend_tokenizer			,     _SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)
				if tokenizer_component_instance:
							__lowerCAmelCase :    Optional[int]				=			json.loads(tokenizer_component_instance.__getstate__())

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
										__lowerCAmelCase :    Tuple				=			tuple(state["sep"])
							if "cls" in state:
										__lowerCAmelCase :    Optional[Any]				=			tuple(state["cls"])

							__lowerCAmelCase :    Any				=			False

							if state.get("add_prefix_space"			,     _SCREAMING_SNAKE_CASE) != add_prefix_space:
										__lowerCAmelCase :    Tuple				=			add_prefix_space
										__lowerCAmelCase :    Union[str, Any]				=			True

							if state.get("trim_offsets"			,     _SCREAMING_SNAKE_CASE) != trim_offsets:
										__lowerCAmelCase :    int				=			trim_offsets
										__lowerCAmelCase :    Any				=			True

							if changes_to_apply:
										__lowerCAmelCase :    int				=			getattr(_SCREAMING_SNAKE_CASE			,     state.pop("type"))
										__lowerCAmelCase :    Optional[int]				=			component_class(**_SCREAMING_SNAKE_CASE)
										setattr(self.backend_tokenizer			,     _SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)

	@property
	def 				_SCREAMING_SNAKE_CASE  (	self:						List[Any])							->			str:

				"""simple docstring"""

				if self._mask_token is None:
							if self.verbose:
										logger.error("Using mask_token, but it is not set yet.")
							return None
				return str(self._mask_token)

	@mask_token.setter
	def 				_SCREAMING_SNAKE_CASE  (	self:						Optional[int]			,     _SCREAMING_SNAKE_CASE:						Optional[Any])							->			Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase :    Tuple				=			AddedToken(_SCREAMING_SNAKE_CASE			,     lstrip=_SCREAMING_SNAKE_CASE			,     rstrip=_SCREAMING_SNAKE_CASE) if isinstance(_SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE) else value
				__lowerCAmelCase :    str				=			value

	def 				_SCREAMING_SNAKE_CASE  (	self:						List[str]			,     *_SCREAMING_SNAKE_CASE:						Dict			,     **_SCREAMING_SNAKE_CASE:						str)							->			BatchEncoding:

				"""simple docstring"""

				__lowerCAmelCase :    int				=			kwargs.get("is_split_into_words"			,     _SCREAMING_SNAKE_CASE)

				if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
							    "to use it with pretokenized inputs.")

				return super()._batch_encode_plus(*_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Tuple			,     *_SCREAMING_SNAKE_CASE:						Tuple			,     **_SCREAMING_SNAKE_CASE:						Optional[Any])							->			BatchEncoding:

				"""simple docstring"""

				__lowerCAmelCase :    Tuple				=			kwargs.get("is_split_into_words"			,     _SCREAMING_SNAKE_CASE)

				if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
							    "to use it with pretokenized inputs.")

				return super()._encode_plus(*_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						int			,     _SCREAMING_SNAKE_CASE:						str			,     _SCREAMING_SNAKE_CASE:						Optional[str] = None)							->			Tuple[str]:

				"""simple docstring"""

				__lowerCAmelCase :    List[str]				=			self._tokenizer.model.save(_SCREAMING_SNAKE_CASE			,     name=_SCREAMING_SNAKE_CASE)
				return tuple(_SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Any			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]			,     _SCREAMING_SNAKE_CASE:						Any=None)							->			Dict:

				"""simple docstring"""

				__lowerCAmelCase :    Optional[Any]				=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
				if token_ids_a is None:
							return output

				return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







	def 				_SCREAMING_SNAKE_CASE  (	self:						Union[str, Any]			,     _SCREAMING_SNAKE_CASE:						List[int]			,     _SCREAMING_SNAKE_CASE:						Optional[List[int]] = None)							->			List[int]:

				"""simple docstring"""

				__lowerCAmelCase :    List[Any]				=			[self.sep_token_id]
				__lowerCAmelCase :    Optional[Any]				=			[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]




"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case				:  Dict =       logging.get_logger(__name__)

__snake_case				:  int =       {
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}



class      A__  (    __SCREAMING_SNAKE_CASE				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      'visual_bert'

	def __init__(	self:						int			,     _SCREAMING_SNAKE_CASE:						Dict=3_0522			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=768			,     _SCREAMING_SNAKE_CASE:						List[str]=512			,     _SCREAMING_SNAKE_CASE:						Optional[int]=12			,     _SCREAMING_SNAKE_CASE:						List[str]=12			,     _SCREAMING_SNAKE_CASE:						Tuple=3072			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]="gelu"			,     _SCREAMING_SNAKE_CASE:						Tuple=0.1			,     _SCREAMING_SNAKE_CASE:						str=0.1			,     _SCREAMING_SNAKE_CASE:						List[Any]=512			,     _SCREAMING_SNAKE_CASE:						Any=2			,     _SCREAMING_SNAKE_CASE:						Optional[int]=0.02			,     _SCREAMING_SNAKE_CASE:						Union[str, Any]=1e-12			,     _SCREAMING_SNAKE_CASE:						List[str]=False			,     _SCREAMING_SNAKE_CASE:						int=True			,     _SCREAMING_SNAKE_CASE:						str=1			,     _SCREAMING_SNAKE_CASE:						Tuple=0			,     _SCREAMING_SNAKE_CASE:						Any=2			,     **_SCREAMING_SNAKE_CASE:						List[str]			,     )							->			str:

				"""simple docstring"""

				super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE			,     bos_token_id=_SCREAMING_SNAKE_CASE			,     eos_token_id=_SCREAMING_SNAKE_CASE			,     **_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase :    str				=			vocab_size
				__lowerCAmelCase :    Tuple				=			max_position_embeddings
				__lowerCAmelCase :    Any				=			hidden_size
				__lowerCAmelCase :    Union[str, Any]				=			visual_embedding_dim
				__lowerCAmelCase :    List[str]				=			num_hidden_layers
				__lowerCAmelCase :    Optional[int]				=			num_attention_heads
				__lowerCAmelCase :    str				=			intermediate_size
				__lowerCAmelCase :    int				=			hidden_act
				__lowerCAmelCase :    Union[str, Any]				=			hidden_dropout_prob
				__lowerCAmelCase :    int				=			attention_probs_dropout_prob
				__lowerCAmelCase :    List[Any]				=			initializer_range
				__lowerCAmelCase :    Dict				=			type_vocab_size
				__lowerCAmelCase :    Optional[int]				=			layer_norm_eps
				__lowerCAmelCase :    Union[str, Any]				=			bypass_transformer
				__lowerCAmelCase :    List[str]				=			special_visual_initialize