

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__a    :Tuple      =			logging.get_logger(__name__)

__a    :Tuple      =			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__a    :Any      =			{
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/tokenizer.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/tokenizer.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json',
        'roberta-base-openai-detector': (
            'https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'
        ),
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'
        ),
    },
}

__a    :Union[str, Any]      =			{
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}

class   _a					(			snake_case_				):




      """simple docstring"""

      _lowerCamelCase       :			Tuple      		=       VOCAB_FILES_NAMES
      _lowerCamelCase       :			int      		=       PRETRAINED_VOCAB_FILES_MAP
      _lowerCamelCase       :			int      		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowerCamelCase       :			Optional[Any]      		=       ['input_ids', 'attention_mask']
      _lowerCamelCase       :			Tuple      		=       RobertaTokenizer





      def __init__(   self							:				int						,			UpperCAmelCase							:				Union[str, Any]=None						,			UpperCAmelCase							:				Union[str, Any]=None						,			UpperCAmelCase							:				Any=None						,			UpperCAmelCase							:				Optional[Any]="replace"						,			UpperCAmelCase							:				Optional[int]="<s>"						,			UpperCAmelCase							:				Tuple="</s>"						,			UpperCAmelCase							:				Tuple="</s>"						,			UpperCAmelCase							:				Optional[Any]="<s>"						,			UpperCAmelCase							:				Optional[int]="<unk>"						,			UpperCAmelCase							:				Any="<pad>"						,			UpperCAmelCase							:				Union[str, Any]="<mask>"						,			UpperCAmelCase							:				Optional[Any]=False						,			UpperCAmelCase							:				Optional[int]=True						,			**UpperCAmelCase							:				Tuple						,			):
            super().__init__(
                UpperCAmelCase						,			UpperCAmelCase						,			tokenizer_file=UpperCAmelCase						,			errors=UpperCAmelCase						,			bos_token=UpperCAmelCase						,			eos_token=UpperCAmelCase						,			sep_token=UpperCAmelCase						,			cls_token=UpperCAmelCase						,			unk_token=UpperCAmelCase						,			pad_token=UpperCAmelCase						,			mask_token=UpperCAmelCase						,			add_prefix_space=UpperCAmelCase						,			trim_offsets=UpperCAmelCase						,			**UpperCAmelCase						,			)

            A_						  =				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
            if pre_tok_state.get("add_prefix_space"						,			UpperCAmelCase	) != add_prefix_space:
                  A_						  =				getattr(UpperCAmelCase						,			pre_tok_state.pop("type"	)	)
                  A_						  =				add_prefix_space
                  A_						  =				pre_tok_class(**UpperCAmelCase	)

            A_						  =				add_prefix_space

            A_						  =				"post_processor"
            A_						  =				getattr(self.backend_tokenizer						,			UpperCAmelCase						,			UpperCAmelCase	)
            if tokenizer_component_instance:
                  A_						  =				json.loads(tokenizer_component_instance.__getstate__()	)

                  # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
                  if "sep" in state:
                        A_						  =				tuple(state["sep"]	)
                  if "cls" in state:
                        A_						  =				tuple(state["cls"]	)

                  A_						  =				False

                  if state.get("add_prefix_space"						,			UpperCAmelCase	) != add_prefix_space:
                        A_						  =				add_prefix_space
                        A_						  =				True

                  if state.get("trim_offsets"						,			UpperCAmelCase	) != trim_offsets:
                        A_						  =				trim_offsets
                        A_						  =				True

                  if changes_to_apply:
                        A_						  =				getattr(UpperCAmelCase						,			state.pop("type"	)	)
                        A_						  =				component_class(**UpperCAmelCase	)
                        setattr(self.backend_tokenizer						,			UpperCAmelCase						,			UpperCAmelCase	)





      @property
      def 		__A	(   self							:				Tuple	):
            if self._mask_token is None:
                  if self.verbose:
                        logger.error("Using mask_token, but it is not set yet."	)
                  return None
            return str(self._mask_token	)





      @mask_token.setter
      def 		__A	(   self							:				List[Any]						,			UpperCAmelCase							:				str	):
            A_						  =				AddedToken(UpperCAmelCase						,			lstrip=UpperCAmelCase						,			rstrip=UpperCAmelCase	) if isinstance(UpperCAmelCase						,			UpperCAmelCase	) else value
            A_						  =				value





      def 		__A	(   self							:				Tuple						,			*UpperCAmelCase							:				Tuple						,			**UpperCAmelCase							:				List[str]	):
            A_						  =				kwargs.get("is_split_into_words"						,			UpperCAmelCase	)
            assert self.add_prefix_space or not is_split_into_words, (
                f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._batch_encode_plus(*UpperCAmelCase						,			**UpperCAmelCase	)





      def 		__A	(   self							:				Tuple						,			*UpperCAmelCase							:				Dict						,			**UpperCAmelCase							:				str	):
            A_						  =				kwargs.get("is_split_into_words"						,			UpperCAmelCase	)

            assert self.add_prefix_space or not is_split_into_words, (
                f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._encode_plus(*UpperCAmelCase						,			**UpperCAmelCase	)





      def 		__A	(   self							:				Any						,			UpperCAmelCase							:				str						,			UpperCAmelCase							:				Optional[str] = None	):
            A_						  =				self._tokenizer.model.save(UpperCAmelCase						,			name=UpperCAmelCase	)
            return tuple(UpperCAmelCase	)





      def 		__A	(   self							:				Any						,			UpperCAmelCase							:				Optional[int]						,			UpperCAmelCase							:				Union[str, Any]=None	):
            A_						  =				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
            if token_ids_a is None:
                  return output

            return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







      def 		__A	(   self							:				Optional[int]						,			UpperCAmelCase							:				List[int]						,			UpperCAmelCase							:				Optional[List[int]] = None	):
            A_						  =				[self.sep_token_id]
            A_						  =				[self.cls_token_id]

            if token_ids_a is None:
                  return len(cls + token_ids_a + sep	) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]



"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__magic_name__						: str    =				logging.get_logger(__name__)

__magic_name__						: List[Any]    =				{
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}




class 			__snake_case   (lowerCamelCase  ):
							__a					 =       '''visual_bert'''




							def __init__(     self:    Any     ,							A_:    List[str]=3_05_22     ,							A_:    Tuple=7_68     ,							A_:    Dict=5_12     ,							A_:    List[Any]=12     ,							A_:    str=12     ,							A_:    Dict=30_72     ,							A_:    str="gelu"     ,							A_:    Union[str, Any]=0.1     ,							A_:    Any=0.1     ,							A_:    List[Any]=5_12     ,							A_:    int=2     ,							A_:    int=0.02     ,							A_:    Any=1E-12     ,							A_:    List[str]=False     ,							A_:    str=True     ,							A_:    Union[str, Any]=1     ,							A_:    Optional[int]=0     ,							A_:    Dict=2     ,							**A_:    Optional[int]     ,							):
										super().__init__(pad_token_id=A_     ,							bos_token_id=A_     ,							eos_token_id=A_     ,							**A_   )

										__lowerCamelCase          =      vocab_size
										__lowerCamelCase          =      max_position_embeddings
										__lowerCamelCase          =      hidden_size
										__lowerCamelCase          =      visual_embedding_dim
										__lowerCamelCase          =      num_hidden_layers
										__lowerCamelCase          =      num_attention_heads
										__lowerCamelCase          =      intermediate_size
										__lowerCamelCase          =      hidden_act
										__lowerCamelCase          =      hidden_dropout_prob
										__lowerCamelCase          =      attention_probs_dropout_prob
										__lowerCamelCase          =      initializer_range
										__lowerCamelCase          =      type_vocab_size
										__lowerCamelCase          =      layer_norm_eps
										__lowerCamelCase          =      bypass_transformer
										__lowerCamelCase          =      special_visual_initialize




