





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


_lowerCamelCase 					=      logging.get_logger(__name__)

_lowerCamelCase 					=      {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase 					=      {
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/tokenizer.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/tokenizer.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json',
        'roberta-base-openai-detector': (
            'https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'
        ),
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'
        ),
    },
}

_lowerCamelCase 					=      {
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}




class 	UpperCamelCase_							(  UpperCamelCase__     ):
							lowerCamelCase_							     =       VOCAB_FILES_NAMES
							lowerCamelCase_							     =       PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase_							     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase_							     =       ["input_ids", "attention_mask"]
							lowerCamelCase_							     =       RobertaTokenizer


							def __init__(				self     :List[Any]				,					__A     :Tuple=None				,					__A     :str=None				,					__A     :Tuple=None				,					__A     :Tuple="replace"				,					__A     :List[Any]="<s>"				,					__A     :Union[str, Any]="</s>"				,					__A     :Any="</s>"				,					__A     :Dict="<s>"				,					__A     :Optional[Any]="<unk>"				,					__A     :List[str]="<pad>"				,					__A     :Any="<mask>"				,					__A     :Dict=False				,					__A     :Tuple=True				,					**__A     :Optional[Any]				,					) -> Optional[Any]:



									"""simple docstring"""



									super().__init__(
									    __A				,					__A				,					tokenizer_file=__A				,					errors=__A				,					bos_token=__A				,					eos_token=__A				,					sep_token=__A				,					cls_token=__A				,					unk_token=__A				,					pad_token=__A				,					mask_token=__A				,					add_prefix_space=__A				,					trim_offsets=__A				,					**__A				,					)

									SCREAMING_SNAKE_CASE__								=   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
									if pre_tok_state.get("""add_prefix_space"""				,					__A		) != add_prefix_space:
											SCREAMING_SNAKE_CASE__								=   getattr(__A				,					pre_tok_state.pop("""type"""		)		)
											SCREAMING_SNAKE_CASE__								=   add_prefix_space
											SCREAMING_SNAKE_CASE__								=   pre_tok_class(**__A		)

									SCREAMING_SNAKE_CASE__								=   add_prefix_space

									SCREAMING_SNAKE_CASE__								=   """post_processor"""
									SCREAMING_SNAKE_CASE__								=   getattr(self.backend_tokenizer				,					__A				,					__A		)
									if tokenizer_component_instance:
											SCREAMING_SNAKE_CASE__								=   json.loads(tokenizer_component_instance.__getstate__()		)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
													SCREAMING_SNAKE_CASE__								=   tuple(state["""sep"""]		)
											if "cls" in state:
													SCREAMING_SNAKE_CASE__								=   tuple(state["""cls"""]		)

											SCREAMING_SNAKE_CASE__								=   False

											if state.get("""add_prefix_space"""				,					__A		) != add_prefix_space:
													SCREAMING_SNAKE_CASE__								=   add_prefix_space
													SCREAMING_SNAKE_CASE__								=   True

											if state.get("""trim_offsets"""				,					__A		) != trim_offsets:
													SCREAMING_SNAKE_CASE__								=   trim_offsets
													SCREAMING_SNAKE_CASE__								=   True

											if changes_to_apply:
													SCREAMING_SNAKE_CASE__								=   getattr(__A				,					state.pop("""type"""		)		)
													SCREAMING_SNAKE_CASE__								=   component_class(**__A		)
													setattr(self.backend_tokenizer				,					__A				,					__A		)


							@property
							def   _snake_case	(				self     :str		) -> str:



									"""simple docstring"""



									if self._mask_token is None:
											if self.verbose:
													logger.error("""Using mask_token, but it is not set yet."""		)
											return None
									return str(self._mask_token		)


							@mask_token.setter
							def   _snake_case	(				self     :List[str]				,					__A     :Tuple		) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   AddedToken(__A				,					lstrip=__A				,					rstrip=__A		) if isinstance(__A				,					__A		) else value
									SCREAMING_SNAKE_CASE__								=   value


							def   _snake_case	(				self     :str				,					*__A     :int				,					**__A     :Tuple		) -> BatchEncoding:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   kwargs.get("""is_split_into_words"""				,					__A		)
									assert self.add_prefix_space or not is_split_into_words, (
									    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
									    "to use it with pretokenized inputs."
									)

									return super()._batch_encode_plus(*__A				,					**__A		)


							def   _snake_case	(				self     :Optional[Any]				,					*__A     :List[Any]				,					**__A     :Dict		) -> BatchEncoding:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   kwargs.get("""is_split_into_words"""				,					__A		)

									assert self.add_prefix_space or not is_split_into_words, (
									    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
									    "to use it with pretokenized inputs."
									)

									return super()._encode_plus(*__A				,					**__A		)


							def   _snake_case	(				self     :int				,					__A     :str				,					__A     :Optional[str] = None		) -> Tuple[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self._tokenizer.model.save(__A				,					name=__A		)
									return tuple(__A		)


							def   _snake_case	(				self     :Dict				,					__A     :Union[str, Any]				,					__A     :Optional[Any]=None		) -> Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
											return output

									return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


							def   _snake_case	(				self     :Optional[int]				,					__A     :List[int]				,					__A     :Optional[List[int]] = None		) -> List[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [self.sep_token_id]
									SCREAMING_SNAKE_CASE__								=   [self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep		) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]






from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 	UpperCamelCase_							(  UpperCamelCase__     ):
							lowerCamelCase_							     =       ["image_processor", "tokenizer"]
							lowerCamelCase_							     =       "AutoImageProcessor"
							lowerCamelCase_							     =       "AutoTokenizer"


							def __init__(				self     :Optional[int]				,					__A     :Optional[Any]				,					__A     :Dict		) -> Dict:



									"""simple docstring"""



									super().__init__(__A				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.image_processor


							def __call__(				self     :int				,					__A     :str=None				,					__A     :int=None				,					__A     :Union[str, Any]=None				,					**__A     :str		) -> Optional[Any]:



									"""simple docstring"""



									if text is None and images is None:
											raise ValueError("""You have to specify either text or images. Both cannot be none."""		)

									if text is not None:
											SCREAMING_SNAKE_CASE__								=   self.tokenizer(__A				,					return_tensors=__A				,					**__A		)

									if images is not None:
											SCREAMING_SNAKE_CASE__								=   self.image_processor(__A				,					return_tensors=__A				,					**__A		)

									if text is not None and images is not None:
											SCREAMING_SNAKE_CASE__								=   image_features.pixel_values
											return encoding
									elif text is not None:
											return encoding
									else:
											return BatchEncoding(data=dict(**__A		)				,					tensor_type=__A		)


							def   _snake_case	(				self     :str				,					*__A     :List[str]				,					**__A     :List[str]		) -> List[Any]:



									"""simple docstring"""



									return self.tokenizer.batch_decode(*__A				,					**__A		)


							def   _snake_case	(				self     :List[str]				,					*__A     :Any				,					**__A     :Any		) -> Tuple:



									"""simple docstring"""



									return self.tokenizer.decode(*__A				,					**__A		)


							@property
							def   _snake_case	(				self     :Dict		) -> List[Any]:



									"""simple docstring"""



									return ["input_ids", "attention_mask", "pixel_values"]