





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


lowercase_		:					str      						=    logging.get_logger(__name__)

lowercase_		:					Dict      						=    {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase_		:					str      						=    {
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase_		:					Tuple      						=    {
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}


class       lowercase					(				a_  ):



						"""simple docstring"""

						_UpperCamelCase	:						Tuple    		=						VOCAB_FILES_NAMES
						_UpperCamelCase	:						List[Any]    		=						PRETRAINED_VOCAB_FILES_MAP
						_UpperCamelCase	:						str    		=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_UpperCamelCase	:						int    		=						["input_ids", "attention_mask"]
						_UpperCamelCase	:						int    		=						RobertaTokenizer







						def __init__(     self      :		Tuple			,					lowerCamelCase_      :		Union[str, Any]=None			,					lowerCamelCase_      :		Tuple=None			,					lowerCamelCase_      :		Any=None			,					lowerCamelCase_      :		Optional[Any]="replace"			,					lowerCamelCase_      :		str="<s>"			,					lowerCamelCase_      :		Tuple="</s>"			,					lowerCamelCase_      :		Any="</s>"			,					lowerCamelCase_      :		Tuple="<s>"			,					lowerCamelCase_      :		Any="<unk>"			,					lowerCamelCase_      :		Tuple="<pad>"			,					lowerCamelCase_      :		List[str]="<mask>"			,					lowerCamelCase_      :		str=False			,					lowerCamelCase_      :		Union[str, Any]=True			,					**lowerCamelCase_      :		int			,					):





										'''simple docstring'''







										super().__init__(
										    _lowerCAmelCase			,					_lowerCAmelCase			,					tokenizer_file=_lowerCAmelCase			,					errors=_lowerCAmelCase			,					bos_token=_lowerCAmelCase			,					eos_token=_lowerCAmelCase			,					sep_token=_lowerCAmelCase			,					cls_token=_lowerCAmelCase			,					unk_token=_lowerCAmelCase			,					pad_token=_lowerCAmelCase			,					mask_token=_lowerCAmelCase			,					add_prefix_space=_lowerCAmelCase			,					trim_offsets=_lowerCAmelCase			,					**_lowerCAmelCase			,					)

										_snake_case			:      Optional[int]	    = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
										if pre_tok_state.get('add_prefix_space'			,					_lowerCAmelCase       ) != add_prefix_space:
														_snake_case			:      str	    = getattr(_lowerCAmelCase			,					pre_tok_state.pop('type'       )       )
														_snake_case			:      Optional[Any]	    = add_prefix_space
														_snake_case			:      Any	    = pre_tok_class(**_lowerCAmelCase       )

										_snake_case			:      List[Any]	    = add_prefix_space

										_snake_case			:      Dict	    = 'post_processor'
										_snake_case			:      List[str]	    = getattr(self.backend_tokenizer			,					_lowerCAmelCase			,					_lowerCAmelCase       )
										if tokenizer_component_instance:
														_snake_case			:      Any	    = json.loads(tokenizer_component_instance.__getstate__()       )

														# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
														if "sep" in state:
																		_snake_case			:      Union[str, Any]	    = tuple(state['sep']       )
														if "cls" in state:
																		_snake_case			:      Any	    = tuple(state['cls']       )

														_snake_case			:      Dict	    = False

														if state.get('add_prefix_space'			,					_lowerCAmelCase       ) != add_prefix_space:
																		_snake_case			:      Tuple	    = add_prefix_space
																		_snake_case			:      Optional[int]	    = True

														if state.get('trim_offsets'			,					_lowerCAmelCase       ) != trim_offsets:
																		_snake_case			:      Optional[Any]	    = trim_offsets
																		_snake_case			:      str	    = True

														if changes_to_apply:
																		_snake_case			:      List[str]	    = getattr(_lowerCAmelCase			,					state.pop('type'       )       )
																		_snake_case			:      Dict	    = component_class(**_lowerCAmelCase       )
																		setattr(self.backend_tokenizer			,					_lowerCAmelCase			,					_lowerCAmelCase       )







						@property
						def 		__UpperCAmelCase							(     self      :		Dict       ):





										'''simple docstring'''







										if self._mask_token is None:
														if self.verbose:
																		logger.error('Using mask_token, but it is not set yet.'       )
														return None
										return str(self._mask_token       )







						@mask_token.setter
						def 		__UpperCAmelCase							(     self      :		Tuple			,					lowerCamelCase_      :		Tuple       ):





										'''simple docstring'''







										_snake_case			:      Optional[int]	    = AddedToken(_lowerCAmelCase			,					lstrip=_lowerCAmelCase			,					rstrip=_lowerCAmelCase       ) if isinstance(_lowerCAmelCase			,					_lowerCAmelCase       ) else value
										_snake_case			:      Tuple	    = value







						def 		__UpperCAmelCase							(     self      :		Dict			,					*lowerCamelCase_      :		Optional[int]			,					**lowerCamelCase_      :		Optional[int]       ):





										'''simple docstring'''







										_snake_case			:      Any	    = kwargs.get('is_split_into_words'			,					_lowerCAmelCase       )
										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*_lowerCAmelCase			,					**_lowerCAmelCase       )







						def 		__UpperCAmelCase							(     self      :		Optional[Any]			,					*lowerCamelCase_      :		str			,					**lowerCamelCase_      :		Any       ):





										'''simple docstring'''







										_snake_case			:      List[str]	    = kwargs.get('is_split_into_words'			,					_lowerCAmelCase       )

										assert self.add_prefix_space or not is_split_into_words, (
										    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*_lowerCAmelCase			,					**_lowerCAmelCase       )







						def 		__UpperCAmelCase							(     self      :		int			,					lowerCamelCase_      :		str			,					lowerCamelCase_      :		Optional[str] = None       ):





										'''simple docstring'''







										_snake_case			:      Any	    = self._tokenizer.model.save(_lowerCAmelCase			,					name=_lowerCAmelCase       )
										return tuple(_lowerCAmelCase       )







						def 		__UpperCAmelCase							(     self      :		Any			,					lowerCamelCase_      :		int			,					lowerCamelCase_      :		List[Any]=None       ):





										'''simple docstring'''







										_snake_case			:      List[Any]	    = [self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
														return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







						def 		__UpperCAmelCase							(     self      :		Union[str, Any]			,					lowerCamelCase_      :		List[int]			,					lowerCamelCase_      :		Optional[List[int]] = None       ):





										'''simple docstring'''







										_snake_case			:      Tuple	    = [self.sep_token_id]
										_snake_case			:      int	    = [self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep       ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]










import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
   import torch_xla.core.xla_model as xm
   import torch_xla.debug.metrics as met


class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							Union[str, Any]     ,					*_lowerCAmelCase							:							Dict     ,					_lowerCAmelCase							:							List[Any]=None     ,					_lowerCAmelCase							:							str=None     ,					**_lowerCAmelCase							:							str	) ->			Dict:




     """simple docstring"""
     super().__init__(*_lowerCAmelCase     ,					**_lowerCAmelCase	)
     snake_case_            =       eval_examples
     snake_case_            =       post_process_function



  def        lowerCAmelCase__	(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							Optional[Dataset] = None     ,					_lowerCAmelCase							:							Any=None     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							str = "eval"     ,					**_lowerCAmelCase							:							Tuple     ,					) ->			Dict[str, float]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()
     snake_case_            =       (
         gen_kwargs["max_length"] if gen_kwargs.get("max_length"	) is not None else self.args.generation_max_length
     )
     snake_case_            =       (
         gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"	) is not None else self.args.generation_num_beams
     )
     snake_case_            =       gen_kwargs

     snake_case_            =       self.eval_dataset if eval_dataset is None else eval_dataset
     snake_case_            =       self.get_eval_dataloader(_lowerCAmelCase	)
     snake_case_            =       self.eval_examples if eval_examples is None else eval_examples

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Evaluation"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics
     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)

     if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
        # Only the main node write the results by default
        snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
        snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()	):
           if not key.startswith(F'''{metric_key_prefix}_'''	):
              snake_case_            =       metrics.pop(_lowerCAmelCase	)

        metrics.update(output.metrics	)
     else:
        snake_case_            =       output.metrics

     if self.args.should_log:
        # Only the main node log the results by default
        self.log(_lowerCAmelCase	)

     if self.args.tpu_metrics_debug or self.args.debug:
        # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
        xm.master_print(met.metrics_report()	)

     snake_case_            =       self.callback_handler.on_evaluate(self.args     ,					self.state     ,					self.control     ,					_lowerCAmelCase	)
     return metrics



  def        lowerCAmelCase__	(  self							:							Optional[Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							List[str]=None     ,					_lowerCAmelCase							:							str = "test"     ,					**_lowerCAmelCase							:							Optional[Any]	) ->			Union[str, Any]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()

     snake_case_            =       self.get_test_dataloader(_lowerCAmelCase	)

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Prediction"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics

     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)
     if self.post_process_function is None or self.compute_metrics is None:
        return output

     snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					"predict"	)
     snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

     # Prefix all keys with metric_key_prefix + '_'
     for key in list(metrics.keys()	):
        if not key.startswith(F'''{metric_key_prefix}_'''	):
           snake_case_            =       metrics.pop(_lowerCAmelCase	)
     metrics.update(output.metrics	)
     return PredictionOutput(predictions=predictions.predictions     ,					label_ids=predictions.label_ids     ,					metrics=_lowerCAmelCase	)




