



'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


SCREAMING_SNAKE_CASE__     :						int	=  logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__     :						Any	=  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  {
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json''',
    },
}

SCREAMING_SNAKE_CASE__     :						List[str]	=  {
    '''RUCAIBox/mvp''': 1_0_2_4,
}

class 							a__( snake_case__     ):
			a_	: Union[str, Any]									=      VOCAB_FILES_NAMES
			a_	: Any									=      PRETRAINED_VOCAB_FILES_MAP
			a_	: Optional[int]									=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a_	: Any									=      ['''input_ids''', '''attention_mask''']
			a_	: Optional[Any]									=      MvpTokenizer






			def __init__(     self					,   _UpperCAmelCase=None					,   _UpperCAmelCase=None					,   _UpperCAmelCase=None					,   _UpperCAmelCase="replace"					,   _UpperCAmelCase="<s>"					,   _UpperCAmelCase="</s>"					,   _UpperCAmelCase="</s>"					,   _UpperCAmelCase="<s>"					,   _UpperCAmelCase="<unk>"					,   _UpperCAmelCase="<pad>"					,   _UpperCAmelCase="<mask>"					,   _UpperCAmelCase=False					,   _UpperCAmelCase=True					,   **_UpperCAmelCase					,   )							->							Tuple:
					super().__init__(
					    _UpperCAmelCase					,   _UpperCAmelCase					,   tokenizer_file=_UpperCAmelCase					,   errors=_UpperCAmelCase					,   bos_token=_UpperCAmelCase					,   eos_token=_UpperCAmelCase					,   sep_token=_UpperCAmelCase					,   cls_token=_UpperCAmelCase					,   unk_token=_UpperCAmelCase					,   pad_token=_UpperCAmelCase					,   mask_token=_UpperCAmelCase					,   add_prefix_space=_UpperCAmelCase					,   trim_offsets=_UpperCAmelCase					,   **_UpperCAmelCase					,   )

					snake_case__	     =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
					if pre_tok_state.get('add_prefix_space'					,   _UpperCAmelCase						) != add_prefix_space:
							snake_case__	     =getattr(_UpperCAmelCase					,   pre_tok_state.pop('type'						)						)
							snake_case__	     =add_prefix_space
							snake_case__	     =pre_tok_class(**_UpperCAmelCase						)

					snake_case__	     =add_prefix_space

					# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
					snake_case__	     ='post_processor'
					snake_case__	     =getattr(self.backend_tokenizer					,   _UpperCAmelCase					,   _UpperCAmelCase						)
					if tokenizer_component_instance:
							snake_case__	     =json.loads(tokenizer_component_instance.__getstate__()						)

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
									snake_case__	     =tuple(state['sep']						)
							if "cls" in state:
									snake_case__	     =tuple(state['cls']						)

							snake_case__	     =False

							if state.get('add_prefix_space'					,   _UpperCAmelCase						) != add_prefix_space:
									snake_case__	     =add_prefix_space
									snake_case__	     =True

							if state.get('trim_offsets'					,   _UpperCAmelCase						) != trim_offsets:
									snake_case__	     =trim_offsets
									snake_case__	     =True

							if changes_to_apply:
									snake_case__	     =getattr(_UpperCAmelCase					,   state.pop('type'						)						)
									snake_case__	     =component_class(**_UpperCAmelCase						)
									setattr(self.backend_tokenizer					,   _UpperCAmelCase					,   _UpperCAmelCase						)






			@property
			def 						_lowercase  (     self						)							->							str:
					if self._mask_token is None:
							if self.verbose:
									logger.error('Using mask_token, but it is not set yet.'						)
							return None
					return str(self._mask_token						)






			@mask_token.setter
			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Optional[Any]:
					snake_case__	     =AddedToken(_UpperCAmelCase					,   lstrip=_UpperCAmelCase					,   rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else value
					snake_case__	     =value






			def 						_lowercase  (     self					,   *_UpperCAmelCase					,   **_UpperCAmelCase						)							->							BatchEncoding:
					snake_case__	     =kwargs.get('is_split_into_words'					,   _UpperCAmelCase						)

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
							    'to use it with pretokenized inputs.'						)

					return super()._batch_encode_plus(*_UpperCAmelCase					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   *_UpperCAmelCase					,   **_UpperCAmelCase						)							->							BatchEncoding:
					snake_case__	     =kwargs.get('is_split_into_words'					,   _UpperCAmelCase						)

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
							    'to use it with pretokenized inputs.'						)

					return super()._encode_plus(*_UpperCAmelCase					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							Tuple[str]:
					snake_case__	     =self._tokenizer.model.save(_UpperCAmelCase					,   name=_UpperCAmelCase						)
					return tuple(_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase=None						)							->							str:
					snake_case__	     =[self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
							return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase = None						)							->							List[int]:
					snake_case__	     =[self.sep_token_id]
					snake_case__	     =[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]









'''simple docstring'''

import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
			from transformers.trainer import DEFAULT_CALLBACKS

			from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel

class 							a__( snake_case__     ):






			def __init__(     self						)							->							Dict:
					snake_case__	     =[]






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[str]:
					self.events.append('on_init_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Dict:
					self.events.append('on_train_begin'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Optional[Any]:
					self.events.append('on_train_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							int:
					self.events.append('on_epoch_begin'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[str]:
					self.events.append('on_epoch_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Optional[int]:
					self.events.append('on_step_begin'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							str:
					self.events.append('on_step_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[Any]:
					self.events.append('on_evaluate'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Dict:
					self.events.append('on_predict'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Any:
					self.events.append('on_save'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Tuple:
					self.events.append('on_log'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[Any]:
					self.events.append('on_prediction_step'						)





@require_torch
class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =tempfile.mkdtemp()






			def 						_lowercase  (     self						)							->							List[str]:
					shutil.rmtree(self.output_dir						)






			def 						_lowercase  (     self					,   _UpperCAmelCase=0					,   _UpperCAmelCase=0					,   _UpperCAmelCase=64					,   _UpperCAmelCase=64					,   _UpperCAmelCase=None					,   _UpperCAmelCase=False					,   **_UpperCAmelCase						)							->							int:
					# disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
					# its set to False since the tests later on depend on its value.
					snake_case__	     =RegressionDataset(length=_UpperCAmelCase						)
					snake_case__	     =RegressionDataset(length=_UpperCAmelCase						)
					snake_case__	     =RegressionModelConfig(a=_UpperCAmelCase					,   b=_UpperCAmelCase						)
					snake_case__	     =RegressionPreTrainedModel(_UpperCAmelCase						)

					snake_case__	     =TrainingArguments(self.output_dir					,   disable_tqdm=_UpperCAmelCase					,   report_to=[]					,   **_UpperCAmelCase						)
					return Trainer(
					    _UpperCAmelCase					,   _UpperCAmelCase					,   train_dataset=_UpperCAmelCase					,   eval_dataset=_UpperCAmelCase					,   callbacks=_UpperCAmelCase					,   )






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							Optional[Any]:
					self.assertEqual(len(_UpperCAmelCase						)					,   len(_UpperCAmelCase						)						)

					# Order doesn't matter
					snake_case__	     =sorted(_UpperCAmelCase					,   key=lambda _UpperCAmelCase						: cb.__name__ if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else cb.__class__.__name__						)
					snake_case__	     =sorted(_UpperCAmelCase					,   key=lambda _UpperCAmelCase						: cb.__name__ if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else cb.__class__.__name__						)

					for cba, cba in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) and isinstance(_UpperCAmelCase					,   _UpperCAmelCase						):
									self.assertEqual(_UpperCAmelCase					,   _UpperCAmelCase						)
							elif isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) and not isinstance(_UpperCAmelCase					,   _UpperCAmelCase						):
									self.assertEqual(_UpperCAmelCase					,   cba.__class__						)
							elif not isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) and isinstance(_UpperCAmelCase					,   _UpperCAmelCase						):
									self.assertEqual(cba.__class__					,   _UpperCAmelCase						)
							else:
									self.assertEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__	     =['on_init_end', 'on_train_begin']
					snake_case__	     =0
					snake_case__	     =len(trainer.get_eval_dataloader()						)
					snake_case__	     =['on_prediction_step'] * len(trainer.get_eval_dataloader()						) + ['on_log', 'on_evaluate']
					for _ in range(trainer.state.num_train_epochs						):
							expected_events.append('on_epoch_begin'						)
							for _ in range(_UpperCAmelCase						):
									step += 1
									expected_events += ["on_step_begin", "on_step_end"]
									if step % trainer.args.logging_steps == 0:
											expected_events.append('on_log'						)
									if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
											expected_events += evaluation_events.copy()
									if step % trainer.args.save_steps == 0:
											expected_events.append('on_save'						)
							expected_events.append('on_epoch_end'						)
							if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
									expected_events += evaluation_events.copy()
					expected_events += ["on_log", "on_train_end"]
					return expected_events






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.get_trainer()
					snake_case__	     =DEFAULT_CALLBACKS.copy() + [ProgressCallback]
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					# Callbacks passed at init are added to the default callbacks
					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]						)
					expected_callbacks.append(_UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					# TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
					snake_case__	     =self.get_trainer(disable_tqdm=_UpperCAmelCase						)
					snake_case__	     =DEFAULT_CALLBACKS.copy() + [PrinterCallback]
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =DEFAULT_CALLBACKS.copy() + [ProgressCallback]
					snake_case__	     =self.get_trainer()

					# We can add, pop, or remove by class name
					trainer.remove_callback(_UpperCAmelCase						)
					expected_callbacks.remove(_UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					snake_case__	     =self.get_trainer()
					snake_case__	     =trainer.pop_callback(_UpperCAmelCase						)
					self.assertEqual(cb.__class__					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					trainer.add_callback(_UpperCAmelCase						)
					expected_callbacks.insert(0					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					# We can also add, pop, or remove by instance
					snake_case__	     =self.get_trainer()
					snake_case__	     =trainer.callback_handler.callbacks[0]
					trainer.remove_callback(_UpperCAmelCase						)
					expected_callbacks.remove(_UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					snake_case__	     =self.get_trainer()
					snake_case__	     =trainer.callback_handler.callbacks[0]
					snake_case__	     =trainer.pop_callback(_UpperCAmelCase						)
					self.assertEqual(_UpperCAmelCase					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					trainer.add_callback(_UpperCAmelCase						)
					expected_callbacks.insert(0					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Dict:
					import warnings

					# XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
					warnings.simplefilter(action='ignore'					,   category=_UpperCAmelCase						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					# Independent log/save/eval
					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   logging_steps=5						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   save_steps=5						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   eval_steps=5					,   evaluation_strategy='steps'						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   evaluation_strategy='epoch'						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					# A bit of everything
					snake_case__	     =self.get_trainer(
					    callbacks=[MyTestTrainerCallback]					,   logging_steps=3					,   save_steps=10					,   eval_steps=5					,   evaluation_strategy='steps'					,   )
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					# warning should be emitted for duplicated callbacks
					with patch('transformers.trainer_callback.logger.warning'						) as warn_mock:
							snake_case__	     =self.get_trainer(
							    callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]					,   )
							assert str(_UpperCAmelCase						) in warn_mock.call_args[0][0]




