



'''simple docstring'''

import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__     :						List[str]	=  get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
			from transformers.models.plbart.modeling_plbart import shift_tokens_right

SCREAMING_SNAKE_CASE__     :						Optional[int]	=  5_0_0_0_3
SCREAMING_SNAKE_CASE__     :						Tuple	=  5_0_0_0_2

@require_sentencepiece
@require_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: Optional[Any]									=      PLBartTokenizer
			a_	: Optional[int]									=      None
			a_	: Optional[Any]									=      False






			def 						_lowercase  (     self						)							->							Tuple:
					super().setUp()

					# We have a SentencePiece fixture for testing
					snake_case__	     =PLBartTokenizer(_UpperCAmelCase					,   language_codes='base'					,   keep_accents=_UpperCAmelCase						)
					tokenizer.save_pretrained(self.tmpdirname						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =PLBartTokenizer(_UpperCAmelCase					,   language_codes='base'					,   keep_accents=_UpperCAmelCase						)

					snake_case__	     =tokenizer.tokenize('This is a test'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁This', '▁is', '▁a', '▁t', 'est']						)

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,   )

					snake_case__	     =tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [
					        SPIECE_UNDERLINE + 'I',
					        SPIECE_UNDERLINE + 'was',
					        SPIECE_UNDERLINE + 'b',
					        'or',
					        'n',
					        SPIECE_UNDERLINE + 'in',
					        SPIECE_UNDERLINE + '',
					        '9',
					        '2',
					        '0',
					        '0',
					        '0',
					        ',',
					        SPIECE_UNDERLINE + 'and',
					        SPIECE_UNDERLINE + 'this',
					        SPIECE_UNDERLINE + 'is',
					        SPIECE_UNDERLINE + 'f',
					        'al',
					        's',
					        'é',
					        '.',
					    ]					,   )
					snake_case__	     =tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]					,   )

					snake_case__	     =tokenizer.convert_ids_to_tokens(_UpperCAmelCase						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [
					        SPIECE_UNDERLINE + 'I',
					        SPIECE_UNDERLINE + 'was',
					        SPIECE_UNDERLINE + 'b',
					        'or',
					        'n',
					        SPIECE_UNDERLINE + 'in',
					        SPIECE_UNDERLINE + '',
					        '<unk>',
					        '2',
					        '0',
					        '0',
					        '0',
					        ',',
					        SPIECE_UNDERLINE + 'and',
					        SPIECE_UNDERLINE + 'this',
					        SPIECE_UNDERLINE + 'is',
					        SPIECE_UNDERLINE + 'f',
					        'al',
					        's',
					        '<unk>',
					        '.',
					    ]					,   )

					snake_case__	     =tokenizer.vocab_size
					snake_case__	     =[tokenizer.convert_ids_to_tokens(_UpperCAmelCase						) for x in range(end - 4					,   _UpperCAmelCase						)]

					self.assertListEqual(_UpperCAmelCase					,   ['__java__', '__python__', '__en_XX__', '<mask>']						)

					snake_case__	     ='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
					snake_case__	     =tokenizer(_UpperCAmelCase						).input_ids
					self.assertEqual(
					    tokenizer.decode(_UpperCAmelCase					,   skip_special_tokens=_UpperCAmelCase					,   clean_up_tokenization_spaces=_UpperCAmelCase						)					,   _UpperCAmelCase					,   )






			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =PLBartTokenizer(_UpperCAmelCase					,   language_codes='multi'					,   keep_accents=_UpperCAmelCase						)

					snake_case__	     =tokenizer.tokenize('This is a test'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁This', '▁is', '▁a', '▁t', 'est']						)

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,   )

					snake_case__	     =tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [
					        SPIECE_UNDERLINE + 'I',
					        SPIECE_UNDERLINE + 'was',
					        SPIECE_UNDERLINE + 'b',
					        'or',
					        'n',
					        SPIECE_UNDERLINE + 'in',
					        SPIECE_UNDERLINE + '',
					        '9',
					        '2',
					        '0',
					        '0',
					        '0',
					        ',',
					        SPIECE_UNDERLINE + 'and',
					        SPIECE_UNDERLINE + 'this',
					        SPIECE_UNDERLINE + 'is',
					        SPIECE_UNDERLINE + 'f',
					        'al',
					        's',
					        'é',
					        '.',
					    ]					,   )
					snake_case__	     =tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]					,   )

					snake_case__	     =tokenizer.convert_ids_to_tokens(_UpperCAmelCase						)
					self.assertListEqual(
					    _UpperCAmelCase					,   [
					        SPIECE_UNDERLINE + 'I',
					        SPIECE_UNDERLINE + 'was',
					        SPIECE_UNDERLINE + 'b',
					        'or',
					        'n',
					        SPIECE_UNDERLINE + 'in',
					        SPIECE_UNDERLINE + '',
					        '<unk>',
					        '2',
					        '0',
					        '0',
					        '0',
					        ',',
					        SPIECE_UNDERLINE + 'and',
					        SPIECE_UNDERLINE + 'this',
					        SPIECE_UNDERLINE + 'is',
					        SPIECE_UNDERLINE + 'f',
					        'al',
					        's',
					        '<unk>',
					        '.',
					    ]					,   )
					snake_case__	     =tokenizer.vocab_size
					snake_case__	     =[tokenizer.convert_ids_to_tokens(_UpperCAmelCase						) for x in range(end - 7					,   _UpperCAmelCase						)]

					self.assertListEqual(
					    _UpperCAmelCase					,   ['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__']						)
					snake_case__	     ='java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
					snake_case__	     =tokenizer(_UpperCAmelCase						).input_ids
					self.assertEqual(
					    tokenizer.decode(_UpperCAmelCase					,   skip_special_tokens=_UpperCAmelCase					,   clean_up_tokenization_spaces=_UpperCAmelCase						)					,   _UpperCAmelCase					,   )





@require_torch
@require_sentencepiece
@require_tokenizers
class 							a__( unittest.TestCase     ):
			a_	: Tuple									=      '''uclanlp/plbart-python-en_XX'''
			a_	: Optional[int]									=      [
			    '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
			    '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
			]
			a_	: Tuple									=      [
			    '''Returns the maximum value of a b c.''',
			    '''Sums the values of a b c.''',
			]
			a_	: Dict									=      [
			    1_3_4,
			    5_4_5_2,
			    3_3_4_6_0,
			    3_3_4_4_1,
			    3_3_4_6_3,
			    3_3_4_6_5,
			    3_3_4_6_3,
			    3_3_4_4_9,
			    9_8_8,
			    2_0,
			    3_3_4_5_6,
			    1_9,
			    3_3_4_5_6,
			    7_7_1,
			    3_9,
			    4_2_5_8,
			    8_8_9,
			    3_3_1_8,
			    3_3_4_4_1,
			    3_3_4_6_3,
			    3_3_4_6_5,
			    3_3_4_6_3,
			    3_3_4_4_9,
			    2_4_7_1,
			    2,
			    PYTHON_CODE,
			]






			@classmethod
			def 						_lowercase  (     cls						)							->							Dict:
					snake_case__	     =PLBartTokenizer.from_pretrained(
					    cls.checkpoint_name					,   language_codes='base'					,   src_lang='python'					,   tgt_lang='en_XX'						)
					snake_case__	     =1
					return cls






			def 						_lowercase  (     self						)							->							Dict:
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']					,   5_0001						)
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']					,   5_0002						)
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']					,   5_0003						)






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
					self.assertListEqual(self.expected_src_tokens					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					self.assertIn(_UpperCAmelCase					,   self.tokenizer.all_special_ids						)
					snake_case__	     =[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
					snake_case__	     =self.tokenizer.decode(_UpperCAmelCase					,   skip_special_tokens=_UpperCAmelCase						)
					snake_case__	     =self.tokenizer.decode(generated_ids[1:]					,   skip_special_tokens=_UpperCAmelCase						)
					self.assertEqual(_UpperCAmelCase					,   _UpperCAmelCase						)
					self.assertNotIn(self.tokenizer.eos_token					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
					self.assertIsInstance(src_text[0]					,   _UpperCAmelCase						)
					snake_case__	     =10
					snake_case__	     =self.tokenizer(_UpperCAmelCase					,   max_length=_UpperCAmelCase					,   truncation=_UpperCAmelCase						).input_ids[0]
					self.assertEqual(ids[-2]					,   2						)
					self.assertEqual(ids[-1]					,   _UpperCAmelCase						)
					self.assertEqual(len(_UpperCAmelCase						)					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__']						)					,   [5_0004, 5_0001]						)






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					snake_case__	     =tempfile.mkdtemp()
					snake_case__	     =self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(_UpperCAmelCase						)
					snake_case__	     =PLBartTokenizer.from_pretrained(_UpperCAmelCase						)
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids					,   _UpperCAmelCase						)






			@require_torch
			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =self.tokenizer(self.src_text					,   text_target=self.tgt_text					,   padding=_UpperCAmelCase					,   return_tensors='pt'						)
					snake_case__	     =shift_tokens_right(batch['labels']					,   self.tokenizer.pad_token_id						)

					# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
					self.assertEqual(batch.input_ids[1][-2:].tolist()					,   [2, PYTHON_CODE]						)
					self.assertEqual(batch.decoder_input_ids[1][0]					,   _UpperCAmelCase						)
					self.assertEqual(batch.decoder_input_ids[1][-1]					,   2						)
					self.assertEqual(batch.labels[1][-2:].tolist()					,   [2, EN_CODE]						)






			@require_torch
			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer(
					    self.src_text					,   text_target=self.tgt_text					,   padding=_UpperCAmelCase					,   truncation=_UpperCAmelCase					,   max_length=len(self.expected_src_tokens						)					,   return_tensors='pt'					,   )
					snake_case__	     =shift_tokens_right(batch['labels']					,   self.tokenizer.pad_token_id						)

					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)

					self.assertEqual((2, 26)					,   batch.input_ids.shape						)
					self.assertEqual((2, 26)					,   batch.attention_mask.shape						)
					snake_case__	     =batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens					,   _UpperCAmelCase						)
					self.assertEqual(2					,   batch.decoder_input_ids[0, -1]						)  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens					,   []						)
					self.assertEqual(self.tokenizer.suffix_tokens					,   [self.tokenizer.eos_token_id, PYTHON_CODE]						)






			def 						_lowercase  (     self						)							->							List[str]:
					snake_case__	     =self.tokenizer(self.src_text					,   padding=_UpperCAmelCase					,   truncation=_UpperCAmelCase					,   max_length=3					,   return_tensors='pt'						)
					snake_case__	     =self.tokenizer(
					    text_target=self.tgt_text					,   padding=_UpperCAmelCase					,   truncation=_UpperCAmelCase					,   max_length=10					,   return_tensors='pt'						)
					snake_case__	     =targets['input_ids']
					snake_case__	     =shift_tokens_right(_UpperCAmelCase					,   self.tokenizer.pad_token_id						)

					self.assertEqual(batch.input_ids.shape[1]					,   3						)
					self.assertEqual(batch.decoder_input_ids.shape[1]					,   10						)






			@require_torch
			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer._build_translation_inputs(
					    'A test'					,   return_tensors='pt'					,   src_lang='en_XX'					,   tgt_lang='java'						)

					self.assertEqual(
					    nested_simplify(_UpperCAmelCase						)					,   {
					        # A, test, EOS, en_XX
					        'input_ids': [[150, 242, 2, 5_0003]],
					        'attention_mask': [[1, 1, 1, 1]],
					        # java
					        'forced_bos_token_id': 5_0001,
					    }					,   )









'''simple docstring'''

import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
			from transformers.trainer import DEFAULT_CALLBACKS

			from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel

class 							a__( snake_case__     ):






			def __init__(     self						)							->							Dict:
					snake_case__	     =[]






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[str]:
					self.events.append('on_init_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Dict:
					self.events.append('on_train_begin'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Optional[Any]:
					self.events.append('on_train_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							int:
					self.events.append('on_epoch_begin'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[str]:
					self.events.append('on_epoch_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Optional[int]:
					self.events.append('on_step_begin'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							str:
					self.events.append('on_step_end'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[Any]:
					self.events.append('on_evaluate'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Dict:
					self.events.append('on_predict'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Any:
					self.events.append('on_save'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							Tuple:
					self.events.append('on_log'						)






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   **_UpperCAmelCase						)							->							List[Any]:
					self.events.append('on_prediction_step'						)





@require_torch
class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =tempfile.mkdtemp()






			def 						_lowercase  (     self						)							->							List[str]:
					shutil.rmtree(self.output_dir						)






			def 						_lowercase  (     self					,   _UpperCAmelCase=0					,   _UpperCAmelCase=0					,   _UpperCAmelCase=64					,   _UpperCAmelCase=64					,   _UpperCAmelCase=None					,   _UpperCAmelCase=False					,   **_UpperCAmelCase						)							->							int:
					# disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
					# its set to False since the tests later on depend on its value.
					snake_case__	     =RegressionDataset(length=_UpperCAmelCase						)
					snake_case__	     =RegressionDataset(length=_UpperCAmelCase						)
					snake_case__	     =RegressionModelConfig(a=_UpperCAmelCase					,   b=_UpperCAmelCase						)
					snake_case__	     =RegressionPreTrainedModel(_UpperCAmelCase						)

					snake_case__	     =TrainingArguments(self.output_dir					,   disable_tqdm=_UpperCAmelCase					,   report_to=[]					,   **_UpperCAmelCase						)
					return Trainer(
					    _UpperCAmelCase					,   _UpperCAmelCase					,   train_dataset=_UpperCAmelCase					,   eval_dataset=_UpperCAmelCase					,   callbacks=_UpperCAmelCase					,   )






			def 						_lowercase  (     self					,   _UpperCAmelCase					,   _UpperCAmelCase						)							->							Optional[Any]:
					self.assertEqual(len(_UpperCAmelCase						)					,   len(_UpperCAmelCase						)						)

					# Order doesn't matter
					snake_case__	     =sorted(_UpperCAmelCase					,   key=lambda _UpperCAmelCase						: cb.__name__ if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else cb.__class__.__name__						)
					snake_case__	     =sorted(_UpperCAmelCase					,   key=lambda _UpperCAmelCase						: cb.__name__ if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) else cb.__class__.__name__						)

					for cba, cba in zip(_UpperCAmelCase					,   _UpperCAmelCase						):
							if isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) and isinstance(_UpperCAmelCase					,   _UpperCAmelCase						):
									self.assertEqual(_UpperCAmelCase					,   _UpperCAmelCase						)
							elif isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) and not isinstance(_UpperCAmelCase					,   _UpperCAmelCase						):
									self.assertEqual(_UpperCAmelCase					,   cba.__class__						)
							elif not isinstance(_UpperCAmelCase					,   _UpperCAmelCase						) and isinstance(_UpperCAmelCase					,   _UpperCAmelCase						):
									self.assertEqual(cba.__class__					,   _UpperCAmelCase						)
							else:
									self.assertEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__	     =['on_init_end', 'on_train_begin']
					snake_case__	     =0
					snake_case__	     =len(trainer.get_eval_dataloader()						)
					snake_case__	     =['on_prediction_step'] * len(trainer.get_eval_dataloader()						) + ['on_log', 'on_evaluate']
					for _ in range(trainer.state.num_train_epochs						):
							expected_events.append('on_epoch_begin'						)
							for _ in range(_UpperCAmelCase						):
									step += 1
									expected_events += ["on_step_begin", "on_step_end"]
									if step % trainer.args.logging_steps == 0:
											expected_events.append('on_log'						)
									if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
											expected_events += evaluation_events.copy()
									if step % trainer.args.save_steps == 0:
											expected_events.append('on_save'						)
							expected_events.append('on_epoch_end'						)
							if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
									expected_events += evaluation_events.copy()
					expected_events += ["on_log", "on_train_end"]
					return expected_events






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.get_trainer()
					snake_case__	     =DEFAULT_CALLBACKS.copy() + [ProgressCallback]
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					# Callbacks passed at init are added to the default callbacks
					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]						)
					expected_callbacks.append(_UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					# TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
					snake_case__	     =self.get_trainer(disable_tqdm=_UpperCAmelCase						)
					snake_case__	     =DEFAULT_CALLBACKS.copy() + [PrinterCallback]
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =DEFAULT_CALLBACKS.copy() + [ProgressCallback]
					snake_case__	     =self.get_trainer()

					# We can add, pop, or remove by class name
					trainer.remove_callback(_UpperCAmelCase						)
					expected_callbacks.remove(_UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					snake_case__	     =self.get_trainer()
					snake_case__	     =trainer.pop_callback(_UpperCAmelCase						)
					self.assertEqual(cb.__class__					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					trainer.add_callback(_UpperCAmelCase						)
					expected_callbacks.insert(0					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					# We can also add, pop, or remove by instance
					snake_case__	     =self.get_trainer()
					snake_case__	     =trainer.callback_handler.callbacks[0]
					trainer.remove_callback(_UpperCAmelCase						)
					expected_callbacks.remove(_UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					snake_case__	     =self.get_trainer()
					snake_case__	     =trainer.callback_handler.callbacks[0]
					snake_case__	     =trainer.pop_callback(_UpperCAmelCase						)
					self.assertEqual(_UpperCAmelCase					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)

					trainer.add_callback(_UpperCAmelCase						)
					expected_callbacks.insert(0					,   _UpperCAmelCase						)
					self.check_callbacks_equality(trainer.callback_handler.callbacks					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Dict:
					import warnings

					# XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
					warnings.simplefilter(action='ignore'					,   category=_UpperCAmelCase						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					# Independent log/save/eval
					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   logging_steps=5						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   save_steps=5						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   eval_steps=5					,   evaluation_strategy='steps'						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					snake_case__	     =self.get_trainer(callbacks=[MyTestTrainerCallback]					,   evaluation_strategy='epoch'						)
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					# A bit of everything
					snake_case__	     =self.get_trainer(
					    callbacks=[MyTestTrainerCallback]					,   logging_steps=3					,   save_steps=10					,   eval_steps=5					,   evaluation_strategy='steps'					,   )
					trainer.train()
					snake_case__	     =trainer.callback_handler.callbacks[-2].events
					self.assertEqual(_UpperCAmelCase					,   self.get_expected_events(_UpperCAmelCase						)						)

					# warning should be emitted for duplicated callbacks
					with patch('transformers.trainer_callback.logger.warning'						) as warn_mock:
							snake_case__	     =self.get_trainer(
							    callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]					,   )
							assert str(_UpperCAmelCase						) in warn_mock.call_args[0][0]




