






import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


SCREAMING_SNAKE_CASE__		:							Dict 				=				get_logger(__name__)







class 				UpperCAmelCase_  (		enum.Enum		):
     __lowerCamelCase      				=  'all_checks'
     __lowerCamelCase      				=  'basic_checks'
     __lowerCamelCase      				=  'no_checks'







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass





def  _lowerCamelCase						(						__lowerCamelCase ,							__lowerCamelCase ,							__lowerCamelCase=None       ) ->			Any:

     '''simple docstring'''
     if expected_checksums is None:
          logger.info("""Unable to verify checksums."""       )
          return
     if len(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       ) > 0:
          raise ExpectedMoreDownloadedFiles(str(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       )       )
     if len(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       ) > 0:
          raise UnexpectedDownloadedFile(str(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       )       )
     UpperCAmelCase__      :  str				      =			[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
     UpperCAmelCase__      :  Dict				      =			""" for """ + verification_name if verification_name is not None else """"""
     if len(__lowerCamelCase       ) > 0:
          raise NonMatchingChecksumError(
              F"Checksums didn't match{for_verification_name}:\n"
              F"{bad_urls}\n"
              """Set `verification_mode='no_checks'` to skip checksums verification and ignore this error"""       )
     logger.info("""All the checksums matched successfully""" + for_verification_name       )







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     pass





def  _lowerCamelCase						(						__lowerCamelCase ,							__lowerCamelCase       ) ->			Union[str, Any]:

     '''simple docstring'''
     if expected_splits is None:
          logger.info("""Unable to verify splits sizes."""       )
          return
     if len(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       ) > 0:
          raise ExpectedMoreSplits(str(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       )       )
     if len(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       ) > 0:
          raise UnexpectedSplits(str(set(__lowerCamelCase       ) - set(__lowerCamelCase       )       )       )
     UpperCAmelCase__      :  List[Any]				      =			[
         {"""expected""": expected_splits[name], """recorded""": recorded_splits[name]}
         for name in expected_splits
         if expected_splits[name].num_examples != recorded_splits[name].num_examples
     ]
     if len(__lowerCamelCase       ) > 0:
          raise NonMatchingSplitsSizesError(str(__lowerCamelCase       )       )
     logger.info("""All the splits matched successfully."""       )





def  _lowerCamelCase						(						__lowerCamelCase ,							__lowerCamelCase = True       ) ->			dict:

     '''simple docstring'''
     if record_checksum:
          UpperCAmelCase__      :  List[Any]				      =			shaaaa()
          with open(__lowerCamelCase ,							"""rb"""       ) as f:
               for chunk in iter(lambda: f.read(1 << 20       ) ,							B""""""       ):
                    m.update(__lowerCamelCase       )
               UpperCAmelCase__      :  int				      =			m.hexdigest()
     else:
          UpperCAmelCase__      :  Dict				      =			None
     return {"num_bytes": os.path.getsize(__lowerCamelCase       ), "checksum": checksum}







def  _lowerCamelCase						(						__lowerCamelCase       ) ->			List[Any]:

     '''simple docstring'''
     if dataset_size and config.IN_MEMORY_MAX_SIZE:
          return dataset_size < config.IN_MEMORY_MAX_SIZE
     else:
          return False














"""simple docstring"""
import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
 from transformers.trainer import DEFAULT_CALLBACKS

 from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel



class        UpperCamelCase__ (				a_):







    """simple docstring"""







    def __init__(		self :	Union[str, Any] ):





           '''simple docstring'''

           __magic_name__      				=    []







    def 				a__			(		self :	Tuple				,    UpperCamelCase_ :	Dict				,    UpperCamelCase_ :	Union[str, Any]				,    UpperCamelCase_ :	int				,    **UpperCamelCase_ :	int ):





           '''simple docstring'''

           self.events.append('on_init_end' )







    def 				a__			(		self :	Any				,    UpperCamelCase_ :	int				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	List[str]				,    **UpperCamelCase_ :	str ):





           '''simple docstring'''

           self.events.append('on_train_begin' )







    def 				a__			(		self :	str				,    UpperCamelCase_ :	Tuple				,    UpperCamelCase_ :	List[str]				,    UpperCamelCase_ :	Optional[int]				,    **UpperCamelCase_ :	Dict ):





           '''simple docstring'''

           self.events.append('on_train_end' )







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	Tuple				,    UpperCamelCase_ :	int				,    UpperCamelCase_ :	Tuple				,    **UpperCamelCase_ :	Optional[Any] ):





           '''simple docstring'''

           self.events.append('on_epoch_begin' )







    def 				a__			(		self :	Optional[int]				,    UpperCamelCase_ :	List[Any]				,    UpperCamelCase_ :	int				,    UpperCamelCase_ :	Tuple				,    **UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           self.events.append('on_epoch_end' )







    def 				a__			(		self :	Union[str, Any]				,    UpperCamelCase_ :	Union[str, Any]				,    UpperCamelCase_ :	Any				,    UpperCamelCase_ :	Union[str, Any]				,    **UpperCamelCase_ :	Optional[Any] ):





           '''simple docstring'''

           self.events.append('on_step_begin' )







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	Optional[Any]				,    UpperCamelCase_ :	int				,    UpperCamelCase_ :	Any				,    **UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           self.events.append('on_step_end' )







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	List[str]				,    UpperCamelCase_ :	int				,    UpperCamelCase_ :	Tuple				,    **UpperCamelCase_ :	Dict ):





           '''simple docstring'''

           self.events.append('on_evaluate' )







    def 				a__			(		self :	str				,    UpperCamelCase_ :	Any				,    UpperCamelCase_ :	Optional[int]				,    UpperCamelCase_ :	Dict				,    **UpperCamelCase_ :	Any ):





           '''simple docstring'''

           self.events.append('on_predict' )







    def 				a__			(		self :	Any				,    UpperCamelCase_ :	Optional[int]				,    UpperCamelCase_ :	Union[str, Any]				,    UpperCamelCase_ :	str				,    **UpperCamelCase_ :	Optional[int] ):





           '''simple docstring'''

           self.events.append('on_save' )







    def 				a__			(		self :	Optional[Any]				,    UpperCamelCase_ :	int				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Union[str, Any]				,    **UpperCamelCase_ :	Optional[int] ):





           '''simple docstring'''

           self.events.append('on_log' )







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	Optional[int]				,    UpperCamelCase_ :	List[Any]				,    UpperCamelCase_ :	str				,    **UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           self.events.append('on_prediction_step' )



@require_torch
class        UpperCamelCase__ (				unittest.TestCase):







    """simple docstring"""







    def 				a__			(		self :	Optional[Any] ):





           '''simple docstring'''

           __magic_name__      				=    tempfile.mkdtemp()







    def 				a__			(		self :	List[Any] ):





           '''simple docstring'''

           shutil.rmtree(self.output_dir )







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	Optional[int]=0				,    UpperCamelCase_ :	List[Any]=0				,    UpperCamelCase_ :	Optional[int]=6_4				,    UpperCamelCase_ :	Optional[int]=6_4				,    UpperCamelCase_ :	List[Any]=None				,    UpperCamelCase_ :	Optional[Any]=False				,    **UpperCamelCase_ :	str ):





           '''simple docstring'''

           __magic_name__      				=    RegressionDataset(length=UpperCamelCase_ )
           __magic_name__      				=    RegressionDataset(length=UpperCamelCase_ )
           __magic_name__      				=    RegressionModelConfig(a=UpperCamelCase_				,    b=UpperCamelCase_ )
           __magic_name__      				=    RegressionPreTrainedModel(UpperCamelCase_ )

           __magic_name__      				=    TrainingArguments(self.output_dir				,    disable_tqdm=UpperCamelCase_				,    report_to=[]				,    **UpperCamelCase_ )
           return Trainer(
               UpperCamelCase_				,    UpperCamelCase_				,    train_dataset=UpperCamelCase_				,    eval_dataset=UpperCamelCase_				,    callbacks=UpperCamelCase_				,    )







    def 				a__			(		self :	Optional[int]				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Dict ):





           '''simple docstring'''

           self.assertEqual(len(UpperCamelCase_ )				,    len(UpperCamelCase_ ) )

           # Order doesn't matter
           __magic_name__      				=    sorted(UpperCamelCase_				,    key=lambda UpperCamelCase_ : cb.__name__ if isinstance(UpperCamelCase_				,    UpperCamelCase_ ) else cb.__class__.__name__ )
           __magic_name__      				=    sorted(UpperCamelCase_				,    key=lambda UpperCamelCase_ : cb.__name__ if isinstance(UpperCamelCase_				,    UpperCamelCase_ ) else cb.__class__.__name__ )

           for cba, cba in zip(UpperCamelCase_				,    UpperCamelCase_ ):
                  if isinstance(UpperCamelCase_				,    UpperCamelCase_ ) and isinstance(UpperCamelCase_				,    UpperCamelCase_ ):
                         self.assertEqual(UpperCamelCase_				,    UpperCamelCase_ )
                  elif isinstance(UpperCamelCase_				,    UpperCamelCase_ ) and not isinstance(UpperCamelCase_				,    UpperCamelCase_ ):
                         self.assertEqual(UpperCamelCase_				,    cba.__class__ )
                  elif not isinstance(UpperCamelCase_				,    UpperCamelCase_ ) and isinstance(UpperCamelCase_				,    UpperCamelCase_ ):
                         self.assertEqual(cba.__class__				,    UpperCamelCase_ )
                  else:
                         self.assertEqual(UpperCamelCase_				,    UpperCamelCase_ )







    def 				a__			(		self :	Union[str, Any]				,    UpperCamelCase_ :	Any ):





           '''simple docstring'''

           __magic_name__      				=    ['on_init_end', 'on_train_begin']
           __magic_name__      				=    0
           __magic_name__      				=    len(trainer.get_eval_dataloader() )
           __magic_name__      				=    ['on_prediction_step'] * len(trainer.get_eval_dataloader() ) + ['on_log', 'on_evaluate']
           for _ in range(trainer.state.num_train_epochs ):
                  expected_events.append('on_epoch_begin' )
                  for _ in range(UpperCamelCase_ ):
                         step += 1
                         expected_events += ["on_step_begin", "on_step_end"]
                         if step % trainer.args.logging_steps == 0:
                                expected_events.append('on_log' )
                         if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                                expected_events += evaluation_events.copy()
                         if step % trainer.args.save_steps == 0:
                                expected_events.append('on_save' )
                  expected_events.append('on_epoch_end' )
                  if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
                         expected_events += evaluation_events.copy()
           expected_events += ["on_log", "on_train_end"]
           return expected_events







    def 				a__			(		self :	List[Any] ):





           '''simple docstring'''

           __magic_name__      				=    self.get_trainer()
           __magic_name__      				=    DEFAULT_CALLBACKS.copy() + [ProgressCallback]
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )

           # Callbacks passed at init are added to the default callbacks
           __magic_name__      				=    self.get_trainer(callbacks=[MyTestTrainerCallback] )
           expected_callbacks.append(UpperCamelCase_ )
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )

           # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
           __magic_name__      				=    self.get_trainer(disable_tqdm=UpperCamelCase_ )
           __magic_name__      				=    DEFAULT_CALLBACKS.copy() + [PrinterCallback]
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )







    def 				a__			(		self :	List[str] ):





           '''simple docstring'''

           __magic_name__      				=    DEFAULT_CALLBACKS.copy() + [ProgressCallback]
           __magic_name__      				=    self.get_trainer()

           # We can add, pop, or remove by class name
           trainer.remove_callback(UpperCamelCase_ )
           expected_callbacks.remove(UpperCamelCase_ )
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )

           __magic_name__      				=    self.get_trainer()
           __magic_name__      				=    trainer.pop_callback(UpperCamelCase_ )
           self.assertEqual(cb.__class__				,    UpperCamelCase_ )
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )

           trainer.add_callback(UpperCamelCase_ )
           expected_callbacks.insert(0				,    UpperCamelCase_ )
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )

           # We can also add, pop, or remove by instance
           __magic_name__      				=    self.get_trainer()
           __magic_name__      				=    trainer.callback_handler.callbacks[0]
           trainer.remove_callback(UpperCamelCase_ )
           expected_callbacks.remove(UpperCamelCase_ )
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )

           __magic_name__      				=    self.get_trainer()
           __magic_name__      				=    trainer.callback_handler.callbacks[0]
           __magic_name__      				=    trainer.pop_callback(UpperCamelCase_ )
           self.assertEqual(UpperCamelCase_				,    UpperCamelCase_ )
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )

           trainer.add_callback(UpperCamelCase_ )
           expected_callbacks.insert(0				,    UpperCamelCase_ )
           self.check_callbacks_equality(trainer.callback_handler.callbacks				,    UpperCamelCase_ )







    def 				a__			(		self :	Any ):





           '''simple docstring'''

           import warnings

           # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
           warnings.simplefilter(action='ignore'				,    category=UpperCamelCase_ )

           __magic_name__      				=    self.get_trainer(callbacks=[MyTestTrainerCallback] )
           trainer.train()
           __magic_name__      				=    trainer.callback_handler.callbacks[-2].events
           self.assertEqual(UpperCamelCase_				,    self.get_expected_events(UpperCamelCase_ ) )

           # Independent log/save/eval
           __magic_name__      				=    self.get_trainer(callbacks=[MyTestTrainerCallback]				,    logging_steps=5 )
           trainer.train()
           __magic_name__      				=    trainer.callback_handler.callbacks[-2].events
           self.assertEqual(UpperCamelCase_				,    self.get_expected_events(UpperCamelCase_ ) )

           __magic_name__      				=    self.get_trainer(callbacks=[MyTestTrainerCallback]				,    save_steps=5 )
           trainer.train()
           __magic_name__      				=    trainer.callback_handler.callbacks[-2].events
           self.assertEqual(UpperCamelCase_				,    self.get_expected_events(UpperCamelCase_ ) )

           __magic_name__      				=    self.get_trainer(callbacks=[MyTestTrainerCallback]				,    eval_steps=5				,    evaluation_strategy='steps' )
           trainer.train()
           __magic_name__      				=    trainer.callback_handler.callbacks[-2].events
           self.assertEqual(UpperCamelCase_				,    self.get_expected_events(UpperCamelCase_ ) )

           __magic_name__      				=    self.get_trainer(callbacks=[MyTestTrainerCallback]				,    evaluation_strategy='epoch' )
           trainer.train()
           __magic_name__      				=    trainer.callback_handler.callbacks[-2].events
           self.assertEqual(UpperCamelCase_				,    self.get_expected_events(UpperCamelCase_ ) )

           # A bit of everything
           __magic_name__      				=    self.get_trainer(
               callbacks=[MyTestTrainerCallback]				,    logging_steps=3				,    save_steps=1_0				,    eval_steps=5				,    evaluation_strategy='steps'				,    )
           trainer.train()
           __magic_name__      				=    trainer.callback_handler.callbacks[-2].events
           self.assertEqual(UpperCamelCase_				,    self.get_expected_events(UpperCamelCase_ ) )

           # warning should be emitted for duplicated callbacks
           with patch('transformers.trainer_callback.logger.warning' ) as warn_mock:
                  __magic_name__      				=    self.get_trainer(
                      callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]				,    )
                  assert str(UpperCamelCase_ ) in warn_mock.call_args[0][0]