





"""simple docstring"""
import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


__UpperCamelCase			:		Optional[Any]		    =   logging.getLogger(__name__)







@dataclass
class 	a				:
 snake_case__    	=    field(
     metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}   )
 snake_case__    	=    field(
     default=a__					,      metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}   )
 snake_case__    	=    field(
     default=a__					,      metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}   )
 snake_case__    	=    field(default=a__					,      metadata={'''help''': '''Set this flag to use fast tokenization.'''}   )
 # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
 # or just modify its tokenizer_config.json.
 snake_case__    	=    field(
     default=a__					,      metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}					,      )


def 						_SCREAMING_SNAKE_CASE       ():
 # See all possible arguments in src/transformers/training_args.py
 # or by passing the --help flag to this script.
 # We now keep distinct sets of args, for a cleaner separation of concerns.

 lowerCAmelCase  							=      HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

 if len(sys.argv       ) == 2 and sys.argv[1].endswith('.json'       ):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
  lowerCAmelCase							,lowerCAmelCase							,lowerCAmelCase  							=      parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]       )       )
 else:
  lowerCAmelCase							,lowerCAmelCase							,lowerCAmelCase  							=      parser.parse_args_into_dataclasses()

 if (
     os.path.exists(training_args.output_dir       )
     and os.listdir(training_args.output_dir       )
     and training_args.do_train
     and not training_args.overwrite_output_dir
 ):
  raise ValueError(
      F'Output directory ({training_args.output_dir}) already exists and is not empty. Use'
      ' --overwrite_output_dir to overcome.'       )

 # Setup logging
 logging.basicConfig(
     format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'   ,			datefmt='%m/%d/%Y %H:%M:%S'   ,			level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN   ,			)
 logger.warning(
     'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'   ,			training_args.local_rank   ,			training_args.device   ,			training_args.n_gpu   ,			bool(training_args.local_rank != -1       )   ,			training_args.fpaa   ,			)
 # Set the verbosity to info of the Transformers logger (on main process only):
 if is_main_process(training_args.local_rank       ):
  transformers.utils.logging.set_verbosity_info()
  transformers.utils.logging.enable_default_handler()
  transformers.utils.logging.enable_explicit_format()
 logger.info('Training/evaluation parameters %s'   ,			_UpperCAmelCase       )

 # Prepare Question-Answering task
 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 lowerCAmelCase  							=      AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path   ,			cache_dir=model_args.cache_dir   ,			)
 lowerCAmelCase  							=      AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path   ,			cache_dir=model_args.cache_dir   ,			use_fast=_UpperCAmelCase   ,			)
 lowerCAmelCase  							=      AutoModelForQuestionAnswering.from_pretrained(
     model_args.model_name_or_path   ,			from_tf=bool('.ckpt' in model_args.model_name_or_path       )   ,			config=_UpperCAmelCase   ,			cache_dir=model_args.cache_dir   ,			)

 # Get datasets
 lowerCAmelCase  							=      hasattr(model.config   ,			'lang2id'       )
 lowerCAmelCase  							=      (
     SquadDataset(
         _UpperCAmelCase   ,			tokenizer=_UpperCAmelCase   ,			is_language_sensitive=_UpperCAmelCase   ,			cache_dir=model_args.cache_dir       )
     if training_args.do_train
     else None
 )
 lowerCAmelCase  							=      (
     SquadDataset(
         _UpperCAmelCase   ,			tokenizer=_UpperCAmelCase   ,			mode='dev'   ,			is_language_sensitive=_UpperCAmelCase   ,			cache_dir=model_args.cache_dir   ,			)
     if training_args.do_eval
     else None
 )

 # Data collator
 lowerCAmelCase  							=      DataCollatorWithPadding(_UpperCAmelCase   ,			pad_to_multiple_of=8       ) if training_args.fpaa else None

 # Initialize our Trainer
 lowerCAmelCase  							=      Trainer(
     model=_UpperCAmelCase   ,			args=_UpperCAmelCase   ,			train_dataset=_UpperCAmelCase   ,			eval_dataset=_UpperCAmelCase   ,			data_collator=_UpperCAmelCase   ,			)

 # Training
 if training_args.do_train:
  trainer.train(
      model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path       ) else None       )
  trainer.save_model()
  # For convenience, we also re-save the tokenizer to the same directory,
  # so that you can share your model easily on huggingface.co/models =)
  if trainer.is_world_master():
   tokenizer.save_pretrained(training_args.output_dir       )

def 						_SCREAMING_SNAKE_CASE       (_UpperCAmelCase :  Any       ):
 # For xla_spawn (TPUs)
 main()


if __name__ == "__main__":
       main()












import unittest
from functools import partial

import torch

from accelerate import Accelerator, debug_launcher
from accelerate.state import AcceleratorState, GradientState
from accelerate.test_utils import require_cpu, require_huggingface_suite
from accelerate.utils import GradientAccumulationPlugin




def        lowerCAmelCase_       (							UpperCamelCase_=2    ,						UpperCamelCase_=True    ,						UpperCamelCase_=False     )   ->						str:
       UpperCamelCase_      				=     Accelerator(step_scheduler_with_optimizer=UpperCamelCase_    ,						split_batches=UpperCamelCase_     )
       UpperCamelCase_      				=     torch.nn.Linear(2    ,						4     )
       UpperCamelCase_      				=     torch.optim.AdamW(model.parameters()    ,						lr=1.0     )
       UpperCamelCase_      				=     torch.optim.lr_scheduler.OneCycleLR(UpperCamelCase_    ,						max_lr=0.01    ,						steps_per_epoch=2    ,						epochs=1     )
       UpperCamelCase_				,       UpperCamelCase_				,       UpperCamelCase_      				=     accelerator.prepare(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_     )

       # Optimizer has stepped
       scheduler.step()
       if step_scheduler_with_optimizer or (num_processes == 1):
              assert (
                  scheduler.scheduler.last_epoch == num_processes
              ), F'''Last Epoch ({scheduler.scheduler.last_epoch}) != Num Processes ({num_processes})'''
       else:
              assert (
                  scheduler.scheduler.last_epoch != num_processes
              ), F'''Last Epoch ({scheduler.scheduler.last_epoch}) == Num Processes ({num_processes})'''




def        lowerCAmelCase_       (							UpperCamelCase_=2    ,						UpperCamelCase_=True    ,						UpperCamelCase_=False     )   ->						List[Any]:
       UpperCamelCase_      				=     Accelerator(step_scheduler_with_optimizer=UpperCamelCase_    ,						split_batches=UpperCamelCase_     )
       UpperCamelCase_      				=     torch.nn.Linear(2    ,						4     )
       UpperCamelCase_      				=     torch.optim.AdamW(model.parameters()    ,						lr=1.0     )
       UpperCamelCase_      				=     torch.optim.lr_scheduler.LambdaLR(UpperCamelCase_    ,						lr_lambda=lambda UpperCamelCase_     : 1 - n / 10     )
       UpperCamelCase_				,       UpperCamelCase_				,       UpperCamelCase_      				=     accelerator.prepare(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_     )

       # Optimizer has stepped
       UpperCamelCase_      				=     False
       scheduler.step()
       UpperCamelCase_      				=     1 - (num_processes if (step_scheduler_with_optimizer and not split_batches) else 1) / 10
       assert (
           scheduler.get_last_lr()[0] == expected_lr
       ), F'''Wrong lr found at first step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}'''

       # Optimizer has not stepped
       UpperCamelCase_      				=     True
       scheduler.step()
       if not step_scheduler_with_optimizer:
              UpperCamelCase_      				=     1 - 2 / 10
       assert (
           scheduler.get_last_lr()[0] == expected_lr
       ), F'''Wrong lr found at second step, expected {expected_lr}, got {scheduler.get_last_lr()[0]}'''




def        lowerCAmelCase_       (							UpperCamelCase_ = 2     )   ->						int:
       from transformers import get_linear_schedule_with_warmup

       UpperCamelCase_      				=     [1, 2, 4]
       for num_steps in steps:
              UpperCamelCase_      				=     GradientAccumulationPlugin(num_steps=UpperCamelCase_    ,						adjust_scheduler=num_steps > 1     )
              UpperCamelCase_      				=     Accelerator(gradient_accumulation_plugin=UpperCamelCase_     )
              UpperCamelCase_      				=     torch.nn.Linear(2    ,						4     )
              UpperCamelCase_      				=     torch.optim.AdamW(model.parameters()    ,						lr=10.0     )
              UpperCamelCase_      				=     get_linear_schedule_with_warmup(optimizer=UpperCamelCase_    ,						num_warmup_steps=0    ,						num_training_steps=20     )

              UpperCamelCase_				,       UpperCamelCase_				,       UpperCamelCase_      				=     accelerator.prepare(UpperCamelCase_    ,						UpperCamelCase_    ,						UpperCamelCase_     )

              for i in range(10 * num_steps     ):
                     with accelerator.accumulate(UpperCamelCase_     ):
                            optimizer.step()
                            scheduler.step()

                     if i == (10 * num_steps - 2):
                            assert (
                                scheduler.get_last_lr()[0] != 0
                            ), F'''Wrong lr found at second-to-last step, expected non-zero, got {scheduler.get_last_lr()[0]}. num_steps: {num_steps}'''
              assert (
                  scheduler.get_last_lr()[0] == 0
              ), F'''Wrong lr found at last step, expected 0, got {scheduler.get_last_lr()[0]}'''
              GradientState._reset_state()







@require_cpu
class 							_UpperCamelCase    (		unittest.TestCase   ):

       def lowercase       (						self:       Dict						)				->       Any:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1						)    ,      num_processes=1						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Dict						)				->       Dict:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1						)    ,      num_processes=1						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Dict						)				->       List[str]:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Any						)				->       int:




              """simple docstring"""


              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Optional[int]						)				->       List[str]:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       Tuple						)				->       Optional[int]:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1    ,      split_batches=_SCREAMING_SNAKE_CASE						)    ,      num_processes=1						)

       def lowercase       (						self:       int						)				->       Tuple:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)						)

       def lowercase       (						self:       Dict						)				->       Dict:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      step_scheduler_with_optimizer=_SCREAMING_SNAKE_CASE						)						)



       @require_huggingface_suite
       def lowercase       (						self:       Dict						)				->       Optional[Any]:




              """simple docstring"""


              AcceleratorState._reset_state(_SCREAMING_SNAKE_CASE						)
              debug_launcher(partial(_SCREAMING_SNAKE_CASE    ,      num_processes=1						)						)
              debug_launcher(_SCREAMING_SNAKE_CASE						)

