





import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


UpperCAmelCase__  :	Tuple     =logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

UpperCAmelCase__  :	Any     ={
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
UpperCAmelCase__  :	Optional[Any]     ={
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
UpperCAmelCase__  :	Optional[int]     =sorted(arg_to_scheduler.keys())
UpperCAmelCase__  :	Union[str, Any]     ='''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''

class 					__A   (  pl.LightningModule			):


				def __init__(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_="base"      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_=None      ,       **UpperCAmelCase_      ,       ):
								super().__init__()
								# TODO: move to self.save_hyperparameters()
								# self.save_hyperparameters()
								# can also expand arguments into trainer signature for easier reading

								self.save_hyperparameters(lowercase_							)
								lowerCamelCase		       =0
								lowerCamelCase		       =Path(self.hparams.output_dir							)
								lowerCamelCase		       =self.hparams.cache_dir if self.hparams.cache_dir else None
								if config is None:
												lowerCamelCase		       =AutoConfig.from_pretrained(
												    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path      ,       **({"""num_labels""": num_labels} if num_labels is not None else {})      ,       cache_dir=lowercase_      ,       **lowercase_      ,       )
								else:
												lowerCamelCase		       =config

								lowerCamelCase		       =("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
								for p in extra_model_params:
												if getattr(self.hparams      ,       lowercase_      ,       lowercase_							):
																assert hasattr(self.config      ,       lowercase_							), f"""model config doesn\'t have a `{p}` attribute"""
																setattr(self.config      ,       lowercase_      ,       getattr(self.hparams      ,       lowercase_							)							)

								if tokenizer is None:
												lowerCamelCase		       =AutoTokenizer.from_pretrained(
												    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path      ,       cache_dir=lowercase_      ,       )
								else:
												lowerCamelCase		       =tokenizer
								lowerCamelCase		       =MODEL_MODES[mode]
								if model is None:
												lowerCamelCase		       =self.model_type.from_pretrained(
												    self.hparams.model_name_or_path      ,       from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path							)      ,       config=self.config      ,       cache_dir=lowercase_      ,       )
								else:
												lowerCamelCase		       =model


				def _snake_case				(						self      ,       *UpperCAmelCase_      ,       **UpperCAmelCase_							):
								lowerCamelCase		       =self.model_type.from_pretrained(*lowercase_      ,       **lowercase_							)


				def _snake_case				(						self							):
								lowerCamelCase		       =arg_to_scheduler[self.hparams.lr_scheduler]
								lowerCamelCase		       =get_schedule_func(
								    self.opt      ,       num_warmup_steps=self.hparams.warmup_steps      ,       num_training_steps=self.total_steps()							)
								lowerCamelCase		       ={"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
								return scheduler


				def _snake_case				(						self							):
								lowerCamelCase		       =self.model
								lowerCamelCase		       =["""bias""", """LayerNorm.weight"""]
								lowerCamelCase		       =[
								    {
								        """params""": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay							)],
								        """weight_decay""": self.hparams.weight_decay,
								    },
								    {
								        """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay							)],
								        """weight_decay""": 0.0,
								    },
								]
								if self.hparams.adafactor:
												lowerCamelCase		       =Adafactor(
												    lowercase_      ,       lr=self.hparams.learning_rate      ,       scale_parameter=lowercase_      ,       relative_step=lowercase_							)

								else:
												lowerCamelCase		       =AdamW(
												    lowercase_      ,       lr=self.hparams.learning_rate      ,       eps=self.hparams.adam_epsilon							)
								lowerCamelCase		       =optimizer

								lowerCamelCase		       =self.get_lr_scheduler()

								return [optimizer], [scheduler]


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_							):
								return self.validation_step(lowercase_      ,       lowercase_							)


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								return self.validation_end(lowercase_							)


				def _snake_case				(						self							):
								lowerCamelCase		       =max(1      ,       self.hparams.gpus							)  # TODO: consider num_tpu_cores
								lowerCamelCase		       =self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
								return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs


				def _snake_case				(						self      ,       UpperCAmelCase_							):

								if mode == "test":
												lowerCamelCase		       =len(self.test_dataloader().dataset							)
								else:
												lowerCamelCase		       =self.get_dataloader("""train"""      ,       self.hparams.train_batch_size      ,       shuffle=lowercase_							)
												lowerCamelCase		       =len(self.train_dataloader().dataset							)


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = False							):
								raise NotImplementedError("""You must implement this for your task"""							)


				def _snake_case				(						self							):
								return self.train_loader


				def _snake_case				(						self							):
								return self.get_dataloader("""dev"""      ,       self.hparams.eval_batch_size      ,       shuffle=lowercase_							)


				def _snake_case				(						self							):
								return self.get_dataloader("""test"""      ,       self.hparams.eval_batch_size      ,       shuffle=lowercase_							)


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								return os.path.join(
								    self.hparams.data_dir      ,       """cached_{}_{}_{}""".format(
								        lowercase_      ,       list(filter(lowercase_      ,       self.hparams.model_name_or_path.split("""/"""							)							)							).pop()      ,       str(self.hparams.max_seq_length							)      ,       )      ,       )


				@pl.utilities.rank_zero_only
				def _snake_case				(						self      ,       UpperCAmelCase_							):
								lowerCamelCase		       =self.output_dir.joinpath("""best_tfmr"""							)
								lowerCamelCase		       =self.step_count
								self.model.save_pretrained(lowercase_							)
								self.tokenizer.save_pretrained(lowercase_							)


				@staticmethod
				def _snake_case				(						UpperCAmelCase_      ,       UpperCAmelCase_							):
								parser.add_argument(
								    """--model_name_or_path"""      ,       default=lowercase_      ,       type=lowercase_      ,       required=lowercase_      ,       help="""Path to pretrained model or model identifier from huggingface.co/models"""      ,       )
								parser.add_argument(
								    """--config_name"""      ,       default=""""""      ,       type=lowercase_      ,       help="""Pretrained config name or path if not the same as model_name"""							)
								parser.add_argument(
								    """--tokenizer_name"""      ,       default=lowercase_      ,       type=lowercase_      ,       help="""Pretrained tokenizer name or path if not the same as model_name"""      ,       )
								parser.add_argument(
								    """--cache_dir"""      ,       default=""""""      ,       type=lowercase_      ,       help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""      ,       )
								parser.add_argument(
								    """--encoder_layerdrop"""      ,       type=lowercase_      ,       help="""Encoder layer dropout probability (Optional). Goes into model.config"""      ,       )
								parser.add_argument(
								    """--decoder_layerdrop"""      ,       type=lowercase_      ,       help="""Decoder layer dropout probability (Optional). Goes into model.config"""      ,       )
								parser.add_argument(
								    """--dropout"""      ,       type=lowercase_      ,       help="""Dropout probability (Optional). Goes into model.config"""      ,       )
								parser.add_argument(
								    """--attention_dropout"""      ,       type=lowercase_      ,       help="""Attention dropout probability (Optional). Goes into model.config"""      ,       )
								parser.add_argument("""--learning_rate"""      ,       default=5E-5      ,       type=lowercase_      ,       help="""The initial learning rate for Adam."""							)
								parser.add_argument(
								    """--lr_scheduler"""      ,       default="""linear"""      ,       choices=lowercase_      ,       metavar=lowercase_      ,       type=lowercase_      ,       help="""Learning rate scheduler"""      ,       )
								parser.add_argument("""--weight_decay"""      ,       default=0.0      ,       type=lowercase_      ,       help="""Weight decay if we apply some."""							)
								parser.add_argument("""--adam_epsilon"""      ,       default=1E-8      ,       type=lowercase_      ,       help="""Epsilon for Adam optimizer."""							)
								parser.add_argument("""--warmup_steps"""      ,       default=0      ,       type=lowercase_      ,       help="""Linear warmup over warmup_steps."""							)
								parser.add_argument("""--num_workers"""      ,       default=4      ,       type=lowercase_      ,       help="""kwarg passed to DataLoader"""							)
								parser.add_argument("""--num_train_epochs"""      ,       dest="""max_epochs"""      ,       default=3      ,       type=lowercase_							)
								parser.add_argument("""--train_batch_size"""      ,       default=32      ,       type=lowercase_							)
								parser.add_argument("""--eval_batch_size"""      ,       default=32      ,       type=lowercase_							)
								parser.add_argument("""--adafactor"""      ,       action="""store_true"""							)




class 					__A   (  pl.Callback			):


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_							):
								lowerCamelCase		       =trainer.lr_schedulers[0]["""scheduler"""]
								lowerCamelCase		       ={f"""lr_group_{i}""": lr for i, lr in enumerate(lr_scheduler.get_lr()							)}
								pl_module.logger.log_metrics(lowercase_							)


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_							):
								rank_zero_info("""***** Validation results *****"""							)
								lowerCamelCase		       =trainer.callback_metrics
								# Log results
								for key in sorted(lowercase_							):
												if key not in ["log", "progress_bar"]:
																rank_zero_info("""{} = {}\n""".format(lowercase_      ,       str(metrics[key]							)							)							)


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_							):
								rank_zero_info("""***** Test results *****"""							)
								lowerCamelCase		       =trainer.callback_metrics
								# Log and save results to file
								lowerCamelCase		       =os.path.join(pl_module.hparams.output_dir      ,       """test_results.txt"""							)
								with open(lowercase_      ,       """w"""							) as writer:
												for key in sorted(lowercase_							):
																if key not in ["log", "progress_bar"]:
																				rank_zero_info("""{} = {}\n""".format(lowercase_      ,       str(metrics[key]							)							)							)
																				writer.write("""{} = {}\n""".format(lowercase_      ,       str(metrics[key]							)							)							)





def    _lowercase    (  _UpperCAmelCase	,					_UpperCAmelCase )  ->     None:
				#  To allow all pl args uncomment the following line
				#  parser = pl.Trainer.add_argparse_args(parser)
				parser.add_argument(
				    """--output_dir"""	,					default=a_	,					type=a_	,					required=a_	,					help="""The output directory where the model predictions and checkpoints will be written."""	,					)
				parser.add_argument(
				    """--fp16"""	,					action="""store_true"""	,					help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""	,					)

				parser.add_argument(
				    """--fp16_opt_level"""	,					type=a_	,					default="""O2"""	,					help=(
				        """For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\']."""
				        """See details at https://nvidia.github.io/apex/amp.html"""
				    )	,					)
				parser.add_argument("""--n_tpu_cores"""	,					dest="""tpu_cores"""	,					type=a_ )
				parser.add_argument("""--max_grad_norm"""	,					dest="""gradient_clip_val"""	,					default=1.0	,					type=a_	,					help="""Max gradient norm""" )
				parser.add_argument("""--do_train"""	,					action="""store_true"""	,					help="""Whether to run training.""" )
				parser.add_argument("""--do_predict"""	,					action="""store_true"""	,					help="""Whether to run predictions on the test set.""" )
				parser.add_argument(
				    """--gradient_accumulation_steps"""	,					dest="""accumulate_grad_batches"""	,					type=a_	,					default=1	,					help="""Number of updates steps to accumulate before performing a backward/update pass."""	,					)
				parser.add_argument("""--seed"""	,					type=a_	,					default=42	,					help="""random seed for initialization""" )
				parser.add_argument(
				    """--data_dir"""	,					default=a_	,					type=a_	,					required=a_	,					help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""	,					)





def    _lowercase    (  _UpperCAmelCase	,					_UpperCAmelCase	,					_UpperCAmelCase=None	,					_UpperCAmelCase=True	,					_UpperCAmelCase=[]	,					_UpperCAmelCase=None	,					_UpperCAmelCase=None	,					**_UpperCAmelCase	,					)  ->     List[str]:
				pl.seed_everything(args.seed )

				# init model
				lowerCamelCase		       =Path(model.hparams.output_dir )
				odir.mkdir(exist_ok=a_ )

				# add custom checkpoints
				if checkpoint_callback is None:
								lowerCamelCase		       =pl.callbacks.ModelCheckpoint(
								    filepath=args.output_dir	,					prefix="""checkpoint"""	,					monitor="""val_loss"""	,					mode="""min"""	,					save_top_k=1 )
				if early_stopping_callback:
								extra_callbacks.append(a_ )
				if logging_callback is None:
								lowerCamelCase		       =LoggingCallback()

				lowerCamelCase		       ={}

				# TODO: remove with PyTorch 1.6 since pl uses native amp
				if args.fpaa:
								lowerCamelCase		       =16
								lowerCamelCase		       =args.fpaa_opt_level

				if args.gpus > 1:
								lowerCamelCase		       ="""ddp"""

				lowerCamelCase		       =args.accumulate_grad_batches
				lowerCamelCase		       =extra_train_kwargs.get("""accelerator"""	,					a_ )
				lowerCamelCase		       =extra_train_kwargs.get("""profiler"""	,					a_ )

				lowerCamelCase		       =pl.Trainer.from_argparse_args(
				    a_	,					weights_summary=a_	,					callbacks=[logging_callback] + extra_callbacks	,					logger=a_	,					checkpoint_callback=a_	,					**a_	,					)

				if args.do_train:
								trainer.fit(a_ )

				return trainer













import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


lowerCamelCase_						     =  logging.get_logger(__name__)




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					Any:
														'''simple docstring'''

														super().__init__()
														if version.parse(torch.__version__   ) < version.parse('1.12.0'   ):
																					raise ImportError(
																					    f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
																					    'PytorchGELUTanh. Please upgrade torch.'   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return nn.functional.gelu(lowercase_			,			approximate='tanh'   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi   ) * (input + 0.04_47_15 * torch.pow(lowercase_			,			3.0   ))   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self			,			lowercase_ = False   )					->					Optional[Any]:
														'''simple docstring'''

														super().__init__()
														if use_gelu_python:
																					lowerCAmelCase_										=     self._gelu_python
														else:
																					lowerCAmelCase_										=     nn.functional.gelu

							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0   )   ))






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return self.act(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1.0 + torch.tanh(input * 0.79_78_84_56_08 * (1.0 + 0.04_47_15 * input * input)   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * torch.sigmoid(1.7_02 * input   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self			,			lowercase_			,			lowercase_   )					->					Optional[int]:
														'''simple docstring'''

														if min > max:
																					raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''   )

														super().__init__()
														lowerCAmelCase_										=     min
														lowerCAmelCase_										=     max






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return torch.clip(gelu(lowercase_   )			,			self.min			,			self.max   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					Union[str, Any]:
														'''simple docstring'''

														super().__init__()
														lowerCAmelCase_										=     math.sqrt(2 / math.pi   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_47_15 * torch.pow(lowercase_			,			3   ))   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return nn.functional.silu(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					List[Any]:
														'''simple docstring'''

														super().__init__()
														if version.parse(torch.__version__   ) < version.parse('1.9.0'   ):
																					lowerCAmelCase_										=     self._mish_python
														else:
																					lowerCAmelCase_										=     nn.functional.mish

							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * torch.tanh(nn.functional.softplus(lowercase_   )   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return self.act(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_			,			lowercase_=0.70_71_07			,			lowercase_=0.28_20_95   )					->					Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     (input - mu).div(sigma * math.sqrt(2.0   )   )
														return 0.5 * (1.0 + torch.erf(lowercase_   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     nn.functional.relu(lowercase_   )
														lowerCAmelCase_										=     torch.square(lowercase_   )
														return squared




class 							a_ (						a_		):


							'''simple docstring'''






							def __getitem__(		self			,			lowercase_   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     super().__getitem__(lowercase_   )
														lowerCAmelCase_						,   lowerCAmelCase_										=     content if isinstance(lowercase_			,			lowercase_   ) else (content, {})
														return cls(**lowercase_   )


lowerCamelCase_						     =  {
    """gelu""": GELUActivation,
    """gelu_10""": (ClippedGELUActivation, {"""min""": -1_0, """max""": 1_0}),
    """gelu_fast""": FastGELUActivation,
    """gelu_new""": NewGELUActivation,
    """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
    """gelu_pytorch_tanh""": PytorchGELUTanh,
    """gelu_accurate""": AccurateGELUActivation,
    """laplace""": LaplaceActivation,
    """linear""": LinearActivation,
    """mish""": MishActivation,
    """quick_gelu""": QuickGELUActivation,
    """relu""": nn.ReLU,
    """relu2""": ReLUSquaredActivation,
    """relu6""": nn.ReLUa,
    """sigmoid""": nn.Sigmoid,
    """silu""": SiLUActivation,
    """swish""": SiLUActivation,
    """tanh""": nn.Tanh,
}
lowerCamelCase_						     =  ClassInstantier(ACTaCLS)
def      lowerCamelCase					(						a_	)	->			str:
							if activation_string in ACTaFN:
														return ACTaFN[activation_string]
							else:
														raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()	)}'''	)


# For backwards compatibility with: from activations import gelu_python
lowerCamelCase_						     =  get_activation("""gelu_python""")
lowerCamelCase_						     =  get_activation("""gelu_new""")
lowerCamelCase_						     =  get_activation("""gelu""")
lowerCamelCase_						     =  get_activation("""gelu_fast""")
lowerCamelCase_						     =  get_activation("""quick_gelu""")
lowerCamelCase_						     =  get_activation("""silu""")
lowerCamelCase_						     =  get_activation("""mish""")



lowerCamelCase_						     =  get_activation("""linear""")



