




"""simple docstring"""
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


a_				:    List[Any]										=     logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

a_				:    Any										=     {
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
a_				:    str										=     {
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
a_				:    Union[str, Any]										=     sorted(arg_to_scheduler.keys())
a_				:    str										=     '''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''

class __lowercase(      pl.LightningModule				):




					'''simple docstring'''


					def __init__(    self						,	__a						,	__a=None						,	__a="base"						,	__a=None						,	__a=None						,	__a=None						,	**__a						,	):
								super().__init__()
								# TODO: move to self.save_hyperparameters()
								# self.save_hyperparameters()
								# can also expand arguments into trainer signature for easier reading

								self.save_hyperparameters(__a     )
								__lowerCamelCase					:	int	       =				0
								__lowerCamelCase					:	Any	       =				Path(self.hparams.output_dir     )
								__lowerCamelCase					:	str	       =				self.hparams.cache_dir if self.hparams.cache_dir else None
								if config is None:
											__lowerCamelCase					:	Tuple	       =				AutoConfig.from_pretrained(
											    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path						,	**({'num_labels': num_labels} if num_labels is not None else {})						,	cache_dir=__a						,	**__a						,	)
								else:
											__lowerCamelCase					:	PretrainedConfig	       =				config

								__lowerCamelCase					:	int	       =				('encoder_layerdrop', 'decoder_layerdrop', 'dropout', 'attention_dropout')
								for p in extra_model_params:
											if getattr(self.hparams						,	__a						,	__a     ):
														assert hasattr(self.config						,	__a     ), f'''model config doesn\'t have a `{p}` attribute'''
														setattr(self.config						,	__a						,	getattr(self.hparams						,	__a     )     )

								if tokenizer is None:
											__lowerCamelCase					:	List[Any]	       =				AutoTokenizer.from_pretrained(
											    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path						,	cache_dir=__a						,	)
								else:
											__lowerCamelCase					:	PreTrainedTokenizer	       =				tokenizer
								__lowerCamelCase					:	Dict	       =				MODEL_MODES[mode]
								if model is None:
											__lowerCamelCase					:	Tuple	       =				self.model_type.from_pretrained(
											    self.hparams.model_name_or_path						,	from_tf=bool('.ckpt' in self.hparams.model_name_or_path     )						,	config=self.config						,	cache_dir=__a						,	)
								else:
											__lowerCamelCase					:	Any	       =				model
					def     snake_case_				(    self						,	*__a						,	**__a     ):
								__lowerCamelCase					:	Tuple	       =				self.model_type.from_pretrained(*__a						,	**__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Dict	       =				arg_to_scheduler[self.hparams.lr_scheduler]
								__lowerCamelCase					:	str	       =				get_schedule_func(
								    self.opt						,	num_warmup_steps=self.hparams.warmup_steps						,	num_training_steps=self.total_steps()     )
								__lowerCamelCase					:	Tuple	       =				{'scheduler': scheduler, 'interval': 'step', 'frequency': 1}
								return scheduler
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	int	       =				self.model
								__lowerCamelCase					:	Tuple	       =				['bias', 'LayerNorm.weight']
								__lowerCamelCase					:	List[Any]	       =				[
								    {
								        'params': [
								            p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay     )
								        ],  # check this named paramters
								        'weight_decay': self.hparams.weight_decay,
								    },
								    {
								        'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay     )],
								        'weight_decay': 0.0,
								    },
								]
								if self.hparams.adafactor:
											__lowerCamelCase					:	str	       =				Adafactor(
											    __a						,	lr=self.hparams.learning_rate						,	scale_parameter=__a						,	relative_step=__a     )

								else:
											__lowerCamelCase					:	str	       =				AdamW(
											    __a						,	lr=self.hparams.learning_rate						,	eps=self.hparams.adam_epsilon     )
								__lowerCamelCase					:	Union[str, Any]	       =				optimizer

								__lowerCamelCase					:	int	       =				self.get_lr_scheduler()

								return [optimizer], [scheduler]
					def     snake_case_				(    self						,	__a						,	__a     ):
								return self.validation_step(__a						,	__a     )
					def     snake_case_				(    self						,	__a     ):
								return self.validation_end(__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[str]	       =				max(1						,	self.hparams.gpus     )  # TODO: consider num_tpu_cores
								__lowerCamelCase					:	Any	       =				self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
								return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
					def     snake_case_				(    self						,	__a     ):
								if stage == "test":
											__lowerCamelCase					:	Dict	       =				len(self.test_dataloader().dataset     )
								else:
											__lowerCamelCase					:	Any	       =				self.get_dataloader('train'						,	self.hparams.train_batch_size						,	shuffle=__a     )
											__lowerCamelCase					:	List[str]	       =				len(self.train_dataloader().dataset     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a = False     ):
								raise NotImplementedError('You must implement this for your task'     )
					def     snake_case_				(    self     ):
								return self.train_loader
					def     snake_case_				(    self     ):
								return self.get_dataloader('dev'						,	self.hparams.eval_batch_size						,	shuffle=__a     )
					def     snake_case_				(    self     ):
								return self.get_dataloader('test'						,	self.hparams.eval_batch_size						,	shuffle=__a     )
					def     snake_case_				(    self						,	__a     ):
								return os.path.join(
								    self.hparams.data_dir						,	'cached_{}_{}_{}'.format(
								        __a						,	list(filter(__a						,	self.hparams.model_name_or_path.split('/'     )     )     ).pop()						,	str(self.hparams.max_seq_length     )						,	)						,	)
					@pl.utilities.rank_zero_only
					def     snake_case_				(    self						,	__a     ):
								__lowerCamelCase					:	Optional[Any]	       =				self.output_dir.joinpath('best_tfmr'     )
								__lowerCamelCase					:	Dict	       =				self.step_count
								self.model.save_pretrained(__a     )
								self.tokenizer.save_pretrained(__a     )





					@staticmethod
					def     snake_case_				(    __a						,	__a     ):
								parser.add_argument(
								    '--model_name_or_path'						,	default=__a						,	type=__a						,	required=__a						,	help='Path to pretrained model or model identifier from huggingface.co/models'						,	)
								parser.add_argument(
								    '--config_name'						,	default=''						,	type=__a						,	help='Pretrained config name or path if not the same as model_name'     )
								parser.add_argument(
								    '--tokenizer_name'						,	default=__a						,	type=__a						,	help='Pretrained tokenizer name or path if not the same as model_name'						,	)
								parser.add_argument(
								    '--cache_dir'						,	default=str(Path(__a     ).parent / 'test_run' / 'cache'     )						,	type=__a						,	help='Where do you want to store the pre-trained models downloaded from huggingface.co'						,	)
								parser.add_argument(
								    '--encoder_layerdrop'						,	type=__a						,	help='Encoder layer dropout probability (Optional). Goes into model.config'						,	)
								parser.add_argument(
								    '--decoder_layerdrop'						,	type=__a						,	help='Decoder layer dropout probability (Optional). Goes into model.config'						,	)
								parser.add_argument(
								    '--dropout'						,	type=__a						,	help='Dropout probability (Optional). Goes into model.config'						,	)
								parser.add_argument(
								    '--attention_dropout'						,	type=__a						,	help='Attention dropout probability (Optional). Goes into model.config'						,	)
								parser.add_argument('--learning_rate'						,	default=5E-5						,	type=__a						,	help='The initial learning rate for Adam.'     )
								parser.add_argument(
								    '--lr_scheduler'						,	default='linear'						,	choices=__a						,	metavar=__a						,	type=__a						,	help='Learning rate scheduler'						,	)
								parser.add_argument('--weight_decay'						,	default=0.0						,	type=__a						,	help='Weight decay if we apply some.'     )
								parser.add_argument('--adam_epsilon'						,	default=1E-8						,	type=__a						,	help='Epsilon for Adam optimizer.'     )
								parser.add_argument('--warmup_steps'						,	default=0						,	type=__a						,	help='Linear warmup over warmup_steps.'     )
								parser.add_argument('--num_workers'						,	default=4						,	type=__a						,	help='kwarg passed to DataLoader'     )
								parser.add_argument('--num_train_epochs'						,	dest='max_epochs'						,	default=3						,	type=__a     )
								parser.add_argument('--train_batch_size'						,	default=32						,	type=__a     )
								parser.add_argument('--eval_batch_size'						,	default=32						,	type=__a     )
								parser.add_argument('--adafactor'						,	action='store_true'     )



class __lowercase(      pl.Callback				):




					'''simple docstring'''


					def     snake_case_				(    self						,	__a						,	__a     ):
								if (
								    trainer.is_global_zero and trainer.global_rank == 0
								):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
											pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.



class __lowercase(      pl.Callback				):




					'''simple docstring'''


					def     snake_case_				(    self						,	__a						,	__a     ):
								# print(pl_module.model.rag)
								for name, param in pl_module.model.rag.named_parameters():
											if param.grad is None:
														print(__a     )



class __lowercase(      pl.Callback				):




					'''simple docstring'''


					def     snake_case_				(    self						,	__a						,	__a     ):
								__lowerCamelCase					:	Union[str, Any]	       =				trainer.lr_schedulers[0]['scheduler']
								__lowerCamelCase					:	Any	       =				{f'''lr_group_{i}''': lr for i, lr in enumerate(lr_scheduler.get_lr()     )}
								pl_module.logger.log_metrics(__a     )
					def     snake_case_				(    self						,	__a						,	__a     ):
								rank_zero_info('***** Validation results *****'     )
								__lowerCamelCase					:	Union[str, Any]	       =				trainer.callback_metrics
								# Log results
								for key in sorted(__a     ):
											if key not in ["log", "progress_bar"]:
														rank_zero_info('{} = {}\n'.format(__a						,	str(metrics[key]     )     )     )





					def     snake_case_				(    self						,	__a						,	__a     ):
								rank_zero_info('***** Test results *****'     )
								__lowerCamelCase					:	Tuple	       =				trainer.callback_metrics
								# Log and save results to file
								__lowerCamelCase					:	Dict	       =				os.path.join(pl_module.hparams.output_dir						,	'test_results.txt'     )
								with open(__a						,	'w'     ) as writer:
											for key in sorted(__a     ):
														if key not in ["log", "progress_bar"]:
																	rank_zero_info('{} = {}\n'.format(__a						,	str(metrics[key]     )     )     )
																	writer.write('{} = {}\n'.format(__a						,	str(metrics[key]     )     )     )





def 						UpperCAmelCase				(			A__:    str							,				A__:    List[Any]     )  ->     None:
			#  To allow all pl args uncomment the following line
			#  parser = pl.Trainer.add_argparse_args(parser)
			parser.add_argument(
			    '--output_dir'							,				default=str(Path(A__     ).parent / 'test_run' / 'model_checkpoints'     )							,				type=A__							,				help='The output directory where the model predictions and checkpoints will be written.'							,				)
			parser.add_argument(
			    '--fp16'							,				action='store_true'							,				help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'							,				)

			parser.add_argument(
			    '--fp16_opt_level'							,				type=A__							,				default='O2'							,				help=(
			        'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
			        'See details at https://nvidia.github.io/apex/amp.html'
			    )							,				)
			parser.add_argument('--n_tpu_cores'							,				dest='tpu_cores'							,				type=A__     )
			parser.add_argument('--max_grad_norm'							,				dest='gradient_clip_val'							,				default=1.0							,				type=A__							,				help='Max gradient norm'     )
			parser.add_argument('--do_train'							,				action='store_true'							,				help='Whether to run training.'     )
			parser.add_argument('--do_predict'							,				action='store_true'							,				help='Whether to run predictions on the test set.'     )
			parser.add_argument(
			    '--gradient_accumulation_steps'							,				dest='accumulate_grad_batches'							,				type=A__							,				default=1							,				help='Number of updates steps to accumulate before performing a backward/update pass.'							,				)
			parser.add_argument('--seed'							,				type=A__							,				default=42							,				help='random seed for initialization'     )
			parser.add_argument(
			    '--data_dir'							,				default=str(Path(A__     ).parent / 'test_run' / 'dummy-train-data'     )							,				type=A__							,				help='The input data dir. Should contain the training files for the CoNLL-2003 NER task.'							,				)






def 						UpperCAmelCase				(			A__:    BaseTransformer							,				A__:    argparse.Namespace							,				A__:    Tuple=None							,				A__:    Optional[Any]=True							,				A__:    int=[]							,				A__:    Any=None							,				A__:    Any=None							,				**A__:    Union[str, Any]							,				)  ->     Tuple:
			pl.seed_everything(args.seed     )

			# init model
			__lowerCamelCase					:	Tuple	       =				Path(model.hparams.output_dir     )
			odir.mkdir(exist_ok=A__     )

			# add custom checkpoints
			if checkpoint_callback is None:
						__lowerCamelCase					:	Dict	       =				pl.callbacks.ModelCheckpoint(
						    filepath=args.output_dir							,				prefix='checkpoint'							,				monitor='val_loss'							,				mode='min'							,				save_top_k=1     )
			if early_stopping_callback:
						extra_callbacks.append(A__     )
			if logging_callback is None:
						__lowerCamelCase					:	int	       =				LoggingCallback()

			__lowerCamelCase					:	str	       =				{}

			if args.fpaa:
						__lowerCamelCase					:	str	       =				16

			if args.gpus > 1:
						__lowerCamelCase					:	int	       =				'auto'
						__lowerCamelCase					:	Any	       =				'ddp'

			__lowerCamelCase					:	Any	       =				args.accumulate_grad_batches
			__lowerCamelCase					:	Optional[Any]	       =				None
			__lowerCamelCase					:	Optional[Any]	       =				'auto'

			__lowerCamelCase					:	Optional[Any]	       =				pl.Trainer.from_argparse_args(
			    A__							,				weights_summary=A__							,				callbacks=[logging_callback] + extra_callbacks + [InitCallback()] + [checkpoint_callback]							,				logger=A__							,				val_check_interval=1							,				num_sanity_val_steps=2							,				**A__							,				)

			if args.do_train:
						trainer.fit(A__     )

			else:
						print('RAG modeling tests with new set functions successfuly executed!'     )
			return trainer









"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_				:    Tuple										=     logging.get_logger(__name__)

a_				:    str										=     {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/config.json'''}

class __lowercase(      lowercase__				):




					'''simple docstring'''


					__a						:						Union[str, Any]     		= 'openai-gpt'
					__a						:						List[Any]     		= {
					    'max_position_embeddings': 'n_positions',
					    'hidden_size': 'n_embd',
					    'num_attention_heads': 'n_head',
					    'num_hidden_layers': 'n_layer',
					}
					def __init__(    self						,	__a=40478						,	__a=512						,	__a=768						,	__a=12						,	__a=12						,	__a="gelu"						,	__a=0.1						,	__a=0.1						,	__a=0.1						,	__a=1E-5						,	__a=0.02						,	__a="cls_index"						,	__a=True						,	__a=None						,	__a=True						,	__a=0.1						,	**__a						,	):
								__lowerCamelCase					:	List[str]	       =				vocab_size
								__lowerCamelCase					:	Union[str, Any]	       =				n_positions
								__lowerCamelCase					:	Optional[Any]	       =				n_embd
								__lowerCamelCase					:	int	       =				n_layer
								__lowerCamelCase					:	List[str]	       =				n_head
								__lowerCamelCase					:	Dict	       =				afn
								__lowerCamelCase					:	Optional[Any]	       =				resid_pdrop
								__lowerCamelCase					:	int	       =				embd_pdrop
								__lowerCamelCase					:	int	       =				attn_pdrop
								__lowerCamelCase					:	Dict	       =				layer_norm_epsilon
								__lowerCamelCase					:	List[str]	       =				initializer_range
								__lowerCamelCase					:	Any	       =				summary_type
								__lowerCamelCase					:	Tuple	       =				summary_use_proj
								__lowerCamelCase					:	List[Any]	       =				summary_activation
								__lowerCamelCase					:	str	       =				summary_first_dropout
								__lowerCamelCase					:	Tuple	       =				summary_proj_to_labels
								super().__init__(**__a     )



