






import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


a__  	=  logging.getLogger(__name__)


class  _lowerCAmelCase      (		a__   ):



				"""simple docstring"""



				_lowercase					: int	     =	"""sequence-classification"""



				def __init__(   self     :     Any      ,		UpperCamelCase__     :     Union[str, Any]):






											'''simple docstring'''







											if type(__a) == dict:
																		snake_case__							=							Namespace(**__a)
											snake_case__							=							glue_output_modes[hparams.task]
											snake_case__							=							glue_tasks_num_labels[hparams.task]

											super().__init__(__a      ,		__a      ,		self.mode)



				def        __magic_name__				(   self     :     List[Any]      ,		**UpperCamelCase__     :     Union[str, Any]):






											'''simple docstring'''







											return self.model(**__a)



				def        __magic_name__				(   self     :     Optional[Any]      ,		UpperCamelCase__     :     List[str]      ,		UpperCamelCase__     :     Union[str, Any]):






											'''simple docstring'''







											snake_case__							=							{"""input_ids""": batch[0], """attention_mask""": batch[1], """labels""": batch[3]}

											if self.config.model_type not in ["distilbert", "bart"]:
																		snake_case__							=							batch[2] if self.config.model_type in ["""bert""", """xlnet""", """albert"""] else None

											snake_case__							=							self(**__a)
											snake_case__							=							outputs[0]

											snake_case__							=							self.trainer.lr_schedulers[0]["""scheduler"""]
											snake_case__							=							{"""loss""": loss, """rate""": lr_scheduler.get_last_lr()[-1]}
											return {"loss": loss, "log": tensorboard_logs}



				def        __magic_name__				(   self     :     Tuple):






											'''simple docstring'''







											snake_case__							=							self.hparams
											snake_case__							=							processors[args.task]()
											snake_case__							=							processor.get_labels()

											for mode in ["train", "dev"]:
																		snake_case__							=							self._feature_file(__a)
																		if os.path.exists(__a) and not args.overwrite_cache:
																									logger.info("""Loading features from cached file %s"""      ,		__a)
																		else:
																									logger.info("""Creating features from dataset file at %s"""      ,		args.data_dir)
																									snake_case__							=							(
																									    processor.get_dev_examples(args.data_dir)
																									    if mode == """dev"""
																									    else processor.get_train_examples(args.data_dir)
																									)
																									snake_case__							=							convert_examples_to_features(
																									    __a      ,		self.tokenizer      ,		max_length=args.max_seq_length      ,		label_list=self.labels      ,		output_mode=args.glue_output_mode      ,		)
																									logger.info("""Saving features into cached file %s"""      ,		__a)
																									torch.save(__a      ,		__a)



				def        __magic_name__				(   self     :     Union[str, Any]      ,		UpperCamelCase__     :     Any      ,		UpperCamelCase__     :     Optional[int]      ,		UpperCamelCase__     :     str = False):






											'''simple docstring'''







											snake_case__							=							"""dev""" if mode == """test""" else mode

											snake_case__							=							self._feature_file(__a)
											logger.info("""Loading features from cached file %s"""      ,		__a)
											snake_case__							=							torch.load(__a)
											snake_case__							=							torch.tensor([f.input_ids for f in features]      ,		dtype=torch.long)
											snake_case__							=							torch.tensor([f.attention_mask for f in features]      ,		dtype=torch.long)
											snake_case__							=							torch.tensor([f.token_type_ids for f in features]      ,		dtype=torch.long)
											if self.hparams.glue_output_mode == "classification":
																		snake_case__							=							torch.tensor([f.label for f in features]      ,		dtype=torch.long)
											elif self.hparams.glue_output_mode == "regression":
																		snake_case__							=							torch.tensor([f.label for f in features]      ,		dtype=torch.float)

											return DataLoader(
											    TensorDataset(__a      ,		__a      ,		__a      ,		__a)      ,		batch_size=__a      ,		shuffle=__a      ,		)



				def        __magic_name__				(   self     :     int      ,		UpperCamelCase__     :     List[str]      ,		UpperCamelCase__     :     Tuple):






											'''simple docstring'''







											snake_case__							=							{"""input_ids""": batch[0], """attention_mask""": batch[1], """labels""": batch[3]}

											if self.config.model_type not in ["distilbert", "bart"]:
																		snake_case__							=							batch[2] if self.config.model_type in ["""bert""", """xlnet""", """albert"""] else None

											snake_case__							=							self(**__a)
											snake_case__							=							outputs[:2]
											snake_case__							=							logits.detach().cpu().numpy()
											snake_case__							=							inputs["""labels"""].detach().cpu().numpy()

											return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}



				def        __magic_name__				(   self     :     Tuple      ,		UpperCamelCase__     :     Tuple):






											'''simple docstring'''







											snake_case__							=							torch.stack([x["""val_loss"""] for x in outputs]).mean().detach().cpu().item()
											snake_case__							=							np.concatenate([x["""pred"""] for x in outputs]      ,		axis=0)

											if self.hparams.glue_output_mode == "classification":
																		snake_case__							=							np.argmax(__a      ,		axis=1)
											elif self.hparams.glue_output_mode == "regression":
																		snake_case__							=							np.squeeze(__a)

											snake_case__							=							np.concatenate([x["""target"""] for x in outputs]      ,		axis=0)
											snake_case__							=							[[] for _ in range(out_label_ids.shape[0])]
											snake_case__							=							[[] for _ in range(out_label_ids.shape[0])]

											snake_case__							=							{**{"""val_loss""": val_loss_mean}, **compute_metrics(self.hparams.task      ,		__a      ,		__a)}

											snake_case__							=							dict(results.items())
											snake_case__							=							results
											return ret, preds_list, out_label_list



				def        __magic_name__				(   self     :     List[Any]      ,		UpperCamelCase__     :     Optional[Any]):






											'''simple docstring'''







											snake_case__							=							self._eval_end(__a)
											snake_case__							=							ret["""log"""]
											return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}



				def        __magic_name__				(   self     :     Tuple      ,		UpperCamelCase__     :     str):






											'''simple docstring'''







											snake_case__							=							self._eval_end(__a)
											snake_case__							=							ret["""log"""]
											# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
											return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}



				@staticmethod
				def        __magic_name__				(   UpperCamelCase__     :     Dict      ,		UpperCamelCase__     :     Union[str, Any]):






											'''simple docstring'''







											BaseTransformer.add_model_specific_args(__a      ,		__a)
											parser.add_argument(
											    """--max_seq_length"""      ,		default=1_2_8      ,		type=__a      ,		help=(
											        """The maximum total input sequence length after tokenization. Sequences longer """
											        """than this will be truncated, sequences shorter will be padded."""
											    )      ,		)

											parser.add_argument(
											    """--task"""      ,		default=""""""      ,		type=__a      ,		required=__a      ,		help="""The GLUE task to run"""      ,		)
											parser.add_argument(
											    """--gpus"""      ,		default=0      ,		type=__a      ,		help="""The number of GPUs allocated for this, it is by default 0 meaning none"""      ,		)

											parser.add_argument(
											    """--overwrite_cache"""      ,		action="""store_true"""      ,		help="""Overwrite the cached training and evaluation sets""")

											return parser


def _UpperCAmelCase (						):
							snake_case__							=							argparse.ArgumentParser()
							add_generic_args(__lowercase	,       os.getcwd()			)
							snake_case__							=							GLUETransformer.add_model_specific_args(__lowercase	,       os.getcwd()			)
							snake_case__							=							parser.parse_args()

							# If output_dir not provided, a folder will be generated in pwd
							if args.output_dir is None:
														snake_case__							=							os.path.join(
														    """./results"""	,       F'''{args.task}_{time.strftime('%Y%m%d_%H%M%S'			)}'''	,       )
														os.makedirs(args.output_dir			)

							snake_case__							=							GLUETransformer(__lowercase			)
							snake_case__							=							generic_train(__lowercase	,       __lowercase			)

							# Optionally, predict on dev set and write to output_dir
							if args.do_predict:
														snake_case__							=							sorted(glob.glob(os.path.join(args.output_dir	,       """checkpoint-epoch=*.ckpt"""			)	,       recursive=__lowercase			)			)
														snake_case__							=							model.load_from_checkpoint(checkpoints[-1]			)
														return trainer.test(__lowercase			)


if __name__ == "__main__":
	main()






'''simple docstring'''







import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase      :   Optional[int]      =       logging.get_logger(__name__)

lowercase      :   Union[str, Any]      =       {
    """microsoft/unispeech-sat-base-100h-libri-ft""": (
        """https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json"""
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     List[Any]          =					"""unispeech-sat"""







     def __init__(				self							,__a=32							,__a=768							,__a=12							,__a=12							,__a=3_072							,__a="gelu"							,__a=0.1							,__a=0.1							,__a=0.1							,__a=0.0							,__a=0.0							,__a=0.1							,__a=0.1							,__a=0.02							,__a=1E-5							,__a="group"							,__a="gelu"							,__a=(512, 512, 512, 512, 512, 512, 512)							,__a=(5, 2, 2, 2, 2, 2, 2)							,__a=(10, 3, 3, 3, 3, 2, 2)							,__a=False							,__a=128							,__a=16							,__a=False							,__a=True							,__a=0.05							,__a=10							,__a=2							,__a=0.0							,__a=10							,__a=0							,__a=320							,__a=2							,__a=0.1							,__a=100							,__a=256							,__a=256							,__a=0.1							,__a="mean"							,__a=False							,__a=False							,__a=256							,__a=(512, 512, 512, 512, 1_500)							,__a=(5, 3, 3, 1, 1)							,__a=(1, 2, 3, 1, 1)							,__a=512							,__a=0							,__a=1							,__a=2							,__a=504							,**__a							,)      ->				str:
          super().__init__(**__a							,pad_token_id=__a							,bos_token_id=__a							,eos_token_id=__a							)
          snake_case			:      List[Any]        =			hidden_size
          snake_case			:      Any        =			feat_extract_norm
          snake_case			:      Dict        =			feat_extract_activation
          snake_case			:      List[Any]        =			list(__a							)
          snake_case			:      int        =			list(__a							)
          snake_case			:      Dict        =			list(__a							)
          snake_case			:      int        =			conv_bias
          snake_case			:      Optional[Any]        =			num_conv_pos_embeddings
          snake_case			:      Any        =			num_conv_pos_embedding_groups
          snake_case			:      Dict        =			len(self.conv_dim							)
          snake_case			:      str        =			num_hidden_layers
          snake_case			:      Optional[Any]        =			intermediate_size
          snake_case			:      Optional[Any]        =			hidden_act
          snake_case			:      Union[str, Any]        =			num_attention_heads
          snake_case			:      str        =			hidden_dropout
          snake_case			:      Tuple        =			attention_dropout
          snake_case			:      Tuple        =			activation_dropout
          snake_case			:      List[Any]        =			feat_proj_dropout
          snake_case			:      List[str]        =			final_dropout
          snake_case			:      Union[str, Any]        =			layerdrop
          snake_case			:      Optional[int]        =			layer_norm_eps
          snake_case			:      int        =			initializer_range
          snake_case			:      List[Any]        =			vocab_size
          snake_case			:      Any        =			num_clusters
          snake_case			:      Dict        =			do_stable_layer_norm
          snake_case			:      int        =			use_weighted_layer_sum

          if (
              (len(self.conv_stride							) != self.num_feat_extract_layers)
              or (len(self.conv_kernel							) != self.num_feat_extract_layers)
              or (len(self.conv_dim							) != self.num_feat_extract_layers)
          ):
               raise ValueError(
                   """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                   """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                   F''' {len(self.conv_dim							)}`, `len(config.conv_stride) = {len(self.conv_stride							)}`,'''
                   F''' `len(config.conv_kernel) = {len(self.conv_kernel							)}`.'''							)

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          snake_case			:      Dict        =			apply_spec_augment
          snake_case			:      Optional[int]        =			mask_time_prob
          snake_case			:      int        =			mask_time_length
          snake_case			:      int        =			mask_time_min_masks
          snake_case			:      int        =			mask_feature_prob
          snake_case			:      Optional[Any]        =			mask_feature_length
          snake_case			:      List[Any]        =			mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          snake_case			:      Optional[Any]        =			num_codevectors_per_group
          snake_case			:      Optional[int]        =			num_codevector_groups
          snake_case			:      Any        =			contrastive_logits_temperature
          snake_case			:      List[str]        =			feat_quantizer_dropout
          snake_case			:      Dict        =			num_negatives
          snake_case			:      int        =			codevector_dim
          snake_case			:      Optional[int]        =			proj_codevector_dim
          snake_case			:      Union[str, Any]        =			diversity_loss_weight

          # ctc loss
          snake_case			:      Union[str, Any]        =			ctc_loss_reduction
          snake_case			:      Optional[int]        =			ctc_zero_infinity

          # SequenceClassification-specific parameter. Feel free to ignore for other classes.
          snake_case			:      Dict        =			classifier_proj_size

          # XVector-specific parameters. Feel free to ignore for other classes.
          snake_case			:      int        =			list(__a							)
          snake_case			:      Tuple        =			list(__a							)
          snake_case			:      List[Any]        =			list(__a							)
          snake_case			:      Dict        =			xvector_output_dim







     @property
     def 			snake_case_      (				self							)      ->				List[str]:
          return functools.reduce(operator.mul							,self.conv_stride							,1							)



