






import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


UpperCAmelCase__      			=						{
    '''distilbert''': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    '''roberta''': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    '''bert''': (BertConfig, BertForMaskedLM, BertTokenizer),
    '''gpt2''': (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}







def     a_   (__A      )				->						Optional[int]:



					"""simple docstring"""
					assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
					assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
					if args.mlm:
										assert os.path.isfile(args.token_counts      )
										assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
					else:
										assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

					assert args.teacher_type == args.student_type or (
					    args.student_type == "distilbert" and args.teacher_type == "bert"
					)
					assert os.path.isfile(args.student_config      )
					if args.student_pretrained_weights is not None:
										assert os.path.isfile(args.student_pretrained_weights      )

					if args.freeze_token_type_embds:
										assert args.student_type in ["roberta"]

					assert args.alpha_ce >= 0.0
					assert args.alpha_mlm >= 0.0
					assert args.alpha_clm >= 0.0
					assert args.alpha_mse >= 0.0
					assert args.alpha_cos >= 0.0
					assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0







def     a_   (__A				,       __A      )				->						Optional[Any]:



					"""simple docstring"""
					if args.student_type == "roberta":
										__a					:						Dict      =					False
					elif args.student_type == "gpt2":
										__a					:						Any      =					False







def     a_   (__A				,       __A      )				->						List[str]:



					"""simple docstring"""
					if args.student_type == "roberta":
										__a					:						Tuple      =					False







def     a_   ()				->						Union[str, Any]:



					"""simple docstring"""
					__a					:						Optional[int]      =					argparse.ArgumentParser(description="Training"      )
					parser.add_argument("--force"				,       action="store_true"				,       help="Overwrite dump_path if it already exists."      )

					parser.add_argument(
					    "--dump_path"				,       type=__A				,       required=__A				,       help="The output directory (log, checkpoints, parameters, etc.)"      )
					parser.add_argument(
					    "--data_file"				,       type=__A				,       required=__A				,       help="The binarized file (tokenized + tokens_to_ids) and grouped by sequence."				,       )

					parser.add_argument(
					    "--student_type"				,       type=__A				,       choices=["distilbert", "roberta", "gpt2"]				,       required=__A				,       help="The student type (DistilBERT, RoBERTa)."				,       )
					parser.add_argument("--student_config"				,       type=__A				,       required=__A				,       help="Path to the student configuration."      )
					parser.add_argument(
					    "--student_pretrained_weights"				,       default=__A				,       type=__A				,       help="Load student initialization checkpoint."      )

					parser.add_argument(
					    "--teacher_type"				,       choices=["bert", "roberta", "gpt2"]				,       required=__A				,       help="Teacher type (BERT, RoBERTa)."      )
					parser.add_argument("--teacher_name"				,       type=__A				,       required=__A				,       help="The teacher model."      )

					parser.add_argument("--temperature"				,       default=2.0				,       type=__A				,       help="Temperature for the softmax temperature."      )
					parser.add_argument(
					    "--alpha_ce"				,       default=0.5				,       type=__A				,       help="Linear weight for the distillation loss. Must be >=0."      )
					parser.add_argument(
					    "--alpha_mlm"				,       default=0.0				,       type=__A				,       help="Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag."				,       )
					parser.add_argument("--alpha_clm"				,       default=0.5				,       type=__A				,       help="Linear weight for the CLM loss. Must be >=0."      )
					parser.add_argument("--alpha_mse"				,       default=0.0				,       type=__A				,       help="Linear weight of the MSE loss. Must be >=0."      )
					parser.add_argument(
					    "--alpha_cos"				,       default=0.0				,       type=__A				,       help="Linear weight of the cosine embedding loss. Must be >=0."      )

					parser.add_argument(
					    "--mlm"				,       action="store_true"				,       help="The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM."      )
					parser.add_argument(
					    "--mlm_mask_prop"				,       default=0.15				,       type=__A				,       help="Proportion of tokens for which we need to make a prediction."				,       )
					parser.add_argument("--word_mask"				,       default=0.8				,       type=__A				,       help="Proportion of tokens to mask out."      )
					parser.add_argument("--word_keep"				,       default=0.1				,       type=__A				,       help="Proportion of tokens to keep."      )
					parser.add_argument("--word_rand"				,       default=0.1				,       type=__A				,       help="Proportion of tokens to randomly replace."      )
					parser.add_argument(
					    "--mlm_smoothing"				,       default=0.7				,       type=__A				,       help="Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec)."				,       )
					parser.add_argument("--token_counts"				,       type=__A				,       help="The token counts in the data_file for MLM."      )

					parser.add_argument(
					    "--restrict_ce_to_mask"				,       action="store_true"				,       help="If true, compute the distillation loss only the [MLM] prediction distribution."				,       )
					parser.add_argument(
					    "--freeze_pos_embs"				,       action="store_true"				,       help="Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only."				,       )
					parser.add_argument(
					    "--freeze_token_type_embds"				,       action="store_true"				,       help="Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only."				,       )

					parser.add_argument("--n_epoch"				,       type=__A				,       default=3				,       help="Number of pass on the whole dataset."      )
					parser.add_argument("--batch_size"				,       type=__A				,       default=5				,       help="Batch size (for each process)."      )
					parser.add_argument(
					    "--group_by_size"				,       action="store_false"				,       help="If true, group sequences that have similar length into the same batch. Default is true."				,       )

					parser.add_argument(
					    "--gradient_accumulation_steps"				,       type=__A				,       default=50				,       help="Gradient accumulation for larger training batches."				,       )
					parser.add_argument("--warmup_prop"				,       default=0.05				,       type=__A				,       help="Linear warmup proportion."      )
					parser.add_argument("--weight_decay"				,       default=0.0				,       type=__A				,       help="Weight decay if we apply some."      )
					parser.add_argument("--learning_rate"				,       default=5e-4				,       type=__A				,       help="The initial learning rate for Adam."      )
					parser.add_argument("--adam_epsilon"				,       default=1e-6				,       type=__A				,       help="Epsilon for Adam optimizer."      )
					parser.add_argument("--max_grad_norm"				,       default=5.0				,       type=__A				,       help="Max gradient norm."      )
					parser.add_argument("--initializer_range"				,       default=0.02				,       type=__A				,       help="Random initialization range."      )

					parser.add_argument(
					    "--fp16"				,       action="store_true"				,       help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"				,       )
					parser.add_argument(
					    "--fp16_opt_level"				,       type=__A				,       default="O1"				,       help=(
					        "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
					        "See details at https://nvidia.github.io/apex/amp.html"
					    )				,       )
					parser.add_argument("--n_gpu"				,       type=__A				,       default=1				,       help="Number of GPUs in the node."      )
					parser.add_argument("--local_rank"				,       type=__A				,       default=-1				,       help="Distributed training - Local rank"      )
					parser.add_argument("--seed"				,       type=__A				,       default=56				,       help="Random seed"      )

					parser.add_argument("--log_interval"				,       type=__A				,       default=500				,       help="Tensorboard logging interval."      )
					parser.add_argument("--checkpoint_interval"				,       type=__A				,       default=4_000				,       help="Checkpoint interval."      )
					__a					:						str      =					parser.parse_args()
					sanity_checks(__A      )

					# ARGS #
					init_gpu_params(__A      )
					set_seed(__A      )
					if args.is_master:
										if os.path.exists(args.dump_path      ):
															if not args.force:
																				raise ValueError(
																				    f'Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'
																				    " itUse `--force` if you want to overwrite it"      )
															else:
																				shutil.rmtree(args.dump_path      )

										if not os.path.exists(args.dump_path      ):
															os.makedirs(args.dump_path      )
										logger.info(f'Experiment will be dumped and logged in {args.dump_path}'      )

										# SAVE PARAMS #
										logger.info(f'Param: {args}'      )
										with open(os.path.join(args.dump_path				,       "parameters.json"      )				,       "w"      ) as f:
															json.dump(vars(__A      )				,       __A				,       indent=4      )
										git_log(args.dump_path      )

					__a      ,			__a      ,			__a					:						List[str]      =					MODEL_CLASSES[args.student_type]
					__a      ,			__a      ,			__a					:						Optional[int]      =					MODEL_CLASSES[args.teacher_type]

					# TOKENIZER #
					__a					:						Dict      =					teacher_tokenizer_class.from_pretrained(args.teacher_name      )
					__a					:						Optional[Any]      =					{}
					for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
										__a					:						Optional[int]      =					tokenizer.all_special_tokens.index(__A      )
										__a					:						Tuple      =					tokenizer.all_special_ids[idx]
					logger.info(f'Special tokens {special_tok_ids}'      )
					__a					:						Tuple      =					special_tok_ids
					__a					:						List[str]      =					tokenizer.max_model_input_sizes[args.teacher_name]

					# DATA LOADER #
					logger.info(f'Loading data from {args.data_file}'      )
					with open(args.data_file				,       "rb"      ) as fp:
										__a					:						Optional[int]      =					pickle.load(__A      )

					if args.mlm:
										logger.info(f'Loading token counts from {args.token_counts} (already pre-computed)'      )
										with open(args.token_counts				,       "rb"      ) as fp:
															__a					:						Tuple      =					pickle.load(__A      )

										__a					:						Tuple      =					np.maximum(__A				,       1      ) ** -args.mlm_smoothing
										for idx in special_tok_ids.values():
															__a					:						Any      =					0.0  # do not predict special tokens
										__a					:						Union[str, Any]      =					torch.from_numpy(__A      )
					else:
										__a					:						Optional[int]      =					None

					__a					:						int      =					LmSeqsDataset(params=__A				,       data=__A      )
					logger.info("Data loader created."      )

					# STUDENT #
					logger.info(f'Loading student config from {args.student_config}'      )
					__a					:						int      =					student_config_class.from_pretrained(args.student_config      )
					__a					:						Union[str, Any]      =					True

					if args.student_pretrained_weights is not None:
										logger.info(f'Loading pretrained weights from {args.student_pretrained_weights}'      )
										__a					:						Tuple      =					student_model_class.from_pretrained(args.student_pretrained_weights				,       config=__A      )
					else:
										__a					:						int      =					student_model_class(__A      )

					if args.n_gpu > 0:
										student.to(f'cuda:{args.local_rank}'      )
					logger.info("Student loaded."      )

					# TEACHER #
					__a					:						Tuple      =					teacher_model_class.from_pretrained(args.teacher_name				,       output_hidden_states=__A      )
					if args.n_gpu > 0:
										teacher.to(f'cuda:{args.local_rank}'      )
					logger.info(f'Teacher loaded from {args.teacher_name}.'      )

					# FREEZING #
					if args.freeze_pos_embs:
										freeze_pos_embeddings(__A				,       __A      )
					if args.freeze_token_type_embds:
										freeze_token_type_embeddings(__A				,       __A      )

					# SANITY CHECKS #
					assert student.config.vocab_size == teacher.config.vocab_size
					assert student.config.hidden_size == teacher.config.hidden_size
					assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
					if args.mlm:
										assert token_probs.size(0      ) == stu_architecture_config.vocab_size

					# DISTILLER #
					torch.cuda.empty_cache()
					__a					:						Union[str, Any]      =					Distiller(
					    params=__A				,       dataset=__A				,       token_probs=__A				,       student=__A				,       teacher=__A      )
					distiller.train()
					logger.info("Let's go get some drinks."      )


if __name__ == "__main__":
			main()













import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						def __init__(self:					str ,					*__UpperCAmelCase:					Optional[int] ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Dict=None ,					**__UpperCAmelCase:					Tuple   )    ->     Dict:




											'''simple docstring'''



											super().__init__(*__UpperCAmelCase ,					**__UpperCAmelCase   )
											__a					:						str      =					eval_examples
											__a					:						List[Any]      =					post_process_function



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					Optional[Dataset] = None ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Optional[List[str]] = None ,					__UpperCAmelCase:					str = "eval" ,					**__UpperCAmelCase:					Any ,					)    ->     Dict[str, float]:




											'''simple docstring'''



											__a					:						Optional[int]      =					gen_kwargs.copy()
											__a					:						Union[str, Any]      =					(
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"   ) is not None else self.args.generation_max_length
											)
											__a					:						Dict      =					(
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"   ) is not None else self.args.generation_num_beams
											)
											__a					:						Optional[Any]      =					gen_kwargs

											__a					:						Union[str, Any]      =					self.eval_dataset if eval_dataset is None else eval_dataset
											__a					:						Optional[int]      =					self.get_eval_dataloader(__UpperCAmelCase   )
											__a					:						Tuple      =					self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						int      =					self.compute_metrics
											__a					:						Dict      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						str      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						str      =					eval_loop(
																    __UpperCAmelCase ,					description="Evaluation" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						Union[str, Any]      =					compute_metrics
											__a					:						str      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__a					:						Union[str, Any]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase   )
																__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()   ):
																					if not key.startswith(f'{metric_key_prefix}_'   ):
																										__a					:						int      =					metrics.pop(__UpperCAmelCase   )

																metrics.update(output.metrics   )
											else:
																__a					:						Optional[int]      =					output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(__UpperCAmelCase   )

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()   )

											__a					:						Tuple      =					self.callback_handler.on_evaluate(self.args ,					self.state ,					self.control ,					__UpperCAmelCase   )
											return metrics



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					Union[str, Any]=None ,					__UpperCAmelCase:					str = "test" ,					**__UpperCAmelCase:					List[Any]   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						Any      =					gen_kwargs.copy()

											__a					:						Tuple      =					self.get_test_dataloader(__UpperCAmelCase   )

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						Optional[Any]      =					self.compute_metrics
											__a					:						int      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						Dict      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						Optional[Any]      =					eval_loop(
																    __UpperCAmelCase ,					description="Prediction" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						List[Any]      =					compute_metrics

											__a					:						Optional[Any]      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__a					:						Optional[int]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					"predict"   )
											__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()   ):
																if not key.startswith(f'{metric_key_prefix}_'   ):
																					__a					:						List[str]      =					metrics.pop(__UpperCAmelCase   )
											metrics.update(output.metrics   )
											return PredictionOutput(predictions=predictions.predictions ,					label_ids=predictions.label_ids ,					metrics=__UpperCAmelCase   )





