


import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


_lowercase       :      str	=	{
    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
    "gpt2": (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}





def   _lowerCAmelCase				(   UpperCamelCase__:     Union[str, Any]						) ->   List[Any]:





	"""simple docstring"""







	assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
	assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
	if args.mlm:
		assert os.path.isfile(args.token_counts						)
		assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
	else:
		assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

	assert args.teacher_type == args.student_type or (
	    args.student_type == "distilbert" and args.teacher_type == "bert"
	)
	assert os.path.isfile(args.student_config						)
	if args.student_pretrained_weights is not None:
		assert os.path.isfile(args.student_pretrained_weights						)

	if args.freeze_token_type_embds:
		assert args.student_type in ["roberta"]

	assert args.alpha_ce >= 0.0
	assert args.alpha_mlm >= 0.0
	assert args.alpha_clm >= 0.0
	assert args.alpha_mse >= 0.0
	assert args.alpha_cos >= 0.0
	assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0





def   _lowerCAmelCase				(   UpperCamelCase__:     Dict							,					UpperCamelCase__:     int						) ->   Any:





	"""simple docstring"""







	if args.student_type == "roberta":
		A								=				False
	elif args.student_type == "gpt2":
		A								=				False





def   _lowerCAmelCase				(   UpperCamelCase__:     Dict							,					UpperCamelCase__:     int						) ->   int:





	"""simple docstring"""







	if args.student_type == "roberta":
		A								=				False





def   _lowerCAmelCase				(   ) ->   Tuple:





	"""simple docstring"""







	A								=				argparse.ArgumentParser(description="""Training"""						)
	parser.add_argument("""--force"""							,					action="""store_true"""							,					help="""Overwrite dump_path if it already exists."""						)

	parser.add_argument(
	    """--dump_path"""							,					type=UpperCamelCase__							,					required=UpperCamelCase__							,					help="""The output directory (log, checkpoints, parameters, etc.)"""						)
	parser.add_argument(
	    """--data_file"""							,					type=UpperCamelCase__							,					required=UpperCamelCase__							,					help="""The binarized file (tokenized + tokens_to_ids) and grouped by sequence."""							,					)

	parser.add_argument(
	    """--student_type"""							,					type=UpperCamelCase__							,					choices=["""distilbert""", """roberta""", """gpt2"""]							,					required=UpperCamelCase__							,					help="""The student type (DistilBERT, RoBERTa)."""							,					)
	parser.add_argument("""--student_config"""							,					type=UpperCamelCase__							,					required=UpperCamelCase__							,					help="""Path to the student configuration."""						)
	parser.add_argument(
	    """--student_pretrained_weights"""							,					default=UpperCamelCase__							,					type=UpperCamelCase__							,					help="""Load student initialization checkpoint."""						)

	parser.add_argument(
	    """--teacher_type"""							,					choices=["""bert""", """roberta""", """gpt2"""]							,					required=UpperCamelCase__							,					help="""Teacher type (BERT, RoBERTa)."""						)
	parser.add_argument("""--teacher_name"""							,					type=UpperCamelCase__							,					required=UpperCamelCase__							,					help="""The teacher model."""						)

	parser.add_argument("""--temperature"""							,					default=2.0							,					type=UpperCamelCase__							,					help="""Temperature for the softmax temperature."""						)
	parser.add_argument(
	    """--alpha_ce"""							,					default=0.5							,					type=UpperCamelCase__							,					help="""Linear weight for the distillation loss. Must be >=0."""						)
	parser.add_argument(
	    """--alpha_mlm"""							,					default=0.0							,					type=UpperCamelCase__							,					help="""Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag."""							,					)
	parser.add_argument("""--alpha_clm"""							,					default=0.5							,					type=UpperCamelCase__							,					help="""Linear weight for the CLM loss. Must be >=0."""						)
	parser.add_argument("""--alpha_mse"""							,					default=0.0							,					type=UpperCamelCase__							,					help="""Linear weight of the MSE loss. Must be >=0."""						)
	parser.add_argument(
	    """--alpha_cos"""							,					default=0.0							,					type=UpperCamelCase__							,					help="""Linear weight of the cosine embedding loss. Must be >=0."""						)

	parser.add_argument(
	    """--mlm"""							,					action="""store_true"""							,					help="""The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM."""						)
	parser.add_argument(
	    """--mlm_mask_prop"""							,					default=0.15							,					type=UpperCamelCase__							,					help="""Proportion of tokens for which we need to make a prediction."""							,					)
	parser.add_argument("""--word_mask"""							,					default=0.8							,					type=UpperCamelCase__							,					help="""Proportion of tokens to mask out."""						)
	parser.add_argument("""--word_keep"""							,					default=0.1							,					type=UpperCamelCase__							,					help="""Proportion of tokens to keep."""						)
	parser.add_argument("""--word_rand"""							,					default=0.1							,					type=UpperCamelCase__							,					help="""Proportion of tokens to randomly replace."""						)
	parser.add_argument(
	    """--mlm_smoothing"""							,					default=0.7							,					type=UpperCamelCase__							,					help="""Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec)."""							,					)
	parser.add_argument("""--token_counts"""							,					type=UpperCamelCase__							,					help="""The token counts in the data_file for MLM."""						)

	parser.add_argument(
	    """--restrict_ce_to_mask"""							,					action="""store_true"""							,					help="""If true, compute the distillation loss only the [MLM] prediction distribution."""							,					)
	parser.add_argument(
	    """--freeze_pos_embs"""							,					action="""store_true"""							,					help="""Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only."""							,					)
	parser.add_argument(
	    """--freeze_token_type_embds"""							,					action="""store_true"""							,					help="""Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only."""							,					)

	parser.add_argument("""--n_epoch"""							,					type=UpperCamelCase__							,					default=3							,					help="""Number of pass on the whole dataset."""						)
	parser.add_argument("""--batch_size"""							,					type=UpperCamelCase__							,					default=5							,					help="""Batch size (for each process)."""						)
	parser.add_argument(
	    """--group_by_size"""							,					action="""store_false"""							,					help="""If true, group sequences that have similar length into the same batch. Default is true."""							,					)

	parser.add_argument(
	    """--gradient_accumulation_steps"""							,					type=UpperCamelCase__							,					default=50							,					help="""Gradient accumulation for larger training batches."""							,					)
	parser.add_argument("""--warmup_prop"""							,					default=0.05							,					type=UpperCamelCase__							,					help="""Linear warmup proportion."""						)
	parser.add_argument("""--weight_decay"""							,					default=0.0							,					type=UpperCamelCase__							,					help="""Weight decay if we apply some."""						)
	parser.add_argument("""--learning_rate"""							,					default=5e-4							,					type=UpperCamelCase__							,					help="""The initial learning rate for Adam."""						)
	parser.add_argument("""--adam_epsilon"""							,					default=1e-6							,					type=UpperCamelCase__							,					help="""Epsilon for Adam optimizer."""						)
	parser.add_argument("""--max_grad_norm"""							,					default=5.0							,					type=UpperCamelCase__							,					help="""Max gradient norm."""						)
	parser.add_argument("""--initializer_range"""							,					default=0.02							,					type=UpperCamelCase__							,					help="""Random initialization range."""						)

	parser.add_argument(
	    """--fp16"""							,					action="""store_true"""							,					help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""							,					)
	parser.add_argument(
	    """--fp16_opt_level"""							,					type=UpperCamelCase__							,					default="""O1"""							,					help=(
	        """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
	        """See details at https://nvidia.github.io/apex/amp.html"""
	    )							,					)
	parser.add_argument("""--n_gpu"""							,					type=UpperCamelCase__							,					default=1							,					help="""Number of GPUs in the node."""						)
	parser.add_argument("""--local_rank"""							,					type=UpperCamelCase__							,					default=-1							,					help="""Distributed training - Local rank"""						)
	parser.add_argument("""--seed"""							,					type=UpperCamelCase__							,					default=56							,					help="""Random seed"""						)

	parser.add_argument("""--log_interval"""							,					type=UpperCamelCase__							,					default=5_00							,					help="""Tensorboard logging interval."""						)
	parser.add_argument("""--checkpoint_interval"""							,					type=UpperCamelCase__							,					default=40_00							,					help="""Checkpoint interval."""						)
	A								=				parser.parse_args()
	sanity_checks(UpperCamelCase__						)

	# ARGS #
	init_gpu_params(UpperCamelCase__						)
	set_seed(UpperCamelCase__						)
	if args.is_master:
		if os.path.exists(args.dump_path						):
			if not args.force:
				raise ValueError(
				    f'Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'
				    """ itUse `--force` if you want to overwrite it"""						)
			else:
				shutil.rmtree(args.dump_path						)

		if not os.path.exists(args.dump_path						):
			os.makedirs(args.dump_path						)
		logger.info(f'Experiment will be dumped and logged in {args.dump_path}'						)

		# SAVE PARAMS #
		logger.info(f'Param: {args}'						)
		with open(os.path.join(args.dump_path							,					"""parameters.json"""						)							,					"""w"""						) as f:
			json.dump(vars(UpperCamelCase__						)							,					UpperCamelCase__							,					indent=4						)
		git_log(args.dump_path						)

	A					,			A					,			A								=				MODEL_CLASSES[args.student_type]
	A					,			A					,			A								=				MODEL_CLASSES[args.teacher_type]

	# TOKENIZER #
	A								=				teacher_tokenizer_class.from_pretrained(args.teacher_name						)
	A								=				{}
	for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
		A								=				tokenizer.all_special_tokens.index(UpperCamelCase__						)
		A								=				tokenizer.all_special_ids[idx]
	logger.info(f'Special tokens {special_tok_ids}'						)
	A								=				special_tok_ids
	A								=				tokenizer.max_model_input_sizes[args.teacher_name]

	# DATA LOADER #
	logger.info(f'Loading data from {args.data_file}'						)
	with open(args.data_file							,					"""rb"""						) as fp:
		A								=				pickle.load(UpperCamelCase__						)

	if args.mlm:
		logger.info(f'Loading token counts from {args.token_counts} (already pre-computed)'						)
		with open(args.token_counts							,					"""rb"""						) as fp:
			A								=				pickle.load(UpperCamelCase__						)

		A								=				np.maximum(UpperCamelCase__							,					1						) ** -args.mlm_smoothing
		for idx in special_tok_ids.values():
			A								=				0.0  # do not predict special tokens
		A								=				torch.from_numpy(UpperCamelCase__						)
	else:
		A								=				None

	A								=				LmSeqsDataset(params=UpperCamelCase__							,					data=UpperCamelCase__						)
	logger.info("""Data loader created."""						)

	# STUDENT #
	logger.info(f'Loading student config from {args.student_config}'						)
	A								=				student_config_class.from_pretrained(args.student_config						)
	A								=				True

	if args.student_pretrained_weights is not None:
		logger.info(f'Loading pretrained weights from {args.student_pretrained_weights}'						)
		A								=				student_model_class.from_pretrained(args.student_pretrained_weights							,					config=UpperCamelCase__						)
	else:
		A								=				student_model_class(UpperCamelCase__						)

	if args.n_gpu > 0:
		student.to(f'cuda:{args.local_rank}'						)
	logger.info("""Student loaded."""						)

	# TEACHER #
	A								=				teacher_model_class.from_pretrained(args.teacher_name							,					output_hidden_states=UpperCamelCase__						)
	if args.n_gpu > 0:
		teacher.to(f'cuda:{args.local_rank}'						)
	logger.info(f'Teacher loaded from {args.teacher_name}.'						)

	# FREEZING #
	if args.freeze_pos_embs:
		freeze_pos_embeddings(UpperCamelCase__							,					UpperCamelCase__						)
	if args.freeze_token_type_embds:
		freeze_token_type_embeddings(UpperCamelCase__							,					UpperCamelCase__						)

	# SANITY CHECKS #
	assert student.config.vocab_size == teacher.config.vocab_size
	assert student.config.hidden_size == teacher.config.hidden_size
	assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
	if args.mlm:
		assert token_probs.size(0						) == stu_architecture_config.vocab_size

	# DISTILLER #
	torch.cuda.empty_cache()
	A								=				Distiller(
	    params=UpperCamelCase__							,					dataset=UpperCamelCase__							,					token_probs=UpperCamelCase__							,					student=UpperCamelCase__							,					teacher=UpperCamelCase__						)
	distiller.train()
	logger.info("""Let's go get some drinks."""						)


if __name__ == "__main__":
				main()







from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase       :      Optional[Any]	=	logging.get_logger(__name__)

_lowercase       :      List[str]	=	{
    "huggingface/informer-tourism-monthly": (
        "https://huggingface.co/huggingface/informer-tourism-monthly/resolve/main/config.json"
    ),
    # See all Informer models at https://huggingface.co/models?filter=informer
}






class       _UpperCamelCase    (					__snake_case				):

				"""simple docstring"""







				lowerCAmelCase 		= 'informer'
				lowerCAmelCase 		= {
				    'hidden_size': 'd_model',
				    'num_attention_heads': 'encoder_attention_heads',
				    'num_hidden_layers': 'encoder_layers',
				}

				def __init__(						self				,   a__ = None				,   a__ = None				,   a__ = "student_t"				,   a__ = "nll"				,   a__ = 1				,   a__ = None				,   a__ = "mean"				,   a__ = 0				,   a__ = 0				,   a__ = 0				,   a__ = 0				,   a__ = None				,   a__ = None				,   a__ = 64				,   a__ = 32				,   a__ = 32				,   a__ = 2				,   a__ = 2				,   a__ = 2				,   a__ = 2				,   a__ = True				,   a__ = "gelu"				,   a__ = 0.05				,   a__ = 0.1				,   a__ = 0.1				,   a__ = 0.1				,   a__ = 0.1				,   a__ = 100				,   a__ = 0.02				,   a__=True				,   a__ = "prob"				,   a__ = 5				,   a__ = True				,   **a__				,   )	->				Union[str, Any]:
					# time series specific configuration
					A								=				prediction_length
					A								=				context_length or prediction_length
					A								=				distribution_output
					A								=				loss
					A								=				input_size
					A								=				num_time_features
					A								=				lags_sequence if lags_sequence is not None else [1, 2, 3, 4, 5, 6, 7]
					A								=				scaling
					A								=				num_dynamic_real_features
					A								=				num_static_real_features
					A								=				num_static_categorical_features

					# set cardinality
					if cardinality and num_static_categorical_features > 0:
						if len(a__   ) != num_static_categorical_features:
							raise ValueError(
							    """The cardinality should be a list of the same length as `num_static_categorical_features`"""   )
						A								=				cardinality
					else:
						A								=				[0]

					# set embedding_dimension
					if embedding_dimension and num_static_categorical_features > 0:
						if len(a__   ) != num_static_categorical_features:
							raise ValueError(
							    """The embedding dimension should be a list of the same length as `num_static_categorical_features`"""   )
						A								=				embedding_dimension
					else:
						A								=				[min(50				,   (cat + 1) // 2   ) for cat in self.cardinality]

					A								=				num_parallel_samples

					# Transformer architecture configuration
					A								=				input_size * len(self.lags_sequence   ) + self._number_of_features
					A								=				d_model
					A								=				encoder_attention_heads
					A								=				decoder_attention_heads
					A								=				encoder_ffn_dim
					A								=				decoder_ffn_dim
					A								=				encoder_layers
					A								=				decoder_layers

					A								=				dropout
					A								=				attention_dropout
					A								=				activation_dropout
					A								=				encoder_layerdrop
					A								=				decoder_layerdrop

					A								=				activation_function
					A								=				init_std

					A								=				use_cache

					# Informer
					A								=				attention_type
					A								=				sampling_factor
					A								=				distil

					super().__init__(is_encoder_decoder=a__				,   **a__   )

				@property
				def      _UpperCAmelCase					(						self   )	->				int:
					return (
					    sum(self.embedding_dimension   )
					    + self.num_dynamic_real_features
					    + self.num_time_features
					    + self.num_static_real_features
					    + self.input_size * 2  # the log1p(abs(loc)) and log(scale) features
					)



