
'''simple docstring'''
import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


a_											=						{
    "cola": 2,
    "mnli": 3,
    "mrpc": 2,
    "sst-2": 2,
    "sts-b": 1,
    "qqp": 2,
    "qnli": 2,
    "rte": 2,
    "wnli": 2,
}


logging.set_verbosity_info()

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE=None       ):
    """simple docstring"""






    snake_case_ :    List[str]     			=						XLNetConfig.from_json_file(lowerCamelCase__       )

    snake_case_ :    Optional[int]     			=						finetuning_task.lower() if finetuning_task is not None else ""
    if finetuning_task in GLUE_TASKS_NUM_LABELS:
        print(f'Building PyTorch XLNetForSequenceClassification model from configuration: {config}'       )
        snake_case_ :    Any     			=						finetuning_task
        snake_case_ :    Optional[int]     			=						GLUE_TASKS_NUM_LABELS[finetuning_task]
        snake_case_ :    str     			=						XLNetForSequenceClassification(lowerCamelCase__       )
    elif "squad" in finetuning_task:
        snake_case_ :    Optional[Any]     			=						finetuning_task
        snake_case_ :    Tuple     			=						XLNetForQuestionAnswering(lowerCamelCase__       )
    else:
        snake_case_ :    int     			=						XLNetLMHeadModel(lowerCamelCase__       )

    # Load weights from tf checkpoint
    load_tf_weights_in_xlnet(lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__       )

    # Save pytorch-model
    snake_case_ :    Optional[int]     			=						os.path.join(lowerCamelCase__,		lowerCamelCase__       )
    snake_case_ :    int     			=						os.path.join(lowerCamelCase__,		lowerCamelCase__       )
    print(f'Save PyTorch model to {os.path.abspath(lowerCamelCase__       )}'       )
    torch.save(model.state_dict(),		lowerCamelCase__       )
    print(f'Save configuration file to {os.path.abspath(lowerCamelCase__       )}'       )
    with open(lowerCamelCase__,		"w",		encoding="utf-8"       ) as f:
        f.write(config.to_json_string()       )


if __name__ == "__main__":
  a_											=						argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
  )
  parser.add_argument(
      "--xlnet_config_file",
      default=None,
      type=str,
      required=True,
      help=(
          "The config json file corresponding to the pre-trained XLNet model. \n"
          "This specifies the model architecture."
      ),
  )
  parser.add_argument(
      "--pytorch_dump_folder_path",
      default=None,
      type=str,
      required=True,
      help="Path to the folder to store the PyTorch model or dataset/vocab.",
  )
  parser.add_argument(
      "--finetuning_task",
      default=None,
      type=str,
      help="Name of a task on which the XLNet TensorFlow model was fine-tuned",
  )
  a_											=						parser.parse_args()
  print(args)

  convert_xlnet_checkpoint_to_pytorch(
      args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
  )






'''simple docstring'''
import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


a_											=						{
    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
    "gpt2": (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
    assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
    if args.mlm:
        assert os.path.isfile(args.token_counts       )
        assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
    else:
        assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

    assert args.teacher_type == args.student_type or (
        args.student_type == "distilbert" and args.teacher_type == "bert"
    )
    assert os.path.isfile(args.student_config       )
    if args.student_pretrained_weights is not None:
        assert os.path.isfile(args.student_pretrained_weights       )

    if args.freeze_token_type_embds:
        assert args.student_type in ["roberta"]

    assert args.alpha_ce >= 0.0
    assert args.alpha_mlm >= 0.0
    assert args.alpha_clm >= 0.0
    assert args.alpha_mse >= 0.0
    assert args.alpha_cos >= 0.0
    assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    if args.student_type == "roberta":
        snake_case_ :    Union[str, Any]     			=						False
    elif args.student_type == "gpt2":
        snake_case_ :    Dict     			=						False

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    if args.student_type == "roberta":
        snake_case_ :    Tuple     			=						False

def 						UpperCamelCase_     (	):
    """simple docstring"""






    snake_case_ :    Union[str, Any]     			=						argparse.ArgumentParser(description="Training"       )
    parser.add_argument("--force",		action="store_true",		help="Overwrite dump_path if it already exists."       )

    parser.add_argument(
        "--dump_path",		type=__SCREAMING_SNAKE_CASE,		required=__SCREAMING_SNAKE_CASE,		help="The output directory (log, checkpoints, parameters, etc.)"       )
    parser.add_argument(
        "--data_file",		type=__SCREAMING_SNAKE_CASE,		required=__SCREAMING_SNAKE_CASE,		help="The binarized file (tokenized + tokens_to_ids) and grouped by sequence.",		)

    parser.add_argument(
        "--student_type",		type=__SCREAMING_SNAKE_CASE,		choices=["distilbert", "roberta", "gpt2"],		required=__SCREAMING_SNAKE_CASE,		help="The student type (DistilBERT, RoBERTa).",		)
    parser.add_argument("--student_config",		type=__SCREAMING_SNAKE_CASE,		required=__SCREAMING_SNAKE_CASE,		help="Path to the student configuration."       )
    parser.add_argument(
        "--student_pretrained_weights",		default=__SCREAMING_SNAKE_CASE,		type=__SCREAMING_SNAKE_CASE,		help="Load student initialization checkpoint."       )

    parser.add_argument(
        "--teacher_type",		choices=["bert", "roberta", "gpt2"],		required=__SCREAMING_SNAKE_CASE,		help="Teacher type (BERT, RoBERTa)."       )
    parser.add_argument("--teacher_name",		type=__SCREAMING_SNAKE_CASE,		required=__SCREAMING_SNAKE_CASE,		help="The teacher model."       )

    parser.add_argument("--temperature",		default=2.0,		type=__SCREAMING_SNAKE_CASE,		help="Temperature for the softmax temperature."       )
    parser.add_argument(
        "--alpha_ce",		default=0.5,		type=__SCREAMING_SNAKE_CASE,		help="Linear weight for the distillation loss. Must be >=0."       )
    parser.add_argument(
        "--alpha_mlm",		default=0.0,		type=__SCREAMING_SNAKE_CASE,		help="Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag.",		)
    parser.add_argument("--alpha_clm",		default=0.5,		type=__SCREAMING_SNAKE_CASE,		help="Linear weight for the CLM loss. Must be >=0."       )
    parser.add_argument("--alpha_mse",		default=0.0,		type=__SCREAMING_SNAKE_CASE,		help="Linear weight of the MSE loss. Must be >=0."       )
    parser.add_argument(
        "--alpha_cos",		default=0.0,		type=__SCREAMING_SNAKE_CASE,		help="Linear weight of the cosine embedding loss. Must be >=0."       )

    parser.add_argument(
        "--mlm",		action="store_true",		help="The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM."       )
    parser.add_argument(
        "--mlm_mask_prop",		default=0.15,		type=__SCREAMING_SNAKE_CASE,		help="Proportion of tokens for which we need to make a prediction.",		)
    parser.add_argument("--word_mask",		default=0.8,		type=__SCREAMING_SNAKE_CASE,		help="Proportion of tokens to mask out."       )
    parser.add_argument("--word_keep",		default=0.1,		type=__SCREAMING_SNAKE_CASE,		help="Proportion of tokens to keep."       )
    parser.add_argument("--word_rand",		default=0.1,		type=__SCREAMING_SNAKE_CASE,		help="Proportion of tokens to randomly replace."       )
    parser.add_argument(
        "--mlm_smoothing",		default=0.7,		type=__SCREAMING_SNAKE_CASE,		help="Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec).",		)
    parser.add_argument("--token_counts",		type=__SCREAMING_SNAKE_CASE,		help="The token counts in the data_file for MLM."       )

    parser.add_argument(
        "--restrict_ce_to_mask",		action="store_true",		help="If true, compute the distillation loss only the [MLM] prediction distribution.",		)
    parser.add_argument(
        "--freeze_pos_embs",		action="store_true",		help="Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only.",		)
    parser.add_argument(
        "--freeze_token_type_embds",		action="store_true",		help="Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only.",		)

    parser.add_argument("--n_epoch",		type=__SCREAMING_SNAKE_CASE,		default=3,		help="Number of pass on the whole dataset."       )
    parser.add_argument("--batch_size",		type=__SCREAMING_SNAKE_CASE,		default=5,		help="Batch size (for each process)."       )
    parser.add_argument(
        "--group_by_size",		action="store_false",		help="If true, group sequences that have similar length into the same batch. Default is true.",		)

    parser.add_argument(
        "--gradient_accumulation_steps",		type=__SCREAMING_SNAKE_CASE,		default=5_0,		help="Gradient accumulation for larger training batches.",		)
    parser.add_argument("--warmup_prop",		default=0.05,		type=__SCREAMING_SNAKE_CASE,		help="Linear warmup proportion."       )
    parser.add_argument("--weight_decay",		default=0.0,		type=__SCREAMING_SNAKE_CASE,		help="Weight decay if we apply some."       )
    parser.add_argument("--learning_rate",		default=5E-4,		type=__SCREAMING_SNAKE_CASE,		help="The initial learning rate for Adam."       )
    parser.add_argument("--adam_epsilon",		default=1E-6,		type=__SCREAMING_SNAKE_CASE,		help="Epsilon for Adam optimizer."       )
    parser.add_argument("--max_grad_norm",		default=5.0,		type=__SCREAMING_SNAKE_CASE,		help="Max gradient norm."       )
    parser.add_argument("--initializer_range",		default=0.02,		type=__SCREAMING_SNAKE_CASE,		help="Random initialization range."       )

    parser.add_argument(
        "--fp16",		action="store_true",		help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",		)
    parser.add_argument(
        "--fp16_opt_level",		type=__SCREAMING_SNAKE_CASE,		default="O1",		help=(
            "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
            "See details at https://nvidia.github.io/apex/amp.html"
        ),		)
    parser.add_argument("--n_gpu",		type=__SCREAMING_SNAKE_CASE,		default=1,		help="Number of GPUs in the node."       )
    parser.add_argument("--local_rank",		type=__SCREAMING_SNAKE_CASE,		default=-1,		help="Distributed training - Local rank"       )
    parser.add_argument("--seed",		type=__SCREAMING_SNAKE_CASE,		default=5_6,		help="Random seed"       )

    parser.add_argument("--log_interval",		type=__SCREAMING_SNAKE_CASE,		default=5_0_0,		help="Tensorboard logging interval."       )
    parser.add_argument("--checkpoint_interval",		type=__SCREAMING_SNAKE_CASE,		default=4_0_0_0,		help="Checkpoint interval."       )
    snake_case_ :    Dict     			=						parser.parse_args()
    sanity_checks(__SCREAMING_SNAKE_CASE       )

    # ARGS #
    init_gpu_params(__SCREAMING_SNAKE_CASE       )
    set_seed(__SCREAMING_SNAKE_CASE       )
    if args.is_master:
        if os.path.exists(args.dump_path       ):
            if not args.force:
                raise ValueError(
                    f'Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'
                    " itUse `--force` if you want to overwrite it"       )
            else:
                shutil.rmtree(args.dump_path       )

        if not os.path.exists(args.dump_path       ):
            os.makedirs(args.dump_path       )
        logger.info(f'Experiment will be dumped and logged in {args.dump_path}'       )

        # SAVE PARAMS #
        logger.info(f'Param: {args}'       )
        with open(os.path.join(args.dump_path,		"parameters.json"       ),		"w"       ) as f:
            json.dump(vars(__SCREAMING_SNAKE_CASE       ),		__SCREAMING_SNAKE_CASE,		indent=4       )
        git_log(args.dump_path       )

    snake_case_			, snake_case_			, snake_case_ :    Optional[int]     			=						MODEL_CLASSES[args.student_type]
    snake_case_			, snake_case_			, snake_case_ :    List[str]     			=						MODEL_CLASSES[args.teacher_type]

    # TOKENIZER #
    snake_case_ :    List[Any]     			=						teacher_tokenizer_class.from_pretrained(args.teacher_name       )
    snake_case_ :    Any     			=						{}
    for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
        snake_case_ :    Dict     			=						tokenizer.all_special_tokens.index(__SCREAMING_SNAKE_CASE       )
        snake_case_ :    str     			=						tokenizer.all_special_ids[idx]
    logger.info(f'Special tokens {special_tok_ids}'       )
    snake_case_ :    List[str]     			=						special_tok_ids
    snake_case_ :    Union[str, Any]     			=						tokenizer.max_model_input_sizes[args.teacher_name]

    # DATA LOADER #
    logger.info(f'Loading data from {args.data_file}'       )
    with open(args.data_file,		"rb"       ) as fp:
        snake_case_ :    Union[str, Any]     			=						pickle.load(__SCREAMING_SNAKE_CASE       )

    if args.mlm:
        logger.info(f'Loading token counts from {args.token_counts} (already pre-computed)'       )
        with open(args.token_counts,		"rb"       ) as fp:
            snake_case_ :    Union[str, Any]     			=						pickle.load(__SCREAMING_SNAKE_CASE       )

        snake_case_ :    str     			=						np.maximum(__SCREAMING_SNAKE_CASE,		1       ) ** -args.mlm_smoothing
        for idx in special_tok_ids.values():
            snake_case_ :    Optional[int]     			=						0.0  # do not predict special tokens
        snake_case_ :    List[str]     			=						torch.from_numpy(__SCREAMING_SNAKE_CASE       )
    else:
        snake_case_ :    Optional[int]     			=						None

    snake_case_ :    Dict     			=						LmSeqsDataset(params=__SCREAMING_SNAKE_CASE,		data=__SCREAMING_SNAKE_CASE       )
    logger.info("Data loader created."       )

    # STUDENT #
    logger.info(f'Loading student config from {args.student_config}'       )
    snake_case_ :    Any     			=						student_config_class.from_pretrained(args.student_config       )
    snake_case_ :    Optional[Any]     			=						True

    if args.student_pretrained_weights is not None:
        logger.info(f'Loading pretrained weights from {args.student_pretrained_weights}'       )
        snake_case_ :    Any     			=						student_model_class.from_pretrained(args.student_pretrained_weights,		config=__SCREAMING_SNAKE_CASE       )
    else:
        snake_case_ :    Any     			=						student_model_class(__SCREAMING_SNAKE_CASE       )

    if args.n_gpu > 0:
        student.to(f'cuda:{args.local_rank}'       )
    logger.info("Student loaded."       )

    # TEACHER #
    snake_case_ :    Optional[Any]     			=						teacher_model_class.from_pretrained(args.teacher_name,		output_hidden_states=__SCREAMING_SNAKE_CASE       )
    if args.n_gpu > 0:
        teacher.to(f'cuda:{args.local_rank}'       )
    logger.info(f'Teacher loaded from {args.teacher_name}.'       )

    # FREEZING #
    if args.freeze_pos_embs:
        freeze_pos_embeddings(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )
    if args.freeze_token_type_embds:
        freeze_token_type_embeddings(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )

    # SANITY CHECKS #
    assert student.config.vocab_size == teacher.config.vocab_size
    assert student.config.hidden_size == teacher.config.hidden_size
    assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
    if args.mlm:
        assert token_probs.size(0       ) == stu_architecture_config.vocab_size

    # DISTILLER #
    torch.cuda.empty_cache()
    snake_case_ :    Any     			=						Distiller(
        params=__SCREAMING_SNAKE_CASE,		dataset=__SCREAMING_SNAKE_CASE,		token_probs=__SCREAMING_SNAKE_CASE,		student=__SCREAMING_SNAKE_CASE,		teacher=__SCREAMING_SNAKE_CASE       )
    distiller.train()
    logger.info("Let's go get some drinks."       )


if __name__ == "__main__":
  main()




