





import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


snake_case 				=       {
    """distilbert""": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    """roberta""": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    """bert""": (BertConfig, BertForMaskedLM, BertTokenizer),
    """gpt2""": (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}

def 							SCREAMING_SNAKE_CASE__       (  snake_case__  :Optional[Any]     )      ->						Any:
  assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
  assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
  if args.mlm:
    assert os.path.isfile(args.token_counts     )
    assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
  else:
    assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

  assert args.teacher_type == args.student_type or (
      args.student_type == "distilbert" and args.teacher_type == "bert"
  )
  assert os.path.isfile(args.student_config     )
  if args.student_pretrained_weights is not None:
    assert os.path.isfile(args.student_pretrained_weights     )

  if args.freeze_token_type_embds:
    assert args.student_type in ["roberta"]

  assert args.alpha_ce >= 0.0
  assert args.alpha_mlm >= 0.0
  assert args.alpha_clm >= 0.0
  assert args.alpha_mse >= 0.0
  assert args.alpha_cos >= 0.0
  assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0

def 							SCREAMING_SNAKE_CASE__       (  snake_case__  :Optional[int]			,							snake_case__  :Tuple     )      ->						int:
  if args.student_type == "roberta":
    _lowercase							= False
  elif args.student_type == "gpt2":
    _lowercase							= False

def 							SCREAMING_SNAKE_CASE__       (  snake_case__  :Tuple			,							snake_case__  :Optional[Any]     )      ->						List[Any]:
  if args.student_type == "roberta":
    _lowercase							= False

def 							SCREAMING_SNAKE_CASE__       (  )      ->						Any:
  _lowercase							= argparse.ArgumentParser(description='Training'     )
  parser.add_argument('--force'			,							action='store_true'			,							help='Overwrite dump_path if it already exists.'     )

  parser.add_argument(
      '--dump_path'			,							type=snake_case__			,							required=snake_case__			,							help='The output directory (log, checkpoints, parameters, etc.)'     )
  parser.add_argument(
      '--data_file'			,							type=snake_case__			,							required=snake_case__			,							help='The binarized file (tokenized + tokens_to_ids) and grouped by sequence.'			,							)

  parser.add_argument(
      '--student_type'			,							type=snake_case__			,							choices=['distilbert', 'roberta', 'gpt2']			,							required=snake_case__			,							help='The student type (DistilBERT, RoBERTa).'			,							)
  parser.add_argument('--student_config'			,							type=snake_case__			,							required=snake_case__			,							help='Path to the student configuration.'     )
  parser.add_argument(
      '--student_pretrained_weights'			,							default=snake_case__			,							type=snake_case__			,							help='Load student initialization checkpoint.'     )

  parser.add_argument(
      '--teacher_type'			,							choices=['bert', 'roberta', 'gpt2']			,							required=snake_case__			,							help='Teacher type (BERT, RoBERTa).'     )
  parser.add_argument('--teacher_name'			,							type=snake_case__			,							required=snake_case__			,							help='The teacher model.'     )

  parser.add_argument('--temperature'			,							default=2.0			,							type=snake_case__			,							help='Temperature for the softmax temperature.'     )
  parser.add_argument(
      '--alpha_ce'			,							default=0.5			,							type=snake_case__			,							help='Linear weight for the distillation loss. Must be >=0.'     )
  parser.add_argument(
      '--alpha_mlm'			,							default=0.0			,							type=snake_case__			,							help='Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag.'			,							)
  parser.add_argument('--alpha_clm'			,							default=0.5			,							type=snake_case__			,							help='Linear weight for the CLM loss. Must be >=0.'     )
  parser.add_argument('--alpha_mse'			,							default=0.0			,							type=snake_case__			,							help='Linear weight of the MSE loss. Must be >=0.'     )
  parser.add_argument(
      '--alpha_cos'			,							default=0.0			,							type=snake_case__			,							help='Linear weight of the cosine embedding loss. Must be >=0.'     )

  parser.add_argument(
      '--mlm'			,							action='store_true'			,							help='The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM.'     )
  parser.add_argument(
      '--mlm_mask_prop'			,							default=0.15			,							type=snake_case__			,							help='Proportion of tokens for which we need to make a prediction.'			,							)
  parser.add_argument('--word_mask'			,							default=0.8			,							type=snake_case__			,							help='Proportion of tokens to mask out.'     )
  parser.add_argument('--word_keep'			,							default=0.1			,							type=snake_case__			,							help='Proportion of tokens to keep.'     )
  parser.add_argument('--word_rand'			,							default=0.1			,							type=snake_case__			,							help='Proportion of tokens to randomly replace.'     )
  parser.add_argument(
      '--mlm_smoothing'			,							default=0.7			,							type=snake_case__			,							help='Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec).'			,							)
  parser.add_argument('--token_counts'			,							type=snake_case__			,							help='The token counts in the data_file for MLM.'     )

  parser.add_argument(
      '--restrict_ce_to_mask'			,							action='store_true'			,							help='If true, compute the distillation loss only the [MLM] prediction distribution.'			,							)
  parser.add_argument(
      '--freeze_pos_embs'			,							action='store_true'			,							help='Freeze positional embeddings during distillation. For student_type in [\'roberta\', \'gpt2\'] only.'			,							)
  parser.add_argument(
      '--freeze_token_type_embds'			,							action='store_true'			,							help='Freeze token type embeddings during distillation if existent. For student_type in [\'roberta\'] only.'			,							)

  parser.add_argument('--n_epoch'			,							type=snake_case__			,							default=3			,							help='Number of pass on the whole dataset.'     )
  parser.add_argument('--batch_size'			,							type=snake_case__			,							default=5			,							help='Batch size (for each process).'     )
  parser.add_argument(
      '--group_by_size'			,							action='store_false'			,							help='If true, group sequences that have similar length into the same batch. Default is true.'			,							)

  parser.add_argument(
      '--gradient_accumulation_steps'			,							type=snake_case__			,							default=50			,							help='Gradient accumulation for larger training batches.'			,							)
  parser.add_argument('--warmup_prop'			,							default=0.05			,							type=snake_case__			,							help='Linear warmup proportion.'     )
  parser.add_argument('--weight_decay'			,							default=0.0			,							type=snake_case__			,							help='Weight decay if we apply some.'     )
  parser.add_argument('--learning_rate'			,							default=5E-4			,							type=snake_case__			,							help='The initial learning rate for Adam.'     )
  parser.add_argument('--adam_epsilon'			,							default=1E-6			,							type=snake_case__			,							help='Epsilon for Adam optimizer.'     )
  parser.add_argument('--max_grad_norm'			,							default=5.0			,							type=snake_case__			,							help='Max gradient norm.'     )
  parser.add_argument('--initializer_range'			,							default=0.02			,							type=snake_case__			,							help='Random initialization range.'     )

  parser.add_argument(
      '--fp16'			,							action='store_true'			,							help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'			,							)
  parser.add_argument(
      '--fp16_opt_level'			,							type=snake_case__			,							default='O1'			,							help=(
          'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
          'See details at https://nvidia.github.io/apex/amp.html'
      )			,							)
  parser.add_argument('--n_gpu'			,							type=snake_case__			,							default=1			,							help='Number of GPUs in the node.'     )
  parser.add_argument('--local_rank'			,							type=snake_case__			,							default=-1			,							help='Distributed training - Local rank'     )
  parser.add_argument('--seed'			,							type=snake_case__			,							default=56			,							help='Random seed'     )

  parser.add_argument('--log_interval'			,							type=snake_case__			,							default=500			,							help='Tensorboard logging interval.'     )
  parser.add_argument('--checkpoint_interval'			,							type=snake_case__			,							default=4000			,							help='Checkpoint interval.'     )
  _lowercase							= parser.parse_args()
  sanity_checks(snake_case__     )

  # ARGS #
  init_gpu_params(snake_case__     )
  set_seed(snake_case__     )
  if args.is_master:
    if os.path.exists(args.dump_path     ):
      if not args.force:
        raise ValueError(
            F"""Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite"""
            ' itUse `--force` if you want to overwrite it'     )
      else:
        shutil.rmtree(args.dump_path     )

    if not os.path.exists(args.dump_path     ):
      os.makedirs(args.dump_path     )
    logger.info(F"""Experiment will be dumped and logged in {args.dump_path}"""     )

    # SAVE PARAMS #
    logger.info(F"""Param: {args}"""     )
    with open(os.path.join(args.dump_path			,							'parameters.json'     )			,							'w'     ) as f:
      json.dump(vars(snake_case__     )			,							snake_case__			,							indent=4     )
    git_log(args.dump_path     )

  _lowercase						,		_lowercase						,		_lowercase							= MODEL_CLASSES[args.student_type]
  _lowercase						,		_lowercase						,		_lowercase							= MODEL_CLASSES[args.teacher_type]

  # TOKENIZER #
  _lowercase							= teacher_tokenizer_class.from_pretrained(args.teacher_name     )
  _lowercase							= {}
  for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
    _lowercase							= tokenizer.all_special_tokens.index(snake_case__     )
    _lowercase							= tokenizer.all_special_ids[idx]
  logger.info(F"""Special tokens {special_tok_ids}"""     )
  _lowercase							= special_tok_ids
  _lowercase							= tokenizer.max_model_input_sizes[args.teacher_name]

  # DATA LOADER #
  logger.info(F"""Loading data from {args.data_file}"""     )
  with open(args.data_file			,							'rb'     ) as fp:
    _lowercase							= pickle.load(snake_case__     )

  if args.mlm:
    logger.info(F"""Loading token counts from {args.token_counts} (already pre-computed)"""     )
    with open(args.token_counts			,							'rb'     ) as fp:
      _lowercase							= pickle.load(snake_case__     )

    _lowercase							= np.maximum(snake_case__			,							1     ) ** -args.mlm_smoothing
    for idx in special_tok_ids.values():
      _lowercase							= 0.0  # do not predict special tokens
    _lowercase							= torch.from_numpy(snake_case__     )
  else:
    _lowercase							= None

  _lowercase							= LmSeqsDataset(params=snake_case__			,							data=snake_case__     )
  logger.info('Data loader created.'     )

  # STUDENT #
  logger.info(F"""Loading student config from {args.student_config}"""     )
  _lowercase							= student_config_class.from_pretrained(args.student_config     )
  _lowercase							= True

  if args.student_pretrained_weights is not None:
    logger.info(F"""Loading pretrained weights from {args.student_pretrained_weights}"""     )
    _lowercase							= student_model_class.from_pretrained(args.student_pretrained_weights			,							config=snake_case__     )
  else:
    _lowercase							= student_model_class(snake_case__     )

  if args.n_gpu > 0:
    student.to(F"""cuda:{args.local_rank}"""     )
  logger.info('Student loaded.'     )

  # TEACHER #
  _lowercase							= teacher_model_class.from_pretrained(args.teacher_name			,							output_hidden_states=snake_case__     )
  if args.n_gpu > 0:
    teacher.to(F"""cuda:{args.local_rank}"""     )
  logger.info(F"""Teacher loaded from {args.teacher_name}."""     )

  # FREEZING #
  if args.freeze_pos_embs:
    freeze_pos_embeddings(snake_case__			,							snake_case__     )
  if args.freeze_token_type_embds:
    freeze_token_type_embeddings(snake_case__			,							snake_case__     )

  # SANITY CHECKS #
  assert student.config.vocab_size == teacher.config.vocab_size
  assert student.config.hidden_size == teacher.config.hidden_size
  assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
  if args.mlm:
    assert token_probs.size(0     ) == stu_architecture_config.vocab_size

  # DISTILLER #
  torch.cuda.empty_cache()
  _lowercase							= Distiller(
      params=snake_case__			,							dataset=snake_case__			,							token_probs=snake_case__			,							student=snake_case__			,							teacher=snake_case__     )
  distiller.train()
  logger.info('Let\'s go get some drinks.'     )


if __name__ == "__main__":
      main()






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
      import torch

      from transformers.generation import DisjunctiveConstraint





@require_torch
class 				A_				(      unittest.TestCase	):







       """simple docstring"""





       def    __UpperCAmelCase (    self	:	Union[str, Any]  )						->					List[Any]:
         # For consistency across different places the DisjunctiveConstraint is called,
         # dc.token_ids is a list of integers. It is also initialized only by integers.

         _lowercase							= [[1, 2, 4], [1, 2, 3, 4]]
         _lowercase							= DisjunctiveConstraint(__A  )
         self.assertTrue(isinstance(dc.token_ids						,__A  )  )

         with self.assertRaises(__A  ):
           DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]  )  )

         with self.assertRaises(__A  ):
           DisjunctiveConstraint([torch.LongTensor([1, 2, 4]  ), torch.LongTensor([1, 2, 3, 4, 5]  )]  )







       def    __UpperCAmelCase (    self	:	List[Any]  )						->					str:
         # We can't have constraints that are complete subsets of another. This leads to a preverse
         # interpretation of "constraint fulfillment": does generating [1,2,3] fulfill the constraint?
         # It would mean that it generated [1,2] which fulfills it, but it's in the middle of potentially
         # fulfilling [1,2,3,4]. If we believe that [1,2,3] does fulfill the constraint, then the algorithm
         # will necessarily never reach [1,2,3,4], giving users a false sense of control (better to just not allow it).
         _lowercase							= [[1, 2], [1, 2, 3, 4]]

         with self.assertRaises(__A  ):
           DisjunctiveConstraint(__A  )  # fails here







       def    __UpperCAmelCase (    self	:	Dict  )						->					Optional[Any]:
         _lowercase							= [[1, 2, 3], [1, 2, 4]]

         _lowercase							= DisjunctiveConstraint(__A  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(1  )
         _lowercase							= stepped is True and completed is False and reset is False
         self.assertTrue(__A  )
         self.assertTrue(not dc.completed  )
         self.assertTrue(dc.current_seq == [1]  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(2  )
         _lowercase							= stepped is True and completed is False and reset is False
         self.assertTrue(__A  )
         self.assertTrue(not dc.completed  )
         self.assertTrue(dc.current_seq == [1, 2]  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(3  )
         _lowercase							= stepped is True and completed is True and reset is False
         self.assertTrue(__A  )
         self.assertTrue(dc.completed  )  # Completed!
         self.assertTrue(dc.current_seq == [1, 2, 3]  )







       def    __UpperCAmelCase (    self	:	List[Any]  )						->					Any:
         _lowercase							= [[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

         _lowercase							= DisjunctiveConstraint(__A  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(1  )
         self.assertTrue(not dc.completed  )
         self.assertTrue(dc.current_seq == [1]  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(2  )
         self.assertTrue(not dc.completed  )
         self.assertTrue(dc.current_seq == [1, 2]  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(4  )
         self.assertTrue(not dc.completed  )
         self.assertTrue(dc.current_seq == [1, 2, 4]  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(5  )
         self.assertTrue(dc.completed  )  # Completed!
         self.assertTrue(dc.current_seq == [1, 2, 4, 5]  )

         dc.reset()

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(1  )
         self.assertTrue(not dc.completed  )
         self.assertTrue(dc.remaining() == 3  )
         self.assertTrue(dc.current_seq == [1]  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(2  )
         self.assertTrue(not dc.completed  )
         self.assertTrue(dc.remaining() == 2  )
         self.assertTrue(dc.current_seq == [1, 2]  )

         _lowercase						,		_lowercase						,		_lowercase							= dc.update(5  )
         self.assertTrue(dc.completed  )  # Completed!
         self.assertTrue(dc.remaining() == 0  )
         self.assertTrue(dc.current_seq == [1, 2, 5]  )