






import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


_SCREAMING_SNAKE_CASE   =  {
    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
    "gpt2": (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}





def        _snake_case   (_snake_case				:       Optional[Any])					->	int:
   assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
   assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
   if args.mlm:
      assert os.path.isfile(args.token_counts)
      assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
   else:
      assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

   assert args.teacher_type == args.student_type or (
       args.student_type == "distilbert" and args.teacher_type == "bert"
   )
   assert os.path.isfile(args.student_config)
   if args.student_pretrained_weights is not None:
      assert os.path.isfile(args.student_pretrained_weights)

   if args.freeze_token_type_embds:
      assert args.student_type in ["roberta"]

   assert args.alpha_ce >= 0.0
   assert args.alpha_mlm >= 0.0
   assert args.alpha_clm >= 0.0
   assert args.alpha_mse >= 0.0
   assert args.alpha_cos >= 0.0
   assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0





def        _snake_case   (_snake_case				:       List[str]      ,    _snake_case				:       int)					->	List[Any]:
   if args.student_type == "roberta":
      _lowercase              =False
   elif args.student_type == "gpt2":
      _lowercase              =False





def        _snake_case   (_snake_case				:       str      ,    _snake_case				:       str)					->	List[Any]:
   if args.student_type == "roberta":
      _lowercase              =False





def        _snake_case   ()					->	Dict:
   _lowercase              =argparse.ArgumentParser(description='Training')
   parser.add_argument('--force'      ,    action='store_true'      ,    help='Overwrite dump_path if it already exists.')

   parser.add_argument(
       '--dump_path'      ,    type=_snake_case      ,    required=_snake_case      ,    help='The output directory (log, checkpoints, parameters, etc.)')
   parser.add_argument(
       '--data_file'      ,    type=_snake_case      ,    required=_snake_case      ,    help='The binarized file (tokenized + tokens_to_ids) and grouped by sequence.'      ,    )

   parser.add_argument(
       '--student_type'      ,    type=_snake_case      ,    choices=['distilbert', 'roberta', 'gpt2']      ,    required=_snake_case      ,    help='The student type (DistilBERT, RoBERTa).'      ,    )
   parser.add_argument('--student_config'      ,    type=_snake_case      ,    required=_snake_case      ,    help='Path to the student configuration.')
   parser.add_argument(
       '--student_pretrained_weights'      ,    default=_snake_case      ,    type=_snake_case      ,    help='Load student initialization checkpoint.')

   parser.add_argument(
       '--teacher_type'      ,    choices=['bert', 'roberta', 'gpt2']      ,    required=_snake_case      ,    help='Teacher type (BERT, RoBERTa).')
   parser.add_argument('--teacher_name'      ,    type=_snake_case      ,    required=_snake_case      ,    help='The teacher model.')

   parser.add_argument('--temperature'      ,    default=2.0      ,    type=_snake_case      ,    help='Temperature for the softmax temperature.')
   parser.add_argument(
       '--alpha_ce'      ,    default=0.5      ,    type=_snake_case      ,    help='Linear weight for the distillation loss. Must be >=0.')
   parser.add_argument(
       '--alpha_mlm'      ,    default=0.0      ,    type=_snake_case      ,    help='Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag.'      ,    )
   parser.add_argument('--alpha_clm'      ,    default=0.5      ,    type=_snake_case      ,    help='Linear weight for the CLM loss. Must be >=0.')
   parser.add_argument('--alpha_mse'      ,    default=0.0      ,    type=_snake_case      ,    help='Linear weight of the MSE loss. Must be >=0.')
   parser.add_argument(
       '--alpha_cos'      ,    default=0.0      ,    type=_snake_case      ,    help='Linear weight of the cosine embedding loss. Must be >=0.')

   parser.add_argument(
       '--mlm'      ,    action='store_true'      ,    help='The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM.')
   parser.add_argument(
       '--mlm_mask_prop'      ,    default=0.15      ,    type=_snake_case      ,    help='Proportion of tokens for which we need to make a prediction.'      ,    )
   parser.add_argument('--word_mask'      ,    default=0.8      ,    type=_snake_case      ,    help='Proportion of tokens to mask out.')
   parser.add_argument('--word_keep'      ,    default=0.1      ,    type=_snake_case      ,    help='Proportion of tokens to keep.')
   parser.add_argument('--word_rand'      ,    default=0.1      ,    type=_snake_case      ,    help='Proportion of tokens to randomly replace.')
   parser.add_argument(
       '--mlm_smoothing'      ,    default=0.7      ,    type=_snake_case      ,    help='Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec).'      ,    )
   parser.add_argument('--token_counts'      ,    type=_snake_case      ,    help='The token counts in the data_file for MLM.')

   parser.add_argument(
       '--restrict_ce_to_mask'      ,    action='store_true'      ,    help='If true, compute the distillation loss only the [MLM] prediction distribution.'      ,    )
   parser.add_argument(
       '--freeze_pos_embs'      ,    action='store_true'      ,    help='Freeze positional embeddings during distillation. For student_type in [\'roberta\', \'gpt2\'] only.'      ,    )
   parser.add_argument(
       '--freeze_token_type_embds'      ,    action='store_true'      ,    help='Freeze token type embeddings during distillation if existent. For student_type in [\'roberta\'] only.'      ,    )

   parser.add_argument('--n_epoch'      ,    type=_snake_case      ,    default=3      ,    help='Number of pass on the whole dataset.')
   parser.add_argument('--batch_size'      ,    type=_snake_case      ,    default=5      ,    help='Batch size (for each process).')
   parser.add_argument(
       '--group_by_size'      ,    action='store_false'      ,    help='If true, group sequences that have similar length into the same batch. Default is true.'      ,    )

   parser.add_argument(
       '--gradient_accumulation_steps'      ,    type=_snake_case      ,    default=50      ,    help='Gradient accumulation for larger training batches.'      ,    )
   parser.add_argument('--warmup_prop'      ,    default=0.05      ,    type=_snake_case      ,    help='Linear warmup proportion.')
   parser.add_argument('--weight_decay'      ,    default=0.0      ,    type=_snake_case      ,    help='Weight decay if we apply some.')
   parser.add_argument('--learning_rate'      ,    default=5E-4      ,    type=_snake_case      ,    help='The initial learning rate for Adam.')
   parser.add_argument('--adam_epsilon'      ,    default=1E-6      ,    type=_snake_case      ,    help='Epsilon for Adam optimizer.')
   parser.add_argument('--max_grad_norm'      ,    default=5.0      ,    type=_snake_case      ,    help='Max gradient norm.')
   parser.add_argument('--initializer_range'      ,    default=0.02      ,    type=_snake_case      ,    help='Random initialization range.')

   parser.add_argument(
       '--fp16'      ,    action='store_true'      ,    help='Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'      ,    )
   parser.add_argument(
       '--fp16_opt_level'      ,    type=_snake_case      ,    default='O1'      ,    help=(
           'For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'
           'See details at https://nvidia.github.io/apex/amp.html'
       )      ,    )
   parser.add_argument('--n_gpu'      ,    type=_snake_case      ,    default=1      ,    help='Number of GPUs in the node.')
   parser.add_argument('--local_rank'      ,    type=_snake_case      ,    default=-1      ,    help='Distributed training - Local rank')
   parser.add_argument('--seed'      ,    type=_snake_case      ,    default=56      ,    help='Random seed')

   parser.add_argument('--log_interval'      ,    type=_snake_case      ,    default=500      ,    help='Tensorboard logging interval.')
   parser.add_argument('--checkpoint_interval'      ,    type=_snake_case      ,    default=4000      ,    help='Checkpoint interval.')
   _lowercase              =parser.parse_args()
   sanity_checks(_snake_case)

   # ARGS #
   init_gpu_params(_snake_case)
   set_seed(_snake_case)
   if args.is_master:
      if os.path.exists(args.dump_path):
         if not args.force:
            raise ValueError(
                f'''Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'''
                ' itUse `--force` if you want to overwrite it')
         else:
            shutil.rmtree(args.dump_path)

      if not os.path.exists(args.dump_path):
         os.makedirs(args.dump_path)
      logger.info(f'''Experiment will be dumped and logged in {args.dump_path}''')

      # SAVE PARAMS #
      logger.info(f'''Param: {args}''')
      with open(os.path.join(args.dump_path      ,    'parameters.json')      ,    'w') as f:
         json.dump(vars(_snake_case)      ,    _snake_case      ,    indent=4)
      git_log(args.dump_path)

   _lowercase       ,       _lowercase       ,       _lowercase              =MODEL_CLASSES[args.student_type]
   _lowercase       ,       _lowercase       ,       _lowercase              =MODEL_CLASSES[args.teacher_type]

   # TOKENIZER #
   _lowercase              =teacher_tokenizer_class.from_pretrained(args.teacher_name)
   _lowercase              ={}
   for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
      _lowercase              =tokenizer.all_special_tokens.index(_snake_case)
      _lowercase              =tokenizer.all_special_ids[idx]
   logger.info(f'''Special tokens {special_tok_ids}''')
   _lowercase              =special_tok_ids
   _lowercase              =tokenizer.max_model_input_sizes[args.teacher_name]

   # DATA LOADER #
   logger.info(f'''Loading data from {args.data_file}''')
   with open(args.data_file      ,    'rb') as fp:
      _lowercase              =pickle.load(_snake_case)

   if args.mlm:
      logger.info(f'''Loading token counts from {args.token_counts} (already pre-computed)''')
      with open(args.token_counts      ,    'rb') as fp:
         _lowercase              =pickle.load(_snake_case)

      _lowercase              =np.maximum(_snake_case      ,    1) ** -args.mlm_smoothing
      for idx in special_tok_ids.values():
         _lowercase              =0.0  # do not predict special tokens
      _lowercase              =torch.from_numpy(_snake_case)
   else:
      _lowercase              =None

   _lowercase              =LmSeqsDataset(params=_snake_case      ,    data=_snake_case)
   logger.info('Data loader created.')

   # STUDENT #
   logger.info(f'''Loading student config from {args.student_config}''')
   _lowercase              =student_config_class.from_pretrained(args.student_config)
   _lowercase              =True

   if args.student_pretrained_weights is not None:
      logger.info(f'''Loading pretrained weights from {args.student_pretrained_weights}''')
      _lowercase              =student_model_class.from_pretrained(args.student_pretrained_weights      ,    config=_snake_case)
   else:
      _lowercase              =student_model_class(_snake_case)

   if args.n_gpu > 0:
      student.to(f'''cuda:{args.local_rank}''')
   logger.info('Student loaded.')

   # TEACHER #
   _lowercase              =teacher_model_class.from_pretrained(args.teacher_name      ,    output_hidden_states=_snake_case)
   if args.n_gpu > 0:
      teacher.to(f'''cuda:{args.local_rank}''')
   logger.info(f'''Teacher loaded from {args.teacher_name}.''')

   # FREEZING #
   if args.freeze_pos_embs:
      freeze_pos_embeddings(_snake_case      ,    _snake_case)
   if args.freeze_token_type_embds:
      freeze_token_type_embeddings(_snake_case      ,    _snake_case)

   # SANITY CHECKS #
   assert student.config.vocab_size == teacher.config.vocab_size
   assert student.config.hidden_size == teacher.config.hidden_size
   assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
   if args.mlm:
      assert token_probs.size(0) == stu_architecture_config.vocab_size

   # DISTILLER #
   torch.cuda.empty_cache()
   _lowercase              =Distiller(
       params=_snake_case      ,    dataset=_snake_case      ,    token_probs=_snake_case      ,    student=_snake_case      ,    teacher=_snake_case)
   distiller.train()
   logger.info('Let\'s go get some drinks.')


if __name__ == "__main__":
    main()










from typing import List, Optional, Union

from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType






class 					SCREAMING_SNAKE_CASE_  (       _a ):
       """simple docstring"""


       __lowerCAmelCase    :		Dict					=['''image_processor''', '''tokenizer''']
       __lowerCAmelCase    :		List[Any]					='''BlipImageProcessor'''
       __lowerCAmelCase    :		Union[str, Any]					=('''BertTokenizer''', '''BertTokenizerFast''')





       def __init__(				self	:Optional[int],      snake_case	:int,      snake_case	:Optional[int]):



          """simple docstring"""

          _lowercase              =False
          super().__init__(snake_case,      snake_case)
          _lowercase              =self.image_processor





       def __call__(				self	:Dict,      snake_case	:ImageInput = None,      snake_case	:Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,      snake_case	:bool = True,      snake_case	:Union[bool, str, PaddingStrategy] = False,      snake_case	:Union[bool, str, TruncationStrategy] = None,      snake_case	:Optional[int] = None,      snake_case	:int = 0,      snake_case	:Optional[int] = None,      snake_case	:Optional[bool] = None,      snake_case	:bool = False,      snake_case	:bool = False,      snake_case	:bool = False,      snake_case	:bool = False,      snake_case	:bool = False,      snake_case	:bool = True,      snake_case	:Optional[Union[str, TensorType]] = None,      **snake_case	:str,      ):



          """simple docstring"""

          if images is None and text is None:
             raise ValueError('You have to specify either images or text.')

          # Get only text
          if images is None:
             _lowercase              =self.tokenizer
             _lowercase              =self.tokenizer(
                 text=snake_case,      add_special_tokens=snake_case,      padding=snake_case,      truncation=snake_case,      max_length=snake_case,      stride=snake_case,      pad_to_multiple_of=snake_case,      return_attention_mask=snake_case,      return_overflowing_tokens=snake_case,      return_special_tokens_mask=snake_case,      return_offsets_mapping=snake_case,      return_token_type_ids=snake_case,      return_length=snake_case,      verbose=snake_case,      return_tensors=snake_case,      **snake_case,      )
             return text_encoding

          # add pixel_values
          _lowercase              =self.image_processor(snake_case,      return_tensors=snake_case)

          if text is not None:
             _lowercase              =self.tokenizer(
                 text=snake_case,      add_special_tokens=snake_case,      padding=snake_case,      truncation=snake_case,      max_length=snake_case,      stride=snake_case,      pad_to_multiple_of=snake_case,      return_attention_mask=snake_case,      return_overflowing_tokens=snake_case,      return_special_tokens_mask=snake_case,      return_offsets_mapping=snake_case,      return_token_type_ids=snake_case,      return_length=snake_case,      verbose=snake_case,      return_tensors=snake_case,      **snake_case,      )
          else:
             _lowercase              =None

          if text_encoding is not None:
             encoding_image_processor.update(snake_case)

          return encoding_image_processor





       def 				UpperCamelCase__						(				self	:List[Any],      *snake_case	:int,      **snake_case	:str):



          """simple docstring"""

          return self.tokenizer.batch_decode(*snake_case,      **snake_case)





       def 				UpperCamelCase__						(				self	:Tuple,      *snake_case	:List[str],      **snake_case	:str):



          """simple docstring"""

          return self.tokenizer.decode(*snake_case,      **snake_case)





       @property
       def 				UpperCamelCase__						(				self	:List[Any]):



          """simple docstring"""

          _lowercase              =self.tokenizer.model_input_names
          _lowercase              =self.image_processor.model_input_names
          return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))


