






'''simple docstring'''



import argparse
import random

import joblib
import numpy as np
import torch
from igf.igf import (
    SecondaryLearner,
    collect_objective_set,
    compute_perplexity,
    generate_datasets,
    load_gpta,
    recopy_gpta,
    set_seed,
    train_secondary_learner,
)
from torch.utils.data import DataLoader, RandomSampler

from transformers import GPTaLMHeadModel


def 				lowercase_					(     __A			:							Tuple=3_2       ,   __A			:							Optional[Any]=1_0       ,   __A			:							Optional[int]=1_0_0       ,   __A			:							str=1_0_2_6       ,   __A			:							List[str]=True       ,   __A			:							Dict="data/tokenized_stories_train_wikitext103.jbl"       ,   __A			:							Tuple="igf_context_pairs.jbl"       ,   )							->   Tuple:



 """simple docstring"""




 set_seed(3      )
 # generate train_data and objective_set
 lowercase	,		lowercase       : str						=generate_datasets(
     __A       ,   __A       ,   number=__A       ,   min_len=1_0_2_6       ,   trim=__A      )
 # keeps model same across runs
 set_seed(4      )
 # model, lm_optimizer, lm_scheduler = recopy_gpt2(model, device, max_steps) # store original model weights
 # can we train on GPU?
 lowercase       : Optional[Any]						=torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''      )

 # load pretrained model
 lowercase       : List[str]						=load_gpta('''gpt2'''      ).to(__A      )
 print('''computing perplexity on objective set'''      )
 lowercase       : Tuple						=compute_perplexity(__A       ,   __A       ,   __A      ).item()
 print('''perplexity on objective set:'''       ,   __A      )

 # collect igf pairs and save to file demo.jbl
 collect_objective_set(__A       ,   __A       ,   __A       ,   __A       ,   __A       ,   __A       ,   __A       ,   __A      )

 # clean up, delete model and data we don't need anymore
 del model, train_data, objective_set
 torch.cuda.empty_cache()


def 				lowercase_					(     __A			:							Optional[Any]       ,   __A			:							int=1_5       ,   __A			:							Union[str, Any]=1_2_8       ,   __A			:							List[Any]=1_0_0       ,   __A			:							Optional[Any]="igf_model.pt"       ,   )							->   Union[str, Any]:



 """simple docstring"""




 set_seed(4_2      )

 # Load pre-trained model
 lowercase       : List[str]						=GPTaLMHeadModel.from_pretrained('''gpt2'''      )

 # Initialize secondary learner to use embedding weights of model
 lowercase       : str						=SecondaryLearner(__A      )

 # Train secondary learner
 lowercase       : Union[str, Any]						=train_secondary_learner(
     __A       ,   __A       ,   max_epochs=__A       ,   batch_size=__A       ,   eval_freq=1_0_0       ,   igf_model_path=__A       ,   )

 del model, secondary_learner_train_data
 torch.cuda.empty_cache()

 return secondary_learner


def 				lowercase_					(     __A			:							Any       ,   __A			:							Dict       ,   __A			:							List[str]       ,   __A			:							List[str]=3_2       ,   __A			:							Any=1_0_0_0       ,   __A			:							Union[str, Any]=1_6       ,   __A			:							int=1.0       ,   __A			:							Optional[Any]=recopy_gpta       ,   __A			:							Optional[Any]=None       ,   __A			:							Optional[Any]=1_0       ,   __A			:							str="gpt2_finetuned.pt"       ,   )							->   Dict:



 """simple docstring"""




 lowercase       : Tuple						=torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''      )
 lowercase       : Tuple						=RandomSampler(__A      )
 lowercase       : List[str]						=DataLoader(__A       ,   sampler=__A      )

 lowercase       : List[str]						=max_steps // (len(__A      )) + 1
 lowercase       : List[Any]						=0
 lowercase       : Any						=torch.zeros((1, context_len)       ,   dtype=torch.long       ,   device=__A      )
 lowercase	,		lowercase	,		lowercase       : Union[str, Any]						=recopy_model(__A       ,   __A       ,   __A      )

 model.train()
 if secondary_learner is not None:
  secondary_learner.to(__A      )
  secondary_learner.eval()
 lowercase       : int						=[]
 lowercase       : List[Any]						=0

 lowercase       : Optional[int]						=[]
 lowercase       : Any						=[]

 # Compute the performance of the transformer model at the beginning
 lowercase       : str						=compute_perplexity(__A       ,   __A       ,   __A      )
 test_perps.append(__A      )
 print('''Test perplexity, step'''       ,   __A       ,   ''':'''       ,   __A      )
 for epoch in range(int(__A      )      ):
  for step, example in enumerate(__A      ):
   torch.cuda.empty_cache()
   lowercase       : int						=random.randint(0       ,   example.size(2      ) - context_len - 1      )
   lowercase       : Optional[int]						=example[0, 0, start : start + context_len]
   lm_optimizer.zero_grad()
   lowercase       : int						=model(__A       ,   labels=__A      )
   lowercase       : List[str]						=True

   if secondary_learner is not None:
    lowercase       : str						=secondary_learner.forward(
        torch.tensor(__A       ,   dtype=torch.long       ,   device=__A      ).unsqueeze(0      )      )[0].item()
    observed_qs.append(float(__A      )      )

    # Here we implement the simple non-constant threshold for the predicted IG(X) value
    # We will decay the selectivity of our secondary learner filter from
    # 1 standard deviation above average to 1 below average after 10 batches.

    if global_step == 1_0:
     lowercase       : List[str]						=-1
    if predicted_q < threshold:
     lowercase       : Union[str, Any]						=False

            # If we passed the filter, add the context to the batch!
   if do_backprop:
    contexts.append(np.array(context.cpu()      )      )
    lowercase       : List[Any]						=outputs[0]
    lm_loss.backward()
    examples += 1

   del outputs

   # Once the batch is filled with enough contexts, backprop on the batch.
   if examples == batch_size:
    torch.cuda.empty_cache()
    lowercase       : Any						=0
    # Do LM backprop
    torch.nn.utils.clip_grad_norm_(model.parameters()       ,   3.0      )
    lm_optimizer.step()
    lm_scheduler.step()  # Update learning rate schedule
    global_step += 1
    # Compute the performance of the transformer model at this batch
    if global_step % eval_interval == 0:
     lowercase       : List[Any]						=compute_perplexity(__A       ,   __A       ,   __A      )
     test_perps.append(__A      )

     print('''Test perplexity, step'''       ,   __A       ,   ''':'''       ,   __A      )
            # Break out of the loop after 60 batches
   if max_steps > 0 and global_step > 6_0:
    break
  if max_steps > 0 and global_step > 6_0:
   break

    # save finetuned transformer model
 torch.save(model.state_dict()       ,   __A      )
 torch.cuda.empty_cache()
 # Do some cleaning up so we can reinitialize for the next run of this function
 del lm_optimizer
 del lm_scheduler
 return model


def 				lowercase_					(     )							->   int:



 """simple docstring"""




 lowercase       : List[str]						=argparse.ArgumentParser(description='''Fine-tune a transformer model with IGF on a language modeling task'''      )

 # Required parameters
 parser.add_argument(
     '''--data_dir'''       ,   default=__A       ,   type=__A       ,   required=__A       ,   help='''The input data dir. Should contain data files for WikiText.'''       ,   )
 parser.add_argument(
     '''--model_name_or_path'''       ,   default=__A       ,   type=__A       ,   required=__A       ,   help='''Path to pretrained model or model identifier from huggingface.co/models'''       ,   )
 parser.add_argument(
     '''--data_file'''       ,   type=__A       ,   default=__A       ,   help=(
         '''A jbl file containing tokenized data which can be split as objective dataset, '''
         '''train_dataset and test_dataset.'''
     )       ,   )

 parser.add_argument(
     '''--igf_data_file'''       ,   type=__A       ,   default=__A       ,   help='''A jbl file containing the context and information gain pairs to train secondary learner.'''       ,   )

 parser.add_argument(
     '''--output_dir'''       ,   default=__A       ,   type=__A       ,   required=__A       ,   help='''The output directory where the final fine-tuned model is stored.'''       ,   )

 parser.add_argument(
     '''--tokenizer_name'''       ,   default=__A       ,   type=__A       ,   help='''Pretrained tokenizer name or path if not the same as model_name'''       ,   )
 parser.add_argument('''--seed'''       ,   type=__A       ,   default=__A       ,   help='''A seed for reproducible training.'''      )

 parser.add_argument(
     '''--context_len'''       ,   default=3_2       ,   type=__A       ,   help=(
         '''The maximum total input sequence length after tokenization. Sequences longer '''
         '''than this will be truncated, sequences shorter will be padded.'''
     )       ,   )

 parser.add_argument(
     '''--size_objective_set'''       ,   default=1_0_0       ,   type=__A       ,   help='''number of articles that are long enough to be used as our objective set'''       ,   )
 parser.add_argument(
     '''--eval_freq'''       ,   default=1_0_0       ,   type=__A       ,   help='''secondary model evaluation is triggered at eval_freq'''      )

 parser.add_argument('''--max_steps'''       ,   default=1_0_0_0       ,   type=__A       ,   help='''To calculate training epochs'''      )

 parser.add_argument(
     '''--secondary_learner_batch_size'''       ,   default=1_2_8       ,   type=__A       ,   help='''batch size of training data for secondary learner'''       ,   )

 parser.add_argument(
     '''--batch_size'''       ,   default=1_6       ,   type=__A       ,   help='''batch size of training data of language model(gpt2) '''      )

 parser.add_argument(
     '''--eval_interval'''       ,   default=1_0       ,   type=__A       ,   help=(
         '''decay the selectivity of our secondary learner filter from'''
         '''1 standard deviation above average to 1 below average after 10 batches'''
     )       ,   )

 parser.add_argument(
     '''--number'''       ,   default=1_0_0       ,   type=__A       ,   help='''The number of examples split to be used as objective_set/test_data'''      )

 parser.add_argument(
     '''--min_len'''       ,   default=1_0_2_6       ,   type=__A       ,   help='''The minimum length of the article to be used as objective set'''      )

 parser.add_argument(
     '''--secondary_learner_max_epochs'''       ,   default=1_5       ,   type=__A       ,   help='''number of epochs to train secondary learner'''      )

 parser.add_argument('''--trim'''       ,   default=__A       ,   type=__A       ,   help='''truncate the example if it exceeds context length'''      )

 parser.add_argument(
     '''--threshold'''       ,   default=1.0       ,   type=__A       ,   help=(
         '''The threshold value used by secondary learner to filter the train_data and allow only'''
         ''' informative data as input to the model'''
     )       ,   )

 parser.add_argument('''--finetuned_model_name'''       ,   default='''gpt2_finetuned.pt'''       ,   type=__A       ,   help='''finetuned_model_name'''      )

 parser.add_argument(
     '''--recopy_model'''       ,   default=__A       ,   type=__A       ,   help='''Reset the model to the original pretrained GPT-2 weights after each iteration'''       ,   )

 # function calls
 # Collecting *n* pairs of context and information gain(X, IG(X)) for training the secondary learner
 generate_n_pairs(
     context_len=3_2       ,   max_steps=1_0       ,   size_objective_set=1_0_0       ,   min_len=1_0_2_6       ,   trim=__A       ,   data_file='''data/tokenized_stories_train_wikitext103.jbl'''       ,   igf_data_file='''igf_context_pairs.jbl'''       ,   )

 # Load train data for secondary learner
 lowercase       : Optional[int]						=joblib.load('''data/IGF_values.jbl'''      )

 # Train secondary learner
 lowercase       : str						=training_secondary_learner(
     __A       ,   secondary_learner_max_epochs=1_5       ,   secondary_learner_batch_size=1_2_8       ,   eval_freq=1_0_0       ,   igf_model_path='''igf_model.pt'''       ,   )

 # load pretrained gpt2 model
 lowercase       : Any						=GPTaLMHeadModel.from_pretrained('''gpt2'''      )
 set_seed(4_2      )

 # Generate train and test data to train and evaluate gpt2 model
 lowercase	,		lowercase       : List[Any]						=generate_datasets(
     context_len=3_2       ,   file='''data/tokenized_stories_train_wikitext103.jbl'''       ,   number=1_0_0       ,   min_len=1_0_2_6       ,   trim=__A      )

 # fine-tuning of the gpt2 model using igf (Information Gain Filtration)
 finetune(
     __A       ,   __A       ,   __A       ,   context_len=3_2       ,   max_steps=1_0_0_0       ,   batch_size=1_6       ,   threshold=1.0       ,   recopy_model=__A       ,   secondary_learner=__A       ,   eval_interval=1_0       ,   finetuned_model_name='''gpt2_finetuned.pt'''       ,   )


if __name__ == "__main__":
      main()










import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase      							=   logging.get_logger(__name__)

_lowerCamelCase      							=   '▁'

_lowerCamelCase      							=   {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
}

_lowerCamelCase      							=   {
    'vocab_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'
        ),
    },
    'spm_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'
        )
    },
}

_lowerCamelCase      							=   {
    'facebook/s2t-small-librispeech-asr': 1024,
}

_lowerCamelCase      							=   ['pt', 'fr', 'ru', 'nl', 'ro', 'it', 'es', 'de']

_lowerCamelCase      							=   {'mustc': MUSTC_LANGS}







class       __A							(   lowerCamelCase__	):




      """simple docstring"""







      UpperCAmelCase__							=   VOCAB_FILES_NAMES
      UpperCAmelCase__							=   PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__							=   MAX_MODEL_INPUT_SIZES
      UpperCAmelCase__							=   ["""input_ids""", """attention_mask"""]

      UpperCAmelCase__							=   []


      def __init__(		self   ,				a__   ,				a__   ,				a__="<s>"   ,				a__="</s>"   ,				a__="<pad>"   ,				a__="<unk>"   ,				a__=False   ,				a__=False   ,				a__=None   ,				a__=None   ,				a__ = None   ,				**a__   ,				):



             """simple docstring"""





             _lowerCamelCase				:  str						=		{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 bos_token=a__   ,				eos_token=a__   ,				unk_token=a__   ,				pad_token=a__   ,				do_upper_case=a__   ,				do_lower_case=a__   ,				tgt_lang=a__   ,				lang_codes=a__   ,				sp_model_kwargs=self.sp_model_kwargs   ,				**a__   ,				)
             _lowerCamelCase				:  Optional[int]						=		do_upper_case
             _lowerCamelCase				:  Optional[Any]						=		do_lower_case

             _lowerCamelCase				:  Tuple						=		load_json(a__)
             _lowerCamelCase				:  Union[str, Any]						=		{v: k for k, v in self.encoder.items()}
             _lowerCamelCase				:  Tuple						=		spm_file
             _lowerCamelCase				:  Any						=		load_spm(a__   ,				self.sp_model_kwargs)

             if lang_codes is not None:
                    _lowerCamelCase				:  List[Any]						=		lang_codes
                    _lowerCamelCase				:  List[str]						=		LANGUAGES[lang_codes]
                    _lowerCamelCase				:  Any						=		[F"""<lang:{lang}>""" for lang in self.langs]
                    _lowerCamelCase				:  Optional[Any]						=		{lang: self.sp_model.PieceToId(F"""<lang:{lang}>""") for lang in self.langs}

                    _lowerCamelCase				:  List[str]						=		self.lang_tokens
                    _lowerCamelCase				:  str						=		tgt_lang if tgt_lang is not None else self.langs[0]

                    self.set_tgt_lang_special_tokens(self._tgt_lang)
             else:
                    _lowerCamelCase				:  Any						=		{}


      @property
      def 					__snake_case      (		self):



             """simple docstring"""





             return len(self.encoder)


      @property
      def 					__snake_case      (		self):



             """simple docstring"""





             return self._tgt_lang


      @tgt_lang.setter
      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  Any						=		new_tgt_lang
             self.set_tgt_lang_special_tokens(a__)


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		self.lang_code_to_id[tgt_lang]
             _lowerCamelCase				:  Any						=		[lang_code_id]


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return self.sp_model.encode(a__   ,				out_type=a__)


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return self.encoder.get(a__   ,				self.encoder[self.unk_token])


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return self.decoder.get(a__   ,				self.unk_token)


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  Union[str, Any]						=		[]
             _lowerCamelCase				:  List[str]						=		''''''
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           _lowerCamelCase				:  List[Any]						=		self.sp_model.decode(a__)
                           out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
                           _lowerCamelCase				:  Optional[int]						=		[]
                    else:
                           current_sub_tokens.append(a__)
             _lowerCamelCase				:  Tuple						=		self.sp_model.decode(a__)
             out_string += decoded.upper() if self.do_upper_case else decoded
             return out_string.strip()


      def 					__snake_case      (		self   ,				a__   ,				a__=None):



             """simple docstring"""





             if token_ids_a is None:
                    return self.prefix_tokens + token_ids_a + [self.eos_token_id]
             # We don't expect to process pairs, but leave the pair logic for API consistency
             return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]


      def 					__snake_case      (		self   ,				a__   ,				a__ = None   ,				a__ = False):



             """simple docstring"""





             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=a__   ,				token_ids_a=a__   ,				already_has_special_tokens=a__)

             _lowerCamelCase				:  Tuple						=		[1] * len(self.prefix_tokens)
             _lowerCamelCase				:  Tuple						=		[1]
             if token_ids_a is None:
                    return prefix_ones + ([0] * len(a__)) + suffix_ones
             return prefix_ones + ([0] * len(a__)) + ([0] * len(a__)) + suffix_ones


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Union[str, Any]						=		self.encoder.copy()
             vocab.update(self.added_tokens_encoder)
             return vocab


      def __getstate__(		self):



             """simple docstring"""





             _lowerCamelCase				:  Union[str, Any]						=		self.__dict__.copy()
             _lowerCamelCase				:  str						=		None
             return state


      def __setstate__(		self   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  Optional[int]						=		d

             # for backward compatibility
             if not hasattr(self   ,				'''sp_model_kwargs'''):
                    _lowerCamelCase				:  List[Any]						=		{}

             _lowerCamelCase				:  Dict						=		load_spm(self.spm_file   ,				self.sp_model_kwargs)



      def 					__snake_case      (		self   ,				a__   ,				a__ = None):



             """simple docstring"""





             _lowerCamelCase				:  str						=		Path(a__)
             assert save_dir.is_dir(), F"""{save_directory} should be a directory"""
             _lowerCamelCase				:  Any						=		save_dir / (
                 (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
             )
             _lowerCamelCase				:  Optional[Any]						=		save_dir / (
                 (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
             )

             save_json(self.encoder   ,				a__)

             if os.path.abspath(self.spm_file) != os.path.abspath(a__) and os.path.isfile(self.spm_file):
                    copyfile(self.spm_file   ,				a__)
             elif not os.path.isfile(self.spm_file):
                    with open(a__   ,				'''wb''') as fi:
                           _lowerCamelCase				:  List[str]						=		self.sp_model.serialized_model_proto()
                           fi.write(a__)

             return (str(a__), str(a__))




def  __UpperCAmelCase(   lowercase_			,     lowercase_   ):
       _lowerCamelCase				:  Optional[Any]						=		sentencepiece.SentencePieceProcessor(**lowercase_   )
       spm.Load(str(lowercase_   )   )
       return spm




def  __UpperCAmelCase(   lowercase_   ):
       with open(lowercase_			,     '''r'''   ) as f:
              return json.load(lowercase_   )




def  __UpperCAmelCase(   lowercase_			,     lowercase_   ):
       with open(lowercase_			,     '''w'''   ) as f:
              json.dump(lowercase_			,     lowercase_			,     indent=2   )

