
import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


lowercase						:      Optional[Any]									= {
    'tiny.en': 'https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt',
    'tiny': 'https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt',
    'base.en': 'https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt',
    'base': 'https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt',
    'small.en': 'https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt',
    'small': 'https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt',
    'medium.en': 'https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt',
    'medium': 'https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt',
    'large': 'https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt',
    'large-v2': 'https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt',
}




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    int)      ->					int:






							'''simple docstring'''





							__UpperCamelCase			:     Optional[Any]         =      ["layers", "blocks"]
							for k in ignore_keys:
														state_dict.pop(_lowerCamelCase		,						_lowerCamelCase)


lowercase						:      Optional[int]									= {
    'blocks': 'layers',
    'mlp.0': 'fc1',
    'mlp.2': 'fc2',
    'mlp_ln': 'final_layer_norm',
    '.attn.query': '.self_attn.q_proj',
    '.attn.key': '.self_attn.k_proj',
    '.attn.value': '.self_attn.v_proj',
    '.attn_ln': '.self_attn_layer_norm',
    '.attn.out': '.self_attn.out_proj',
    '.cross_attn.query': '.encoder_attn.q_proj',
    '.cross_attn.key': '.encoder_attn.k_proj',
    '.cross_attn.value': '.encoder_attn.v_proj',
    '.cross_attn_ln': '.encoder_attn_layer_norm',
    '.cross_attn.out': '.encoder_attn.out_proj',
    'decoder.ln.': 'decoder.layer_norm.',
    'encoder.ln.': 'encoder.layer_norm.',
    'token_embedding': 'embed_tokens',
    'encoder.positional_embedding': 'encoder.embed_positions.weight',
    'decoder.positional_embedding': 'decoder.embed_positions.weight',
    'ln_post': 'layer_norm',
}




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    Any)      ->					str:






							'''simple docstring'''





							__UpperCamelCase			:     Optional[int]         =      list(s_dict.keys())
							for key in keys:
														__UpperCamelCase			:     Tuple         =      key
														for k, v in WHISPER_MAPPING.items():
																					if k in key:
																												__UpperCamelCase			:     Union[str, Any]         =      new_key.replace(_lowerCamelCase		,						_lowerCamelCase)

														print(F'{key} -> {new_key}')

														__UpperCamelCase			:     str         =      s_dict.pop(_lowerCamelCase)
							return s_dict




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    str)      ->					str:






							'''simple docstring'''





							__UpperCamelCase     ,		__UpperCamelCase			:     Union[str, Any]         =      emb.weight.shape
							__UpperCamelCase			:     Dict         =      nn.Linear(_lowerCamelCase		,						_lowerCamelCase		,						bias=_lowerCamelCase)
							__UpperCamelCase			:     List[str]         =      emb.weight.data
							return lin_layer




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    str		,						_lowerCamelCase				:    str)      ->					bytes:






							'''simple docstring'''





							os.makedirs(_lowerCamelCase		,						exist_ok=_lowerCamelCase)
							__UpperCamelCase			:     str         =      os.path.basename(_lowerCamelCase)

							__UpperCamelCase			:     Optional[Any]         =      url.split("/")[-2]
							__UpperCamelCase			:     Dict         =      os.path.join(_lowerCamelCase		,						_lowerCamelCase)

							if os.path.exists(_lowerCamelCase) and not os.path.isfile(_lowerCamelCase):
														raise RuntimeError(F'{download_target} exists and is not a regular file')

							if os.path.isfile(_lowerCamelCase):
														__UpperCamelCase			:     List[str]         =      open(_lowerCamelCase		,						"rb").read()
														if hashlib.shaaaa(_lowerCamelCase).hexdigest() == expected_shaaaa:
																					return model_bytes
														else:
																					warnings.warn(F'{download_target} exists, but the SHA256 checksum does not match; re-downloading the file')

							with urllib.request.urlopen(_lowerCamelCase) as source, open(_lowerCamelCase		,						"wb") as output:
														with tqdm(
														    total=int(source.info().get("Content-Length"))		,						ncols=80		,						unit="iB"		,						unit_scale=_lowerCamelCase		,						unit_divisor=1_024) as loop:
																					while True:
																												__UpperCamelCase			:     Union[str, Any]         =      source.read(8_192)
																												if not buffer:
																																			break

																												output.write(_lowerCamelCase)
																												loop.update(len(_lowerCamelCase))

							__UpperCamelCase			:     List[str]         =      open(_lowerCamelCase		,						"rb").read()
							if hashlib.shaaaa(_lowerCamelCase).hexdigest() != expected_shaaaa:
														raise RuntimeError(
														    "Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model.")

							return model_bytes




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    Union[str, Any]		,						_lowerCamelCase				:    List[Any])      ->					List[str]:






							'''simple docstring'''





							if ".pt" not in checkpoint_path:
														__UpperCamelCase			:     str         =      _download(_MODELS[checkpoint_path])
							else:
														__UpperCamelCase			:     Union[str, Any]         =      torch.load(_lowerCamelCase		,						map_location="cpu")
							__UpperCamelCase			:     Optional[int]         =      original_checkpoint["dims"]
							__UpperCamelCase			:     int         =      original_checkpoint["model_state_dict"]
							__UpperCamelCase			:     Optional[int]         =      state_dict["decoder.token_embedding.weight"]
							remove_ignore_keys_(_lowerCamelCase)
							rename_keys(_lowerCamelCase)
							__UpperCamelCase			:     List[Any]         =      True
							__UpperCamelCase			:     Union[str, Any]         =      state_dict["decoder.layers.0.fc1.weight"].shape[0]

							__UpperCamelCase			:     List[str]         =      WhisperConfig(
							    vocab_size=dimensions["n_vocab"]		,						encoder_ffn_dim=_lowerCamelCase		,						decoder_ffn_dim=_lowerCamelCase		,						num_mel_bins=dimensions["n_mels"]		,						d_model=dimensions["n_audio_state"]		,						max_target_positions=dimensions["n_text_ctx"]		,						encoder_layers=dimensions["n_audio_layer"]		,						encoder_attention_heads=dimensions["n_audio_head"]		,						decoder_layers=dimensions["n_text_layer"]		,						decoder_attention_heads=dimensions["n_text_state"]		,						max_source_positions=dimensions["n_audio_ctx"]		,						)

							__UpperCamelCase			:     List[str]         =      WhisperForConditionalGeneration(_lowerCamelCase)
							__UpperCamelCase     ,		__UpperCamelCase			:     List[str]         =      model.model.load_state_dict(_lowerCamelCase		,						strict=_lowerCamelCase)
							if len(_lowerCamelCase) > 0 and not set(_lowerCamelCase) <= {
							    "encoder.embed_positions.weights",
							    "decoder.embed_positions.weights",
							}:
														raise ValueError(
														    "Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,"
														    F' but all the following weights are missing {missing}')

							if tie_embeds:
														__UpperCamelCase			:     str         =      make_linear_from_emb(model.model.decoder.embed_tokens)
							else:
														__UpperCamelCase			:     Union[str, Any]         =      proj_out_weights

							model.save_pretrained(_lowerCamelCase)


if __name__ == "__main__":
		lowercase						:      List[str]									= argparse.ArgumentParser()
		# # Required parameters
		parser.add_argument('--checkpoint_path', type=str, help='Patht to the downloaded checkpoints')
		parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
		lowercase						:      Any									= parser.parse_args()

		convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)

import argparse
import random

import joblib
import numpy as np
import torch
from igf.igf import (
    SecondaryLearner,
    collect_objective_set,
    compute_perplexity,
    generate_datasets,
    load_gpta,
    recopy_gpta,
    set_seed,
    train_secondary_learner,
)
from torch.utils.data import DataLoader, RandomSampler

from transformers import GPTaLMHeadModel




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    int=32		,						_lowerCamelCase				:    str=10		,						_lowerCamelCase				:    Dict=100		,						_lowerCamelCase				:    int=1_026		,						_lowerCamelCase				:    Union[str, Any]=True		,						_lowerCamelCase				:    str="data/tokenized_stories_train_wikitext103.jbl"		,						_lowerCamelCase				:    Any="igf_context_pairs.jbl"		,						)      ->					str:






							'''simple docstring'''





							set_seed(3)
							# generate train_data and objective_set
							__UpperCamelCase     ,		__UpperCamelCase			:     Union[str, Any]         =      generate_datasets(
							    _lowerCamelCase		,						_lowerCamelCase		,						number=_lowerCamelCase		,						min_len=1_026		,						trim=_lowerCamelCase)
							# keeps model same across runs
							set_seed(4)
							# model, lm_optimizer, lm_scheduler = recopy_gpt2(model, device, max_steps) # store original model weights
							# can we train on GPU?
							__UpperCamelCase			:     Dict         =      torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

							# load pretrained model
							__UpperCamelCase			:     str         =      load_gpta("gpt2").to(_lowerCamelCase)
							print("computing perplexity on objective set")
							__UpperCamelCase			:     Union[str, Any]         =      compute_perplexity(_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase).item()
							print("perplexity on objective set:"		,						_lowerCamelCase)

							# collect igf pairs and save to file demo.jbl
							collect_objective_set(_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase)

							# clean up, delete model and data we don't need anymore
							del model, train_data, objective_set
							torch.cuda.empty_cache()




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    List[Any]		,						_lowerCamelCase				:    Any=15		,						_lowerCamelCase				:    Union[str, Any]=128		,						_lowerCamelCase				:    Any=100		,						_lowerCamelCase				:    List[Any]="igf_model.pt"		,						)      ->					Any:






							'''simple docstring'''





							set_seed(42)

							# Load pre-trained model
							__UpperCamelCase			:     int         =      GPTaLMHeadModel.from_pretrained("gpt2")

							# Initialize secondary learner to use embedding weights of model
							__UpperCamelCase			:     Any         =      SecondaryLearner(_lowerCamelCase)

							# Train secondary learner
							__UpperCamelCase			:     Union[str, Any]         =      train_secondary_learner(
							    _lowerCamelCase		,						_lowerCamelCase		,						max_epochs=_lowerCamelCase		,						batch_size=_lowerCamelCase		,						eval_freq=100		,						igf_model_path=_lowerCamelCase		,						)

							del model, secondary_learner_train_data
							torch.cuda.empty_cache()

							return secondary_learner




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    List[str]		,						_lowerCamelCase				:    Tuple		,						_lowerCamelCase				:    Optional[Any]		,						_lowerCamelCase				:    int=32		,						_lowerCamelCase				:    Tuple=1_000		,						_lowerCamelCase				:    Dict=16		,						_lowerCamelCase				:    Union[str, Any]=1.0		,						_lowerCamelCase				:    Optional[Any]=recopy_gpta		,						_lowerCamelCase				:    List[Any]=None		,						_lowerCamelCase				:    Optional[Any]=10		,						_lowerCamelCase				:    Union[str, Any]="gpt2_finetuned.pt"		,						)      ->					Union[str, Any]:






							'''simple docstring'''





							__UpperCamelCase			:     Optional[Any]         =      torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
							__UpperCamelCase			:     List[Any]         =      RandomSampler(_lowerCamelCase)
							__UpperCamelCase			:     Any         =      DataLoader(_lowerCamelCase		,						sampler=_lowerCamelCase)

							__UpperCamelCase			:     Tuple         =      max_steps // (len(_lowerCamelCase)) + 1
							__UpperCamelCase			:     List[Any]         =      0
							__UpperCamelCase			:     List[Any]         =      torch.zeros((1, context_len)		,						dtype=torch.long		,						device=_lowerCamelCase)
							__UpperCamelCase     ,		__UpperCamelCase     ,		__UpperCamelCase			:     List[Any]         =      recopy_model(_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase)

							model.train()
							if secondary_learner is not None:
														secondary_learner.to(_lowerCamelCase)
														secondary_learner.eval()
							__UpperCamelCase			:     Union[str, Any]         =      []
							__UpperCamelCase			:     Any         =      0

							__UpperCamelCase			:     List[Any]         =      []
							__UpperCamelCase			:     Any         =      []

							# Compute the performance of the transformer model at the beginning
							__UpperCamelCase			:     str         =      compute_perplexity(_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase)
							test_perps.append(_lowerCamelCase)
							print("Test perplexity, step"		,						_lowerCamelCase		,						":"		,						_lowerCamelCase)
							for epoch in range(int(_lowerCamelCase)):
														for step, example in enumerate(_lowerCamelCase):
																					torch.cuda.empty_cache()
																					__UpperCamelCase			:     Optional[Any]         =      random.randint(0		,						example.size(2) - context_len - 1)
																					__UpperCamelCase			:     Optional[Any]         =      example[0, 0, start : start + context_len]
																					lm_optimizer.zero_grad()
																					__UpperCamelCase			:     List[Any]         =      model(_lowerCamelCase		,						labels=_lowerCamelCase)
																					__UpperCamelCase			:     int         =      True

																					if secondary_learner is not None:
																												__UpperCamelCase			:     Optional[int]         =      secondary_learner.forward(
																												    torch.tensor(_lowerCamelCase		,						dtype=torch.long		,						device=_lowerCamelCase).unsqueeze(0))[0].item()
																												observed_qs.append(float(_lowerCamelCase))

																												# Here we implement the simple non-constant threshold for the predicted IG(X) value
																												# We will decay the selectivity of our secondary learner filter from
																												# 1 standard deviation above average to 1 below average after 10 batches.

																												if global_step == 10:
																																			__UpperCamelCase			:     List[str]         =      -1
																												if predicted_q < threshold:
																																			__UpperCamelCase			:     Optional[int]         =      False

            # If we passed the filter, add the context to the batch!
																					if do_backprop:
																												contexts.append(np.array(context.cpu()))
																												__UpperCamelCase			:     Optional[Any]         =      outputs[0]
																												lm_loss.backward()
																												examples += 1

																					del outputs

																					# Once the batch is filled with enough contexts, backprop on the batch.
																					if examples == batch_size:
																												torch.cuda.empty_cache()
																												__UpperCamelCase			:     str         =      0
																												# Do LM backprop
																												torch.nn.utils.clip_grad_norm_(model.parameters()		,						3.0)
																												lm_optimizer.step()
																												lm_scheduler.step()  # Update learning rate schedule
																												global_step += 1
																												# Compute the performance of the transformer model at this batch
																												if global_step % eval_interval == 0:
																																			__UpperCamelCase			:     List[Any]         =      compute_perplexity(_lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase)
																																			test_perps.append(_lowerCamelCase)

																																			print("Test perplexity, step"		,						_lowerCamelCase		,						":"		,						_lowerCamelCase)
            # Break out of the loop after 60 batches
																					if max_steps > 0 and global_step > 60:
																												break
														if max_steps > 0 and global_step > 60:
																					break

    # save finetuned transformer model
							torch.save(model.state_dict()		,						_lowerCamelCase)
							torch.cuda.empty_cache()
							# Do some cleaning up so we can reinitialize for the next run of this function
							del lm_optimizer
							del lm_scheduler
							return model




def 			_SCREAMING_SNAKE_CASE      (	)      ->					Optional[Any]:






							'''simple docstring'''





							__UpperCamelCase			:     Tuple         =      argparse.ArgumentParser(description="Fine-tune a transformer model with IGF on a language modeling task")

							# Required parameters
							parser.add_argument(
							    "--data_dir"		,						default=_lowerCamelCase		,						type=_lowerCamelCase		,						required=_lowerCamelCase		,						help="The input data dir. Should contain data files for WikiText."		,						)
							parser.add_argument(
							    "--model_name_or_path"		,						default=_lowerCamelCase		,						type=_lowerCamelCase		,						required=_lowerCamelCase		,						help="Path to pretrained model or model identifier from huggingface.co/models"		,						)
							parser.add_argument(
							    "--data_file"		,						type=_lowerCamelCase		,						default=_lowerCamelCase		,						help=(
							        "A jbl file containing tokenized data which can be split as objective dataset, "
							        "train_dataset and test_dataset."
							    )		,						)

							parser.add_argument(
							    "--igf_data_file"		,						type=_lowerCamelCase		,						default=_lowerCamelCase		,						help="A jbl file containing the context and information gain pairs to train secondary learner."		,						)

							parser.add_argument(
							    "--output_dir"		,						default=_lowerCamelCase		,						type=_lowerCamelCase		,						required=_lowerCamelCase		,						help="The output directory where the final fine-tuned model is stored."		,						)

							parser.add_argument(
							    "--tokenizer_name"		,						default=_lowerCamelCase		,						type=_lowerCamelCase		,						help="Pretrained tokenizer name or path if not the same as model_name"		,						)
							parser.add_argument("--seed"		,						type=_lowerCamelCase		,						default=_lowerCamelCase		,						help="A seed for reproducible training.")

							parser.add_argument(
							    "--context_len"		,						default=32		,						type=_lowerCamelCase		,						help=(
							        "The maximum total input sequence length after tokenization. Sequences longer "
							        "than this will be truncated, sequences shorter will be padded."
							    )		,						)

							parser.add_argument(
							    "--size_objective_set"		,						default=100		,						type=_lowerCamelCase		,						help="number of articles that are long enough to be used as our objective set"		,						)
							parser.add_argument(
							    "--eval_freq"		,						default=100		,						type=_lowerCamelCase		,						help="secondary model evaluation is triggered at eval_freq")

							parser.add_argument("--max_steps"		,						default=1_000		,						type=_lowerCamelCase		,						help="To calculate training epochs")

							parser.add_argument(
							    "--secondary_learner_batch_size"		,						default=128		,						type=_lowerCamelCase		,						help="batch size of training data for secondary learner"		,						)

							parser.add_argument(
							    "--batch_size"		,						default=16		,						type=_lowerCamelCase		,						help="batch size of training data of language model(gpt2) ")

							parser.add_argument(
							    "--eval_interval"		,						default=10		,						type=_lowerCamelCase		,						help=(
							        "decay the selectivity of our secondary learner filter from"
							        "1 standard deviation above average to 1 below average after 10 batches"
							    )		,						)

							parser.add_argument(
							    "--number"		,						default=100		,						type=_lowerCamelCase		,						help="The number of examples split to be used as objective_set/test_data")

							parser.add_argument(
							    "--min_len"		,						default=1_026		,						type=_lowerCamelCase		,						help="The minimum length of the article to be used as objective set")

							parser.add_argument(
							    "--secondary_learner_max_epochs"		,						default=15		,						type=_lowerCamelCase		,						help="number of epochs to train secondary learner")

							parser.add_argument("--trim"		,						default=_lowerCamelCase		,						type=_lowerCamelCase		,						help="truncate the example if it exceeds context length")

							parser.add_argument(
							    "--threshold"		,						default=1.0		,						type=_lowerCamelCase		,						help=(
							        "The threshold value used by secondary learner to filter the train_data and allow only"
							        " informative data as input to the model"
							    )		,						)

							parser.add_argument("--finetuned_model_name"		,						default="gpt2_finetuned.pt"		,						type=_lowerCamelCase		,						help="finetuned_model_name")

							parser.add_argument(
							    "--recopy_model"		,						default=_lowerCamelCase		,						type=_lowerCamelCase		,						help="Reset the model to the original pretrained GPT-2 weights after each iteration"		,						)

							# function calls
							# Collecting *n* pairs of context and information gain(X, IG(X)) for training the secondary learner
							generate_n_pairs(
							    context_len=32		,						max_steps=10		,						size_objective_set=100		,						min_len=1_026		,						trim=_lowerCamelCase		,						data_file="data/tokenized_stories_train_wikitext103.jbl"		,						igf_data_file="igf_context_pairs.jbl"		,						)

							# Load train data for secondary learner
							__UpperCamelCase			:     Any         =      joblib.load("data/IGF_values.jbl")

							# Train secondary learner
							__UpperCamelCase			:     Optional[Any]         =      training_secondary_learner(
							    _lowerCamelCase		,						secondary_learner_max_epochs=15		,						secondary_learner_batch_size=128		,						eval_freq=100		,						igf_model_path="igf_model.pt"		,						)

							# load pretrained gpt2 model
							__UpperCamelCase			:     int         =      GPTaLMHeadModel.from_pretrained("gpt2")
							set_seed(42)

							# Generate train and test data to train and evaluate gpt2 model
							__UpperCamelCase     ,		__UpperCamelCase			:     Union[str, Any]         =      generate_datasets(
							    context_len=32		,						file="data/tokenized_stories_train_wikitext103.jbl"		,						number=100		,						min_len=1_026		,						trim=_lowerCamelCase)

							# fine-tuning of the gpt2 model using igf (Information Gain Filtration)
							finetune(
							    _lowerCamelCase		,						_lowerCamelCase		,						_lowerCamelCase		,						context_len=32		,						max_steps=1_000		,						batch_size=16		,						threshold=1.0		,						recopy_model=_lowerCamelCase		,						secondary_learner=_lowerCamelCase		,						eval_interval=10		,						finetuned_model_name="gpt2_finetuned.pt"		,						)


if __name__ == "__main__":
		main()