






import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


A__      :     Dict       =					{
    'tiny.en': 'https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt',
    'tiny': 'https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt',
    'base.en': 'https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt',
    'base': 'https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt',
    'small.en': 'https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt',
    'small': 'https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt',
    'medium.en': 'https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt',
    'medium': 'https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt',
    'large': 'https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt',
    'large-v2': 'https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt',
}






def     a     (		lowerCamelCase_							):






				'''simple docstring'''
				lowercase__  					=     ['''layers''', '''blocks''']
				for k in ignore_keys:
								state_dict.pop(lowerCamelCase_						,  lowerCamelCase_							)


A__      :     List[str]       =					{
    'blocks': 'layers',
    'mlp.0': 'fc1',
    'mlp.2': 'fc2',
    'mlp_ln': 'final_layer_norm',
    '.attn.query': '.self_attn.q_proj',
    '.attn.key': '.self_attn.k_proj',
    '.attn.value': '.self_attn.v_proj',
    '.attn_ln': '.self_attn_layer_norm',
    '.attn.out': '.self_attn.out_proj',
    '.cross_attn.query': '.encoder_attn.q_proj',
    '.cross_attn.key': '.encoder_attn.k_proj',
    '.cross_attn.value': '.encoder_attn.v_proj',
    '.cross_attn_ln': '.encoder_attn_layer_norm',
    '.cross_attn.out': '.encoder_attn.out_proj',
    'decoder.ln.': 'decoder.layer_norm.',
    'encoder.ln.': 'encoder.layer_norm.',
    'token_embedding': 'embed_tokens',
    'encoder.positional_embedding': 'encoder.embed_positions.weight',
    'decoder.positional_embedding': 'decoder.embed_positions.weight',
    'ln_post': 'layer_norm',
}






def     a     (		lowerCamelCase_							):






				'''simple docstring'''
				lowercase__  					=     list(s_dict.keys()							)
				for key in keys:
								lowercase__  					=     key
								for k, v in WHISPER_MAPPING.items():
												if k in key:
																lowercase__  					=     new_key.replace(lowerCamelCase_						,  lowerCamelCase_							)

								print(F"""{key} -> {new_key}"""							)

								lowercase__  					=     s_dict.pop(lowerCamelCase_							)
				return s_dict






def     a     (		lowerCamelCase_							):






				'''simple docstring'''
				lowercase__					,		lowercase__  					=     emb.weight.shape
				lowercase__  					=     nn.Linear(lowerCamelCase_						,  lowerCamelCase_						,  bias=lowerCamelCase_							)
				lowercase__  					=     emb.weight.data
				return lin_layer






def     a     (		lowerCamelCase_						,  lowerCamelCase_							):






				'''simple docstring'''
				os.makedirs(lowerCamelCase_						,  exist_ok=lowerCamelCase_							)
				lowercase__  					=     os.path.basename(lowerCamelCase_							)

				lowercase__  					=     url.split('''/'''							)[-2]
				lowercase__  					=     os.path.join(lowerCamelCase_						,  lowerCamelCase_							)

				if os.path.exists(lowerCamelCase_							) and not os.path.isfile(lowerCamelCase_							):
								raise RuntimeError(F"""{download_target} exists and is not a regular file"""							)

				if os.path.isfile(lowerCamelCase_							):
								lowercase__  					=     open(lowerCamelCase_						,  '''rb'''							).read()
								if hashlib.shaaaa(lowerCamelCase_							).hexdigest() == expected_shaaaa:
												return model_bytes
								else:
												warnings.warn(F"""{download_target} exists, but the SHA256 checksum does not match; re-downloading the file"""							)

				with urllib.request.urlopen(lowerCamelCase_							) as source, open(lowerCamelCase_						,  '''wb'''							) as output:
								with tqdm(
								    total=int(source.info().get('''Content-Length'''							)							)						,  ncols=80						,  unit='''iB'''						,  unit_scale=lowerCamelCase_						,  unit_divisor=1024							) as loop:
												while True:
																lowercase__  					=     source.read(8192							)
																if not buffer:
																				break

																output.write(lowerCamelCase_							)
																loop.update(len(lowerCamelCase_							)							)

				lowercase__  					=     open(lowerCamelCase_						,  '''rb'''							).read()
				if hashlib.shaaaa(lowerCamelCase_							).hexdigest() != expected_shaaaa:
								raise RuntimeError(
								    '''Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model.'''							)

				return model_bytes






def     a     (		lowerCamelCase_						,  lowerCamelCase_							):






				'''simple docstring'''
				if ".pt" not in checkpoint_path:
								lowercase__  					=     _download(_MODELS[checkpoint_path]							)
				else:
								lowercase__  					=     torch.load(lowerCamelCase_						,  map_location='''cpu'''							)
				lowercase__  					=     original_checkpoint['''dims''']
				lowercase__  					=     original_checkpoint['''model_state_dict''']
				lowercase__  					=     state_dict['''decoder.token_embedding.weight''']
				remove_ignore_keys_(lowerCamelCase_							)
				rename_keys(lowerCamelCase_							)
				lowercase__  					=     True
				lowercase__  					=     state_dict['''decoder.layers.0.fc1.weight'''].shape[0]

				lowercase__  					=     WhisperConfig(
				    vocab_size=dimensions['''n_vocab''']						,  encoder_ffn_dim=lowerCamelCase_						,  decoder_ffn_dim=lowerCamelCase_						,  num_mel_bins=dimensions['''n_mels''']						,  d_model=dimensions['''n_audio_state''']						,  max_target_positions=dimensions['''n_text_ctx''']						,  encoder_layers=dimensions['''n_audio_layer''']						,  encoder_attention_heads=dimensions['''n_audio_head''']						,  decoder_layers=dimensions['''n_text_layer''']						,  decoder_attention_heads=dimensions['''n_text_state''']						,  max_source_positions=dimensions['''n_audio_ctx''']						,  )

				lowercase__  					=     WhisperForConditionalGeneration(lowerCamelCase_							)
				lowercase__					,		lowercase__  					=     model.model.load_state_dict(lowerCamelCase_						,  strict=lowerCamelCase_							)
				if len(lowerCamelCase_							) > 0 and not set(lowerCamelCase_							) <= {
				    "encoder.embed_positions.weights",
				    "decoder.embed_positions.weights",
				}:
								raise ValueError(
								    '''Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,'''
								    F""" but all the following weights are missing {missing}"""							)

				if tie_embeds:
								lowercase__  					=     make_linear_from_emb(model.model.decoder.embed_tokens							)
				else:
								lowercase__  					=     proj_out_weights

				model.save_pretrained(lowerCamelCase_							)


if __name__ == "__main__":
							A__      :     List[Any]       =					argparse.ArgumentParser()
							# # Required parameters
							parser.add_argument('--checkpoint_path', type=str, help='Patht to the downloaded checkpoints')
							parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
							A__      :     Optional[int]       =					parser.parse_args()

							convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)












import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_pegasus import PegasusTokenizer
else:
							A__      :     Optional[Any]       =					None


A__      :     str       =					logging.get_logger(__name__)


A__      :     Optional[int]       =					'▁'

A__      :     List[str]       =					{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

A__      :     Dict       =					{
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'},
    'tokenizer_file': {
        'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'
    },
}

A__      :     int       =					{
    'google/pegasus-xsum': 5_12,
}
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				lowercase__											=			VOCAB_FILES_NAMES
				lowercase__											=			PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=			PegasusTokenizer
				lowercase__											=			["""input_ids""", """attention_mask"""]




				def __init__(	self		:       Dict,    lowerCamelCase		:       str=None,    lowerCamelCase		:       List[str]=None,    lowerCamelCase		:       int="<pad>",    lowerCamelCase		:       str="</s>",    lowerCamelCase		:       Optional[Any]="<unk>",    lowerCamelCase		:       Any="<mask_2>",    lowerCamelCase		:       Optional[Any]="<mask_1>",    lowerCamelCase		:       int=None,    lowerCamelCase		:       Dict=103,    **lowerCamelCase		:       List[Any],    ):

								'''simple docstring'''
								lowercase__  					=     offset

								if additional_special_tokens is not None:
												if not isinstance(lowerCamelCase,    lowerCamelCase    ):
																raise TypeError(
																    F"""additional_special_tokens should be of type {type(lowerCamelCase    )}, but is"""
																    F""" {type(lowerCamelCase    )}"""    )

												lowercase__  					=     (
												    ([mask_token_sent] + additional_special_tokens)
												    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
												    else additional_special_tokens
												)
												# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
												additional_special_tokens_extended += [
												    F"""<unk_{i}>""" for i in range(len(lowerCamelCase    ),    self.offset - 1    )
												]

												if len(set(lowerCamelCase    )    ) != len(lowerCamelCase    ):
																raise ValueError(
																    '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
																    F""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""    )
												lowercase__  					=     additional_special_tokens_extended
								else:
												lowercase__  					=     [mask_token_sent] if mask_token_sent is not None else []
												additional_special_tokens += [F"""<unk_{i}>""" for i in range(2,    self.offset    )]

								super().__init__(
								    lowerCamelCase,    tokenizer_file=lowerCamelCase,    pad_token=lowerCamelCase,    eos_token=lowerCamelCase,    unk_token=lowerCamelCase,    mask_token=lowerCamelCase,    mask_token_sent=lowerCamelCase,    offset=lowerCamelCase,    additional_special_tokens=lowerCamelCase,    **lowerCamelCase,    )
								lowercase__  					=     vocab_file
								lowercase__  					=     False if not self.vocab_file else True




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       int    ):

								'''simple docstring'''
								lowercase__  					=     set(self.all_special_ids    )  # call it once instead of inside list comp
								all_special_ids.remove(self.unk_token_id    )  # <unk> is only sometimes special

								if all_special_ids != set(range(len(self.additional_special_tokens    ) + 3    )    ):
												raise ValueError(
												    '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
												    F""" {len(self.additional_special_tokens    )} additional_special_tokens, but got {all_special_ids}"""    )

								return [1 if x in all_special_ids else 0 for x in seq]




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       List,    lowerCamelCase		:       Optional[List] = None,    lowerCamelCase		:       bool = False    ):

								'''simple docstring'''
								if already_has_special_tokens:
												return self._special_token_mask(lowerCamelCase    )
								elif token_ids_a is None:
												return self._special_token_mask(lowerCamelCase    ) + [1]
								else:
												return self._special_token_mask(token_ids_a + token_ids_a    ) + [1]




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       Optional[int],    lowerCamelCase		:       Optional[Any]=None    ):

								'''simple docstring'''
								if token_ids_a is None:
												return token_ids_a + [self.eos_token_id]
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return token_ids_a + token_ids_a + [self.eos_token_id]




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       str,    lowerCamelCase		:       Optional[str] = None    ):

								'''simple docstring'''
								if not self.can_save_slow_tokenizer:
												raise ValueError(
												    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
												    '''tokenizer.'''    )

								if not os.path.isdir(lowerCamelCase    ):
												logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
												return
								lowercase__  					=     os.path.join(
								    lowerCamelCase,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

								if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase    ):
												copyfile(self.vocab_file,    lowerCamelCase    )

								return (out_vocab_file,)




