




import os

import jsonlines
import numpy as np
from tqdm import tqdm


UpperCAmelCase_		:					Dict          =  2048
UpperCAmelCase_		:					int          =  4096
UpperCAmelCase_		:					Any          =  42
UpperCAmelCase_		:					Optional[int]          =  os.environ.pop("PROCESS_TRAIN", "false")
UpperCAmelCase_		:					str          =  {"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}




def 		lowerCAmelCase_ (						lowerCamelCase ):
				def choose_first(lowerCamelCase						,  lowerCamelCase=False ):
								assert isinstance(lowerCamelCase						,  lowerCamelCase )
								if len(lowerCamelCase ) == 1:
												__magic_name__						:						List[str]										=answer[0]
												return {k: [answer[k]] for k in answer} if is_long_answer else answer
								for a in answer:
												if is_long_answer:
																__magic_name__						:						Tuple										={k: [a[k]] for k in a}
												if len(a["""start_token"""] ) > 0:
																break
								return a

				__magic_name__						:						str										={"""id""": example["""id"""]}
				__magic_name__						:						List[Any]										=example["""annotations"""]
				__magic_name__						:						List[str]										=annotation["""yes_no_answer"""]
				if 0 in yes_no_answer or 1 in yes_no_answer:
								__magic_name__						:						Optional[int]										=["""yes"""] if 1 in yes_no_answer else ["""no"""]
								__magic_name__						:						List[str]										=[]
								__magic_name__						:						Dict										=[]
								__magic_name__						:						str										=["""<cls>"""]
				else:
								__magic_name__						:						Tuple										=["""short"""]
								__magic_name__						:						Optional[int]										=choose_first(annotation["""short_answers"""] )
								if len(out["""start_token"""] ) == 0:
												# answer will be long if short is not available
												__magic_name__						:						Tuple										=["""long"""]
												__magic_name__						:						Tuple										=choose_first(annotation["""long_answer"""]						,  is_long_answer=lowerCamelCase )
												__magic_name__						:						List[Any]										=[]
								answer.update(lowerCamelCase )

				# disregard some samples
				if len(answer["""start_token"""] ) > 1 or answer["start_token"] == answer["end_token"]:
								__magic_name__						:						Any										=True
				else:
								__magic_name__						:						List[str]										=False

				__magic_name__						:						int										=["""start_token""", """end_token""", """start_byte""", """end_byte""", """text"""]
				if not all(isinstance(answer[k]						,  lowerCamelCase ) for k in cols ):
								raise ValueError("""Issue in ID"""						,  example["""id"""] )

				return answer




def 		lowerCAmelCase_ (						lowerCamelCase						,  lowerCamelCase=False ):
				__magic_name__						:						Optional[int]										=_get_single_answer(lowerCamelCase )
				# bytes are of no use
				del answer["start_byte"]
				del answer["end_byte"]

				# handle yes_no answers explicitly
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								__magic_name__						:						Any										=example["""document"""]["""tokens"""]
								__magic_name__						:						str										=[]
								for i in range(len(doc["""token"""] ) ):
												if not doc["is_html"][i]:
																context.append(doc["""token"""][i] )
								return {
								    "context": " ".join(lowerCamelCase ),
								    "answer": {
								        "start_token": -100,  # ignore index in cross-entropy
								        "end_token": -100,  # ignore index in cross-entropy
								        "category": answer["category"],
								        "span": answer["category"],  # extra
								    },
								}

				# later, help in removing all no answers
				if answer["start_token"] == [-1]:
								return {
								    "context": "None",
								    "answer": {
								        "start_token": -1,
								        "end_token": -1,
								        "category": "null",
								        "span": "None",  # extra
								    },
								}

				# handling normal samples

				__magic_name__						:						Dict										=["""start_token""", """end_token"""]
				answer.update({k: answer[k][0] if len(answer[k] ) > 0 else answer[k] for k in cols} )  # e.g. [10] == 10

				__magic_name__						:						Tuple										=example["""document"""]["""tokens"""]
				__magic_name__						:						Optional[int]										=answer["""start_token"""]
				__magic_name__						:						List[Any]										=answer["""end_token"""]

				__magic_name__						:						Optional[Any]										=[]
				for i in range(len(doc["""token"""] ) ):
								if not doc["is_html"][i]:
												context.append(doc["""token"""][i] )
								else:
												if answer["start_token"] > i:
																start_token -= 1
												if answer["end_token"] > i:
																end_token -= 1
				__magic_name__						:						Optional[int]										=""" """.join(context[start_token:end_token] )

				# checking above code
				if assertion:
								__magic_name__						:						List[str]										=doc["""is_html"""][answer["""start_token"""] : answer["""end_token"""]]
								__magic_name__						:						str										=doc["""token"""][answer["""start_token"""] : answer["""end_token"""]]
								__magic_name__						:						Dict										=""" """.join([old[i] for i in range(len(lowerCamelCase ) ) if not is_html[i]] )
								if new != old:
												print("""ID:"""						,  example["""id"""] )
												print("""New:"""						,  lowerCamelCase						,  end="""\n""" )
												print("""Old:"""						,  lowerCamelCase						,  end="""\n\n""" )

				return {
				    "context": " ".join(lowerCamelCase ),
				    "answer": {
				        "start_token": start_token,
				        "end_token": end_token - 1,  # this makes it inclusive
				        "category": answer["category"],  # either long or short
				        "span": new,  # extra
				    },
				}




def 		lowerCAmelCase_ (						lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase=2048						,  lowerCamelCase=4096						,  lowerCamelCase=True ):
				# overlap will be of doc_stride - q_len

				__magic_name__						:						Any										=get_context_and_ans(lowerCamelCase						,  assertion=lowerCamelCase )
				__magic_name__						:						Union[str, Any]										=out["""answer"""]

				# later, removing these samples
				if answer["start_token"] == -1:
								return {
								    "example_id": example["id"],
								    "input_ids": [[-1]],
								    "labels": {
								        "start_token": [-1],
								        "end_token": [-1],
								        "category": ["null"],
								    },
								}

				__magic_name__						:						List[Any]										=tokenizer(example["""question"""]["""text"""]						,  out["""context"""] ).input_ids
				__magic_name__						:						Dict										=input_ids.index(tokenizer.sep_token_id ) + 1

				# return yes/no
				if answer["category"][0] in ["yes", "no"]:  # category is list with one element
								__magic_name__						:						List[str]										=[]
								__magic_name__						:						int										=[]
								__magic_name__						:						List[str]										=input_ids[:q_len]
								__magic_name__						:						Dict										=range(lowerCamelCase						,  len(lowerCamelCase )						,  max_length - doc_stride )
								for i in doc_start_indices:
												__magic_name__						:						List[Any]										=i + max_length - q_len
												__magic_name__						:						Tuple										=input_ids[i:end_index]
												inputs.append(q_indices + slice )
												category.append(answer["""category"""][0] )
												if slice[-1] == tokenizer.sep_token_id:
																break

								return {
								    "example_id": example["id"],
								    "input_ids": inputs,
								    "labels": {
								        "start_token": [-100] * len(lowerCamelCase ),
								        "end_token": [-100] * len(lowerCamelCase ),
								        "category": category,
								    },
								}

				__magic_name__						:						int										=out["""context"""].split()
				__magic_name__						:						Any										=splitted_context[answer["""end_token"""]]
				__magic_name__						:						str										=len(
				    tokenizer(
				        """ """.join(splitted_context[: answer["""start_token"""]] )						,  add_special_tokens=lowerCamelCase						,  ).input_ids )
				__magic_name__						:						Optional[int]										=len(
				    tokenizer(""" """.join(splitted_context[: answer["""end_token"""]] )						,  add_special_tokens=lowerCamelCase ).input_ids )

				answer["start_token"] += q_len
				answer["end_token"] += q_len

				# fixing end token
				__magic_name__						:						Union[str, Any]										=len(tokenizer(lowerCamelCase						,  add_special_tokens=lowerCamelCase ).input_ids )
				if num_sub_tokens > 1:
								answer["end_token"] += num_sub_tokens - 1

				__magic_name__						:						str										=input_ids[answer["""start_token"""] : answer["""end_token"""] + 1]  # right & left are inclusive
				__magic_name__						:						Dict										=answer["""start_token"""]
				__magic_name__						:						int										=answer["""end_token"""]

				if assertion:
								__magic_name__						:						Any										=tokenizer.decode(lowerCamelCase )
								if answer["span"] != new:
												print("""ISSUE IN TOKENIZATION""" )
												print("""OLD:"""						,  answer["""span"""] )
												print("""NEW:"""						,  lowerCamelCase						,  end="""\n\n""" )

				if len(lowerCamelCase ) <= max_length:
								return {
								    "example_id": example["id"],
								    "input_ids": [input_ids],
								    "labels": {
								        "start_token": [answer["start_token"]],
								        "end_token": [answer["end_token"]],
								        "category": answer["category"],
								    },
								}

				__magic_name__						:						Any										=input_ids[:q_len]
				__magic_name__						:						Union[str, Any]										=range(lowerCamelCase						,  len(lowerCamelCase )						,  max_length - doc_stride )

				__magic_name__						:						Any										=[]
				__magic_name__						:						List[str]										=[]
				__magic_name__						:						List[str]										=[]
				__magic_name__						:						str										=[]  # null, yes, no, long, short
				for i in doc_start_indices:
								__magic_name__						:						List[Any]										=i + max_length - q_len
								__magic_name__						:						Dict										=input_ids[i:end_index]
								inputs.append(q_indices + slice )
								assert len(inputs[-1] ) <= max_length, "Issue in truncating length"

								if start_token >= i and end_token <= end_index - 1:
												__magic_name__						:						List[Any]										=start_token - i + q_len
												__magic_name__						:						Optional[Any]										=end_token - i + q_len
												answers_category.append(answer["""category"""][0] )  # ["short"] -> "short"
								else:
												__magic_name__						:						Optional[Any]										=-100
												__magic_name__						:						Optional[Any]										=-100
												answers_category.append("""null""" )
								__magic_name__						:						Optional[int]										=inputs[-1][start_token : end_token + 1]

								answers_start_token.append(lowerCamelCase )
								answers_end_token.append(lowerCamelCase )
								if assertion:
												if new != old and new != [tokenizer.cls_token_id]:
																print("""ISSUE in strided for ID:"""						,  example["""id"""] )
																print("""New:"""						,  tokenizer.decode(lowerCamelCase ) )
																print("""Old:"""						,  tokenizer.decode(lowerCamelCase )						,  end="""\n\n""" )
								if slice[-1] == tokenizer.sep_token_id:
												break

				return {
				    "example_id": example["id"],
				    "input_ids": inputs,
				    "labels": {
				        "start_token": answers_start_token,
				        "end_token": answers_end_token,
				        "category": answers_category,
				    },
				}




def 		lowerCAmelCase_ (						lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase=2048						,  lowerCamelCase=4096						,  lowerCamelCase=False ):
				__magic_name__						:						List[Any]										=get_strided_contexts_and_ans(
				    lowerCamelCase						,  lowerCamelCase						,  doc_stride=lowerCamelCase						,  max_length=lowerCamelCase						,  assertion=lowerCamelCase						,  )

				return example




def 		lowerCAmelCase_ (						lowerCamelCase						,  lowerCamelCase ):
				with jsonlines.open(lowerCamelCase						,  """a""" ) as writer:
								for example in tqdm(lowerCamelCase						,  total=len(lowerCamelCase )						,  desc="""Saving samples ... """ ):
												__magic_name__						:						int										=example["""labels"""]
												for ids, start, end, cat in zip(
												    example["""input_ids"""]						,  labels["""start_token"""]						,  labels["""end_token"""]						,  labels["""category"""]						,  ):
																if start == -1 and end == -1:
																				continue  # leave waste samples with no answer
																if cat == "null" and np.random.rand() < 0.6:
																				continue  # removing 50 % samples
																writer.write(
																    {
																        """input_ids""": ids,
																        """start_token""": start,
																        """end_token""": end,
																        """category""": CATEGORY_MAPPING[cat],
																    } )


if __name__ == "__main__":
							from datasets import load_dataset

							from transformers import BigBirdTokenizer

							UpperCAmelCase_		:					Optional[int]          =  load_dataset("natural_questions")
							UpperCAmelCase_		:					Optional[int]          =  BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")

							UpperCAmelCase_		:					str          =  data["train" if PROCESS_TRAIN == "true" else "validation"]

							UpperCAmelCase_		:					Optional[int]          =  {
							    "tokenizer": tokenizer,
							    "doc_stride": DOC_STRIDE,
							    "max_length": MAX_LENGTH,
							    "assertion": False,
							}
							UpperCAmelCase_		:					int          =  data.map(prepare_inputs, fn_kwargs=fn_kwargs)
							UpperCAmelCase_		:					Optional[Any]          =  data.remove_columns(["annotations", "document", "id", "question"])
							print(data)

							np.random.seed(SEED)
							UpperCAmelCase_		:					int          =  "nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
							save_to_disk(data, file_name=cache_file_name)





"""simple docstring"""



import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


_UpperCamelCase								=		{
    "tiny.en": "https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt",
    "tiny": "https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt",
    "base.en": "https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt",
    "base": "https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt",
    "small.en": "https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt",
    "small": "https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt",
    "medium.en": "https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt",
    "medium": "https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt",
    "large": "https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt",
    "large-v2": "https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt",
}

def  _A(		lowerCAmelCase							):
							A__       :     Any								=			["""layers""", """blocks"""]
							for k in ignore_keys:
														state_dict.pop(lowerCAmelCase			,  lowerCAmelCase							)


_UpperCamelCase								=		{
    "blocks": "layers",
    "mlp.0": "fc1",
    "mlp.2": "fc2",
    "mlp_ln": "final_layer_norm",
    ".attn.query": ".self_attn.q_proj",
    ".attn.key": ".self_attn.k_proj",
    ".attn.value": ".self_attn.v_proj",
    ".attn_ln": ".self_attn_layer_norm",
    ".attn.out": ".self_attn.out_proj",
    ".cross_attn.query": ".encoder_attn.q_proj",
    ".cross_attn.key": ".encoder_attn.k_proj",
    ".cross_attn.value": ".encoder_attn.v_proj",
    ".cross_attn_ln": ".encoder_attn_layer_norm",
    ".cross_attn.out": ".encoder_attn.out_proj",
    "decoder.ln.": "decoder.layer_norm.",
    "encoder.ln.": "encoder.layer_norm.",
    "token_embedding": "embed_tokens",
    "encoder.positional_embedding": "encoder.embed_positions.weight",
    "decoder.positional_embedding": "decoder.embed_positions.weight",
    "ln_post": "layer_norm",
}

def  _A(		lowerCAmelCase							):
							A__       :     str								=			list(s_dict.keys()							)
							for key in keys:
														A__       :     Optional[Any]								=			key
														for k, v in WHISPER_MAPPING.items():
																					if k in key:
																												A__       :     Optional[Any]								=			new_key.replace(lowerCAmelCase			,  lowerCAmelCase							)

														print(F'''{key} -> {new_key}'''							)

														A__       :     Any								=			s_dict.pop(lowerCAmelCase							)
							return s_dict

def  _A(		lowerCAmelCase							):
							A__				,							A__       :     Union[str, Any]								=			emb.weight.shape
							A__       :     Optional[Any]								=			nn.Linear(lowerCAmelCase			,  lowerCAmelCase			,  bias=lowerCAmelCase							)
							A__       :     int								=			emb.weight.data
							return lin_layer

def  _A(		lowerCAmelCase			,  lowerCAmelCase							):
							os.makedirs(lowerCAmelCase			,  exist_ok=lowerCAmelCase							)
							A__       :     Union[str, Any]								=			os.path.basename(lowerCAmelCase							)

							A__       :     str								=			url.split("""/"""							)[-2]
							A__       :     Union[str, Any]								=			os.path.join(lowerCAmelCase			,  lowerCAmelCase							)

							if os.path.exists(lowerCAmelCase							) and not os.path.isfile(lowerCAmelCase							):
														raise RuntimeError(F'''{download_target} exists and is not a regular file'''							)

							if os.path.isfile(lowerCAmelCase							):
														A__       :     Tuple								=			open(lowerCAmelCase			,  """rb"""							).read()
														if hashlib.shaaaa(lowerCAmelCase							).hexdigest() == expected_shaaaa:
																					return model_bytes
														else:
																					warnings.warn(F'''{download_target} exists, but the SHA256 checksum does not match; re-downloading the file'''							)

							with urllib.request.urlopen(lowerCAmelCase							) as source, open(lowerCAmelCase			,  """wb"""							) as output:
														with tqdm(
														    total=int(source.info().get("""Content-Length"""							)							)			,  ncols=80			,  unit="""iB"""			,  unit_scale=lowerCAmelCase			,  unit_divisor=1024							) as loop:
																					while True:
																												A__       :     Dict								=			source.read(8192							)
																												if not buffer:
																																			break

																												output.write(lowerCAmelCase							)
																												loop.update(len(lowerCAmelCase							)							)

							A__       :     List[str]								=			open(lowerCAmelCase			,  """rb"""							).read()
							if hashlib.shaaaa(lowerCAmelCase							).hexdigest() != expected_shaaaa:
														raise RuntimeError(
														    """Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model."""							)

							return model_bytes

def  _A(		lowerCAmelCase			,  lowerCAmelCase							):
							if ".pt" not in checkpoint_path:
														A__       :     str								=			_download(_MODELS[checkpoint_path]							)
							else:
														A__       :     Optional[int]								=			torch.load(lowerCAmelCase			,  map_location="""cpu"""							)
							A__       :     List[str]								=			original_checkpoint["""dims"""]
							A__       :     List[Any]								=			original_checkpoint["""model_state_dict"""]
							A__       :     Dict								=			state_dict["""decoder.token_embedding.weight"""]
							remove_ignore_keys_(lowerCAmelCase							)
							rename_keys(lowerCAmelCase							)
							A__       :     Any								=			True
							A__       :     List[str]								=			state_dict["""decoder.layers.0.fc1.weight"""].shape[0]

							A__       :     str								=			WhisperConfig(
							    vocab_size=dimensions["""n_vocab"""]			,  encoder_ffn_dim=lowerCAmelCase			,  decoder_ffn_dim=lowerCAmelCase			,  num_mel_bins=dimensions["""n_mels"""]			,  d_model=dimensions["""n_audio_state"""]			,  max_target_positions=dimensions["""n_text_ctx"""]			,  encoder_layers=dimensions["""n_audio_layer"""]			,  encoder_attention_heads=dimensions["""n_audio_head"""]			,  decoder_layers=dimensions["""n_text_layer"""]			,  decoder_attention_heads=dimensions["""n_text_state"""]			,  max_source_positions=dimensions["""n_audio_ctx"""]			,  )

							A__       :     int								=			WhisperForConditionalGeneration(lowerCAmelCase							)
							A__				,							A__       :     List[Any]								=			model.model.load_state_dict(lowerCAmelCase			,  strict=lowerCAmelCase							)
							if len(lowerCAmelCase							) > 0 and not set(lowerCAmelCase							) <= {
							    "encoder.embed_positions.weights",
							    "decoder.embed_positions.weights",
							}:
														raise ValueError(
														    """Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,"""
														    F''' but all the following weights are missing {missing}'''							)

							if tie_embeds:
														A__       :     Tuple								=			make_linear_from_emb(model.model.decoder.embed_tokens							)
							else:
														A__       :     Any								=			proj_out_weights

							model.save_pretrained(lowerCAmelCase							)


if __name__ == "__main__":
				_UpperCamelCase								=		argparse.ArgumentParser()
				# # Required parameters
				parser.add_argument("--checkpoint_path", type=str, help="Patht to the downloaded checkpoints")
				parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
				_UpperCamelCase								=		parser.parse_args()

				convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)





