



'''simple docstring'''

import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode






def 		_snake_case		(					_SCREAMING_SNAKE_CASE     :						List[str]							,		**_SCREAMING_SNAKE_CASE     :						List[str]							)			->						Union[str, Any]:







	"""simple docstring"""
	return RunBeamCommand(
	    args.dataset							,		args.name							,		args.cache_dir							,		args.beam_pipeline_options							,		args.data_dir							,		args.all_configs							,		args.save_info or args.save_infos							,		args.ignore_verifications							,		args.force_redownload							,		**_SCREAMING_SNAKE_CASE							,		)





class __snake_case(      _lowerCAmelCase					):


						'''simple docstring'''



						@staticmethod
						def 				__snake_case    (					A_      )	->	str:
							lowerCAmelCase    =	parser.add_parser("""run_beam"""	,       help="""Run a Beam dataset processing pipeline"""      )
							run_beam_parser.add_argument("""dataset"""	,       type=A_	,       help="""Name of the dataset to download"""      )
							run_beam_parser.add_argument("""--name"""	,       type=A_	,       default=A_	,       help="""Dataset config name"""      )
							run_beam_parser.add_argument(
							    """--cache_dir"""	,       type=A_	,       default=A_	,       help="""Cache directory where the datasets are stored"""	,       )
							run_beam_parser.add_argument(
							    """--beam_pipeline_options"""	,       type=A_	,       default=""""""	,       help="""Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`"""	,       )
							run_beam_parser.add_argument(
							    """--data_dir"""	,       type=A_	,       default=A_	,       help="""Can be used to specify a manual directory to get the files from"""	,       )
							run_beam_parser.add_argument("""--all_configs"""	,       action="""store_true"""	,       help="""Test all dataset configurations"""      )
							run_beam_parser.add_argument("""--save_info"""	,       action="""store_true"""	,       help="""Save the dataset infos file"""      )
							run_beam_parser.add_argument(
							    """--ignore_verifications"""	,       action="""store_true"""	,       help="""Run the test without checksums and splits checks"""      )
							run_beam_parser.add_argument("""--force_redownload"""	,       action="""store_true"""	,       help="""Force dataset redownload"""      )
							# aliases
							run_beam_parser.add_argument("""--save_infos"""	,       action="""store_true"""	,       help="""alias for save_info"""      )
							run_beam_parser.set_defaults(func=A_      )


						def __init__(					self	,       A_	,       A_	,       A_	,       A_	,       A_	,       A_	,       A_	,       A_	,       A_	,       **A_	,       )	->	Any:
							lowerCAmelCase    =	dataset
							lowerCAmelCase    =	name
							lowerCAmelCase    =	cache_dir
							lowerCAmelCase    =	beam_pipeline_options
							lowerCAmelCase    =	data_dir
							lowerCAmelCase    =	all_configs
							lowerCAmelCase    =	save_infos
							lowerCAmelCase    =	ignore_verifications
							lowerCAmelCase    =	force_redownload
							lowerCAmelCase    =	config_kwargs




						def 				__snake_case    (					self      )	->	int:
							import apache_beam as beam

							if self._name is not None and self._all_configs:
								print("""Both parameters `name` and `all_configs` can't be used at once."""      )
								exit(1      )
							lowerCAmelCase,   lowerCAmelCase    =	self._dataset, self._name
							lowerCAmelCase    =	dataset_module_factory(A_      )
							lowerCAmelCase    =	import_main_class(dataset_module.module_path      )
							lowerCAmelCase    =	[]
							if self._beam_pipeline_options:
								lowerCAmelCase    =	beam.options.pipeline_options.PipelineOptions(
								    flags=[f'--{opt.strip()}' for opt in self._beam_pipeline_options.split(""","""      ) if opt]      )
							else:
								lowerCAmelCase    =	None
							if self._all_configs and len(builder_cls.BUILDER_CONFIGS      ) > 0:
								for builder_config in builder_cls.BUILDER_CONFIGS:
									builders.append(
									    builder_cls(
									        config_name=builder_config.name	,       data_dir=self._data_dir	,       hash=dataset_module.hash	,       beam_options=A_	,       cache_dir=self._cache_dir	,       base_path=dataset_module.builder_kwargs.get("""base_path"""      )	,       )      )
							else:
								builders.append(
								    builder_cls(
								        config_name=A_	,       data_dir=self._data_dir	,       beam_options=A_	,       cache_dir=self._cache_dir	,       base_path=dataset_module.builder_kwargs.get("""base_path"""      )	,       **self._config_kwargs	,       )      )

							for builder in builders:
								builder.download_and_prepare(
								    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
								    if not self._force_redownload
								    else DownloadMode.FORCE_REDOWNLOAD	,       download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH      )	,       verification_mode=VerificationMode.NO_CHECKS
								    if self._ignore_verifications
								    else VerificationMode.ALL_CHECKS	,       try_from_hf_gcs=A_	,       )
								if self._save_infos:
									builder._save_infos()

							print("""Apache beam run successful."""      )

							# If save_infos=True, the dataset infos file is created next to the loaded module file.
							# Let's move it to the original directory of the dataset script, to allow the user to
							# upload them on S3 at the same time afterwards.
							if self._save_infos:
								lowerCAmelCase    =	os.path.join(builder_cls.get_imported_module_dir()	,       config.DATASETDICT_INFOS_FILENAME      )

								lowerCAmelCase    =	Path(A_      ).name + """.py"""

								lowerCAmelCase    =	os.path.join(A_	,       A_      )
								if os.path.isfile(A_      ):
									lowerCAmelCase    =	os.path.dirname(A_      )
								elif os.path.isfile(A_      ):
									lowerCAmelCase    =	path
								else:  # in case of a remote dataset
									print(f'Dataset Infos file saved at {dataset_infos_path}'      )
									exit(1      )

								# Move datasetinfo back to the user
								lowerCAmelCase    =	os.path.join(A_	,       config.DATASETDICT_INFOS_FILENAME      )
								copyfile(A_	,       A_      )
								print(f'Dataset Infos file saved at {user_dataset_infos_path}'      )




'''simple docstring'''

import argparse

# utilities to import the model weights and config file
import os
import pickle as pkl

# PyTorch + new model classes
import torch
from torch import nn

from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM


# import the EncoderLayer class used to pretrain
# !! NOTE !! this requires the version of fairseq that is built when you install the Mega source
try:
				from fairseq.modules.mega_layer import MegaEncoderLayer
except ImportError:
				raise ImportError('You need to install the version of fairseq from the Mega repo!')

class __snake_case(      nn.Module					):


						'''simple docstring'''



						def __init__(					self	,       A_	,       A_	,       A_      )	->	Dict:
							super().__init__()
							lowerCAmelCase    =	mega_args
							lowerCAmelCase    =	nn.Embedding(A_	,       self.mega_args.encoder_embed_dim      )
							lowerCAmelCase    =	nn.ModuleList([MegaEncoderLayer(self.mega_args      ) for _ in range(A_      )]      )
							lowerCAmelCase    =	depth




						def 				__snake_case    (					self	,       A_	,       A_	,       A_=True	,       A_=0      )	->	List[Any]:

							# Mega expects embeddings to be (time, batch, embedding size), but
							# Hugging Face returns tokens as (batch, time)
							if batch_first:
								lowerCAmelCase    =	input_ids.T

							# to make things more confusing, Mega expects the attention mask to
							# be (batch, time), but with values of 0 (normal token) and 1 (ignore token)
							# which is the opposite of what HF returns
							if ignore_mask_value == 0:
								lowerCAmelCase    =	1 - attention_mask

							# get token embeddings from IDs
							lowerCAmelCase    =	self.embedding_layer(A_      )

							# pass through the Mega layers
							# input is (time, batch, encoder dim) and output is the same
							for encoder in self.encoders:
								lowerCAmelCase    =	encoder(A_	,       A_      )

							# return according to the shape specified
							if batch_first:
								# (T, B, H) --> (B, T, H)
								return torch.transpose(A_	,       0	,       1      )
							else:
								return embeds







class __snake_case(      nn.Module					):


						'''simple docstring'''



						def __init__(					self	,       A_	,       A_	,       A_      )	->	Optional[Any]:
							super().__init__()
							lowerCAmelCase    =	MegaLM(A_	,       A_	,       A_      )
							lowerCAmelCase    =	nn.Linear(mega_args.encoder_embed_dim	,       A_      )
							lowerCAmelCase    =	nn.Dropout(p=0.1      )




						def 				__snake_case    (					self	,       A_	,       A_	,       A_=True	,       A_=0      )	->	List[Any]:
							lowerCAmelCase    =	self.mega(A_	,       A_	,       A_	,       A_      )
							return self.mlm_head(self.dropout(A_      )      )







def 		_snake_case		(					_SCREAMING_SNAKE_CASE     :						Optional[int]							,		_SCREAMING_SNAKE_CASE     :						List[str]							,		_SCREAMING_SNAKE_CASE     :						Union[str, Any]							)			->						List[Any]:







	"""simple docstring"""
	with open(os.path.join(_SCREAMING_SNAKE_CASE							,		"""model_args.pkl"""							)							,		"""rb"""							) as f:
		lowerCAmelCase    =	pkl.load(_SCREAMING_SNAKE_CASE							)

	# load the original encoder
	lowerCAmelCase    =	OriginalMegaForMaskedLM(**_SCREAMING_SNAKE_CASE							).eval()

	# load its weights
	print(
	    """Original Mega encoder:"""							,		original_mlm.mega.load_state_dict(
	        torch.load(os.path.join(_SCREAMING_SNAKE_CASE							,		"""encoder_weights.pt"""							)							,		map_location="""cpu"""							)							)							,		)
	print(
	    """Original Mega MLM layer:"""							,		original_mlm.mlm_head.load_state_dict(
	        torch.load(os.path.join(_SCREAMING_SNAKE_CASE							,		"""mlm_head_weights.pt"""							)							,		map_location="""cpu"""							)							)							,		)

	# create a new config from the old one
	lowerCAmelCase    =	MegaConfig(
	    num_hidden_layers=mega_original_args["""depth"""]							,		vocab_size=mega_original_args["""vocab_size"""]							,		hidden_size=mega_original_args["""mega_args"""].encoder_embed_dim							,		shared_representation_size=mega_original_args["""mega_args"""].encoder_z_dim							,		intermediate_size=mega_original_args["""mega_args"""].encoder_hidden_dim							,		ema_projection_size=mega_original_args["""mega_args"""].encoder_n_dim							,		dropout_prob=mega_original_args["""mega_args"""].dropout							,		attention_probs_dropout_prob=mega_original_args["""mega_args"""].attention_dropout							,		hidden_dropout_prob=mega_original_args["""mega_args"""].hidden_dropout							,		activation=mega_original_args["""mega_args"""].activation_fn							,		attention_activation=mega_original_args["""mega_args"""].attention_activation_fn							,		bidirectional=mega_original_args["""mega_args"""].bidirectional							,		use_chunking=mega_original_args["""mega_args"""].encoder_chunk_size > 0							,		chunk_size=mega_original_args["""mega_args"""].encoder_chunk_size							,		truncation=mega_original_args["""mega_args"""].truncation_length							,		normalization_type=mega_original_args["""mega_args"""].normalization_type							,		normalize_before_mega=_SCREAMING_SNAKE_CASE							,		norm_affine=_SCREAMING_SNAKE_CASE							,		use_feature_dropout=mega_original_args["""mega_args"""].feature_dropout							,		relative_positional_bias=mega_original_args["""mega_args"""].rel_pos_bias							,		max_positions=mega_original_args["""mega_args"""].max_source_positions							,		nffn_hidden_size=mega_original_args["""mega_args"""].encoder_ffn_embed_dim							,		normalize_before_ffn=mega_original_args["""mega_args"""].normalize_before							,		nffn_activation_dropout_prob=0.0							,		add_token_type_embeddings=_SCREAMING_SNAKE_CASE							,		add_lm_hidden_dense_layer=_SCREAMING_SNAKE_CASE							,		)

	lowerCAmelCase    =	MegaForMaskedLM(_SCREAMING_SNAKE_CASE							).eval()

	# the originl checkpoint just uses nn.Embedding for the word embeddings
	# we use a wrapper module for embeddings to add support for positional embeddings
	lowerCAmelCase    =	original_mlm.mega.embedding_layer.weight

	# modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face
	# ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained,
	# also renaming previously confusing parameter names
	lowerCAmelCase    =	original_mlm.mega.encoders.state_dict()
	lowerCAmelCase    =	{}
	for module_name in original_state_dict.keys():
		lowerCAmelCase    =	None
		# have to handle gamma, beta, and alpha differently due to their use
		# in multiple modules within the original repository;
		# beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights
		# the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here
		if "beta" in module_name:
			# EMA sub-layers were always called "move" in the original repo
			if "move.beta" in module_name:
				lowerCAmelCase    =	module_name.replace("""move.beta"""							,		"""ema_gate.ema_expansion_matrix"""							)
			elif "mega_layer.beta" in module_name:
				lowerCAmelCase    =	module_name.replace("""beta"""							,		"""qk_bias"""							)
			else:
				lowerCAmelCase    =	module_name.replace("""beta"""							,		"""b_param"""							)
        # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights
		elif "gamma" in module_name:
			if "move.gamma" in module_name:
				lowerCAmelCase    =	module_name.replace("""move.gamma"""							,		"""ema_gate.kernel_projection_matrix"""							)
			elif "mega_layer.gamma" in module_name:
				lowerCAmelCase    =	module_name.replace("""gamma"""							,		"""qk_weight"""							)
			else:
				lowerCAmelCase    =	module_name.replace("""gamma"""							,		"""g_param"""							)
        # alpha is used in EMA and positional bias; renaming to improve readability
		elif "move.alpha" in module_name:
			lowerCAmelCase    =	module_name.replace("""move.alpha"""							,		"""ema_gate.decay_factor"""							)
		# delta is only used in EMA; renaming to improve readability
		elif "move.delta" in module_name:
			lowerCAmelCase    =	module_name.replace("""move.delta"""							,		"""ema_gate.damping_factor"""							)
		# omega is only used in EMA; renaming to improve readability
		elif "omega" in module_name:
			lowerCAmelCase    =	module_name.replace("""move.omega"""							,		"""ema_gate.residual_weight"""							)

		if new_module_name:
			lowerCAmelCase    =	new_module_name

	if len(_SCREAMING_SNAKE_CASE							) != 0:
		print(f'Renaming these keys: {updated_keys.keys()}'							)
	else:
		print("""No need to rename state dict entries"""							)
	for old, new in updated_keys.items():
		lowerCAmelCase    =	original_state_dict.pop(_SCREAMING_SNAKE_CASE							)

	# now attempt to load the state dictionary with updated names
	# note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style
	print("""HF Mega encoder:"""							,		hf_mlm.mega.layers.load_state_dict(_SCREAMING_SNAKE_CASE							)							)

	# load the MLM head weights directly
	print(
	    """HF Mega MLM layer:"""							,		hf_mlm.mlm_head.load_state_dict(
	        torch.load(os.path.join(_SCREAMING_SNAKE_CASE							,		"""mlm_head_weights.pt"""							)							,		map_location="""cpu"""							)							)							,		)

	# test on a randomly generated input sequence
	lowerCAmelCase    =	torch.randint(0							,		hf_config.vocab_size							,		size=(4, 256)							)
	lowerCAmelCase    =	torch.ones_like(_SCREAMING_SNAKE_CASE							)
	# mask a few tokens to make sure masking is applied appropriately :)
	lowerCAmelCase    =	0

	# run forward passes
	lowerCAmelCase    =	original_mlm(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		batch_first=_SCREAMING_SNAKE_CASE							,		ignore_mask_value=0							)
	lowerCAmelCase    =	hf_mlm(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)[0]

	# print shapes and diff
	print(f'original output {original_output.shape}'							)
	print(f'hf output {hf_output.shape}'							)
	print(f'max diff: {(original_output - hf_output).max()}'							)  # 0.0
	lowerCAmelCase    =	torch.allclose(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		atol=1E-3							)

	if success:
		print("""Yay!"""							)
		hf_mlm.save_pretrained(_SCREAMING_SNAKE_CASE							)
	else:
		raise RuntimeError(f'Something\'s broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}'							)

	if includes_tokenizer:
		print("""Transferring tokenizer"""							)
		lowerCAmelCase    =	AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
		tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
				UpperCAmelCase			=    argparse.ArgumentParser()

				parser.add_argument(
				    '--pretrained_checkpoint_path',
				    default=None,
				    type=str,
				    required=True,
				    help='Point to the directory containing your model weights using the official Mega repo',
				)

				parser.add_argument(
				    '--output_path', default=None, type=str, required=True, help='Location to save the Hugging Face version'
				)

				parser.add_argument(
				    '--includes_tokenizer',
				    action='store_true',
				    help='Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo',
				)

				UpperCAmelCase			=    parser.parse_args()

				convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer)