






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCamelCase__	=							logging.getLogger(__name__)


def 							__A()		->				Any:

		"""simple docstring"""






		_UpperCamelCase						    =	argparse.ArgumentParser(
		    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.""" )
		parser.add_argument(
		    """--dataset_name"""						,  type=_lowercase						,  default="""wikitext"""						,  help="""Name of the training. Explore datasets at: hf.co/datasets."""						,  )
		parser.add_argument(
		    """--dataset_config"""						,  type=_lowercase						,  default="""wikitext-103-raw-v1"""						,  help="""Configuration name of the dataset.""" )
		parser.add_argument(
		    """--tokenizer_name_or_path"""						,  type=_lowercase						,  default="""sayakpaul/unigram-tokenizer-wikitext"""						,  help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""						,  )
		parser.add_argument(
		    """--shard_size"""						,  type=_lowercase						,  default=1_0_0_0						,  help="""Number of entries to go in a single shard."""						,  )
		parser.add_argument("""--split"""						,  type=_lowercase						,  default="""train"""						,  choices=["""train""", """test""", """validation"""] )
		parser.add_argument(
		    """--limit"""						,  default=_lowercase						,  type=_lowercase						,  help="""Limit the number of shards (used for debugging)."""						,  )
		parser.add_argument(
		    """--max_length"""						,  type=_lowercase						,  default=5_1_2						,  help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
		    """ sequence length that is a multiple of 8."""						,  )
		parser.add_argument(
		    """--output_dir"""						,  default="""tf-tpu"""						,  type=_lowercase						,  help="""Output directory where the TFRecord shards will be saved. If the"""
		    """ path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"""
		    """ shards will be directly saved to a Google Cloud Storage bucket."""						,  )

		_UpperCamelCase						    =	parser.parse_args()
		return args


def 							__A(lowerCAmelCase )		->				List[str]:

		"""simple docstring"""






		def fn(lowerCAmelCase ):
				return tokenizer(examples["""text"""] )

		return fn


def 							__A(lowerCAmelCase )		->				List[Any]:

		"""simple docstring"""






		_UpperCamelCase						    =	[]
		for i in range(len(tokenized_data["""input_ids"""] ) ):
				_UpperCamelCase						    =	{
				    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i] ) ),
				    """attention_mask""": tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i] ) ),
				}
				_UpperCamelCase						    =	tf.train.Features(feature=_lowercase )
				_UpperCamelCase						    =	tf.train.Example(features=_lowercase )
				_UpperCamelCase						    =	example.SerializeToString()
				records.append(_lowercase )
		return records


def 							__A(lowerCAmelCase )		->				List[str]:

		"""simple docstring"""






		_UpperCamelCase						    =	datasets.load_dataset(args.dataset_name						,  args.dataset_config						,  split=args.split )

		if args.limit is not None:
				_UpperCamelCase						    =	min(len(_lowercase )						,  args.limit )
				_UpperCamelCase						    =	dataset.select(range(_lowercase ) )
				print(F'Limiting the dataset to {args.limit} entries.' )

		_UpperCamelCase						    =	AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir ):
						os.makedirs(args.output_dir )
				_UpperCamelCase						    =	os.path.join(args.output_dir						,  args.split )
				if not os.path.exists(_lowercase ):
						os.makedirs(_lowercase )
		else:
				_UpperCamelCase						    =	os.path.join(args.output_dir						,  args.split )

		# Tokenize the whole dataset at once.
		_UpperCamelCase						    =	tokenize_function(_lowercase )
		_UpperCamelCase						    =	dataset.map(_lowercase						,  batched=_lowercase						,  num_proc=4						,  remove_columns=["""text"""] )

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(lowerCAmelCase ):
				# Concatenate all texts.
				_UpperCamelCase						    =	{k: sum(examples[k]						,  [] ) for k in examples.keys()}
				_UpperCamelCase						    =	len(concatenated_examples[list(examples.keys() )[0]] )
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				_UpperCamelCase						    =	(total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				_UpperCamelCase						    =	{
				    k: [t[i : i + args.max_length] for i in range(0						,  _lowercase						,  args.max_length )]
				    for k, t in concatenated_examples.items()
				}
				return result

		_UpperCamelCase						    =	dataset_tokenized.map(_lowercase						,  batched=_lowercase						,  batch_size=1_0_0_0						,  num_proc=4 )

		_UpperCamelCase						    =	0
		_UpperCamelCase						    =	0
		for shard in range(0						,  len(_lowercase )						,  args.shard_size ):
				_UpperCamelCase						    =	grouped_dataset[shard : shard + args.shard_size]
				_UpperCamelCase						    =	len(dataset_snapshot["""input_ids"""] )
				_UpperCamelCase						    =	os.path.join(_lowercase						,  F'dataset-{shard_count}-{records_containing}.tfrecord' )
				_UpperCamelCase						    =	get_serialized_examples(_lowercase )

				with tf.io.TFRecordWriter(_lowercase ) as out_file:
						for i in range(len(_lowercase ) ):
								_UpperCamelCase						    =	serialized_examples[i]
								out_file.write(_lowercase )
						print("""Wrote file {} containing {} records""".format(_lowercase						,  _lowercase ) )

				shard_count += 1
				total_records += records_containing

		with open(F'split-{args.split}-records-count.txt'						,  """w""" ) as f:
				print(F'Total {args.split} records: {total_records}'						,  file=_lowercase )


if __name__ == "__main__":
		lowerCamelCase__	=							parse_args()
		main(args)





import argparse
from argparse import Namespace

import torch
from torch import nn

from transformers import XGLMConfig, XGLMForCausalLM


def 							__A(lowerCAmelCase )		->				Any:

		"""simple docstring"""






		_UpperCamelCase						    =	[
		    """decoder.version""",
		    """decoder.output_projection.weight""",
		    """_float_tensor""",
		    """decoder.embed_positions._float_tensor""",
		]
		for k in ignore_keys:
				state_dict.pop(lowerCAmelCase						,  lowerCAmelCase )


def 							__A(lowerCAmelCase )		->				Optional[int]:

		"""simple docstring"""






		_UpperCamelCase    ,	_UpperCamelCase						    =	emb.weight.shape
		_UpperCamelCase						    =	nn.Linear(lowerCAmelCase						,  lowerCAmelCase						,  bias=lowerCAmelCase )
		_UpperCamelCase						    =	emb.weight.data
		return lin_layer


def 							__A(lowerCAmelCase )		->				Tuple:

		"""simple docstring"""






		_UpperCamelCase						    =	torch.load(lowerCAmelCase						,  map_location="""cpu""" )
		_UpperCamelCase						    =	Namespace(**checkpoint["""cfg"""]["""model"""] )
		_UpperCamelCase						    =	checkpoint["""model"""]
		remove_ignore_keys_(lowerCAmelCase )
		_UpperCamelCase						    =	state_dict["""decoder.embed_tokens.weight"""].shape[0]

		_UpperCamelCase						    =	{key.replace("""decoder"""						,  """model""" ): val for key, val in state_dict.items()}

		_UpperCamelCase						    =	XGLMConfig(
		    vocab_size=lowerCAmelCase						,  max_position_embeddings=args.max_target_positions						,  num_layers=args.decoder_layers						,  attention_heads=args.decoder_attention_heads						,  ffn_dim=args.decoder_ffn_embed_dim						,  d_model=args.decoder_embed_dim						,  layerdrop=args.decoder_layerdrop						,  dropout=args.dropout						,  attention_dropout=args.attention_dropout						,  activation_dropout=args.activation_dropout						,  activation_function="""gelu"""						,  scale_embedding=not args.no_scale_embedding						,  tie_word_embeddings=args.share_decoder_input_output_embed						,  )

		_UpperCamelCase						    =	XGLMForCausalLM(lowerCAmelCase )
		_UpperCamelCase						    =	model.load_state_dict(lowerCAmelCase						,  strict=lowerCAmelCase )
		print(lowerCAmelCase )
		_UpperCamelCase						    =	make_linear_from_emb(model.model.embed_tokens )

		return model


if __name__ == "__main__":
		lowerCamelCase__	=							argparse.ArgumentParser()
		# Required parameters
		parser.add_argument("fairseq_path", type=str, help="path to a model.pt on local filesystem.")
		parser.add_argument("pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
		lowerCamelCase__	=							parser.parse_args()
		lowerCamelCase__	=							convert_fairseq_xglm_checkpoint_from_disk(args.fairseq_path)
		model.save_pretrained(args.pytorch_dump_folder_path)



