





'''simple docstring'''






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase									=   logging.getLogger(__name__)






def   __A    (				):
		lowerCAmelCase  :     Any       						= argparse.ArgumentParser(
		    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."						)
		parser.add_argument(
		    "--dataset_name"   ,type=a_   ,default="wikitext"   ,help="Name of the training. Explore datasets at: hf.co/datasets."   ,)
		parser.add_argument(
		    "--dataset_config"   ,type=a_   ,default="wikitext-103-raw-v1"   ,help="Configuration name of the dataset."						)
		parser.add_argument(
		    "--tokenizer_name_or_path"   ,type=a_   ,default="sayakpaul/unigram-tokenizer-wikitext"   ,help="Tokenizer identifier. Can be a local filepath or a Hub identifier."   ,)
		parser.add_argument(
		    "--shard_size"   ,type=a_   ,default=1_0_0_0   ,help="Number of entries to go in a single shard."   ,)
		parser.add_argument("--split"   ,type=a_   ,default="train"   ,choices=["train", "test", "validation"]						)
		parser.add_argument(
		    "--limit"   ,default=a_   ,type=a_   ,help="Limit the number of shards (used for debugging)."   ,)
		parser.add_argument(
		    "--max_length"   ,type=a_   ,default=5_1_2   ,help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
		    " sequence length that is a multiple of 8."   ,)
		parser.add_argument(
		    "--output_dir"   ,default="tf-tpu"   ,type=a_   ,help="Output directory where the TFRecord shards will be saved. If the"
		    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
		    " shards will be directly saved to a Google Cloud Storage bucket."   ,)

		lowerCAmelCase  :     List[Any]       						= parser.parse_args()
		return args






def   __A    (				a_    :	List[Any]						):
		def fn(a_    :	str						):
				return tokenizer(examples["text"]						)

		return fn






def   __A    (				a_    :	int						):
		lowerCAmelCase  :     Union[str, Any]       						= []
		for i in range(len(tokenized_data["input_ids"]						)						):
				lowerCAmelCase  :     str       						= {
				    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]						)						),
				    "attention_mask": tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]						)						),
				}
				lowerCAmelCase  :     List[str]       						= tf.train.Features(feature=a_						)
				lowerCAmelCase  :     Tuple       						= tf.train.Example(features=a_						)
				lowerCAmelCase  :     List[Any]       						= example.SerializeToString()
				records.append(a_						)
		return records






def   __A    (				a_    :	Tuple						):
		lowerCAmelCase  :     Dict       						= datasets.load_dataset(args.dataset_name   ,args.dataset_config   ,split=args.split						)

		if args.limit is not None:
				lowerCAmelCase  :     Dict       						= min(len(a_						)   ,args.limit						)
				lowerCAmelCase  :     Tuple       						= dataset.select(range(a_						)						)
				print(f'''Limiting the dataset to {args.limit} entries.'''						)

		lowerCAmelCase  :     Optional[Any]       						= AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir						):
						os.makedirs(args.output_dir						)
				lowerCAmelCase  :     List[Any]       						= os.path.join(args.output_dir   ,args.split						)
				if not os.path.exists(a_						):
						os.makedirs(a_						)
		else:
				lowerCAmelCase  :     Tuple       						= os.path.join(args.output_dir   ,args.split						)

		# Tokenize the whole dataset at once.
		lowerCAmelCase  :     int       						= tokenize_function(a_						)
		lowerCAmelCase  :     Tuple       						= dataset.map(a_   ,batched=a_   ,num_proc=4   ,remove_columns=["text"]						)

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(a_    :	Any						):
				# Concatenate all texts.
				lowerCAmelCase  :     str       						= {k: sum(examples[k]   ,[]						) for k in examples.keys()}
				lowerCAmelCase  :     List[str]       						= len(concatenated_examples[list(examples.keys()						)[0]]						)
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				lowerCAmelCase  :     Tuple       						= (total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				lowerCAmelCase  :     int       						= {
				    k: [t[i : i + args.max_length] for i in range(0   ,a_   ,args.max_length						)]
				    for k, t in concatenated_examples.items()
				}
				return result

		lowerCAmelCase  :     Dict       						= dataset_tokenized.map(a_   ,batched=a_   ,batch_size=1_0_0_0   ,num_proc=4						)

		lowerCAmelCase  :     Tuple       						= 0
		lowerCAmelCase  :     Any       						= 0
		for shard in range(0   ,len(a_						)   ,args.shard_size						):
				lowerCAmelCase  :     Optional[int]       						= grouped_dataset[shard : shard + args.shard_size]
				lowerCAmelCase  :     Tuple       						= len(dataset_snapshot["input_ids"]						)
				lowerCAmelCase  :     Optional[int]       						= os.path.join(a_   ,f'''dataset-{shard_count}-{records_containing}.tfrecord'''						)
				lowerCAmelCase  :     Dict       						= get_serialized_examples(a_						)

				with tf.io.TFRecordWriter(a_						) as out_file:
						for i in range(len(a_						)						):
								lowerCAmelCase  :     Any       						= serialized_examples[i]
								out_file.write(a_						)
						print("Wrote file {} containing {} records".format(a_   ,a_						)						)

				shard_count += 1
				total_records += records_containing

		with open(f'''split-{args.split}-records-count.txt'''   ,"w"						) as f:
				print(f'''Total {args.split} records: {total_records}'''   ,file=a_						)


if __name__ == "__main__":
					lowerCAmelCase									=   parse_args()
					main(args)










'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase									=   logging.get_logger(__name__)

lowerCAmelCase									=   {
    """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json""",
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}
class 		lowerCamelCase	(   _A							):
	snake_case_   = "gpt_neox"



	def __init__( self		,   a_=50_432		,   a_=6_144		,   a_=44		,   a_=64		,   a_=24_576		,   a_="gelu"		,   a_=0.25		,   a_=10_000		,   a_=0.0		,   a_=0.0		,   a_=0.1		,   a_=2_048		,   a_=0.02		,   a_=1e-5		,   a_=True		,   a_=0		,   a_=2		,   a_=False		,   a_=True		,   a_=None		,   **a_		,   ):
			super().__init__(bos_token_id=a_		,   eos_token_id=a_		,   **a_						)
			lowerCAmelCase  :     Dict       						= vocab_size
			lowerCAmelCase  :     str       						= max_position_embeddings
			lowerCAmelCase  :     int       						= hidden_size
			lowerCAmelCase  :     Optional[Any]       						= num_hidden_layers
			lowerCAmelCase  :     int       						= num_attention_heads
			lowerCAmelCase  :     Any       						= intermediate_size
			lowerCAmelCase  :     Tuple       						= hidden_act
			lowerCAmelCase  :     Any       						= rotary_pct
			lowerCAmelCase  :     str       						= rotary_emb_base
			lowerCAmelCase  :     Tuple       						= attention_dropout
			lowerCAmelCase  :     Optional[int]       						= hidden_dropout
			lowerCAmelCase  :     int       						= classifier_dropout
			lowerCAmelCase  :     str       						= initializer_range
			lowerCAmelCase  :     str       						= layer_norm_eps
			lowerCAmelCase  :     List[Any]       						= use_cache
			lowerCAmelCase  :     Dict       						= tie_word_embeddings
			lowerCAmelCase  :     Any       						= use_parallel_residual
			lowerCAmelCase  :     Union[str, Any]       						= rope_scaling
			self._rope_scaling_validation()

			if self.hidden_size % self.num_attention_heads != 0:
					raise ValueError(
					    "The hidden size is not divisble by the number of attention heads! Make sure to update them!"						)





	def 	_lowerCamelCase     ( self						):
			if self.rope_scaling is None:
					return

			if not isinstance(self.rope_scaling		,   a_						) or len(self.rope_scaling						) != 2:
					raise ValueError(
					    "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
					    F'''got {self.rope_scaling}'''						)
			lowerCAmelCase  :     int       						= self.rope_scaling.get("type"		,   a_						)
			lowerCAmelCase  :     int       						= self.rope_scaling.get("factor"		,   a_						)
			if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
					raise ValueError(
					    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''						)
			if rope_scaling_factor is None or not isinstance(a_		,   a_						) or rope_scaling_factor <= 1.0:
					raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''						)



