


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_snake_case	:						Union[str, Any]         =							logging.getLogger(__name__)







def 							lowerCAmelCase_	(   ):
							__snake_case						:     int									=  argparse.ArgumentParser(
							    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."      )
							parser.add_argument(
							    "--dataset_name"				,      type=__lowerCamelCase				,      default="wikitext"				,      help="Name of the training. Explore datasets at: hf.co/datasets."				,      )
							parser.add_argument(
							    "--dataset_config"				,      type=__lowerCamelCase				,      default="wikitext-103-raw-v1"				,      help="Configuration name of the dataset."      )
							parser.add_argument(
							    "--tokenizer_name_or_path"				,      type=__lowerCamelCase				,      default="sayakpaul/unigram-tokenizer-wikitext"				,      help="Tokenizer identifier. Can be a local filepath or a Hub identifier."				,      )
							parser.add_argument(
							    "--shard_size"				,      type=__lowerCamelCase				,      default=1_0_0_0				,      help="Number of entries to go in a single shard."				,      )
							parser.add_argument("--split"				,      type=__lowerCamelCase				,      default="train"				,      choices=["train", "test", "validation"]      )
							parser.add_argument(
							    "--limit"				,      default=__lowerCamelCase				,      type=__lowerCamelCase				,      help="Limit the number of shards (used for debugging)."				,      )
							parser.add_argument(
							    "--max_length"				,      type=__lowerCamelCase				,      default=5_1_2				,      help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
							    " sequence length that is a multiple of 8."				,      )
							parser.add_argument(
							    "--output_dir"				,      default="tf-tpu"				,      type=__lowerCamelCase				,      help="Output directory where the TFRecord shards will be saved. If the"
							    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
							    " shards will be directly saved to a Google Cloud Storage bucket."				,      )

							__snake_case						:     List[str]									=  parser.parse_args()
							return args







def 							lowerCAmelCase_	(   __lowerCamelCase      ):
							def fn(__lowerCamelCase      ):
														return tokenizer(examples["text"]      )

							return fn







def 							lowerCAmelCase_	(   __lowerCamelCase      ):
							__snake_case						:     Tuple									=  []
							for i in range(len(tokenized_data["input_ids"]      )      ):
														__snake_case						:     Tuple									=  {
														    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]      )      ),
														    "attention_mask": tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]      )      ),
														}
														__snake_case						:     List[Any]									=  tf.train.Features(feature=__lowerCamelCase      )
														__snake_case						:     str									=  tf.train.Example(features=__lowerCamelCase      )
														__snake_case						:     List[str]									=  example.SerializeToString()
														records.append(__lowerCamelCase      )
							return records







def 							lowerCAmelCase_	(   __lowerCamelCase      ):
							__snake_case						:     Optional[int]									=  datasets.load_dataset(args.dataset_name				,      args.dataset_config				,      split=args.split      )

							if args.limit is not None:
														__snake_case						:     Optional[Any]									=  min(len(__lowerCamelCase      )				,      args.limit      )
														__snake_case						:     Dict									=  dataset.select(range(__lowerCamelCase      )      )
														print(F'Limiting the dataset to {args.limit} entries.'      )

							__snake_case						:     Dict									=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir      ):
																					os.makedirs(args.output_dir      )
														__snake_case						:     Dict									=  os.path.join(args.output_dir				,      args.split      )
														if not os.path.exists(__lowerCamelCase      ):
																					os.makedirs(__lowerCamelCase      )
							else:
														__snake_case						:     str									=  os.path.join(args.output_dir				,      args.split      )

							# Tokenize the whole dataset at once.
							__snake_case						:     Any									=  tokenize_function(__lowerCamelCase      )
							__snake_case						:     Optional[Any]									=  dataset.map(__lowerCamelCase				,      batched=__lowerCamelCase				,      num_proc=4				,      remove_columns=["text"]      )

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(__lowerCamelCase      ):
														# Concatenate all texts.
														__snake_case						:     List[str]									=  {k: sum(examples[k]				,      []      ) for k in examples.keys()}
														__snake_case						:     List[Any]									=  len(concatenated_examples[list(examples.keys()      )[0]]      )
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														__snake_case						:     Any									=  (total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														__snake_case						:     int									=  {
														    k: [t[i : i + args.max_length] for i in range(0				,      __lowerCamelCase				,      args.max_length      )]
														    for k, t in concatenated_examples.items()
														}
														return result

							__snake_case						:     Any									=  dataset_tokenized.map(__lowerCamelCase				,      batched=__lowerCamelCase				,      batch_size=1_0_0_0				,      num_proc=4      )

							__snake_case						:     Optional[Any]									=  0
							__snake_case						:     Optional[Any]									=  0
							for shard in range(0				,      len(__lowerCamelCase      )				,      args.shard_size      ):
														__snake_case						:     List[str]									=  grouped_dataset[shard : shard + args.shard_size]
														__snake_case						:     Any									=  len(dataset_snapshot["input_ids"]      )
														__snake_case						:     List[Any]									=  os.path.join(__lowerCamelCase				,      F'dataset-{shard_count}-{records_containing}.tfrecord'      )
														__snake_case						:     Optional[Any]									=  get_serialized_examples(__lowerCamelCase      )

														with tf.io.TFRecordWriter(__lowerCamelCase      ) as out_file:
																					for i in range(len(__lowerCamelCase      )      ):
																												__snake_case						:     Union[str, Any]									=  serialized_examples[i]
																												out_file.write(__lowerCamelCase      )
																					print("Wrote file {} containing {} records".format(__lowerCamelCase				,      __lowerCamelCase      )      )

														shard_count += 1
														total_records += records_containing

							with open(F'split-{args.split}-records-count.txt'				,      "w"      ) as f:
														print(F'Total {args.split} records: {total_records}'				,      file=__lowerCamelCase      )


if __name__ == "__main__":
			_snake_case	:						List[Any]         =							parse_args()
			main(args)









class 					UpperCamelCase_ :



	'''simple docstring'''




	def __init__(			self			,     a   )			->       Tuple:
								snake_case_           =				n
								snake_case_           =				[None] * self.n
								snake_case_           =				0  # index of the first element
								snake_case_           =				0
								snake_case_           =				0




	def __len__(			self   )			->       int:
								return self.size




	def 				_UpperCamelCase     (			self   )			->       bool:
								return self.size == 0




	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								return False if self.is_empty() else self.array[self.front]




	def 				_UpperCamelCase     (			self			,     a   )			->       Dict:
								if self.size >= self.n:
															raise Exception('QUEUE IS FULL'   )

								snake_case_           =				data
								snake_case_           =				(self.rear + 1) % self.n
								self.size += 1
								return self






	def 				_UpperCamelCase     (			self   )			->       List[Any]:
								if self.size == 0:
															raise Exception('UNDERFLOW'   )

								snake_case_           =				self.array[self.front]
								snake_case_           =				None
								snake_case_           =				(self.front + 1) % self.n
								self.size -= 1
								return temp





