"""simple docstring"""


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


a				=logging.getLogger(__name__)





def      lowerCamelCase_					(   )				->   Union[str, Any]:




						'''simple docstring'''



						lowerCamelCase__   =argparse.ArgumentParser(
						    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."							)
						parser.add_argument(
						    "--dataset_name"  ,       type=__lowerCAmelCase  ,       default="wikitext"  ,       help="Name of the training. Explore datasets at: hf.co/datasets."  ,       )
						parser.add_argument(
						    "--dataset_config"  ,       type=__lowerCAmelCase  ,       default="wikitext-103-raw-v1"  ,       help="Configuration name of the dataset."							)
						parser.add_argument(
						    "--tokenizer_name_or_path"  ,       type=__lowerCAmelCase  ,       default="sayakpaul/unigram-tokenizer-wikitext"  ,       help="Tokenizer identifier. Can be a local filepath or a Hub identifier."  ,       )
						parser.add_argument(
						    "--shard_size"  ,       type=__lowerCAmelCase  ,       default=1000  ,       help="Number of entries to go in a single shard."  ,       )
						parser.add_argument("--split"  ,       type=__lowerCAmelCase  ,       default="train"  ,       choices=["train", "test", "validation"]							)
						parser.add_argument(
						    "--limit"  ,       default=__lowerCAmelCase  ,       type=__lowerCAmelCase  ,       help="Limit the number of shards (used for debugging)."  ,       )
						parser.add_argument(
						    "--max_length"  ,       type=__lowerCAmelCase  ,       default=512  ,       help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
						    " sequence length that is a multiple of 8."  ,       )
						parser.add_argument(
						    "--output_dir"  ,       default="tf-tpu"  ,       type=__lowerCAmelCase  ,       help="Output directory where the TFRecord shards will be saved. If the"
						    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
						    " shards will be directly saved to a Google Cloud Storage bucket."  ,       )

						lowerCamelCase__   =parser.parse_args()
						return args





def      lowerCamelCase_					(   __lowerCAmelCase							)				->   List[str]:




						'''simple docstring'''



						def fn(__lowerCAmelCase							):
												return tokenizer(examples["text"]							)

						return fn





def      lowerCamelCase_					(   __lowerCAmelCase							)				->   Optional[int]:




						'''simple docstring'''



						lowerCamelCase__   =[]
						for i in range(len(tokenized_data["input_ids"]							)							):
												lowerCamelCase__   ={
												    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]							)							),
												    "attention_mask": tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]							)							),
												}
												lowerCamelCase__   =tf.train.Features(feature=__lowerCAmelCase							)
												lowerCamelCase__   =tf.train.Example(features=__lowerCAmelCase							)
												lowerCamelCase__   =example.SerializeToString()
												records.append(__lowerCAmelCase							)
						return records





def      lowerCamelCase_					(   __lowerCAmelCase							)				->   str:




						'''simple docstring'''



						lowerCamelCase__   =datasets.load_dataset(args.dataset_name  ,       args.dataset_config  ,       split=args.split							)

						if args.limit is not None:
												lowerCamelCase__   =min(len(__lowerCAmelCase							)  ,       args.limit							)
												lowerCamelCase__   =dataset.select(range(__lowerCAmelCase							)							)
												print(F'''Limiting the dataset to {args.limit} entries.'''							)

						lowerCamelCase__   =AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir							):
																		os.makedirs(args.output_dir							)
												lowerCamelCase__   =os.path.join(args.output_dir  ,       args.split							)
												if not os.path.exists(__lowerCAmelCase							):
																		os.makedirs(__lowerCAmelCase							)
						else:
												lowerCamelCase__   =os.path.join(args.output_dir  ,       args.split							)

						# Tokenize the whole dataset at once.
						lowerCamelCase__   =tokenize_function(__lowerCAmelCase							)
						lowerCamelCase__   =dataset.map(__lowerCAmelCase  ,       batched=__lowerCAmelCase  ,       num_proc=4  ,       remove_columns=["text"]							)

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(__lowerCAmelCase							):
												# Concatenate all texts.
												lowerCamelCase__   ={k: sum(examples[k]  ,       []							) for k in examples.keys()}
												lowerCamelCase__   =len(concatenated_examples[list(examples.keys()							)[0]]							)
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												lowerCamelCase__   =(total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												lowerCamelCase__   ={
												    k: [t[i : i + args.max_length] for i in range(0  ,       __lowerCAmelCase  ,       args.max_length							)]
												    for k, t in concatenated_examples.items()
												}
												return result

						lowerCamelCase__   =dataset_tokenized.map(__lowerCAmelCase  ,       batched=__lowerCAmelCase  ,       batch_size=1000  ,       num_proc=4							)

						lowerCamelCase__   =0
						lowerCamelCase__   =0
						for shard in range(0  ,       len(__lowerCAmelCase							)  ,       args.shard_size							):
												lowerCamelCase__   =grouped_dataset[shard : shard + args.shard_size]
												lowerCamelCase__   =len(dataset_snapshot["input_ids"]							)
												lowerCamelCase__   =os.path.join(__lowerCAmelCase  ,       F'''dataset-{shard_count}-{records_containing}.tfrecord'''							)
												lowerCamelCase__   =get_serialized_examples(__lowerCAmelCase							)

												with tf.io.TFRecordWriter(__lowerCAmelCase							) as out_file:
																		for i in range(len(__lowerCAmelCase							)							):
																								lowerCamelCase__   =serialized_examples[i]
																								out_file.write(__lowerCAmelCase							)
																		print("Wrote file {} containing {} records".format(__lowerCAmelCase  ,       __lowerCAmelCase							)							)

												shard_count += 1
												total_records += records_containing

						with open(F'''split-{args.split}-records-count.txt'''  ,       "w"							) as f:
												print(F'''Total {args.split} records: {total_records}'''  ,       file=__lowerCAmelCase							)


if __name__ == "__main__":
	a				=parse_args()
	main(args)



"""simple docstring"""


def      lowerCamelCase_					(   __lowerCAmelCase							)				->   int:




						'''simple docstring'''



						if not isinstance(__lowerCAmelCase  ,       __lowerCAmelCase							):
												raise TypeError("only integers accepted as input"							)
						else:
												lowerCamelCase__   =str(abs(__lowerCAmelCase							)							)
												lowerCamelCase__   =[list(__lowerCAmelCase							) for char in range(len(__lowerCAmelCase							)							)]
												for index in range(len(__lowerCAmelCase							)							):
																		num_transpositions[index].pop(__lowerCAmelCase							)
												return max(
												    int("".join(list(__lowerCAmelCase							)							)							) for transposition in num_transpositions							)


if __name__ == "__main__":
	__import__('doctest').testmod()


