



"""simple docstring"""






import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_lowercase			:			Tuple										=						logging.getLogger(__name__)






def 					snake_case__       (		):
			"""simple docstring"""
			lowerCamelCase__		:		Any		     =argparse.ArgumentParser(
			    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''     )
			parser.add_argument(
			    '''--dataset_name'''							,		type=__lowerCamelCase							,		default='''wikitext'''							,		help='''Name of the training. Explore datasets at: hf.co/datasets.'''							,		)
			parser.add_argument(
			    '''--dataset_config'''							,		type=__lowerCamelCase							,		default='''wikitext-103-raw-v1'''							,		help='''Configuration name of the dataset.'''     )
			parser.add_argument(
			    '''--tokenizer_name_or_path'''							,		type=__lowerCamelCase							,		default='''sayakpaul/unigram-tokenizer-wikitext'''							,		help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''							,		)
			parser.add_argument(
			    '''--shard_size'''							,		type=__lowerCamelCase							,		default=1000							,		help='''Number of entries to go in a single shard.'''							,		)
			parser.add_argument('''--split'''							,		type=__lowerCamelCase							,		default='''train'''							,		choices=['''train''', '''test''', '''validation''']     )
			parser.add_argument(
			    '''--limit'''							,		default=__lowerCamelCase							,		type=__lowerCamelCase							,		help='''Limit the number of shards (used for debugging).'''							,		)
			parser.add_argument(
			    '''--max_length'''							,		type=__lowerCamelCase							,		default=512							,		help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
			    ''' sequence length that is a multiple of 8.'''							,		)
			parser.add_argument(
			    '''--output_dir'''							,		default='''tf-tpu'''							,		type=__lowerCamelCase							,		help='''Output directory where the TFRecord shards will be saved. If the'''
			    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
			    ''' shards will be directly saved to a Google Cloud Storage bucket.'''							,		)

			lowerCamelCase__		:		Optional[Any]		     =parser.parse_args()
			return args






def 					snake_case__       (		__lowerCamelCase    :   str     ):
			"""simple docstring"""
			def fn(__lowerCamelCase    :   Optional[Any]     ):
						return tokenizer(examples['''text''']     )

			return fn






def 					snake_case__       (		__lowerCamelCase    :   List[str]     ):
			"""simple docstring"""
			lowerCamelCase__		:		Union[str, Any]		     =[]
			for i in range(len(tokenized_data['''input_ids''']     )     ):
						lowerCamelCase__		:		int		     ={
						    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]     )     ),
						    '''attention_mask''': tf.train.Feature(
						        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]     )     ),
						}
						lowerCamelCase__		:		Dict		     =tf.train.Features(feature=__lowerCamelCase     )
						lowerCamelCase__		:		Optional[int]		     =tf.train.Example(features=__lowerCamelCase     )
						lowerCamelCase__		:		Union[str, Any]		     =example.SerializeToString()
						records.append(__lowerCamelCase     )
			return records






def 					snake_case__       (		__lowerCamelCase    :   Union[str, Any]     ):
			"""simple docstring"""
			lowerCamelCase__		:		Union[str, Any]		     =datasets.load_dataset(args.dataset_name							,		args.dataset_config							,		split=args.split     )

			if args.limit is not None:
						lowerCamelCase__		:		Tuple		     =min(len(__lowerCamelCase     )							,		args.limit     )
						lowerCamelCase__		:		List[str]		     =dataset.select(range(__lowerCamelCase     )     )
						print(f'''Limiting the dataset to {args.limit} entries.'''     )

			lowerCamelCase__		:		Dict		     =AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

			# Handle output directory creation.
			# For serializing into a Google Cloud Storage Bucket, one needs to first
			# create a bucket.
			if "gs" not in args.output_dir:
						if not os.path.exists(args.output_dir     ):
									os.makedirs(args.output_dir     )
						lowerCamelCase__		:		Dict		     =os.path.join(args.output_dir							,		args.split     )
						if not os.path.exists(__lowerCamelCase     ):
									os.makedirs(__lowerCamelCase     )
			else:
						lowerCamelCase__		:		Union[str, Any]		     =os.path.join(args.output_dir							,		args.split     )

			# Tokenize the whole dataset at once.
			lowerCamelCase__		:		Optional[int]		     =tokenize_function(__lowerCamelCase     )
			lowerCamelCase__		:		int		     =dataset.map(__lowerCamelCase							,		batched=__lowerCamelCase							,		num_proc=4							,		remove_columns=['''text''']     )

			# We need to concatenate all our texts together, and then split the result
			# into chunks of a fixed size, which we will call block_size. To do this, we
			# will use the map method again, with the option batched=True. When we use batched=True,
			# the function we pass to map() will be passed multiple inputs at once, allowing us
			# to group them into more or fewer examples than we had in the input.
			# This allows us to create our new fixed-length samples. The advantage of this
			# method is that we don't lose a whole lot of content from the dataset compared to the
			# case where we simply tokenize with a pre-defined max_length.

			def group_texts(__lowerCamelCase    :   List[str]     ):
						# Concatenate all texts.
						lowerCamelCase__		:		Any		     ={k: sum(examples[k]							,		[]     ) for k in examples.keys()}
						lowerCamelCase__		:		List[Any]		     =len(concatenated_examples[list(examples.keys()     )[0]]     )
						# We drop the small remainder, though you could add padding instead if the model supports it
						# In this, as in all things, we advise you to follow your heart 🫀
						lowerCamelCase__		:		Any		     =(total_length // args.max_length) * args.max_length
						# Split by chunks of max_len.
						lowerCamelCase__		:		Dict		     ={
						    k: [t[i : i + args.max_length] for i in range(0							,		__lowerCamelCase							,		args.max_length     )]
						    for k, t in concatenated_examples.items()
						}
						return result

			lowerCamelCase__		:		List[str]		     =dataset_tokenized.map(__lowerCamelCase							,		batched=__lowerCamelCase							,		batch_size=1000							,		num_proc=4     )

			lowerCamelCase__		:		int		     =0
			lowerCamelCase__		:		Optional[Any]		     =0
			for shard in range(0							,		len(__lowerCamelCase     )							,		args.shard_size     ):
						lowerCamelCase__		:		str		     =grouped_dataset[shard : shard + args.shard_size]
						lowerCamelCase__		:		Tuple		     =len(dataset_snapshot['''input_ids''']     )
						lowerCamelCase__		:		Optional[int]		     =os.path.join(__lowerCamelCase							,		f'''dataset-{shard_count}-{records_containing}.tfrecord'''     )
						lowerCamelCase__		:		str		     =get_serialized_examples(__lowerCamelCase     )

						with tf.io.TFRecordWriter(__lowerCamelCase     ) as out_file:
									for i in range(len(__lowerCamelCase     )     ):
												lowerCamelCase__		:		Optional[int]		     =serialized_examples[i]
												out_file.write(__lowerCamelCase     )
									print('''Wrote file {} containing {} records'''.format(__lowerCamelCase							,		__lowerCamelCase     )     )

						shard_count += 1
						total_records += records_containing

			with open(f'''split-{args.split}-records-count.txt'''							,		'''w'''     ) as f:
						print(f'''Total {args.split} records: {total_records}'''							,		file=__lowerCamelCase     )


if __name__ == "__main__":
		_lowercase			:			Optional[Any]										=						parse_args()
		main(args)









"""simple docstring"""






from ..utils import DummyObject, requires_backends

class 	__SCREAMING_SNAKE_CASE     (				metaclass=lowerCAmelCase_       ):


					'''simple docstring'''





					_a             =							['torch', 'torchsde']



					def __init__(	self							:  Union[str, Any],  *lowerCamelCase							:  str,  **lowerCamelCase							:  int					)-> Tuple:
								requires_backends(self,  ['''torch''', '''torchsde''']					)



					@classmethod
					def 		snake_case	(	cls							:  List[str],  *lowerCamelCase							:  Optional[Any],  **lowerCamelCase							:  Dict					)-> str:
								requires_backends(cls,  ['''torch''', '''torchsde''']					)




					@classmethod
					def 		snake_case	(	cls							:  Tuple,  *lowerCamelCase							:  Dict,  **lowerCamelCase							:  Tuple					)-> List[str]:
								requires_backends(cls,  ['''torch''', '''torchsde''']					)




