
'''simple docstring'''

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_UpperCAmelCase :   Optional[int]			    =	logging.getLogger(__name__)







def     snake_case__						(   )			->   int:
		_UpperCamelCase							:			Dict     =   argparse.ArgumentParser(
		    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''						)
		parser.add_argument(
		    '''--dataset_name'''      ,type=UpperCamelCase_      ,default='''wikitext'''      ,help='''Name of the training. Explore datasets at: hf.co/datasets.'''      ,)
		parser.add_argument(
		    '''--dataset_config'''      ,type=UpperCamelCase_      ,default='''wikitext-103-raw-v1'''      ,help='''Configuration name of the dataset.'''						)
		parser.add_argument(
		    '''--tokenizer_name_or_path'''      ,type=UpperCamelCase_      ,default='''sayakpaul/unigram-tokenizer-wikitext'''      ,help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''      ,)
		parser.add_argument(
		    '''--shard_size'''      ,type=UpperCamelCase_      ,default=10_00      ,help='''Number of entries to go in a single shard.'''      ,)
		parser.add_argument('''--split'''      ,type=UpperCamelCase_      ,default='''train'''      ,choices=['''train''', '''test''', '''validation''']						)
		parser.add_argument(
		    '''--limit'''      ,default=UpperCamelCase_      ,type=UpperCamelCase_      ,help='''Limit the number of shards (used for debugging).'''      ,)
		parser.add_argument(
		    '''--max_length'''      ,type=UpperCamelCase_      ,default=5_12      ,help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
		    ''' sequence length that is a multiple of 8.'''      ,)
		parser.add_argument(
		    '''--output_dir'''      ,default='''tf-tpu'''      ,type=UpperCamelCase_      ,help='''Output directory where the TFRecord shards will be saved. If the'''
		    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
		    ''' shards will be directly saved to a Google Cloud Storage bucket.'''      ,)

		_UpperCamelCase							:			Union[str, Any]     =   parser.parse_args()
		return args







def     snake_case__						(   UpperCamelCase						)			->   Tuple:
		def fn(UpperCamelCase						):
				return tokenizer(examples['''text''']						)

		return fn







def     snake_case__						(   UpperCamelCase						)			->   int:
		_UpperCamelCase							:			List[str]     =   []
		for i in range(len(tokenized_data['''input_ids''']						)						):
				_UpperCamelCase							:			Optional[Any]     =   {
				    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]						)						),
				    '''attention_mask''': tf.train.Feature(
				        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]						)						),
				}
				_UpperCamelCase							:			Tuple     =   tf.train.Features(feature=UpperCamelCase_						)
				_UpperCamelCase							:			str     =   tf.train.Example(features=UpperCamelCase_						)
				_UpperCamelCase							:			List[str]     =   example.SerializeToString()
				records.append(UpperCamelCase_						)
		return records







def     snake_case__						(   UpperCamelCase						)			->   Dict:
		_UpperCamelCase							:			List[str]     =   datasets.load_dataset(args.dataset_name      ,args.dataset_config      ,split=args.split						)

		if args.limit is not None:
				_UpperCamelCase							:			Optional[Any]     =   min(len(UpperCamelCase_						)      ,args.limit						)
				_UpperCamelCase							:			List[Any]     =   dataset.select(range(UpperCamelCase_						)						)
				print(f'''Limiting the dataset to {args.limit} entries.'''						)

		_UpperCamelCase							:			str     =   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path						)

		# Handle output directory creation.
		# For serializing into a Google Cloud Storage Bucket, one needs to first
		# create a bucket.
		if "gs" not in args.output_dir:
				if not os.path.exists(args.output_dir						):
						os.makedirs(args.output_dir						)
				_UpperCamelCase							:			str     =   os.path.join(args.output_dir      ,args.split						)
				if not os.path.exists(UpperCamelCase_						):
						os.makedirs(UpperCamelCase_						)
		else:
				_UpperCamelCase							:			str     =   os.path.join(args.output_dir      ,args.split						)

		# Tokenize the whole dataset at once.
		_UpperCamelCase							:			Tuple     =   tokenize_function(UpperCamelCase_						)
		_UpperCamelCase							:			Optional[int]     =   dataset.map(UpperCamelCase_      ,batched=UpperCamelCase_      ,num_proc=4      ,remove_columns=['''text''']						)

		# We need to concatenate all our texts together, and then split the result
		# into chunks of a fixed size, which we will call block_size. To do this, we
		# will use the map method again, with the option batched=True. When we use batched=True,
		# the function we pass to map() will be passed multiple inputs at once, allowing us
		# to group them into more or fewer examples than we had in the input.
		# This allows us to create our new fixed-length samples. The advantage of this
		# method is that we don't lose a whole lot of content from the dataset compared to the
		# case where we simply tokenize with a pre-defined max_length.

		def group_texts(UpperCamelCase						):
				# Concatenate all texts.
				_UpperCamelCase							:			Dict     =   {k: sum(examples[k]      ,[]						) for k in examples.keys()}
				_UpperCamelCase							:			Optional[int]     =   len(concatenated_examples[list(examples.keys()						)[0]]						)
				# We drop the small remainder, though you could add padding instead if the model supports it
				# In this, as in all things, we advise you to follow your heart 🫀
				_UpperCamelCase							:			List[Any]     =   (total_length // args.max_length) * args.max_length
				# Split by chunks of max_len.
				_UpperCamelCase							:			str     =   {
				    k: [t[i : i + args.max_length] for i in range(0      ,UpperCamelCase_      ,args.max_length						)]
				    for k, t in concatenated_examples.items()
				}
				return result

		_UpperCamelCase							:			Tuple     =   dataset_tokenized.map(UpperCamelCase_      ,batched=UpperCamelCase_      ,batch_size=10_00      ,num_proc=4						)

		_UpperCamelCase							:			Tuple     =   0
		_UpperCamelCase							:			Tuple     =   0
		for shard in range(0      ,len(UpperCamelCase_						)      ,args.shard_size						):
				_UpperCamelCase							:			List[Any]     =   grouped_dataset[shard : shard + args.shard_size]
				_UpperCamelCase							:			Union[str, Any]     =   len(dataset_snapshot['''input_ids''']						)
				_UpperCamelCase							:			int     =   os.path.join(UpperCamelCase_      ,f'''dataset-{shard_count}-{records_containing}.tfrecord'''						)
				_UpperCamelCase							:			List[str]     =   get_serialized_examples(UpperCamelCase_						)

				with tf.io.TFRecordWriter(UpperCamelCase_						) as out_file:
						for i in range(len(UpperCamelCase_						)						):
								_UpperCamelCase							:			List[str]     =   serialized_examples[i]
								out_file.write(UpperCamelCase_						)
						print('''Wrote file {} containing {} records'''.format(UpperCamelCase_      ,UpperCamelCase_						)						)

				shard_count += 1
				total_records += records_containing

		with open(f'''split-{args.split}-records-count.txt'''      ,'''w'''						) as f:
				print(f'''Total {args.split} records: {total_records}'''      ,file=UpperCamelCase_						)


if __name__ == "__main__":
	_UpperCAmelCase :   Optional[Any]			    =	parse_args()
	main(args)






_SCREAMING_SNAKE_CASE 		=      9.8_06_65





def 			lowercase(	UpperCamelCase_							,					UpperCamelCase_							,					UpperCamelCase_ = g       )		->	float:







 '''simple docstring'''

 if fluid_density <= 0:
  raise ValueError("""Impossible fluid density"""       )
 if volume < 0:
  raise ValueError("""Impossible Object volume"""       )
 if gravity <= 0:
  raise ValueError("""Impossible Gravity"""       )

 return fluid_density * gravity * volume


if __name__ == "__main__":
     import doctest

     # run doctest
     doctest.testmod()
