


'''simple docstring'''



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__lowercase     :						Union[str, Any]	=     logging.getLogger(__name__)






def   lowerCamelCase							():
					__a      :	List[Any]    =  argparse.ArgumentParser(
					    description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'       )
					parser.add_argument(
					    '--dataset_name'						,				type=_SCREAMING_SNAKE_CASE						,				default='wikitext'						,				help='Name of the training. Explore datasets at: hf.co/datasets.'						,				)
					parser.add_argument(
					    '--dataset_config'						,				type=_SCREAMING_SNAKE_CASE						,				default='wikitext-103-raw-v1'						,				help='Configuration name of the dataset.'       )
					parser.add_argument(
					    '--tokenizer_name_or_path'						,				type=_SCREAMING_SNAKE_CASE						,				default='sayakpaul/unigram-tokenizer-wikitext'						,				help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'						,				)
					parser.add_argument(
					    '--shard_size'						,				type=_SCREAMING_SNAKE_CASE						,				default=1_000						,				help='Number of entries to go in a single shard.'						,				)
					parser.add_argument('--split'						,				type=_SCREAMING_SNAKE_CASE						,				default='train'						,				choices=['train', 'test', 'validation']       )
					parser.add_argument(
					    '--limit'						,				default=_SCREAMING_SNAKE_CASE						,				type=_SCREAMING_SNAKE_CASE						,				help='Limit the number of shards (used for debugging).'						,				)
					parser.add_argument(
					    '--max_length'						,				type=_SCREAMING_SNAKE_CASE						,				default=512						,				help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
					    ' sequence length that is a multiple of 8.'						,				)
					parser.add_argument(
					    '--output_dir'						,				default='tf-tpu'						,				type=_SCREAMING_SNAKE_CASE						,				help='Output directory where the TFRecord shards will be saved. If the'
					    ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
					    ' shards will be directly saved to a Google Cloud Storage bucket.'						,				)

					__a      :	List[Any]    =  parser.parse_args()
					return args






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Optional[int]       ):
					def fn(_SCREAMING_SNAKE_CASE				:       List[Any]       ):
										return tokenizer(examples['text']       )

					return fn






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Tuple       ):
					__a      :	List[str]    =  []
					for i in range(len(tokenized_data['input_ids']       )       ):
										__a      :	int    =  {
										    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]       )       ),
										    'attention_mask': tf.train.Feature(
										        intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]       )       ),
										}
										__a      :	str    =  tf.train.Features(feature=_SCREAMING_SNAKE_CASE       )
										__a      :	List[str]    =  tf.train.Example(features=_SCREAMING_SNAKE_CASE       )
										__a      :	Optional[Any]    =  example.SerializeToString()
										records.append(_SCREAMING_SNAKE_CASE       )
					return records






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Any       ):
					__a      :	str    =  datasets.load_dataset(args.dataset_name						,				args.dataset_config						,				split=args.split       )

					if args.limit is not None:
										__a      :	Optional[int]    =  min(len(_SCREAMING_SNAKE_CASE       )						,				args.limit       )
										__a      :	int    =  dataset.select(range(_SCREAMING_SNAKE_CASE       )       )
										print(F"""Limiting the dataset to {args.limit} entries."""       )

					__a      :	str    =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path       )

					# Handle output directory creation.
					# For serializing into a Google Cloud Storage Bucket, one needs to first
					# create a bucket.
					if "gs" not in args.output_dir:
										if not os.path.exists(args.output_dir       ):
															os.makedirs(args.output_dir       )
										__a      :	List[str]    =  os.path.join(args.output_dir						,				args.split       )
										if not os.path.exists(_SCREAMING_SNAKE_CASE       ):
															os.makedirs(_SCREAMING_SNAKE_CASE       )
					else:
										__a      :	Any    =  os.path.join(args.output_dir						,				args.split       )

					# Tokenize the whole dataset at once.
					__a      :	List[Any]    =  tokenize_function(_SCREAMING_SNAKE_CASE       )
					__a      :	Any    =  dataset.map(_SCREAMING_SNAKE_CASE						,				batched=_SCREAMING_SNAKE_CASE						,				num_proc=4						,				remove_columns=['text']       )

					# We need to concatenate all our texts together, and then split the result
					# into chunks of a fixed size, which we will call block_size. To do this, we
					# will use the map method again, with the option batched=True. When we use batched=True,
					# the function we pass to map() will be passed multiple inputs at once, allowing us
					# to group them into more or fewer examples than we had in the input.
					# This allows us to create our new fixed-length samples. The advantage of this
					# method is that we don't lose a whole lot of content from the dataset compared to the
					# case where we simply tokenize with a pre-defined max_length.

					def group_texts(_SCREAMING_SNAKE_CASE				:       Optional[int]       ):
										# Concatenate all texts.
										__a      :	Optional[Any]    =  {k: sum(examples[k]						,				[]       ) for k in examples.keys()}
										__a      :	str    =  len(concatenated_examples[list(examples.keys()       )[0]]       )
										# We drop the small remainder, though you could add padding instead if the model supports it
										# In this, as in all things, we advise you to follow your heart 🫀
										__a      :	str    =  (total_length // args.max_length) * args.max_length
										# Split by chunks of max_len.
										__a      :	Tuple    =  {
										    k: [t[i : i + args.max_length] for i in range(0						,				_SCREAMING_SNAKE_CASE						,				args.max_length       )]
										    for k, t in concatenated_examples.items()
										}
										return result

					__a      :	Optional[int]    =  dataset_tokenized.map(_SCREAMING_SNAKE_CASE						,				batched=_SCREAMING_SNAKE_CASE						,				batch_size=1_000						,				num_proc=4       )

					__a      :	Optional[int]    =  0
					__a      :	Any    =  0
					for shard in range(0						,				len(_SCREAMING_SNAKE_CASE       )						,				args.shard_size       ):
										__a      :	Dict    =  grouped_dataset[shard : shard + args.shard_size]
										__a      :	Tuple    =  len(dataset_snapshot['input_ids']       )
										__a      :	Dict    =  os.path.join(_SCREAMING_SNAKE_CASE						,				F"""dataset-{shard_count}-{records_containing}.tfrecord"""       )
										__a      :	str    =  get_serialized_examples(_SCREAMING_SNAKE_CASE       )

										with tf.io.TFRecordWriter(_SCREAMING_SNAKE_CASE       ) as out_file:
															for i in range(len(_SCREAMING_SNAKE_CASE       )       ):
																				__a      :	List[str]    =  serialized_examples[i]
																				out_file.write(_SCREAMING_SNAKE_CASE       )
															print('Wrote file {} containing {} records'.format(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       )       )

										shard_count += 1
										total_records += records_containing

					with open(F"""split-{args.split}-records-count.txt"""						,				'w'       ) as f:
										print(F"""Total {args.split} records: {total_records}"""						,				file=_SCREAMING_SNAKE_CASE       )


if __name__ == "__main__":
	__lowercase     :						Any	=     parse_args()
	main(args)







'''simple docstring'''



def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int						,				_SCREAMING_SNAKE_CASE				:       int       ):
					if not isinstance(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       ):
										raise ValueError('iterations must be defined as integers'       )
					if not isinstance(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       ) or not number >= 1:
										raise ValueError(
										    'starting number must be\n                         and integer and be more than 0'       )
					if not iterations >= 1:
										raise ValueError('Iterations must be done more than 0 times to play FizzBuzz'       )

					__a      :	Dict    =  ''
					while number <= iterations:
										if number % 3 == 0:
															out += "Fizz"
										if number % 5 == 0:
															out += "Buzz"
										if 0 not in (number % 3, number % 5):
															out += str(_SCREAMING_SNAKE_CASE       )

										# print(out)
										number += 1
										out += " "
					return out


if __name__ == "__main__":
	import doctest

	doctest.testmod()



