





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowerCAmelCase							:       str					 =	logging.getLogger(__name__)






def 		lowerCAmelCase  (	)      ->    str:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Any           =   argparse.ArgumentParser(
						    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''					)
						parser.add_argument(
						    '''--dataset_name'''       ,			type=UpperCamelCase__       ,			default='''wikitext'''       ,			help='''Name of the training. Explore datasets at: hf.co/datasets.'''       ,			)
						parser.add_argument(
						    '''--dataset_config'''       ,			type=UpperCamelCase__       ,			default='''wikitext-103-raw-v1'''       ,			help='''Configuration name of the dataset.'''					)
						parser.add_argument(
						    '''--tokenizer_name_or_path'''       ,			type=UpperCamelCase__       ,			default='''sayakpaul/unigram-tokenizer-wikitext'''       ,			help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''       ,			)
						parser.add_argument(
						    '''--shard_size'''       ,			type=UpperCamelCase__       ,			default=1_000       ,			help='''Number of entries to go in a single shard.'''       ,			)
						parser.add_argument('''--split'''       ,			type=UpperCamelCase__       ,			default='''train'''       ,			choices=['''train''', '''test''', '''validation''']					)
						parser.add_argument(
						    '''--limit'''       ,			default=UpperCamelCase__       ,			type=UpperCamelCase__       ,			help='''Limit the number of shards (used for debugging).'''       ,			)
						parser.add_argument(
						    '''--max_length'''       ,			type=UpperCamelCase__       ,			default=512       ,			help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
						    ''' sequence length that is a multiple of 8.'''       ,			)
						parser.add_argument(
						    '''--output_dir'''       ,			default='''tf-tpu'''       ,			type=UpperCamelCase__       ,			help='''Output directory where the TFRecord shards will be saved. If the'''
						    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
						    ''' shards will be directly saved to a Google Cloud Storage bucket.'''       ,			)

						__SCREAMING_SNAKE_CASE:				List[str]           =   parser.parse_args()
						return args






def 		lowerCAmelCase  (	UpperCamelCase__   :   int					)      ->    Union[str, Any]:

						"""simple docstring"""





						def fn(UpperCamelCase__   :   Optional[Any]					):
												return tokenizer(examples['''text''']					)

						return fn






def 		lowerCAmelCase  (	UpperCamelCase__   :   List[Any]					)      ->    Optional[Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Any           =   []
						for i in range(len(tokenized_data['''input_ids''']					)					):
												__SCREAMING_SNAKE_CASE:				List[Any]           =   {
												    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]					)					),
												    '''attention_mask''': tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]					)					),
												}
												__SCREAMING_SNAKE_CASE:				List[Any]           =   tf.train.Features(feature=UpperCamelCase__					)
												__SCREAMING_SNAKE_CASE:				int           =   tf.train.Example(features=UpperCamelCase__					)
												__SCREAMING_SNAKE_CASE:				List[str]           =   example.SerializeToString()
												records.append(UpperCamelCase__					)
						return records






def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[Any]					)      ->    Any:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				int           =   datasets.load_dataset(args.dataset_name       ,			args.dataset_config       ,			split=args.split					)

						if args.limit is not None:
												__SCREAMING_SNAKE_CASE:				Dict           =   min(len(UpperCamelCase__					)       ,			args.limit					)
												__SCREAMING_SNAKE_CASE:				int           =   dataset.select(range(UpperCamelCase__					)					)
												print(F"""Limiting the dataset to {args.limit} entries."""					)

						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path					)

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir					):
																		os.makedirs(args.output_dir					)
												__SCREAMING_SNAKE_CASE:				Optional[int]           =   os.path.join(args.output_dir       ,			args.split					)
												if not os.path.exists(UpperCamelCase__					):
																		os.makedirs(UpperCamelCase__					)
						else:
												__SCREAMING_SNAKE_CASE:				List[str]           =   os.path.join(args.output_dir       ,			args.split					)

						# Tokenize the whole dataset at once.
						__SCREAMING_SNAKE_CASE:				str           =   tokenize_function(UpperCamelCase__					)
						__SCREAMING_SNAKE_CASE:				List[Any]           =   dataset.map(UpperCamelCase__       ,			batched=UpperCamelCase__       ,			num_proc=4       ,			remove_columns=['''text''']					)

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(UpperCamelCase__   :   Tuple					):
												# Concatenate all texts.
												__SCREAMING_SNAKE_CASE:				Any           =   {k: sum(examples[k]       ,			[]					) for k in examples.keys()}
												__SCREAMING_SNAKE_CASE:				Dict           =   len(concatenated_examples[list(examples.keys()					)[0]]					)
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												__SCREAMING_SNAKE_CASE:				List[Any]           =   (total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												__SCREAMING_SNAKE_CASE:				Tuple           =   {
												    k: [t[i : i + args.max_length] for i in range(0       ,			UpperCamelCase__       ,			args.max_length					)]
												    for k, t in concatenated_examples.items()
												}
												return result

						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   dataset_tokenized.map(UpperCamelCase__       ,			batched=UpperCamelCase__       ,			batch_size=1_000       ,			num_proc=4					)

						__SCREAMING_SNAKE_CASE:				Optional[int]           =   0
						__SCREAMING_SNAKE_CASE:				Optional[int]           =   0
						for shard in range(0       ,			len(UpperCamelCase__					)       ,			args.shard_size					):
												__SCREAMING_SNAKE_CASE:				str           =   grouped_dataset[shard : shard + args.shard_size]
												__SCREAMING_SNAKE_CASE:				Dict           =   len(dataset_snapshot['''input_ids''']					)
												__SCREAMING_SNAKE_CASE:				Dict           =   os.path.join(UpperCamelCase__       ,			F"""dataset-{shard_count}-{records_containing}.tfrecord"""					)
												__SCREAMING_SNAKE_CASE:				str           =   get_serialized_examples(UpperCamelCase__					)

												with tf.io.TFRecordWriter(UpperCamelCase__					) as out_file:
																		for i in range(len(UpperCamelCase__					)					):
																								__SCREAMING_SNAKE_CASE:				Tuple           =   serialized_examples[i]
																								out_file.write(UpperCamelCase__					)
																		print('''Wrote file {} containing {} records'''.format(UpperCamelCase__       ,			UpperCamelCase__					)					)

												shard_count += 1
												total_records += records_containing

						with open(F"""split-{args.split}-records-count.txt"""       ,			'''w'''					) as f:
												print(F"""Total {args.split} records: {total_records}"""       ,			file=UpperCamelCase__					)


if __name__ == "__main__":
							lowerCAmelCase							:       Union[str, Any]					 =	parse_args()
							main(args)










from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream





class 				a			(      __lowercase			):






				def __init__(  self			, _lowerCAmelCase			, _lowerCAmelCase = None			, _lowerCAmelCase = None			, _lowerCAmelCase = False			, _lowerCAmelCase = False			, _lowerCAmelCase = None			, _lowerCAmelCase = None			, **_lowerCAmelCase			, ):

										"""simple docstring"""




										super().__init__(
										    features=_lowerCAmelCase			, cache_dir=_lowerCAmelCase			, keep_in_memory=_lowerCAmelCase			, streaming=_lowerCAmelCase			, num_proc=_lowerCAmelCase			, **_lowerCAmelCase			, )
										__SCREAMING_SNAKE_CASE:				Any           =   Generator(
										    cache_dir=_lowerCAmelCase			, features=_lowerCAmelCase			, generator=_lowerCAmelCase			, gen_kwargs=_lowerCAmelCase			, **_lowerCAmelCase			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										if self.streaming:
																__SCREAMING_SNAKE_CASE:				List[str]           =   self.builder.as_streaming_dataset(split='''train'''					)
										# Build regular (map-style) dataset
										else:
																__SCREAMING_SNAKE_CASE:				str           =   None
																__SCREAMING_SNAKE_CASE:				List[Any]           =   None
																__SCREAMING_SNAKE_CASE:				Tuple           =   None
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   None

																self.builder.download_and_prepare(
																    download_config=_lowerCAmelCase			, download_mode=_lowerCAmelCase			, verification_mode=_lowerCAmelCase			, base_path=_lowerCAmelCase			, num_proc=self.num_proc			, )
																__SCREAMING_SNAKE_CASE:				List[str]           =   self.builder.as_dataset(
																    split='''train'''			, verification_mode=_lowerCAmelCase			, in_memory=self.keep_in_memory					)
										return dataset



