




'''simple docstring'''


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_snake_case : int  				=						logging.getLogger(__name__)
def 						snake_case_       ():





							'''simple docstring'''


							_a      	=							argparse.ArgumentParser(
							    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.''' )
							parser.add_argument(
							    '''--dataset_name'''					,		type=__snake_case					,		default='''wikitext'''					,		help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,		)
							parser.add_argument(
							    '''--dataset_config'''					,		type=__snake_case					,		default='''wikitext-103-raw-v1'''					,		help='''Configuration name of the dataset.''' )
							parser.add_argument(
							    '''--tokenizer_name_or_path'''					,		type=__snake_case					,		default='''sayakpaul/unigram-tokenizer-wikitext'''					,		help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''					,		)
							parser.add_argument(
							    '''--shard_size'''					,		type=__snake_case					,		default=1000					,		help='''Number of entries to go in a single shard.'''					,		)
							parser.add_argument('''--split'''					,		type=__snake_case					,		default='''train'''					,		choices=['''train''', '''test''', '''validation'''] )
							parser.add_argument(
							    '''--limit'''					,		default=__snake_case					,		type=__snake_case					,		help='''Limit the number of shards (used for debugging).'''					,		)
							parser.add_argument(
							    '''--max_length'''					,		type=__snake_case					,		default=512					,		help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
							    ''' sequence length that is a multiple of 8.'''					,		)
							parser.add_argument(
							    '''--output_dir'''					,		default='''tf-tpu'''					,		type=__snake_case					,		help='''Output directory where the TFRecord shards will be saved. If the'''
							    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
							    ''' shards will be directly saved to a Google Cloud Storage bucket.'''					,		)

							_a      	=							parser.parse_args()
							return args
def 						snake_case_       (UpperCamelCase			:    List[Any] ):





							'''simple docstring'''


							def fn(UpperCamelCase			:    List[str] ):
														return tokenizer(examples['''text'''] )

							return fn
def 						snake_case_       (UpperCamelCase			:    Dict ):





							'''simple docstring'''


							_a      	=							[]
							for i in range(len(tokenized_data['''input_ids'''] ) ):
														_a      	=							{
														    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i] ) ),
														    '''attention_mask''': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i] ) ),
														}
														_a      	=							tf.train.Features(feature=__snake_case )
														_a      	=							tf.train.Example(features=__snake_case )
														_a      	=							example.SerializeToString()
														records.append(__snake_case )
							return records
def 						snake_case_       (UpperCamelCase			:    str ):





							'''simple docstring'''


							_a      	=							datasets.load_dataset(args.dataset_name					,		args.dataset_config					,		split=args.split )

							if args.limit is not None:
														_a      	=							min(len(__snake_case )					,		args.limit )
														_a      	=							dataset.select(range(__snake_case ) )
														print(f'Limiting the dataset to {args.limit} entries.' )

							_a      	=							AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir ):
																					os.makedirs(args.output_dir )
														_a      	=							os.path.join(args.output_dir					,		args.split )
														if not os.path.exists(__snake_case ):
																					os.makedirs(__snake_case )
							else:
														_a      	=							os.path.join(args.output_dir					,		args.split )

							# Tokenize the whole dataset at once.
							_a      	=							tokenize_function(__snake_case )
							_a      	=							dataset.map(__snake_case					,		batched=__snake_case					,		num_proc=4					,		remove_columns=['''text'''] )

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(UpperCamelCase			:    Optional[Any] ):
														# Concatenate all texts.
														_a      	=							{k: sum(examples[k]					,		[] ) for k in examples.keys()}
														_a      	=							len(concatenated_examples[list(examples.keys() )[0]] )
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														_a      	=							(total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														_a      	=							{
														    k: [t[i : i + args.max_length] for i in range(0					,		__snake_case					,		args.max_length )]
														    for k, t in concatenated_examples.items()
														}
														return result

							_a      	=							dataset_tokenized.map(__snake_case					,		batched=__snake_case					,		batch_size=1000					,		num_proc=4 )

							_a      	=							0
							_a      	=							0
							for shard in range(0					,		len(__snake_case )					,		args.shard_size ):
														_a      	=							grouped_dataset[shard : shard + args.shard_size]
														_a      	=							len(dataset_snapshot['''input_ids'''] )
														_a      	=							os.path.join(__snake_case					,		f'dataset-{shard_count}-{records_containing}.tfrecord' )
														_a      	=							get_serialized_examples(__snake_case )

														with tf.io.TFRecordWriter(__snake_case ) as out_file:
																					for i in range(len(__snake_case ) ):
																												_a      	=							serialized_examples[i]
																												out_file.write(__snake_case )
																					print('''Wrote file {} containing {} records'''.format(__snake_case					,		__snake_case ) )

														shard_count += 1
														total_records += records_containing

							with open(f'split-{args.split}-records-count.txt'					,		'''w''' ) as f:
														print(f'Total {args.split} records: {total_records}'					,		file=__snake_case )


if __name__ == "__main__":
				_snake_case : Optional[int]  				=						parse_args()
				main(args)



"""simple docstring"""





import fire

from utils import calculate_rouge, save_json



def  lowerCamelCase__						(     __snake_case,	__snake_case,	__snake_case=None,	**__snake_case      )				->						Optional[int]:


					"""simple docstring"""





					_UpperCamelCase									=    [x.strip() for x in open(__snake_case      ).readlines()]
					_UpperCamelCase									=    [x.strip() for x in open(__snake_case      ).readlines()][: len(__snake_case      )]
					_UpperCamelCase									=    calculate_rouge(__snake_case,	__snake_case,	**__snake_case      )
					if save_path is not None:
										save_json(__snake_case,	__snake_case,	indent=__snake_case      )
					return metrics  # these print nicely


if __name__ == "__main__":
				fire.Fire(calculate_rouge_path)






