


"""simple docstring"""


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


A  :     str    					=    logging.getLogger(__name__)


def _lowerCamelCase			(	):

			'''simple docstring'''




			__lowerCAmelCase       					=    argparse.ArgumentParser(
			    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."   )
			parser.add_argument(
			    "--dataset_name"					,				type=_UpperCamelCase					,				default="wikitext"					,				help="Name of the training. Explore datasets at: hf.co/datasets."					,				)
			parser.add_argument(
			    "--dataset_config"					,				type=_UpperCamelCase					,				default="wikitext-103-raw-v1"					,				help="Configuration name of the dataset."   )
			parser.add_argument(
			    "--tokenizer_name_or_path"					,				type=_UpperCamelCase					,				default="sayakpaul/unigram-tokenizer-wikitext"					,				help="Tokenizer identifier. Can be a local filepath or a Hub identifier."					,				)
			parser.add_argument(
			    "--shard_size"					,				type=_UpperCamelCase					,				default=1000					,				help="Number of entries to go in a single shard."					,				)
			parser.add_argument("--split"					,				type=_UpperCamelCase					,				default="train"					,				choices=["train", "test", "validation"]   )
			parser.add_argument(
			    "--limit"					,				default=_UpperCamelCase					,				type=_UpperCamelCase					,				help="Limit the number of shards (used for debugging)."					,				)
			parser.add_argument(
			    "--max_length"					,				type=_UpperCamelCase					,				default=512					,				help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
			    " sequence length that is a multiple of 8."					,				)
			parser.add_argument(
			    "--output_dir"					,				default="tf-tpu"					,				type=_UpperCamelCase					,				help="Output directory where the TFRecord shards will be saved. If the"
			    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
			    " shards will be directly saved to a Google Cloud Storage bucket."					,				)

			__lowerCAmelCase       					=    parser.parse_args()
			return args


def _lowerCamelCase			(	_UpperCamelCase   ):

			'''simple docstring'''




			def fn(_UpperCamelCase   ):
						return tokenizer(examples["text"]   )

			return fn


def _lowerCamelCase			(	_UpperCamelCase   ):

			'''simple docstring'''




			__lowerCAmelCase       					=    []
			for i in range(len(tokenized_data["input_ids"]   )   ):
						__lowerCAmelCase       					=    {
						    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]   )   ),
						    "attention_mask": tf.train.Feature(
						        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]   )   ),
						}
						__lowerCAmelCase       					=    tf.train.Features(feature=_UpperCamelCase   )
						__lowerCAmelCase       					=    tf.train.Example(features=_UpperCamelCase   )
						__lowerCAmelCase       					=    example.SerializeToString()
						records.append(_UpperCamelCase   )
			return records


def _lowerCamelCase			(	_UpperCamelCase   ):

			'''simple docstring'''




			__lowerCAmelCase       					=    datasets.load_dataset(args.dataset_name					,				args.dataset_config					,				split=args.split   )

			if args.limit is not None:
						__lowerCAmelCase       					=    min(len(_UpperCamelCase   )					,				args.limit   )
						__lowerCAmelCase       					=    dataset.select(range(_UpperCamelCase   )   )
						print(f"Limiting the dataset to {args.limit} entries."   )

			__lowerCAmelCase       					=    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path   )

			# Handle output directory creation.
			# For serializing into a Google Cloud Storage Bucket, one needs to first
			# create a bucket.
			if "gs" not in args.output_dir:
						if not os.path.exists(args.output_dir   ):
									os.makedirs(args.output_dir   )
						__lowerCAmelCase       					=    os.path.join(args.output_dir					,				args.split   )
						if not os.path.exists(_UpperCamelCase   ):
									os.makedirs(_UpperCamelCase   )
			else:
						__lowerCAmelCase       					=    os.path.join(args.output_dir					,				args.split   )

			# Tokenize the whole dataset at once.
			__lowerCAmelCase       					=    tokenize_function(_UpperCamelCase   )
			__lowerCAmelCase       					=    dataset.map(_UpperCamelCase					,				batched=_UpperCamelCase					,				num_proc=4					,				remove_columns=["text"]   )

			# We need to concatenate all our texts together, and then split the result
			# into chunks of a fixed size, which we will call block_size. To do this, we
			# will use the map method again, with the option batched=True. When we use batched=True,
			# the function we pass to map() will be passed multiple inputs at once, allowing us
			# to group them into more or fewer examples than we had in the input.
			# This allows us to create our new fixed-length samples. The advantage of this
			# method is that we don't lose a whole lot of content from the dataset compared to the
			# case where we simply tokenize with a pre-defined max_length.

			def group_texts(_UpperCamelCase   ):
						# Concatenate all texts.
						__lowerCAmelCase       					=    {k: sum(examples[k]					,				[]   ) for k in examples.keys()}
						__lowerCAmelCase       					=    len(concatenated_examples[list(examples.keys()   )[0]]   )
						# We drop the small remainder, though you could add padding instead if the model supports it
						# In this, as in all things, we advise you to follow your heart 🫀
						__lowerCAmelCase       					=    (total_length // args.max_length) * args.max_length
						# Split by chunks of max_len.
						__lowerCAmelCase       					=    {
						    k: [t[i : i + args.max_length] for i in range(0					,				_UpperCamelCase					,				args.max_length   )]
						    for k, t in concatenated_examples.items()
						}
						return result

			__lowerCAmelCase       					=    dataset_tokenized.map(_UpperCamelCase					,				batched=_UpperCamelCase					,				batch_size=1000					,				num_proc=4   )

			__lowerCAmelCase       					=    0
			__lowerCAmelCase       					=    0
			for shard in range(0					,				len(_UpperCamelCase   )					,				args.shard_size   ):
						__lowerCAmelCase       					=    grouped_dataset[shard : shard + args.shard_size]
						__lowerCAmelCase       					=    len(dataset_snapshot["input_ids"]   )
						__lowerCAmelCase       					=    os.path.join(_UpperCamelCase					,				f"dataset-{shard_count}-{records_containing}.tfrecord"   )
						__lowerCAmelCase       					=    get_serialized_examples(_UpperCamelCase   )

						with tf.io.TFRecordWriter(_UpperCamelCase   ) as out_file:
									for i in range(len(_UpperCamelCase   )   ):
												__lowerCAmelCase       					=    serialized_examples[i]
												out_file.write(_UpperCamelCase   )
									print("Wrote file {} containing {} records".format(_UpperCamelCase					,				_UpperCamelCase   )   )

						shard_count += 1
						total_records += records_containing

			with open(f"split-{args.split}-records-count.txt"					,				"w"   ) as f:
						print(f"Total {args.split} records: {total_records}"					,				file=_UpperCamelCase   )


if __name__ == "__main__":
	A  :     int    					=    parse_args()
	main(args)








"""simple docstring"""


def _lowerCamelCase			(	_UpperCamelCase					,				_UpperCamelCase   ):

			'''simple docstring'''




			__lowerCAmelCase       					=    len(_UpperCamelCase   )
			print("The following activities are selected:"   )

			# The first activity is always selected
			__lowerCAmelCase       					=    0
			print(_UpperCamelCase					,				end=","   )

			# Consider rest of the activities
			for j in range(_UpperCamelCase   ):
						# If this activity has start time greater than
						# or equal to the finish time of previously
						# selected activity, then select it
						if start[j] >= finish[i]:
									print(_UpperCamelCase					,				end=","   )
									__lowerCAmelCase       					=    j


if __name__ == "__main__":
	import doctest

	doctest.testmod()

	A  :     Dict    					=    [1, 3, 0, 5, 8, 5]
	A  :     int    					=    [2, 4, 6, 7, 9, 9]
	print_max_activities(start, finish)




