






"""simple docstring"""







import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_lowerCAmelCase            =	logging.getLogger(__name__)


def      UpperCamelCase       ( )     ->       int:
				lowercase    :      List[str]       = argparse.ArgumentParser(
				    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.""" )
				parser.add_argument(
				    """--dataset_name"""  ,				type=_A  ,				default="""wikitext"""  ,				help="""Name of the training. Explore datasets at: hf.co/datasets."""  ,				)
				parser.add_argument(
				    """--dataset_config"""  ,				type=_A  ,				default="""wikitext-103-raw-v1"""  ,				help="""Configuration name of the dataset.""" )
				parser.add_argument(
				    """--tokenizer_name_or_path"""  ,				type=_A  ,				default="""sayakpaul/unigram-tokenizer-wikitext"""  ,				help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""  ,				)
				parser.add_argument(
				    """--shard_size"""  ,				type=_A  ,				default=1_000  ,				help="""Number of entries to go in a single shard."""  ,				)
				parser.add_argument("""--split"""  ,				type=_A  ,				default="""train"""  ,				choices=["""train""", """test""", """validation"""] )
				parser.add_argument(
				    """--limit"""  ,				default=_A  ,				type=_A  ,				help="""Limit the number of shards (used for debugging)."""  ,				)
				parser.add_argument(
				    """--max_length"""  ,				type=_A  ,				default=512  ,				help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
				    """ sequence length that is a multiple of 8."""  ,				)
				parser.add_argument(
				    """--output_dir"""  ,				default="""tf-tpu"""  ,				type=_A  ,				help="""Output directory where the TFRecord shards will be saved. If the"""
				    """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
				    """ shards will be directly saved to a Google Cloud Storage bucket."""  ,				)

				lowercase    :      str       = parser.parse_args()
				return args


def      UpperCamelCase       ( _A )     ->       Dict:
				def fn(_A ):
								return tokenizer(examples["""text"""] )

				return fn


def      UpperCamelCase       ( _A )     ->       Optional[int]:
				lowercase    :      int       = []
				for i in range(len(tokenized_data["""input_ids"""] ) ):
								lowercase    :      Dict       = {
								    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i] ) ),
								    """attention_mask""": tf.train.Feature(
								        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i] ) ),
								}
								lowercase    :      Optional[int]       = tf.train.Features(feature=_A )
								lowercase    :      Optional[Any]       = tf.train.Example(features=_A )
								lowercase    :      int       = example.SerializeToString()
								records.append(_A )
				return records


def      UpperCamelCase       ( _A )     ->       Any:
				lowercase    :      List[Any]       = datasets.load_dataset(args.dataset_name  ,				args.dataset_config  ,				split=args.split )

				if args.limit is not None:
								lowercase    :      Union[str, Any]       = min(len(_A )  ,				args.limit )
								lowercase    :      str       = dataset.select(range(_A ) )
								print(F"""Limiting the dataset to {args.limit} entries.""" )

				lowercase    :      List[Any]       = AutoTokenizer.from_pretrained(args.tokenizer_name_or_path )

				# Handle output directory creation.
				# For serializing into a Google Cloud Storage Bucket, one needs to first
				# create a bucket.
				if "gs" not in args.output_dir:
								if not os.path.exists(args.output_dir ):
												os.makedirs(args.output_dir )
								lowercase    :      Optional[int]       = os.path.join(args.output_dir  ,				args.split )
								if not os.path.exists(_A ):
												os.makedirs(_A )
				else:
								lowercase    :      Any       = os.path.join(args.output_dir  ,				args.split )

				# Tokenize the whole dataset at once.
				lowercase    :      List[Any]       = tokenize_function(_A )
				lowercase    :      Dict       = dataset.map(_A  ,				batched=_A  ,				num_proc=4  ,				remove_columns=["""text"""] )

				# We need to concatenate all our texts together, and then split the result
				# into chunks of a fixed size, which we will call block_size. To do this, we
				# will use the map method again, with the option batched=True. When we use batched=True,
				# the function we pass to map() will be passed multiple inputs at once, allowing us
				# to group them into more or fewer examples than we had in the input.
				# This allows us to create our new fixed-length samples. The advantage of this
				# method is that we don't lose a whole lot of content from the dataset compared to the
				# case where we simply tokenize with a pre-defined max_length.

				def group_texts(_A ):
								# Concatenate all texts.
								lowercase    :      Optional[int]       = {k: sum(examples[k]  ,				[] ) for k in examples.keys()}
								lowercase    :      Dict       = len(concatenated_examples[list(examples.keys() )[0]] )
								# We drop the small remainder, though you could add padding instead if the model supports it
								# In this, as in all things, we advise you to follow your heart 🫀
								lowercase    :      Optional[int]       = (total_length // args.max_length) * args.max_length
								# Split by chunks of max_len.
								lowercase    :      Optional[Any]       = {
								    k: [t[i : i + args.max_length] for i in range(0  ,				_A  ,				args.max_length )]
								    for k, t in concatenated_examples.items()
								}
								return result

				lowercase    :      int       = dataset_tokenized.map(_A  ,				batched=_A  ,				batch_size=1_000  ,				num_proc=4 )

				lowercase    :      List[Any]       = 0
				lowercase    :      List[Any]       = 0
				for shard in range(0  ,				len(_A )  ,				args.shard_size ):
								lowercase    :      Any       = grouped_dataset[shard : shard + args.shard_size]
								lowercase    :      List[Any]       = len(dataset_snapshot["""input_ids"""] )
								lowercase    :      Dict       = os.path.join(_A  ,				F"""dataset-{shard_count}-{records_containing}.tfrecord""" )
								lowercase    :      List[str]       = get_serialized_examples(_A )

								with tf.io.TFRecordWriter(_A ) as out_file:
												for i in range(len(_A ) ):
																lowercase    :      str       = serialized_examples[i]
																out_file.write(_A )
												print("""Wrote file {} containing {} records""".format(_A  ,				_A ) )

								shard_count += 1
								total_records += records_containing

				with open(F"""split-{args.split}-records-count.txt"""  ,				"""w""" ) as f:
								print(F"""Total {args.split} records: {total_records}"""  ,				file=_A )


if __name__ == "__main__":
				_lowerCAmelCase            =	parse_args()
				main(args)








"""simple docstring"""







def      UpperCamelCase       ( _A  ,				_A )     ->       int:
				lowercase    :      int       = 1  # To kept the Calculated Value
				# Since C(n, k) = C(n, n-k)
				if k > (n - k):
								lowercase    :      List[Any]       = n - k
				# Calculate C(n,k)
				for i in range(_A ):
								result *= n - i
								result //= i + 1
				return result


def      UpperCamelCase       ( _A )     ->       int:
				return binomial_coefficient(2 * node_count  ,				_A ) // (node_count + 1)


def      UpperCamelCase       ( _A )     ->       int:
				if n < 0:
								raise ValueError("""factorial() not defined for negative values""" )
				lowercase    :      Union[str, Any]       = 1
				for i in range(1  ,				n + 1 ):
								result *= i
				return result


def      UpperCamelCase       ( _A )     ->       int:
				return catalan_number(_A ) * factorial(_A )


if __name__ == "__main__":
				_lowerCAmelCase            =	int(input('Enter the number of nodes: ').strip() or 0)
				if node_count <= 0:
								raise ValueError('We need some nodes to work with.')
				print(
				    F'Given {node_count} nodes, there are {binary_tree_count(node_count)} '
				    F'binary trees and {catalan_number(node_count)} binary search trees.'
				)
