



import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__A							       =		logging.getLogger(__name__)



def 			__a	(       ) ->					Union[str, Any]:






						'''simple docstring'''



						UpperCAmelCase_=  argparse.ArgumentParser(
						    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""							)
						parser.add_argument(
						    """--dataset_name"""     ,type=__lowerCAmelCase     ,default="""wikitext"""     ,help="""Name of the training. Explore datasets at: hf.co/datasets."""     ,)
						parser.add_argument(
						    """--dataset_config"""     ,type=__lowerCAmelCase     ,default="""wikitext-103-raw-v1"""     ,help="""Configuration name of the dataset."""							)
						parser.add_argument(
						    """--tokenizer_name_or_path"""     ,type=__lowerCAmelCase     ,default="""sayakpaul/unigram-tokenizer-wikitext"""     ,help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""     ,)
						parser.add_argument(
						    """--shard_size"""     ,type=__lowerCAmelCase     ,default=10_00     ,help="""Number of entries to go in a single shard."""     ,)
						parser.add_argument("""--split"""     ,type=__lowerCAmelCase     ,default="""train"""     ,choices=["""train""", """test""", """validation"""]							)
						parser.add_argument(
						    """--limit"""     ,default=__lowerCAmelCase     ,type=__lowerCAmelCase     ,help="""Limit the number of shards (used for debugging)."""     ,)
						parser.add_argument(
						    """--max_length"""     ,type=__lowerCAmelCase     ,default=5_12     ,help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
						    """ sequence length that is a multiple of 8."""     ,)
						parser.add_argument(
						    """--output_dir"""     ,default="""tf-tpu"""     ,type=__lowerCAmelCase     ,help="""Output directory where the TFRecord shards will be saved. If the"""
						    """ path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"""
						    """ shards will be directly saved to a Google Cloud Storage bucket."""     ,)

						UpperCAmelCase_=  parser.parse_args()
						return args



def 			__a	(       lowerCAmelCase_  :   Tuple							) ->					List[str]:






						'''simple docstring'''



						def fn(lowerCAmelCase_  :   Dict							):
												return tokenizer(examples["""text"""]							)

						return fn



def 			__a	(       lowerCAmelCase_  :   str							) ->					Optional[int]:






						'''simple docstring'''



						UpperCAmelCase_=  []
						for i in range(len(tokenized_data["""input_ids"""]							)							):
												UpperCAmelCase_=  {
												    """input_ids""": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]							)							),
												    """attention_mask""": tf.train.Feature(
												        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]							)							),
												}
												UpperCAmelCase_=  tf.train.Features(feature=__lowerCAmelCase							)
												UpperCAmelCase_=  tf.train.Example(features=__lowerCAmelCase							)
												UpperCAmelCase_=  example.SerializeToString()
												records.append(__lowerCAmelCase							)
						return records



def 			__a	(       lowerCAmelCase_  :   List[str]							) ->					str:






						'''simple docstring'''



						UpperCAmelCase_=  datasets.load_dataset(args.dataset_name     ,args.dataset_config     ,split=args.split							)

						if args.limit is not None:
												UpperCAmelCase_=  min(len(__lowerCAmelCase							)     ,args.limit							)
												UpperCAmelCase_=  dataset.select(range(__lowerCAmelCase							)							)
												print(F"""Limiting the dataset to {args.limit} entries."""							)

						UpperCAmelCase_=  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path							)

						# Handle output directory creation.
						# For serializing into a Google Cloud Storage Bucket, one needs to first
						# create a bucket.
						if "gs" not in args.output_dir:
												if not os.path.exists(args.output_dir							):
																		os.makedirs(args.output_dir							)
												UpperCAmelCase_=  os.path.join(args.output_dir     ,args.split							)
												if not os.path.exists(__lowerCAmelCase							):
																		os.makedirs(__lowerCAmelCase							)
						else:
												UpperCAmelCase_=  os.path.join(args.output_dir     ,args.split							)

						# Tokenize the whole dataset at once.
						UpperCAmelCase_=  tokenize_function(__lowerCAmelCase							)
						UpperCAmelCase_=  dataset.map(__lowerCAmelCase     ,batched=__lowerCAmelCase     ,num_proc=4     ,remove_columns=["""text"""]							)

						# We need to concatenate all our texts together, and then split the result
						# into chunks of a fixed size, which we will call block_size. To do this, we
						# will use the map method again, with the option batched=True. When we use batched=True,
						# the function we pass to map() will be passed multiple inputs at once, allowing us
						# to group them into more or fewer examples than we had in the input.
						# This allows us to create our new fixed-length samples. The advantage of this
						# method is that we don't lose a whole lot of content from the dataset compared to the
						# case where we simply tokenize with a pre-defined max_length.

						def group_texts(lowerCAmelCase_  :   str							):
												# Concatenate all texts.
												UpperCAmelCase_=  {k: sum(examples[k]     ,[]							) for k in examples.keys()}
												UpperCAmelCase_=  len(concatenated_examples[list(examples.keys()							)[0]]							)
												# We drop the small remainder, though you could add padding instead if the model supports it
												# In this, as in all things, we advise you to follow your heart 🫀
												UpperCAmelCase_=  (total_length // args.max_length) * args.max_length
												# Split by chunks of max_len.
												UpperCAmelCase_=  {
												    k: [t[i : i + args.max_length] for i in range(0     ,__lowerCAmelCase     ,args.max_length							)]
												    for k, t in concatenated_examples.items()
												}
												return result

						UpperCAmelCase_=  dataset_tokenized.map(__lowerCAmelCase     ,batched=__lowerCAmelCase     ,batch_size=10_00     ,num_proc=4							)

						UpperCAmelCase_=  0
						UpperCAmelCase_=  0
						for shard in range(0     ,len(__lowerCAmelCase							)     ,args.shard_size							):
												UpperCAmelCase_=  grouped_dataset[shard : shard + args.shard_size]
												UpperCAmelCase_=  len(dataset_snapshot["""input_ids"""]							)
												UpperCAmelCase_=  os.path.join(__lowerCAmelCase     ,F"""dataset-{shard_count}-{records_containing}.tfrecord"""							)
												UpperCAmelCase_=  get_serialized_examples(__lowerCAmelCase							)

												with tf.io.TFRecordWriter(__lowerCAmelCase							) as out_file:
																		for i in range(len(__lowerCAmelCase							)							):
																								UpperCAmelCase_=  serialized_examples[i]
																								out_file.write(__lowerCAmelCase							)
																		print("""Wrote file {} containing {} records""".format(__lowerCAmelCase     ,__lowerCAmelCase							)							)

												shard_count += 1
												total_records += records_containing

						with open(F"""split-{args.split}-records-count.txt"""     ,"""w"""							) as f:
												print(F"""Total {args.split} records: {total_records}"""     ,file=__lowerCAmelCase							)


if __name__ == "__main__":
	__A							       =		parse_args()
	main(args)





"""simple docstring"""


a				=range(2, 20 + 1)
a				=[10**k for k in range(ks[-1] + 1)]
a				={}





def      lowerCamelCase_					(   __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)				->   Any:




						'''simple docstring'''



						lowerCamelCase__   =sum(a_i[j] for j in range(__lowerCAmelCase  ,       len(__lowerCAmelCase							)							)							)
						lowerCamelCase__   =sum(a_i[j] * base[j] for j in range(min(len(__lowerCAmelCase							)  ,       __lowerCAmelCase							)							)							)

						lowerCamelCase__   , lowerCamelCase__   =0, 0
						lowerCamelCase__   =n - i

						lowerCamelCase__   =memo.get(__lowerCAmelCase							)

						if sub_memo is not None:
												lowerCamelCase__   =sub_memo.get(__lowerCAmelCase							)

												if jumps is not None and len(__lowerCAmelCase							) > 0:
																		# find and make the largest jump without going over
																		lowerCamelCase__   =-1
																		for _k in range(len(__lowerCAmelCase							) - 1  ,       -1  ,       -1							):
																								if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
																														lowerCamelCase__   =_k
																														break

																		if max_jump >= 0:
																								lowerCamelCase__   , lowerCamelCase__   , lowerCamelCase__   =jumps[max_jump]
																								# since the difference between jumps is cached, add c
																								lowerCamelCase__   =diff + c
																								for j in range(min(__lowerCAmelCase  ,       len(__lowerCAmelCase							)							)							):
																														lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)
																								if new_c > 0:
																														add(__lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)

												else:
																		lowerCamelCase__   =[]
						else:
												lowerCamelCase__   ={c: []}
												lowerCamelCase__   =sub_memo

						if dn >= max_dn or c + diff >= base[k]:
												return diff, dn

						if k > ks[0]:
												while True:
																		# keep doing smaller jumps
																		lowerCamelCase__   , lowerCamelCase__   =next_term(__lowerCAmelCase  ,       k - 1  ,       i + dn  ,       __lowerCAmelCase							)
																		diff += _diff
																		dn += terms_jumped

																		if dn >= max_dn or c + diff >= base[k]:
																								break
						else:
												# would be too small a jump, just compute sequential terms instead
												lowerCamelCase__   , lowerCamelCase__   =compute(__lowerCAmelCase  ,       __lowerCAmelCase  ,       i + dn  ,       __lowerCAmelCase							)
												diff += _diff
												dn += terms_jumped

						lowerCamelCase__   =sub_memo[c]

						# keep jumps sorted by # of terms skipped
						lowerCamelCase__   =0
						while j < len(__lowerCAmelCase							):
												if jumps[j][1] > dn:
																		break
												j += 1

						# cache the jump for this value digitsum(b) and c
						sub_memo[c].insert(__lowerCAmelCase  ,       (diff, dn, k)							)
						return (diff, dn)





def      lowerCamelCase_					(   __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)				->   List[str]:




						'''simple docstring'''



						if i >= n:
												return 0, i
						if k > len(__lowerCAmelCase							):
												a_i.extend([0 for _ in range(k - len(__lowerCAmelCase							)							)]							)

						# note: a_i -> b * 10^k + c
						# ds_b -> digitsum(b)
						# ds_c -> digitsum(c)
						lowerCamelCase__   =i
						lowerCamelCase__   , lowerCamelCase__   , lowerCamelCase__   =0, 0, 0
						for j in range(len(__lowerCAmelCase							)							):
												if j >= k:
																		ds_b += a_i[j]
												else:
																		ds_c += a_i[j]

						while i < n:
												i += 1
												lowerCamelCase__   =ds_c + ds_b
												diff += addend
												lowerCamelCase__   =0
												for j in range(__lowerCAmelCase							):
																		lowerCamelCase__   =a_i[j] + addend
																		lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)

																		ds_c += a_i[j]

												if addend > 0:
																		break

						if addend > 0:
												add(__lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)
						return diff, i - start_i





def      lowerCamelCase_					(   __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)				->   int:




						'''simple docstring'''



						for j in range(__lowerCAmelCase  ,       len(__lowerCAmelCase							)							):
												lowerCamelCase__   =digits[j] + addend
												if s >= 10:
																		lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)
																		lowerCamelCase__   =addend // 10 + quotient
												else:
																		lowerCamelCase__   =s
																		lowerCamelCase__   =addend // 10

												if addend == 0:
																		break

						while addend > 0:
												lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)
												digits.append(__lowerCAmelCase							)





def      lowerCamelCase_					(   __lowerCAmelCase = 10**15							)				->   int:




						'''simple docstring'''



						lowerCamelCase__   =[1]
						lowerCamelCase__   =1
						lowerCamelCase__   =0
						while True:
												lowerCamelCase__   , lowerCamelCase__   =next_term(__lowerCAmelCase  ,       20  ,       i + dn  ,       __lowerCAmelCase							)
												dn += terms_jumped
												if dn == n - i:
																		break

						lowerCamelCase__   =0
						for j in range(len(__lowerCAmelCase							)							):
												a_n += digits[j] * 10**j
						return a_n


if __name__ == "__main__":
	print(F'''{solution() = }''')


