





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__lowerCAmelCase :str     =    logging.getLogger(__name__)







def    A				(							):
					_snake_case       :					int				       =   argparse.ArgumentParser(
					    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."				)
					parser.add_argument(
					    "--dataset_name"  ,  type=UpperCAmelCase  ,  default="wikitext"  ,  help="Name of the training. Explore datasets at: hf.co/datasets."  ,  )
					parser.add_argument(
					    "--dataset_config"  ,  type=UpperCAmelCase  ,  default="wikitext-103-raw-v1"  ,  help="Configuration name of the dataset."				)
					parser.add_argument(
					    "--tokenizer_name_or_path"  ,  type=UpperCAmelCase  ,  default="sayakpaul/unigram-tokenizer-wikitext"  ,  help="Tokenizer identifier. Can be a local filepath or a Hub identifier."  ,  )
					parser.add_argument(
					    "--shard_size"  ,  type=UpperCAmelCase  ,  default=1_000  ,  help="Number of entries to go in a single shard."  ,  )
					parser.add_argument("--split"  ,  type=UpperCAmelCase  ,  default="train"  ,  choices=["train", "test", "validation"]				)
					parser.add_argument(
					    "--limit"  ,  default=UpperCAmelCase  ,  type=UpperCAmelCase  ,  help="Limit the number of shards (used for debugging)."  ,  )
					parser.add_argument(
					    "--max_length"  ,  type=UpperCAmelCase  ,  default=512  ,  help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
					    " sequence length that is a multiple of 8."  ,  )
					parser.add_argument(
					    "--output_dir"  ,  default="tf-tpu"  ,  type=UpperCAmelCase  ,  help="Output directory where the TFRecord shards will be saved. If the"
					    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
					    " shards will be directly saved to a Google Cloud Storage bucket."  ,  )

					_snake_case       :					Dict				       =   parser.parse_args()
					return args







def    A				(							UpperCAmelCase				):
					def fn(UpperCAmelCase				):
										return tokenizer(examples["text"]				)

					return fn







def    A				(							UpperCAmelCase				):
					_snake_case       :					List[str]				       =   []
					for i in range(len(tokenized_data["input_ids"]				)				):
										_snake_case       :					int				       =   {
										    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]				)				),
										    "attention_mask": tf.train.Feature(
										        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]				)				),
										}
										_snake_case       :					List[str]				       =   tf.train.Features(feature=UpperCAmelCase				)
										_snake_case       :					Dict				       =   tf.train.Example(features=UpperCAmelCase				)
										_snake_case       :					Any				       =   example.SerializeToString()
										records.append(UpperCAmelCase				)
					return records







def    A				(							UpperCAmelCase				):
					_snake_case       :					Optional[int]				       =   datasets.load_dataset(args.dataset_name  ,  args.dataset_config  ,  split=args.split				)

					if args.limit is not None:
										_snake_case       :					Dict				       =   min(len(UpperCAmelCase				)  ,  args.limit				)
										_snake_case       :					List[str]				       =   dataset.select(range(UpperCAmelCase				)				)
										print(F"""Limiting the dataset to {args.limit} entries."""				)

					_snake_case       :					Any				       =   AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

					# Handle output directory creation.
					# For serializing into a Google Cloud Storage Bucket, one needs to first
					# create a bucket.
					if "gs" not in args.output_dir:
										if not os.path.exists(args.output_dir				):
															os.makedirs(args.output_dir				)
										_snake_case       :					str				       =   os.path.join(args.output_dir  ,  args.split				)
										if not os.path.exists(UpperCAmelCase				):
															os.makedirs(UpperCAmelCase				)
					else:
										_snake_case       :					Union[str, Any]				       =   os.path.join(args.output_dir  ,  args.split				)

					# Tokenize the whole dataset at once.
					_snake_case       :					Optional[Any]				       =   tokenize_function(UpperCAmelCase				)
					_snake_case       :					Optional[int]				       =   dataset.map(UpperCAmelCase  ,  batched=UpperCAmelCase  ,  num_proc=4  ,  remove_columns=["text"]				)

					# We need to concatenate all our texts together, and then split the result
					# into chunks of a fixed size, which we will call block_size. To do this, we
					# will use the map method again, with the option batched=True. When we use batched=True,
					# the function we pass to map() will be passed multiple inputs at once, allowing us
					# to group them into more or fewer examples than we had in the input.
					# This allows us to create our new fixed-length samples. The advantage of this
					# method is that we don't lose a whole lot of content from the dataset compared to the
					# case where we simply tokenize with a pre-defined max_length.

					def group_texts(UpperCAmelCase				):
										# Concatenate all texts.
										_snake_case       :					Optional[int]				       =   {k: sum(examples[k]  ,  []				) for k in examples.keys()}
										_snake_case       :					Union[str, Any]				       =   len(concatenated_examples[list(examples.keys()				)[0]]				)
										# We drop the small remainder, though you could add padding instead if the model supports it
										# In this, as in all things, we advise you to follow your heart 🫀
										_snake_case       :					Tuple				       =   (total_length // args.max_length) * args.max_length
										# Split by chunks of max_len.
										_snake_case       :					List[Any]				       =   {
										    k: [t[i : i + args.max_length] for i in range(0  ,  UpperCAmelCase  ,  args.max_length				)]
										    for k, t in concatenated_examples.items()
										}
										return result

					_snake_case       :					Optional[int]				       =   dataset_tokenized.map(UpperCAmelCase  ,  batched=UpperCAmelCase  ,  batch_size=1_000  ,  num_proc=4				)

					_snake_case       :					Dict				       =   0
					_snake_case       :					Optional[Any]				       =   0
					for shard in range(0  ,  len(UpperCAmelCase				)  ,  args.shard_size				):
										_snake_case       :					Any				       =   grouped_dataset[shard : shard + args.shard_size]
										_snake_case       :					Optional[Any]				       =   len(dataset_snapshot["input_ids"]				)
										_snake_case       :					Optional[int]				       =   os.path.join(UpperCAmelCase  ,  F"""dataset-{shard_count}-{records_containing}.tfrecord"""				)
										_snake_case       :					Tuple				       =   get_serialized_examples(UpperCAmelCase				)

										with tf.io.TFRecordWriter(UpperCAmelCase				) as out_file:
															for i in range(len(UpperCAmelCase				)				):
																				_snake_case       :					int				       =   serialized_examples[i]
																				out_file.write(UpperCAmelCase				)
															print("Wrote file {} containing {} records".format(UpperCAmelCase  ,  UpperCAmelCase				)				)

										shard_count += 1
										total_records += records_containing

					with open(F"""split-{args.split}-records-count.txt"""  ,  "w"				) as f:
										print(F"""Total {args.split} records: {total_records}"""  ,  file=UpperCAmelCase				)


if __name__ == "__main__":
							__lowerCAmelCase :List[Any]     =    parse_args()
							main(args)






import sys
import turtle







def    A				(							UpperCAmelCase  ,  UpperCAmelCase				):
					return (pa[0] + pa[0]) / 2, (pa[1] + pa[1]) / 2







def    A				(							UpperCAmelCase  ,  UpperCAmelCase  ,  UpperCAmelCase  ,  UpperCAmelCase  ,  ):
					my_pen.up()
					my_pen.goto(vertexa[0]  ,  vertexa[1]				)
					my_pen.down()
					my_pen.goto(vertexa[0]  ,  vertexa[1]				)
					my_pen.goto(vertexa[0]  ,  vertexa[1]				)
					my_pen.goto(vertexa[0]  ,  vertexa[1]				)

					if depth == 0:
										return

					triangle(UpperCAmelCase  ,  get_mid(UpperCAmelCase  ,  UpperCAmelCase				)  ,  get_mid(UpperCAmelCase  ,  UpperCAmelCase				)  ,  depth - 1				)
					triangle(UpperCAmelCase  ,  get_mid(UpperCAmelCase  ,  UpperCAmelCase				)  ,  get_mid(UpperCAmelCase  ,  UpperCAmelCase				)  ,  depth - 1				)
					triangle(UpperCAmelCase  ,  get_mid(UpperCAmelCase  ,  UpperCAmelCase				)  ,  get_mid(UpperCAmelCase  ,  UpperCAmelCase				)  ,  depth - 1				)


if __name__ == "__main__":
							if len(sys.argv) != 2:
														raise ValueError(
														    'Correct format for using this script: '
														    'python fractals.py <int:depth_for_fractal>'
														)
							__lowerCAmelCase :Any     =    turtle.Turtle()
							my_pen.ht()
							my_pen.speed(5)
							my_pen.pencolor('red')

							__lowerCAmelCase :Optional[int]     =    [(-1_75, -1_25), (0, 1_75), (1_75, -1_25)]  # vertices of triangle
							triangle(vertices[0], vertices[1], vertices[2], int(sys.argv[1]))