


import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__magic_name__  	=     logging.getLogger(__name__)







def 							__magic_name__    (     ):
	'''simple docstring'''


	lowerCamelCase_					:				Dict              =  argparse.ArgumentParser(
	    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.")
	parser.add_argument(
	    "--dataset_name"     ,	type=a_     ,	default="wikitext"     ,	help="Name of the training. Explore datasets at: hf.co/datasets."     ,	)
	parser.add_argument(
	    "--dataset_config"     ,	type=a_     ,	default="wikitext-103-raw-v1"     ,	help="Configuration name of the dataset.")
	parser.add_argument(
	    "--tokenizer_name_or_path"     ,	type=a_     ,	default="sayakpaul/unigram-tokenizer-wikitext"     ,	help="Tokenizer identifier. Can be a local filepath or a Hub identifier."     ,	)
	parser.add_argument(
	    "--shard_size"     ,	type=a_     ,	default=1000     ,	help="Number of entries to go in a single shard."     ,	)
	parser.add_argument("--split"     ,	type=a_     ,	default="train"     ,	choices=["train", "test", "validation"])
	parser.add_argument(
	    "--limit"     ,	default=a_     ,	type=a_     ,	help="Limit the number of shards (used for debugging)."     ,	)
	parser.add_argument(
	    "--max_length"     ,	type=a_     ,	default=512     ,	help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
	    " sequence length that is a multiple of 8."     ,	)
	parser.add_argument(
	    "--output_dir"     ,	default="tf-tpu"     ,	type=a_     ,	help="Output directory where the TFRecord shards will be saved. If the"
	    " path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"
	    " shards will be directly saved to a Google Cloud Storage bucket."     ,	)

	lowerCamelCase_					:				List[Any]              =  parser.parse_args()
	return args







def 							__magic_name__    (     lowerCAmelCase_):
	'''simple docstring'''


	def fn(lowerCAmelCase_):
		return tokenizer(examples["text"])

	return fn







def 							__magic_name__    (     lowerCAmelCase_):
	'''simple docstring'''


	lowerCamelCase_					:				Union[str, Any]              =  []
	for i in range(len(tokenized_data["input_ids"])):
		lowerCamelCase_					:				Union[str, Any]              =  {
		    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i])),
		    "attention_mask": tf.train.Feature(
		        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i])),
		}
		lowerCamelCase_					:				Optional[int]              =  tf.train.Features(feature=a_)
		lowerCamelCase_					:				Tuple              =  tf.train.Example(features=a_)
		lowerCamelCase_					:				Optional[Any]              =  example.SerializeToString()
		records.append(a_)
	return records







def 							__magic_name__    (     lowerCAmelCase_):
	'''simple docstring'''


	lowerCamelCase_					:				List[Any]              =  datasets.load_dataset(args.dataset_name     ,	args.dataset_config     ,	split=args.split)

	if args.limit is not None:
		lowerCamelCase_					:				Optional[Any]              =  min(len(a_)     ,	args.limit)
		lowerCamelCase_					:				Tuple              =  dataset.select(range(a_))
		print(F"""Limiting the dataset to {args.limit} entries.""")

	lowerCamelCase_					:				str              =  AutoTokenizer.from_pretrained(args.tokenizer_name_or_path)

	# Handle output directory creation.
	# For serializing into a Google Cloud Storage Bucket, one needs to first
	# create a bucket.
	if "gs" not in args.output_dir:
		if not os.path.exists(args.output_dir):
			os.makedirs(args.output_dir)
		lowerCamelCase_					:				str              =  os.path.join(args.output_dir     ,	args.split)
		if not os.path.exists(a_):
			os.makedirs(a_)
	else:
		lowerCamelCase_					:				int              =  os.path.join(args.output_dir     ,	args.split)

	# Tokenize the whole dataset at once.
	lowerCamelCase_					:				str              =  tokenize_function(a_)
	lowerCamelCase_					:				List[Any]              =  dataset.map(a_     ,	batched=a_     ,	num_proc=4     ,	remove_columns=["text"])

	# We need to concatenate all our texts together, and then split the result
	# into chunks of a fixed size, which we will call block_size. To do this, we
	# will use the map method again, with the option batched=True. When we use batched=True,
	# the function we pass to map() will be passed multiple inputs at once, allowing us
	# to group them into more or fewer examples than we had in the input.
	# This allows us to create our new fixed-length samples. The advantage of this
	# method is that we don't lose a whole lot of content from the dataset compared to the
	# case where we simply tokenize with a pre-defined max_length.

	def group_texts(lowerCAmelCase_):
		# Concatenate all texts.
		lowerCamelCase_					:				List[Any]              =  {k: sum(examples[k]     ,	[]) for k in examples.keys()}
		lowerCamelCase_					:				int              =  len(concatenated_examples[list(examples.keys())[0]])
		# We drop the small remainder, though you could add padding instead if the model supports it
		# In this, as in all things, we advise you to follow your heart 🫀
		lowerCamelCase_					:				List[str]              =  (total_length // args.max_length) * args.max_length
		# Split by chunks of max_len.
		lowerCamelCase_					:				Optional[Any]              =  {
		    k: [t[i : i + args.max_length] for i in range(0     ,	a_     ,	args.max_length)]
		    for k, t in concatenated_examples.items()
		}
		return result

	lowerCamelCase_					:				List[Any]              =  dataset_tokenized.map(a_     ,	batched=a_     ,	batch_size=1000     ,	num_proc=4)

	lowerCamelCase_					:				Tuple              =  0
	lowerCamelCase_					:				Optional[int]              =  0
	for shard in range(0     ,	len(a_)     ,	args.shard_size):
		lowerCamelCase_					:				int              =  grouped_dataset[shard : shard + args.shard_size]
		lowerCamelCase_					:				str              =  len(dataset_snapshot["input_ids"])
		lowerCamelCase_					:				Any              =  os.path.join(a_     ,	F"""dataset-{shard_count}-{records_containing}.tfrecord""")
		lowerCamelCase_					:				Optional[Any]              =  get_serialized_examples(a_)

		with tf.io.TFRecordWriter(a_) as out_file:
			for i in range(len(a_)):
				lowerCamelCase_					:				Optional[Any]              =  serialized_examples[i]
				out_file.write(a_)
			print("Wrote file {} containing {} records".format(a_     ,	a_))

		shard_count += 1
		total_records += records_containing

	with open(F"""split-{args.split}-records-count.txt"""     ,	"w") as f:
		print(F"""Total {args.split} records: {total_records}"""     ,	file=a_)


if __name__ == "__main__":
							__magic_name__  	=     parse_args()
							main(args)







'''simple docstring'''


import warnings
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase_					=   logging.get_logger(__name__)

UpperCAmelCase_					=   {
    "nvidia/segformer-b0-finetuned-ade-512-512": (
        "https://huggingface.co/nvidia/segformer-b0-finetuned-ade-512-512/resolve/main/config.json"
    ),
    # See all SegFormer models at https://huggingface.co/models?filter=segformer
}





class  __lowercase    (	__magic_name__			):
   _a    						=   """segformer"""







   def __init__(						self   ,				UpperCamelCase=3   ,				UpperCamelCase=4   ,				UpperCamelCase=[2, 2, 2, 2]   ,				UpperCamelCase=[8, 4, 2, 1]   ,				UpperCamelCase=[32, 64, 160, 256]   ,				UpperCamelCase=[7, 3, 3, 3]   ,				UpperCamelCase=[4, 2, 2, 2]   ,				UpperCamelCase=[1, 2, 5, 8]   ,				UpperCamelCase=[4, 4, 4, 4]   ,				UpperCamelCase="gelu"   ,				UpperCamelCase=0.0   ,				UpperCamelCase=0.0   ,				UpperCamelCase=0.1   ,				UpperCamelCase=0.02   ,				UpperCamelCase=0.1   ,				UpperCamelCase=1e-6   ,				UpperCamelCase=256   ,				UpperCamelCase=255   ,				**UpperCamelCase   ,				)					->     int:
     super().__init__(**UpperCamelCase   )

     if "reshape_last_stage" in kwargs and kwargs["reshape_last_stage"] is False:
       warnings.warn(
           'Reshape_last_stage is set to False in this config. This argument is deprecated and will soon be'
           ' removed, as the behaviour will default to that of reshape_last_stage = True.'   ,				UpperCamelCase   ,				)

     __a								=       num_channels
     __a								=       num_encoder_blocks
     __a								=       depths
     __a								=       sr_ratios
     __a								=       hidden_sizes
     __a								=       patch_sizes
     __a								=       strides
     __a								=       mlp_ratios
     __a								=       num_attention_heads
     __a								=       hidden_act
     __a								=       hidden_dropout_prob
     __a								=       attention_probs_dropout_prob
     __a								=       classifier_dropout_prob
     __a								=       initializer_range
     __a								=       drop_path_rate
     __a								=       layer_norm_eps
     __a								=       decoder_hidden_size
     __a								=       kwargs.get('reshape_last_stage'   ,				UpperCamelCase   )
     __a								=       semantic_loss_ignore_index





class  __lowercase    (	__magic_name__			):
   _a    						=   version.parse("""1.11"""			)







   @property
   def 		UpperCamelCase__      (						self   )					->     Mapping[str, Mapping[int, str]]:
     return OrderedDict(
         [
             ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
         ]   )







   @property
   def 		UpperCamelCase__      (						self   )					->     float:
     return 1e-4







   @property
   def 		UpperCamelCase__      (						self   )					->     int:
     return 12



