
"""simple docstring"""





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_a								=						logging.getLogger(__name__)



def  lowerCamelCase__						(     )				->						str:


					"""simple docstring"""





					_UpperCamelCase									=    argparse.ArgumentParser(
					    description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''      )
					parser.add_argument(
					    '''--dataset_name''',	type=__snake_case,	default='''wikitext''',	help='''Name of the training. Explore datasets at: hf.co/datasets.''',	)
					parser.add_argument(
					    '''--dataset_config''',	type=__snake_case,	default='''wikitext-103-raw-v1''',	help='''Configuration name of the dataset.'''      )
					parser.add_argument(
					    '''--tokenizer_name_or_path''',	type=__snake_case,	default='''sayakpaul/unigram-tokenizer-wikitext''',	help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.''',	)
					parser.add_argument(
					    '''--shard_size''',	type=__snake_case,	default=10_00,	help='''Number of entries to go in a single shard.''',	)
					parser.add_argument('''--split''',	type=__snake_case,	default='''train''',	choices=['''train''', '''test''', '''validation''']      )
					parser.add_argument(
					    '''--limit''',	default=__snake_case,	type=__snake_case,	help='''Limit the number of shards (used for debugging).''',	)
					parser.add_argument(
					    '''--max_length''',	type=__snake_case,	default=5_12,	help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
					    ''' sequence length that is a multiple of 8.''',	)
					parser.add_argument(
					    '''--output_dir''',	default='''tf-tpu''',	type=__snake_case,	help='''Output directory where the TFRecord shards will be saved. If the'''
					    ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
					    ''' shards will be directly saved to a Google Cloud Storage bucket.''',	)

					_UpperCamelCase									=    parser.parse_args()
					return args



def  lowerCamelCase__						(     __snake_case      )				->						Union[str, Any]:


					"""simple docstring"""





					def fn(__snake_case      ):
										return tokenizer(examples['''text''']      )

					return fn



def  lowerCamelCase__						(     __snake_case      )				->						str:


					"""simple docstring"""





					_UpperCamelCase									=    []
					for i in range(len(tokenized_data['''input_ids''']      )      ):
										_UpperCamelCase									=    {
										    '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]      )      ),
										    '''attention_mask''': tf.train.Feature(
										        intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]      )      ),
										}
										_UpperCamelCase									=    tf.train.Features(feature=__snake_case      )
										_UpperCamelCase									=    tf.train.Example(features=__snake_case      )
										_UpperCamelCase									=    example.SerializeToString()
										records.append(__snake_case      )
					return records



def  lowerCamelCase__						(     __snake_case      )				->						str:


					"""simple docstring"""





					_UpperCamelCase									=    datasets.load_dataset(args.dataset_name,	args.dataset_config,	split=args.split      )

					if args.limit is not None:
										_UpperCamelCase									=    min(len(__snake_case      ),	args.limit      )
										_UpperCamelCase									=    dataset.select(range(__snake_case      )      )
										print(F'''Limiting the dataset to {args.limit} entries.'''      )

					_UpperCamelCase									=    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

					# Handle output directory creation.
					# For serializing into a Google Cloud Storage Bucket, one needs to first
					# create a bucket.
					if "gs" not in args.output_dir:
										if not os.path.exists(args.output_dir      ):
															os.makedirs(args.output_dir      )
										_UpperCamelCase									=    os.path.join(args.output_dir,	args.split      )
										if not os.path.exists(__snake_case      ):
															os.makedirs(__snake_case      )
					else:
										_UpperCamelCase									=    os.path.join(args.output_dir,	args.split      )

					# Tokenize the whole dataset at once.
					_UpperCamelCase									=    tokenize_function(__snake_case      )
					_UpperCamelCase									=    dataset.map(__snake_case,	batched=__snake_case,	num_proc=4,	remove_columns=['''text''']      )

					# We need to concatenate all our texts together, and then split the result
					# into chunks of a fixed size, which we will call block_size. To do this, we
					# will use the map method again, with the option batched=True. When we use batched=True,
					# the function we pass to map() will be passed multiple inputs at once, allowing us
					# to group them into more or fewer examples than we had in the input.
					# This allows us to create our new fixed-length samples. The advantage of this
					# method is that we don't lose a whole lot of content from the dataset compared to the
					# case where we simply tokenize with a pre-defined max_length.

					def group_texts(__snake_case      ):
										# Concatenate all texts.
										_UpperCamelCase									=    {k: sum(examples[k],	[]      ) for k in examples.keys()}
										_UpperCamelCase									=    len(concatenated_examples[list(examples.keys()      )[0]]      )
										# We drop the small remainder, though you could add padding instead if the model supports it
										# In this, as in all things, we advise you to follow your heart 🫀
										_UpperCamelCase									=    (total_length // args.max_length) * args.max_length
										# Split by chunks of max_len.
										_UpperCamelCase									=    {
										    k: [t[i : i + args.max_length] for i in range(0,	__snake_case,	args.max_length      )]
										    for k, t in concatenated_examples.items()
										}
										return result

					_UpperCamelCase									=    dataset_tokenized.map(__snake_case,	batched=__snake_case,	batch_size=10_00,	num_proc=4      )

					_UpperCamelCase									=    0
					_UpperCamelCase									=    0
					for shard in range(0,	len(__snake_case      ),	args.shard_size      ):
										_UpperCamelCase									=    grouped_dataset[shard : shard + args.shard_size]
										_UpperCamelCase									=    len(dataset_snapshot['''input_ids''']      )
										_UpperCamelCase									=    os.path.join(__snake_case,	F'''dataset-{shard_count}-{records_containing}.tfrecord'''      )
										_UpperCamelCase									=    get_serialized_examples(__snake_case      )

										with tf.io.TFRecordWriter(__snake_case      ) as out_file:
															for i in range(len(__snake_case      )      ):
																				_UpperCamelCase									=    serialized_examples[i]
																				out_file.write(__snake_case      )
															print('''Wrote file {} containing {} records'''.format(__snake_case,	__snake_case      )      )

										shard_count += 1
										total_records += records_containing

					with open(F'''split-{args.split}-records-count.txt''',	'''w'''      ) as f:
										print(F'''Total {args.split} records: {total_records}''',	file=__snake_case      )


if __name__ == "__main__":
				_a								=						parse_args()
				main(args)










'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


_UpperCamelCase 							=  {"""configuration_vit""": ["""VIT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ViTConfig""", """ViTOnnxConfig"""]}

try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  ["""ViTFeatureExtractor"""]
      _UpperCamelCase 							=  ["""ViTImageProcessor"""]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  [
          """VIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """ViTForImageClassification""",
          """ViTForMaskedImageModeling""",
          """ViTModel""",
          """ViTPreTrainedModel""",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  [
          """TFViTForImageClassification""",
          """TFViTModel""",
          """TFViTPreTrainedModel""",
      ]

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  [
          """FlaxViTForImageClassification""",
          """FlaxViTModel""",
          """FlaxViTPreTrainedModel""",
      ]

if TYPE_CHECKING:
      from .configuration_vit import VIT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTConfig, ViTOnnxConfig

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .feature_extraction_vit import ViTFeatureExtractor
            from .image_processing_vit import ViTImageProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_vit import (
                VIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                ViTForImageClassification,
                ViTForMaskedImageModeling,
                ViTModel,
                ViTPreTrainedModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel, FlaxViTPreTrainedModel


else:
      import sys

      _UpperCamelCase 							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

