
import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


_snake_case      = logging.getLogger(__name__)

def  __lowerCamelCase					( )       ->      Union[str, Any]:
     UpperCamelCase						    =					argparse.ArgumentParser(
         description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'			)
     parser.add_argument(
         '--dataset_name'						,     type=_lowercase						,     default='wikitext'						,     help='Name of the training. Explore datasets at: hf.co/datasets.'						,     )
     parser.add_argument(
         '--dataset_config'						,     type=_lowercase						,     default='wikitext-103-raw-v1'						,     help='Configuration name of the dataset.'			)
     parser.add_argument(
         '--tokenizer_name_or_path'						,     type=_lowercase						,     default='sayakpaul/unigram-tokenizer-wikitext'						,     help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'						,     )
     parser.add_argument(
         '--shard_size'						,     type=_lowercase						,     default=1000						,     help='Number of entries to go in a single shard.'						,     )
     parser.add_argument('--split'						,     type=_lowercase						,     default='train'						,     choices=['train', 'test', 'validation']			)
     parser.add_argument(
         '--limit'						,     default=_lowercase						,     type=_lowercase						,     help='Limit the number of shards (used for debugging).'						,     )
     parser.add_argument(
         '--max_length'						,     type=_lowercase						,     default=512						,     help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
         ' sequence length that is a multiple of 8.'						,     )
     parser.add_argument(
         '--output_dir'						,     default='tf-tpu'						,     type=_lowercase						,     help='Output directory where the TFRecord shards will be saved. If the'
         ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
         ' shards will be directly saved to a Google Cloud Storage bucket.'						,     )

     UpperCamelCase						    =					parser.parse_args()
     return args

def  __lowerCamelCase					( _lowercase			)       ->      List[str]:
     def fn(_lowercase			):
          return tokenizer(examples['text']			)

     return fn

def  __lowerCamelCase					( _lowercase			)       ->      Dict:
     UpperCamelCase						    =					[]
     for i in range(len(tokenized_data['input_ids']			)			):
          UpperCamelCase						    =					{
              'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]			)			),
              'attention_mask': tf.train.Feature(
                  intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]			)			),
          }
          UpperCamelCase						    =					tf.train.Features(feature=_lowercase			)
          UpperCamelCase						    =					tf.train.Example(features=_lowercase			)
          UpperCamelCase						    =					example.SerializeToString()
          records.append(_lowercase			)
     return records

def  __lowerCamelCase					( _lowercase			)       ->      Any:
     UpperCamelCase						    =					datasets.load_dataset(args.dataset_name						,     args.dataset_config						,     split=args.split			)

     if args.limit is not None:
          UpperCamelCase						    =					min(len(_lowercase			)						,     args.limit			)
          UpperCamelCase						    =					dataset.select(range(_lowercase			)			)
          print(F'Limiting the dataset to {args.limit} entries.'			)

     UpperCamelCase						    =					AutoTokenizer.from_pretrained(args.tokenizer_name_or_path			)

     # Handle output directory creation.
     # For serializing into a Google Cloud Storage Bucket, one needs to first
     # create a bucket.
     if "gs" not in args.output_dir:
          if not os.path.exists(args.output_dir			):
               os.makedirs(args.output_dir			)
          UpperCamelCase						    =					os.path.join(args.output_dir						,     args.split			)
          if not os.path.exists(_lowercase			):
               os.makedirs(_lowercase			)
     else:
          UpperCamelCase						    =					os.path.join(args.output_dir						,     args.split			)

     # Tokenize the whole dataset at once.
     UpperCamelCase						    =					tokenize_function(_lowercase			)
     UpperCamelCase						    =					dataset.map(_lowercase						,     batched=_lowercase						,     num_proc=4						,     remove_columns=['text']			)

     # We need to concatenate all our texts together, and then split the result
     # into chunks of a fixed size, which we will call block_size. To do this, we
     # will use the map method again, with the option batched=True. When we use batched=True,
     # the function we pass to map() will be passed multiple inputs at once, allowing us
     # to group them into more or fewer examples than we had in the input.
     # This allows us to create our new fixed-length samples. The advantage of this
     # method is that we don't lose a whole lot of content from the dataset compared to the
     # case where we simply tokenize with a pre-defined max_length.

     def group_texts(_lowercase			):
          # Concatenate all texts.
          UpperCamelCase						    =					{k: sum(examples[k]						,     []			) for k in examples.keys()}
          UpperCamelCase						    =					len(concatenated_examples[list(examples.keys()			)[0]]			)
          # We drop the small remainder, though you could add padding instead if the model supports it
          # In this, as in all things, we advise you to follow your heart 🫀
          UpperCamelCase						    =					(total_length // args.max_length) * args.max_length
          # Split by chunks of max_len.
          UpperCamelCase						    =					{
              k: [t[i : i + args.max_length] for i in range(0						,     _lowercase						,     args.max_length			)]
              for k, t in concatenated_examples.items()
          }
          return result

     UpperCamelCase						    =					dataset_tokenized.map(_lowercase						,     batched=_lowercase						,     batch_size=1000						,     num_proc=4			)

     UpperCamelCase						    =					0
     UpperCamelCase						    =					0
     for shard in range(0						,     len(_lowercase			)						,     args.shard_size			):
          UpperCamelCase						    =					grouped_dataset[shard : shard + args.shard_size]
          UpperCamelCase						    =					len(dataset_snapshot['input_ids']			)
          UpperCamelCase						    =					os.path.join(_lowercase						,     F'dataset-{shard_count}-{records_containing}.tfrecord'			)
          UpperCamelCase						    =					get_serialized_examples(_lowercase			)

          with tf.io.TFRecordWriter(_lowercase			) as out_file:
               for i in range(len(_lowercase			)			):
                    UpperCamelCase						    =					serialized_examples[i]
                    out_file.write(_lowercase			)
               print('Wrote file {} containing {} records'.format(_lowercase						,     _lowercase			)			)

          shard_count += 1
          total_records += records_containing

     with open(F'split-{args.split}-records-count.txt'						,     'w'			) as f:
          print(F'Total {args.split} records: {total_records}'						,     file=_lowercase			)


if __name__ == "__main__":
 _snake_case      = parse_args()
 main(args)




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_snake_case      = logging.get_logger(__name__)

_snake_case      = {
    '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/config.json''',
    '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/config.json''',
    '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/config.json''',
    '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/config.json''',
    '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/config.json''',
    '''roberta-large-openai-detector''': '''https://huggingface.co/roberta-large-openai-detector/resolve/main/config.json''',
}
class       _lowerCAmelCase  (		__magic_name__							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					Dict       						="roberta"

 def __init__(						self			:			Union[str, Any]					, SCREAMING_SNAKE_CASE__			:			Tuple=5_02_65					, SCREAMING_SNAKE_CASE__			:			List[str]=7_68					, SCREAMING_SNAKE_CASE__			:			List[str]=12					, SCREAMING_SNAKE_CASE__			:			Any=12					, SCREAMING_SNAKE_CASE__			:			Tuple=30_72					, SCREAMING_SNAKE_CASE__			:			str="gelu"					, SCREAMING_SNAKE_CASE__			:			Union[str, Any]=0.1					, SCREAMING_SNAKE_CASE__			:			int=0.1					, SCREAMING_SNAKE_CASE__			:			int=5_12					, SCREAMING_SNAKE_CASE__			:			Optional[Any]=2					, SCREAMING_SNAKE_CASE__			:			Dict=0.02					, SCREAMING_SNAKE_CASE__			:			List[str]=1e-12					, SCREAMING_SNAKE_CASE__			:			Any=1					, SCREAMING_SNAKE_CASE__			:			Optional[Any]=0					, SCREAMING_SNAKE_CASE__			:			str=2					, SCREAMING_SNAKE_CASE__			:			Any="absolute"					, SCREAMING_SNAKE_CASE__			:			Any=True					, SCREAMING_SNAKE_CASE__			:			List[Any]=None					, **SCREAMING_SNAKE_CASE__			:			Optional[int]					, ):






      """simple docstring"""



      super().__init__(pad_token_id=SCREAMING_SNAKE_CASE__					, bos_token_id=SCREAMING_SNAKE_CASE__					, eos_token_id=SCREAMING_SNAKE_CASE__					, **SCREAMING_SNAKE_CASE__   )

      UpperCamelCase						    =					vocab_size
      UpperCamelCase						    =					hidden_size
      UpperCamelCase						    =					num_hidden_layers
      UpperCamelCase						    =					num_attention_heads
      UpperCamelCase						    =					hidden_act
      UpperCamelCase						    =					intermediate_size
      UpperCamelCase						    =					hidden_dropout_prob
      UpperCamelCase						    =					attention_probs_dropout_prob
      UpperCamelCase						    =					max_position_embeddings
      UpperCamelCase						    =					type_vocab_size
      UpperCamelCase						    =					initializer_range
      UpperCamelCase						    =					layer_norm_eps
      UpperCamelCase						    =					position_embedding_type
      UpperCamelCase						    =					use_cache
      UpperCamelCase						    =					classifier_dropout




class       _lowerCAmelCase  (		__magic_name__							):

 """simple docstring"""

 @property
 def      __lowerCAmelCase  (						self			:			Optional[int]   ):






      """simple docstring"""



      if self.task == "multiple-choice":
           UpperCamelCase						    =					{0: 'batch', 1: 'choice', 2: 'sequence'}
      else:
           UpperCamelCase						    =					{0: 'batch', 1: 'sequence'}
      return OrderedDict(
          [
              ('input_ids', dynamic_axis),
              ('attention_mask', dynamic_axis),
          ]   )


