




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


a_   		=     logging.getLogger(__name__)




def 					__SCREAMING_SNAKE_CASE   (  )       ->	Union[str, Any]:

    """simple docstring"""


    __UpperCamelCase  		=						argparse.ArgumentParser(
        description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''      )
    parser.add_argument(
        '''--dataset_name'''					,     type=lowercase_					,     default='''wikitext'''					,     help='''Name of the training. Explore datasets at: hf.co/datasets.'''					,     )
    parser.add_argument(
        '''--dataset_config'''					,     type=lowercase_					,     default='''wikitext-103-raw-v1'''					,     help='''Configuration name of the dataset.'''      )
    parser.add_argument(
        '''--tokenizer_name_or_path'''					,     type=lowercase_					,     default='''sayakpaul/unigram-tokenizer-wikitext'''					,     help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''					,     )
    parser.add_argument(
        '''--shard_size'''					,     type=lowercase_					,     default=10_00					,     help='''Number of entries to go in a single shard.'''					,     )
    parser.add_argument('''--split'''					,     type=lowercase_					,     default='''train'''					,     choices=['''train''', '''test''', '''validation''']      )
    parser.add_argument(
        '''--limit'''					,     default=lowercase_					,     type=lowercase_					,     help='''Limit the number of shards (used for debugging).'''					,     )
    parser.add_argument(
        '''--max_length'''					,     type=lowercase_					,     default=5_12					,     help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
        ''' sequence length that is a multiple of 8.'''					,     )
    parser.add_argument(
        '''--output_dir'''					,     default='''tf-tpu'''					,     type=lowercase_					,     help='''Output directory where the TFRecord shards will be saved. If the'''
        ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
        ''' shards will be directly saved to a Google Cloud Storage bucket.'''					,     )

    __UpperCamelCase  		=						parser.parse_args()
    return args




def 					__SCREAMING_SNAKE_CASE   (  lowercase_      )       ->	Optional[int]:

    """simple docstring"""


    def fn(lowercase_      ):
        return tokenizer(examples['''text''']      )

    return fn




def 					__SCREAMING_SNAKE_CASE   (  lowercase_      )       ->	List[Any]:

    """simple docstring"""


    __UpperCamelCase  		=						[]
    for i in range(len(tokenized_data['''input_ids''']      )      ):
        __UpperCamelCase  		=						{
            '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]      )      ),
            '''attention_mask''': tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]      )      ),
        }
        __UpperCamelCase  		=						tf.train.Features(feature=lowercase_      )
        __UpperCamelCase  		=						tf.train.Example(features=lowercase_      )
        __UpperCamelCase  		=						example.SerializeToString()
        records.append(lowercase_      )
    return records




def 					__SCREAMING_SNAKE_CASE   (  lowercase_      )       ->	str:

    """simple docstring"""


    __UpperCamelCase  		=						datasets.load_dataset(args.dataset_name					,     args.dataset_config					,     split=args.split      )

    if args.limit is not None:
        __UpperCamelCase  		=						min(len(lowercase_      )					,     args.limit      )
        __UpperCamelCase  		=						dataset.select(range(lowercase_      )      )
        print(F"Limiting the dataset to {args.limit} entries."      )

    __UpperCamelCase  		=						AutoTokenizer.from_pretrained(args.tokenizer_name_or_path      )

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir      ):
            os.makedirs(args.output_dir      )
        __UpperCamelCase  		=						os.path.join(args.output_dir					,     args.split      )
        if not os.path.exists(lowercase_      ):
            os.makedirs(lowercase_      )
    else:
        __UpperCamelCase  		=						os.path.join(args.output_dir					,     args.split      )

    # Tokenize the whole dataset at once.
    __UpperCamelCase  		=						tokenize_function(lowercase_      )
    __UpperCamelCase  		=						dataset.map(lowercase_					,     batched=lowercase_					,     num_proc=4					,     remove_columns=['''text''']      )

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(lowercase_      ):
        # Concatenate all texts.
        __UpperCamelCase  		=						{k: sum(examples[k]					,     []      ) for k in examples.keys()}
        __UpperCamelCase  		=						len(concatenated_examples[list(examples.keys()      )[0]]      )
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        __UpperCamelCase  		=						(total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        __UpperCamelCase  		=						{
            k: [t[i : i + args.max_length] for i in range(0					,     lowercase_					,     args.max_length      )]
            for k, t in concatenated_examples.items()
        }
        return result

    __UpperCamelCase  		=						dataset_tokenized.map(lowercase_					,     batched=lowercase_					,     batch_size=10_00					,     num_proc=4      )

    __UpperCamelCase  		=						0
    __UpperCamelCase  		=						0
    for shard in range(0					,     len(lowercase_      )					,     args.shard_size      ):
        __UpperCamelCase  		=						grouped_dataset[shard : shard + args.shard_size]
        __UpperCamelCase  		=						len(dataset_snapshot['''input_ids''']      )
        __UpperCamelCase  		=						os.path.join(lowercase_					,     F"dataset-{shard_count}-{records_containing}.tfrecord"      )
        __UpperCamelCase  		=						get_serialized_examples(lowercase_      )

        with tf.io.TFRecordWriter(lowercase_      ) as out_file:
            for i in range(len(lowercase_      )      ):
                __UpperCamelCase  		=						serialized_examples[i]
                out_file.write(lowercase_      )
            print('''Wrote file {} containing {} records'''.format(lowercase_					,     lowercase_      )      )

        shard_count += 1
        total_records += records_containing

    with open(F"split-{args.split}-records-count.txt"					,     '''w'''      ) as f:
        print(F"Total {args.split} records: {total_records}"					,     file=lowercase_      )


if __name__ == "__main__":
 a_   		=     parse_args()
 main(args)












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_   		=     {
    "configuration_longformer": [
        "LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "LongformerConfig",
        "LongformerOnnxConfig",
    ],
    "tokenization_longformer": ["LongformerTokenizer"],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   		=     ["LongformerTokenizerFast"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   		=     [
     "LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
     "LongformerForMaskedLM",
     "LongformerForMultipleChoice",
     "LongformerForQuestionAnswering",
     "LongformerForSequenceClassification",
     "LongformerForTokenClassification",
     "LongformerModel",
     "LongformerPreTrainedModel",
     "LongformerSelfAttention",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   		=     [
     "TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFLongformerForMaskedLM",
     "TFLongformerForMultipleChoice",
     "TFLongformerForQuestionAnswering",
     "TFLongformerForSequenceClassification",
     "TFLongformerForTokenClassification",
     "TFLongformerModel",
     "TFLongformerPreTrainedModel",
     "TFLongformerSelfAttention",
 ]


if TYPE_CHECKING:
 from .configuration_longformer import (
     LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
     LongformerConfig,
     LongformerOnnxConfig,
 )
 from .tokenization_longformer import LongformerTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_longformer_fast import LongformerTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_longformer import (
      LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
      LongformerForMaskedLM,
      LongformerForMultipleChoice,
      LongformerForQuestionAnswering,
      LongformerForSequenceClassification,
      LongformerForTokenClassification,
      LongformerModel,
      LongformerPreTrainedModel,
      LongformerSelfAttention,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_longformer import (
      TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFLongformerForMaskedLM,
      TFLongformerForMultipleChoice,
      TFLongformerForQuestionAnswering,
      TFLongformerForSequenceClassification,
      TFLongformerForTokenClassification,
      TFLongformerModel,
      TFLongformerPreTrainedModel,
      TFLongformerSelfAttention,
  )

else:
 import sys

 a_   		=     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






