

import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


lowercase_			=  logging.getLogger(__name__)
def 						_snake_case(   )							->				Tuple:





    '''simple docstring'''






    A__   				=       argparse.ArgumentParser(
        description='Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'				)
    parser.add_argument(
        '--dataset_name'   ,      type=SCREAMING_SNAKE_CASE__   ,      default='wikitext'   ,      help='Name of the training. Explore datasets at: hf.co/datasets.'   ,      )
    parser.add_argument(
        '--dataset_config'   ,      type=SCREAMING_SNAKE_CASE__   ,      default='wikitext-103-raw-v1'   ,      help='Configuration name of the dataset.'				)
    parser.add_argument(
        '--tokenizer_name_or_path'   ,      type=SCREAMING_SNAKE_CASE__   ,      default='sayakpaul/unigram-tokenizer-wikitext'   ,      help='Tokenizer identifier. Can be a local filepath or a Hub identifier.'   ,      )
    parser.add_argument(
        '--shard_size'   ,      type=SCREAMING_SNAKE_CASE__   ,      default=1000   ,      help='Number of entries to go in a single shard.'   ,      )
    parser.add_argument('--split'   ,      type=SCREAMING_SNAKE_CASE__   ,      default='train'   ,      choices=['train', 'test', 'validation']				)
    parser.add_argument(
        '--limit'   ,      default=SCREAMING_SNAKE_CASE__   ,      type=SCREAMING_SNAKE_CASE__   ,      help='Limit the number of shards (used for debugging).'   ,      )
    parser.add_argument(
        '--max_length'   ,      type=SCREAMING_SNAKE_CASE__   ,      default=512   ,      help='Maximum sequence length. For training on TPUs, it helps to have a maximum'
        ' sequence length that is a multiple of 8.'   ,      )
    parser.add_argument(
        '--output_dir'   ,      default='tf-tpu'   ,      type=SCREAMING_SNAKE_CASE__   ,      help='Output directory where the TFRecord shards will be saved. If the'
        ' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'
        ' shards will be directly saved to a Google Cloud Storage bucket.'   ,      )

    A__   				=       parser.parse_args()
    return args
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     int				)							->				int:





    '''simple docstring'''






    def fn(SCREAMING_SNAKE_CASE__      :     Optional[Any]				):
        return tokenizer(examples['text']				)

    return fn
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     Tuple				)							->				List[Any]:





    '''simple docstring'''






    A__   				=       []
    for i in range(len(tokenized_data['input_ids']				)				):
        A__   				=       {
            'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['input_ids'][i]				)				),
            'attention_mask': tf.train.Feature(
                intaa_list=tf.train.IntaaList(value=tokenized_data['attention_mask'][i]				)				),
        }
        A__   				=       tf.train.Features(feature=SCREAMING_SNAKE_CASE__				)
        A__   				=       tf.train.Example(features=SCREAMING_SNAKE_CASE__				)
        A__   				=       example.SerializeToString()
        records.append(SCREAMING_SNAKE_CASE__				)
    return records
def 						_snake_case(   SCREAMING_SNAKE_CASE__      :     Optional[int]				)							->				List[str]:





    '''simple docstring'''






    A__   				=       datasets.load_dataset(args.dataset_name   ,      args.dataset_config   ,      split=args.split				)

    if args.limit is not None:
        A__   				=       min(len(SCREAMING_SNAKE_CASE__				)   ,      args.limit				)
        A__   				=       dataset.select(range(SCREAMING_SNAKE_CASE__				)				)
        print(f'Limiting the dataset to {args.limit} entries.'				)

    A__   				=       AutoTokenizer.from_pretrained(args.tokenizer_name_or_path				)

    # Handle output directory creation.
    # For serializing into a Google Cloud Storage Bucket, one needs to first
    # create a bucket.
    if "gs" not in args.output_dir:
        if not os.path.exists(args.output_dir				):
            os.makedirs(args.output_dir				)
        A__   				=       os.path.join(args.output_dir   ,      args.split				)
        if not os.path.exists(SCREAMING_SNAKE_CASE__				):
            os.makedirs(SCREAMING_SNAKE_CASE__				)
    else:
        A__   				=       os.path.join(args.output_dir   ,      args.split				)

    # Tokenize the whole dataset at once.
    A__   				=       tokenize_function(SCREAMING_SNAKE_CASE__				)
    A__   				=       dataset.map(SCREAMING_SNAKE_CASE__   ,      batched=SCREAMING_SNAKE_CASE__   ,      num_proc=4   ,      remove_columns=['text']				)

    # We need to concatenate all our texts together, and then split the result
    # into chunks of a fixed size, which we will call block_size. To do this, we
    # will use the map method again, with the option batched=True. When we use batched=True,
    # the function we pass to map() will be passed multiple inputs at once, allowing us
    # to group them into more or fewer examples than we had in the input.
    # This allows us to create our new fixed-length samples. The advantage of this
    # method is that we don't lose a whole lot of content from the dataset compared to the
    # case where we simply tokenize with a pre-defined max_length.

    def group_texts(SCREAMING_SNAKE_CASE__      :     List[str]				):
        # Concatenate all texts.
        A__   				=       {k: sum(examples[k]   ,      []				) for k in examples.keys()}
        A__   				=       len(concatenated_examples[list(examples.keys()				)[0]]				)
        # We drop the small remainder, though you could add padding instead if the model supports it
        # In this, as in all things, we advise you to follow your heart 🫀
        A__   				=       (total_length // args.max_length) * args.max_length
        # Split by chunks of max_len.
        A__   				=       {
            k: [t[i : i + args.max_length] for i in range(0   ,      SCREAMING_SNAKE_CASE__   ,      args.max_length				)]
            for k, t in concatenated_examples.items()
        }
        return result

    A__   				=       dataset_tokenized.map(SCREAMING_SNAKE_CASE__   ,      batched=SCREAMING_SNAKE_CASE__   ,      batch_size=1000   ,      num_proc=4				)

    A__   				=       0
    A__   				=       0
    for shard in range(0   ,      len(SCREAMING_SNAKE_CASE__				)   ,      args.shard_size				):
        A__   				=       grouped_dataset[shard : shard + args.shard_size]
        A__   				=       len(dataset_snapshot['input_ids']				)
        A__   				=       os.path.join(SCREAMING_SNAKE_CASE__   ,      f'dataset-{shard_count}-{records_containing}.tfrecord'				)
        A__   				=       get_serialized_examples(SCREAMING_SNAKE_CASE__				)

        with tf.io.TFRecordWriter(SCREAMING_SNAKE_CASE__				) as out_file:
            for i in range(len(SCREAMING_SNAKE_CASE__				)				):
                A__   				=       serialized_examples[i]
                out_file.write(SCREAMING_SNAKE_CASE__				)
            print('Wrote file {} containing {} records'.format(SCREAMING_SNAKE_CASE__   ,      SCREAMING_SNAKE_CASE__				)				)

        shard_count += 1
        total_records += records_containing

    with open(f'split-{args.split}-records-count.txt'   ,      'w'				) as f:
        print(f'Total {args.split} records: {total_records}'   ,      file=SCREAMING_SNAKE_CASE__				)


if __name__ == "__main__":
     lowercase_			=  parse_args()
     main(args)




import datasets

from .evaluate import evaluate


lowercase_			=  "\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n"

lowercase_			=  "\nThis metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n"

lowercase_			=  "\nComputes SQuAD scores (F1 and EM).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair as given in the references (see below)\n        - 'prediction_text': the text of the answer\n    references: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair (see above),\n        - 'answers': a Dict in the SQuAD dataset format\n            {\n                'text': list of possible texts for the answer, as a list of strings\n                'answer_start': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    'exact_match': Exact match (the normalized answer exactly match the gold answer)\n    'f1': The F-score of predicted tokens versus the gold answer\nExamples:\n\n    >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}]\n    >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}]\n    >>> squad_metric = datasets.load_metric(\"squad\")\n    >>> results = squad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 100.0, 'f1': 100.0}\n"


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,							_KWARGS_DESCRIPTION			)
class 			A						(							datasets.Metric			):







    """simple docstring"""





    def 						snake_case__					(				self			:	List[Any]		)->   Tuple:





        '''simple docstring'''







        return datasets.MetricInfo(
            description=_DESCRIPTION,citation=_CITATION,inputs_description=_KWARGS_DESCRIPTION,features=datasets.Features(
                {
                    'predictions': {'id': datasets.Value('string'		), 'prediction_text': datasets.Value('string'		)},
                    'references': {
                        'id': datasets.Value('string'		),
                        'answers': datasets.features.Sequence(
                            {
                                'text': datasets.Value('string'		),
                                'answer_start': datasets.Value('int32'		),
                            }		),
                    },
                }		),codebase_urls=['https://rajpurkar.github.io/SQuAD-explorer/'],reference_urls=['https://rajpurkar.github.io/SQuAD-explorer/'],)


    def 						snake_case__					(				self			:	str,lowercase_			:	List[Any],lowercase_			:	Dict		)->   List[str]:





        '''simple docstring'''







        A__   				=       {prediction['id']: prediction['prediction_text'] for prediction in predictions}
        A__   				=       [
            {
                'paragraphs': [
                    {
                        'qas': [
                            {
                                'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
                                'id': ref['id'],
                            }
                            for ref in references
                        ]
                    }
                ]
            }
        ]
        A__   				=       evaluate(dataset=lowercase_,predictions=lowercase_		)
        return score

