


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_fnet import FNetTokenizer
else:
	snake_case__			 =						None

snake_case__			 =						logging.get_logger(__name__)
snake_case__			 =						{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case__			 =						{
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json''',
    },
}

snake_case__			 =						{
    '''google/fnet-base''': 512,
    '''google/fnet-large''': 512,
}

snake_case__			 =						'''▁'''


class 					lowerCAmelCase_ (       _a):
						lowerCamelCase_         =     VOCAB_FILES_NAMES
						lowerCamelCase_         =     PRETRAINED_VOCAB_FILES_MAP
						lowerCamelCase_         =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCamelCase_         =     ['input_ids', 'token_type_ids']
						lowerCamelCase_         =     FNetTokenizer




						def __init__(	self						:   Optional[int]   ,     __A						:   str=None   ,     __A						:   Optional[Any]=None   ,     __A						:   Dict=False   ,     __A						:   Dict=True   ,     __A						:   Optional[int]=True   ,     __A						:   Optional[Any]="<unk>"   ,     __A						:   Any="[SEP]"   ,     __A						:   Dict="<pad>"   ,     __A						:   Tuple="[CLS]"   ,     __A						:   List[str]="[MASK]"   ,     **__A						:   Any   ,     )      ->Optional[Any]:






									"""simple docstring"""






									a__ :Any						  =    (
									    AddedToken(__A   ,     lstrip=__A   ,     rstrip=__A   ,     normalized=__A						)
									    if isinstance(__A   ,     __A						)
									    else mask_token
									)

									super().__init__(
									    __A   ,     tokenizer_file=__A   ,     do_lower_case=__A   ,     remove_space=__A   ,     keep_accents=__A   ,     unk_token=__A   ,     sep_token=__A   ,     pad_token=__A   ,     cls_token=__A   ,     mask_token=__A   ,     **__A   ,     )

									a__ :Any						  =    do_lower_case
									a__ :str						  =    remove_space
									a__ :Tuple						  =    keep_accents
									a__ :Optional[int]						  =    vocab_file
									a__ :Optional[Any]						  =    False if not self.vocab_file else True




						def  _snake_case					(	self						:   int   ,     __A						:   List[int]   ,     __A						:   Optional[List[int]] = None						)      ->List[int]:






									"""simple docstring"""






									a__ :Dict						  =    [self.sep_token_id]
									a__ :Optional[int]						  =    [self.cls_token_id]
									if token_ids_a is None:
												return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep




						def  _snake_case					(	self						:   List[Any]   ,     __A						:   List[int]   ,     __A						:   Optional[List[int]] = None						)      ->List[int]:






									"""simple docstring"""






									a__ :Optional[int]						  =    [self.sep_token_id]
									a__ :str						  =    [self.cls_token_id]

									if token_ids_a is None:
												return len(cls + token_ids_a + sep						) * [0]
									return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






						def  _snake_case					(	self						:   Dict   ,     __A						:   str   ,     __A						:   Optional[str] = None						)      ->Tuple[str]:






									"""simple docstring"""






									if not os.path.isdir(__A						):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
												return
									a__ :str						  =    os.path.join(
									    __A   ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

									if os.path.abspath(self.vocab_file						) != os.path.abspath(__A						):
												copyfile(self.vocab_file   ,     __A						)

									return (out_vocab_file,)






import datasets

from .evaluate import evaluate


snake_case__			 =						'''\
@inproceedings{Rajpurkar2016SQuAD10,
  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},
  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
  booktitle={EMNLP},
  year={2016}
}
'''

snake_case__			 =						'''
This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).

Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by
crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,
from the corresponding reading passage, or the question might be unanswerable.
'''

snake_case__			 =						'''
Computes SQuAD scores (F1 and EM).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair as given in the references (see below)
        - \'prediction_text\': the text of the answer
    references: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair (see above),
        - \'answers\': a Dict in the SQuAD dataset format
            {
                \'text\': list of possible texts for the answer, as a list of strings
                \'answer_start\': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
Examples:

    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]
    >>> squad_metric = datasets.load_metric("squad")
    >>> results = squad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 100.0, \'f1\': 100.0}
'''


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,_KWARGS_DESCRIPTION)
class 					lowerCAmelCase_ (       datasets.Metric):




						def  _snake_case					(	self						:   List[str]						)      ->Union[str, Any]:






									"""simple docstring"""






									return datasets.MetricInfo(
									    description=_DESCRIPTION   ,     citation=_CITATION   ,     inputs_description=_KWARGS_DESCRIPTION   ,     features=datasets.Features(
									        {
									            "predictions": {"id": datasets.Value("string"						), "prediction_text": datasets.Value("string"						)},
									            "references": {
									                "id": datasets.Value("string"						),
									                "answers": datasets.features.Sequence(
									                    {
									                        "text": datasets.Value("string"						),
									                        "answer_start": datasets.Value("int32"						),
									                    }						),
									            },
									        }						)   ,     codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"]   ,     reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"]   ,     )






						def  _snake_case					(	self						:   Dict   ,     __A						:   List[Any]   ,     __A						:   Optional[int]						)      ->str:






									"""simple docstring"""






									a__ :Optional[int]						  =    {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
									a__ :Optional[Any]						  =    [
									    {
									        "paragraphs": [
									            {
									                "qas": [
									                    {
									                        "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
									                        "id": ref["id"],
									                    }
									                    for ref in references
									                ]
									            }
									        ]
									    }
									]
									a__ :Union[str, Any]						  =    evaluate(dataset=__A   ,     predictions=__A						)
									return score


