


import datasets

from .evaluate import evaluate


lowerCAmelCase__			:      List[str]								='\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n'

lowerCAmelCase__			:      Any								='\nThis metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n'

lowerCAmelCase__			:      Tuple								='\nComputes SQuAD scores (F1 and EM).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair as given in the references (see below)\n        - \'prediction_text\': the text of the answer\n    references: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair (see above),\n        - \'answers\': a Dict in the SQuAD dataset format\n            {\n                \'text\': list of possible texts for the answer, as a list of strings\n                \'answer_start\': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)\n    \'f1\': The F-score of predicted tokens versus the gold answer\nExamples:\n\n    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> squad_metric = datasets.load_metric("squad")\n    >>> results = squad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 100.0, \'f1\': 100.0}\n'







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,       _KWARGS_DESCRIPTION  )
class 	__lowercase    (datasets.Metric  ):





							"""simple docstring"""







							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										return datasets.MetricInfo(
										    description=_DESCRIPTION			,						citation=_CITATION			,						inputs_description=_KWARGS_DESCRIPTION			,						features=datasets.Features(
										        {
										            'predictions': {'id': datasets.Value('string'       ), 'prediction_text': datasets.Value('string'       )},
										            'references': {
										                'id': datasets.Value('string'       ),
										                'answers': datasets.features.Sequence(
										                    {
										                        'text': datasets.Value('string'       ),
										                        'answer_start': datasets.Value('int32'       ),
										                    }       ),
										            },
										        }       )			,						codebase_urls=['https://rajpurkar.github.io/SQuAD-explorer/']			,						reference_urls=['https://rajpurkar.github.io/SQuAD-explorer/']			,						)






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					{prediction['id']: prediction['prediction_text'] for prediction in predictions}
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					[
										    {
										        'paragraphs': [
										            {
										                'qas': [
										                    {
										                        'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
										                        'id': ref['id'],
										                    }
										                    for ref in references
										                ]
										            }
										        ]
										    }
										]
										SCREAMING_SNAKE_CASE_		:     Tuple					=					evaluate(dataset=_lowerCamelCase			,						predictions=_lowerCamelCase       )
										return score







"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": 512 for name in _model_names}
__SCREAMING_SNAKE_CASE							=     {F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      FunnelTokenizer
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      2



     def __init__(							self :str      ,  _lowerCamelCase :int=None      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Optional[Any]="<unk>"      ,  _lowerCamelCase :Optional[Any]="<sep>"      ,  _lowerCamelCase :List[Any]="<pad>"      ,  _lowerCamelCase :List[Any]="<cls>"      ,  _lowerCamelCase :Union[str, Any]="<mask>"      ,  _lowerCamelCase :Any="<s>"      ,  _lowerCamelCase :Union[str, Any]="</s>"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Dict=True      ,  _lowerCamelCase :Tuple=None      ,  _lowerCamelCase :Tuple="##"      ,  **_lowerCamelCase :int      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  bos_token=_lowerCamelCase      ,  eos_token=_lowerCamelCase      ,  clean_text=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  wordpieces_prefix=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					str   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          normalizer_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or normalizer_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'      ,  _lowerCamelCase   ) != tokenize_chinese_chars
      ):
       UpperCamelCase_     :					int   	=getattr(_lowerCamelCase      ,  normalizer_state.pop('type'   )   )
       UpperCamelCase_     :					Dict   	=do_lower_case
       UpperCamelCase_     :					Tuple   	=strip_accents
       UpperCamelCase_     :					int   	=tokenize_chinese_chars
       UpperCamelCase_     :					Tuple   	=normalizer_class(**_lowerCamelCase   )

      UpperCamelCase_     :					Dict   	=do_lower_case



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[int]=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[int]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0]
      return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )
