






"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_lowerCAmelCase       =				logging.get_logger(__name__)

_lowerCAmelCase       =				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase       =				[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_lowerCAmelCase       =				{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_lowerCAmelCase       =				{F'funnel-transformer/{name}': 512 for name in _model_names}
_lowerCAmelCase       =				{F'funnel-transformer/{name}': {"do_lower_case": True} for name in _model_names}



class   SCREAMING_SNAKE_CASE	(					_SCREAMING_SNAKE_CASE    ):





	"""simple docstring"""




	_A							:					Any 			=					VOCAB_FILES_NAMES
	_A							:					Optional[Any] 			=					PRETRAINED_VOCAB_FILES_MAP
	_A							:					Any 			=					PRETRAINED_INIT_CONFIGURATION
	_A							:					List[Any] 			=					FunnelTokenizer
	_A							:					List[Any] 			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A							:					int 			=					2





	def __init__(self  ,  lowerCAmelCase_=None  ,  lowerCAmelCase_=None  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_="<unk>"  ,  lowerCAmelCase_="<sep>"  ,  lowerCAmelCase_="<pad>"  ,  lowerCAmelCase_="<cls>"  ,  lowerCAmelCase_="<mask>"  ,  lowerCAmelCase_="<s>"  ,  lowerCAmelCase_="</s>"  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=None  ,  lowerCAmelCase_="##"  ,  **lowerCAmelCase_  ,  ):
		super().__init__(
		    lowerCAmelCase_  ,  tokenizer_file=lowerCAmelCase_  ,  do_lower_case=lowerCAmelCase_  ,  unk_token=lowerCAmelCase_  ,  sep_token=lowerCAmelCase_  ,  pad_token=lowerCAmelCase_  ,  cls_token=lowerCAmelCase_  ,  mask_token=lowerCAmelCase_  ,  bos_token=lowerCAmelCase_  ,  eos_token=lowerCAmelCase_  ,  clean_text=lowerCAmelCase_  ,  tokenize_chinese_chars=lowerCAmelCase_  ,  strip_accents=lowerCAmelCase_  ,  wordpieces_prefix=lowerCAmelCase_  ,  **lowerCAmelCase_  ,  )

		A_		:    List[Any]						     = json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
		if (
		    normalizer_state.get("""lowercase"""  ,  lowerCAmelCase_				) != do_lower_case
		    or normalizer_state.get("""strip_accents"""  ,  lowerCAmelCase_				) != strip_accents
		    or normalizer_state.get("""handle_chinese_chars"""  ,  lowerCAmelCase_				) != tokenize_chinese_chars
		):
			A_		:    str						     = getattr(lowerCAmelCase_  ,  normalizer_state.pop("""type"""				)				)
			A_		:    Any						     = do_lower_case
			A_		:    str						     = strip_accents
			A_		:    Any						     = tokenize_chinese_chars
			A_		:    int						     = normalizer_class(**lowerCAmelCase_				)

		A_		:    List[str]						     = do_lower_case





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_=None				):
		A_		:    Tuple						     = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

		if token_ids_a:
			output += token_ids_a + [self.sep_token_id]

		return output





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None				):
		A_		:    List[str]						     = [self.sep_token_id]
		A_		:    Optional[int]						     = [self.cls_token_id]
		if token_ids_a is None:
			return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
		return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None				):
		A_		:    Optional[Any]						     = self._tokenizer.model.save(lowerCAmelCase_  ,  name=lowerCAmelCase_				)
		return tuple(lowerCAmelCase_				)









"""simple docstring"""
import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


_lowerCAmelCase       =				"\\n@article{hendrycksmath2021,\n  title={Measuring Mathematical Problem Solving With the MATH Dataset},\n  author={Dan Hendrycks\n    and Collin Burns\n    and Saurav Kadavath\n    and Akul Arora\n    and Steven Basart\n    and Eric Tang\n    and Dawn Song\n    and Jacob Steinhardt},\n  journal={arXiv preprint arXiv:2103.03874},\n  year={2021}\n}\n"


_lowerCAmelCase       =				"\\nThis metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.\nIt first canonicalizes the inputs (e.g., converting \"1/2\" to \"\\frac{1}{2}\") and then computes accuracy.\n"


_lowerCAmelCase       =				r"\nCalculates accuracy after canonicalizing inputs.\n\nArgs:\n    predictions: list of predictions to score. Each prediction\n        is a string that contains natural language and LaTex.\n    references: list of reference for each prediction. Each\n        reference is a string that contains natural language\n        and LaTex.\nReturns:\n    accuracy: accuracy after canonicalizing inputs\n        (e.g., converting \"1/2\" to \"\\frac{1}{2}\")\n\nExamples:\n    >>> metric = datasets.load_metric(\"competition_math\")\n    >>> results = metric.compute(references=[\"\\frac{1}{2}\"], predictions=[\"1/2\"])\n    >>> print(results)\n    {'accuracy': 1.0}\n"



@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION							,							_KWARGS_DESCRIPTION    )
class   SCREAMING_SNAKE_CASE	(					datasets.Metric    ):





	"""simple docstring"""




	def  lowerCamelCase(self				):
		return datasets.MetricInfo(
		    description=_DESCRIPTION  ,  citation=_CITATION  ,  inputs_description=_KWARGS_DESCRIPTION  ,  features=datasets.Features(
		        {
		            """predictions""": datasets.Value("""string"""				),
		            """references""": datasets.Value("""string"""				),
		        }				)  ,  homepage="""https://github.com/hendrycks/math"""  ,  codebase_urls=["""https://github.com/hendrycks/math"""]  ,  )





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    Optional[Any]						     = 0.0
		for i, j in zip(lowerCAmelCase_  ,  lowerCAmelCase_				):
			n_correct += 1.0 if math_equivalence.is_equiv(lowerCAmelCase_  ,  lowerCAmelCase_				) else 0.0
		A_		:    List[str]						     = n_correct / len(lowerCAmelCase_				)
		return {
		    "accuracy": accuracy,
		}

