


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__UpperCamelCase     		=							logging.get_logger(__name__)

__UpperCamelCase     		=							{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCamelCase     		=							{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

__UpperCamelCase     		=							{
    'distilbert-base-uncased': 5_1_2,
    'distilbert-base-uncased-distilled-squad': 5_1_2,
    'distilbert-base-cased': 5_1_2,
    'distilbert-base-cased-distilled-squad': 5_1_2,
    'distilbert-base-german-cased': 5_1_2,
    'distilbert-base-multilingual-cased': 5_1_2,
}


__UpperCamelCase     		=							{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}







class 							lowerCamelCase__						(				UpperCAmelCase   ):






			"""simple docstring"""






			_UpperCamelCase		:       List[str] 			=		VOCAB_FILES_NAMES
			_UpperCamelCase		:       Optional[int] 			=		PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase		:       int 			=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase		:       Tuple 			=		PRETRAINED_INIT_CONFIGURATION
			_UpperCamelCase		:       str 			=		['input_ids', 'attention_mask']
			_UpperCamelCase		:       Union[str, Any] 			=		DistilBertTokenizer




			def __init__(				self   ,    snake_case=None   ,    snake_case=None   ,    snake_case=True   ,    snake_case="[UNK]"   ,    snake_case="[SEP]"   ,    snake_case="[PAD]"   ,    snake_case="[CLS]"   ,    snake_case="[MASK]"   ,    snake_case=True   ,    snake_case=None   ,    **snake_case   ,    ):

						'''simple docstring'''


						super().__init__(
						    snake_case   ,    tokenizer_file=snake_case   ,    do_lower_case=snake_case   ,    unk_token=snake_case   ,    sep_token=snake_case   ,    pad_token=snake_case   ,    cls_token=snake_case   ,    mask_token=snake_case   ,    tokenize_chinese_chars=snake_case   ,    strip_accents=snake_case   ,    **snake_case   ,    )

						UpperCamelCase__       =							json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
						if (
						    normalizer_state.get("lowercase"   ,    snake_case  ) != do_lower_case
						    or normalizer_state.get("strip_accents"   ,    snake_case  ) != strip_accents
						    or normalizer_state.get("handle_chinese_chars"   ,    snake_case  ) != tokenize_chinese_chars
						):
									UpperCamelCase__       =							getattr(snake_case   ,    normalizer_state.pop("type"  )  )
									UpperCamelCase__       =							do_lower_case
									UpperCamelCase__       =							strip_accents
									UpperCamelCase__       =							tokenize_chinese_chars
									UpperCamelCase__       =							normalizer_class(**snake_case  )

						UpperCamelCase__       =							do_lower_case




			def 					snake_case__					(				self   ,    snake_case   ,    snake_case=None  ):

						'''simple docstring'''


						UpperCamelCase__       =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output




			def 					snake_case__					(				self   ,    snake_case   ,    snake_case = None  ):

						'''simple docstring'''


						UpperCamelCase__       =							[self.sep_token_id]
						UpperCamelCase__       =							[self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep  ) * [0]
						return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







			def 					snake_case__					(				self   ,    snake_case   ,    snake_case = None  ):

						'''simple docstring'''


						UpperCamelCase__       =							self._tokenizer.model.save(snake_case   ,    name=snake_case  )
						return tuple(snake_case  )






import re
import string

import numpy as np

import datasets


__UpperCamelCase     		=							'\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

__UpperCamelCase     		=							'\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

__UpperCamelCase     		=							'\n'







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,						_KWARGS_DESCRIPTION   )
class 							lowerCamelCase__						(				datasets.Metric   ):






			"""simple docstring"""






			def 					snake_case__					(				self  ):

						'''simple docstring'''


						return datasets.MetricInfo(
						    description=_DESCRIPTION   ,    citation=_CITATION   ,    inputs_description=_KWARGS_DESCRIPTION   ,    features=datasets.Features(
						        {
						            "predictions": datasets.Value("string"   ,    id="sequence"  ),
						            "references": datasets.Value("string"   ,    id="sequence"  ),
						        }  )   ,    reference_urls=[]   ,    )







			def 					snake_case__					(				self   ,    snake_case   ,    snake_case   ,    snake_case=None   ,    snake_case=False   ,    snake_case=False   ,    snake_case=False   ,    ):

						'''simple docstring'''


						if regexes_to_ignore is not None:
									for s in regexes_to_ignore:
												UpperCamelCase__       =							np.array([re.sub(snake_case   ,    ""   ,    snake_case  ) for x in predictions]  )
												UpperCamelCase__       =							np.array([re.sub(snake_case   ,    ""   ,    snake_case  ) for x in references]  )
						else:
									UpperCamelCase__       =							np.asarray(snake_case  )
									UpperCamelCase__       =							np.asarray(snake_case  )

						if ignore_case:
									UpperCamelCase__       =							np.char.lower(snake_case  )
									UpperCamelCase__       =							np.char.lower(snake_case  )

						if ignore_punctuation:
									UpperCamelCase__       =							string.punctuation.maketrans(""   ,    ""   ,    string.punctuation  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )

						if ignore_numbers:
									UpperCamelCase__       =							string.digits.maketrans(""   ,    ""   ,    string.digits  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )

						UpperCamelCase__       =							predictions == references

						return {"exact_match": np.mean(snake_case  ) * 100}


