




import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_ta import TaTokenizer
else:
			__SCREAMING_SNAKE_CASE 						=None


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE 						={"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__SCREAMING_SNAKE_CASE 						={
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/tokenizer.json""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/tokenizer.json""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/tokenizer.json""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/tokenizer.json""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/tokenizer.json""",
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__SCREAMING_SNAKE_CASE 						={
    """t5-small""": 512,
    """t5-base""": 512,
    """t5-large""": 512,
    """t5-3b""": 512,
    """t5-11b""": 512,
}






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							Dict	      =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__					:							List[Any]	      =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__					:							Union[str, Any]	      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__					:							Optional[Any]	      =					["input_ids", "attention_mask"]
				SCREAMING_SNAKE_CASE__					:							Any	      =					TaTokenizer

				SCREAMING_SNAKE_CASE__					:							List[int]	      =					[]


				def __init__(							self:   Dict		,					_lowerCamelCase:   Dict=None		,					_lowerCamelCase:   Dict=None		,					_lowerCamelCase:   Optional[int]="</s>"		,					_lowerCamelCase:   Any="<unk>"		,					_lowerCamelCase:   Optional[Any]="<pad>"		,					_lowerCamelCase:   Tuple=1_00		,					_lowerCamelCase:   Tuple=None		,					**_lowerCamelCase:   str		,					):
											# Add extra_ids to the special token list
											if extra_ids > 0 and additional_special_tokens is None:
																		SCREAMING_SNAKE_CASE_   	=    [f"<extra_id_{i}>" for i in range(_lowerCamelCase		)]
											elif extra_ids > 0 and additional_special_tokens is not None:
																		# Check that we have the right number of extra special tokens
																		SCREAMING_SNAKE_CASE_   	=    len(set(filter(lambda _lowerCamelCase		: bool('''extra_id_''' in str(_lowerCamelCase		)		)		,					_lowerCamelCase		)		)		)
																		if extra_tokens != extra_ids:
																									raise ValueError(
																									    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
																									    ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
																									    ''' tokens'''		)

											super().__init__(
											    _lowerCamelCase		,					tokenizer_file=_lowerCamelCase		,					eos_token=_lowerCamelCase		,					unk_token=_lowerCamelCase		,					pad_token=_lowerCamelCase		,					extra_ids=_lowerCamelCase		,					additional_special_tokens=_lowerCamelCase		,					**_lowerCamelCase		,					)

											SCREAMING_SNAKE_CASE_   	=    vocab_file
											SCREAMING_SNAKE_CASE_   	=    False if not self.vocab_file else True
											SCREAMING_SNAKE_CASE_   	=    extra_ids


				@staticmethod
				def     _A							(							_lowerCamelCase:   Dict		,					_lowerCamelCase:   Optional[int]		,					_lowerCamelCase:   int		):
											if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
																		SCREAMING_SNAKE_CASE_   	=    TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
																		if init_max_model_length is not None and init_max_model_length != max_model_length:
																									return init_max_model_length
																		elif init_max_model_length is None:
																									warnings.warn(
																									    '''This tokenizer was incorrectly instantiated with a model max length of'''
																									    f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
																									    ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
																									    ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
																									    f" {pretrained_model_name_or_path} automatically truncating your input to"
																									    f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
																									    f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
																									    ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
																									    ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''		,					_lowerCamelCase		,					)

											return max_model_length


				def     _A							(							self:   Dict		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[str] = None		):
											if not self.can_save_slow_tokenizer:
																		raise ValueError(
																		    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																		    '''tokenizer.'''		)

											if not os.path.isdir(_lowerCamelCase		):
																		logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
																		return
											SCREAMING_SNAKE_CASE_   	=    os.path.join(
											    _lowerCamelCase		,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(_lowerCamelCase		):
																		copyfile(self.vocab_file		,					_lowerCamelCase		)
																		logger.info(f"Copy vocab file to {out_vocab_file}"		)

											return (out_vocab_file,)


				def     _A							(							self:   Optional[Any]		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   Optional[List[int]] = None		):
											SCREAMING_SNAKE_CASE_   	=    token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																		return self.prefix_tokens + token_ids_a
											else:
																		SCREAMING_SNAKE_CASE_   	=    token_ids_a + [self.eos_token_id]
																		return self.prefix_tokens + token_ids_a + token_ids_a


				def     _A							(							self:   str		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   Optional[List[int]] = None		):
											SCREAMING_SNAKE_CASE_   	=    [self.eos_token_id]

											if token_ids_a is None:
																		return len(token_ids_a + eos		) * [0]
											return len(token_ids_a + eos + token_ids_a + eos		) * [0]


				def     _A							(							self:   Union[str, Any]		):
											return list(
											    set(filter(lambda _lowerCamelCase		: bool(re.search(R'''<extra_id_\d+>'''		,					_lowerCamelCase		)		) is not None		,					self.additional_special_tokens		)		)		)





				def     _A							(							self:   Any		):
											return [self.convert_tokens_to_ids(_lowerCamelCase		) for token in self.get_sentinel_tokens()]








import re
import string

import numpy as np

import datasets


__SCREAMING_SNAKE_CASE 						="""
Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.
"""

__SCREAMING_SNAKE_CASE 						="""
Args:
    predictions: List of predicted texts.
    references: List of reference texts.
    regexes_to_ignore: List, defaults to None. Regex expressions of characters to
        ignore when calculating the exact matches. Note: these regexes are removed
        from the input data before the changes based on the options below (e.g. ignore_case,
        ignore_punctuation, ignore_numbers) are applied.
    ignore_case: Boolean, defaults to False. If true, turns everything
        to lowercase so that capitalization differences are ignored.
    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before
        comparing predictions and references.
    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before
        comparing predictions and references.
Returns:
    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.
Examples:
    >>> exact_match = datasets.load_metric(\"exact_match\")
    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]
    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]
    >>> results = exact_match.compute(references=refs, predictions=preds)
    >>> print(round(results[\"exact_match\"], 1))
    25.0

    >>> exact_match = datasets.load_metric(\"exact_match\")
    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]
    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]
    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\"], ignore_case=True, ignore_punctuation=True)
    >>> print(round(results[\"exact_match\"], 1))
    50.0


    >>> exact_match = datasets.load_metric(\"exact_match\")
    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]
    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]
    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\", \"YELL\"], ignore_case=True, ignore_punctuation=True)
    >>> print(round(results[\"exact_match\"], 1))
    75.0

    >>> exact_match = datasets.load_metric(\"exact_match\")
    >>> refs = [\"the cat\", \"theater\", \"YELLING\", \"agent007\"]
    >>> preds = [\"cat?\", \"theater\", \"yelling\", \"agent\"]
    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=[\"the \", \"yell\", \"YELL\"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)
    >>> print(round(results[\"exact_match\"], 1))
    100.0

    >>> exact_match = datasets.load_metric(\"exact_match\")
    >>> refs = [\"The cat sat on the mat.\", \"Theaters are great.\", \"It's like comparing oranges and apples.\"]
    >>> preds = [\"The cat sat on the mat?\", \"Theaters are great.\", \"It's like comparing apples and oranges.\"]
    >>> results = exact_match.compute(references=refs, predictions=preds)
    >>> print(round(results[\"exact_match\"], 1))
    33.3

"""

__SCREAMING_SNAKE_CASE 						="""
"""






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,  _KWARGS_DESCRIPTION)
class  __magic_name__						(  datasets.Metric):
				'''simple docstring'''







				def     _A							(							self:   Optional[int]		):
											return datasets.MetricInfo(
											    description=_DESCRIPTION		,					citation=_CITATION		,					inputs_description=_KWARGS_DESCRIPTION		,					features=datasets.Features(
											        {
											            '''predictions''': datasets.Value('''string'''		,					id='''sequence'''		),
											            '''references''': datasets.Value('''string'''		,					id='''sequence'''		),
											        }		)		,					reference_urls=[]		,					)





				def     _A							(							self:   Tuple		,					_lowerCamelCase:   int		,					_lowerCamelCase:   int		,					_lowerCamelCase:   Tuple=None		,					_lowerCamelCase:   str=False		,					_lowerCamelCase:   List[str]=False		,					_lowerCamelCase:   List[str]=False		,					):
											if regexes_to_ignore is not None:
																		for s in regexes_to_ignore:
																									SCREAMING_SNAKE_CASE_   	=    np.array([re.sub(_lowerCamelCase		,					''''''		,					_lowerCamelCase		) for x in predictions]		)
																									SCREAMING_SNAKE_CASE_   	=    np.array([re.sub(_lowerCamelCase		,					''''''		,					_lowerCamelCase		) for x in references]		)
											else:
																		SCREAMING_SNAKE_CASE_   	=    np.asarray(_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    np.asarray(_lowerCamelCase		)

											if ignore_case:
																		SCREAMING_SNAKE_CASE_   	=    np.char.lower(_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    np.char.lower(_lowerCamelCase		)

											if ignore_punctuation:
																		SCREAMING_SNAKE_CASE_   	=    string.punctuation.maketrans(''''''		,					''''''		,					string.punctuation		)
																		SCREAMING_SNAKE_CASE_   	=    np.char.translate(_lowerCamelCase		,					table=_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    np.char.translate(_lowerCamelCase		,					table=_lowerCamelCase		)

											if ignore_numbers:
																		SCREAMING_SNAKE_CASE_   	=    string.digits.maketrans(''''''		,					''''''		,					string.digits		)
																		SCREAMING_SNAKE_CASE_   	=    np.char.translate(_lowerCamelCase		,					table=_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    np.char.translate(_lowerCamelCase		,					table=_lowerCamelCase		)

											SCREAMING_SNAKE_CASE_   	=    predictions == references

											return {"exact_match": np.mean(_lowerCamelCase		) * 1_00}


