






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A        =			logging.get_logger(__name__)

A        =			'▁'

A        =			{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

A        =			{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

A        =			{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

A        =			{
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

A        =			{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}


class 							__a							(  __A						):





		'''simple docstring'''



		UpperCAmelCase__      :				List[str]				    =						["input_ids"]

		UpperCAmelCase__      :				Dict				    =						VOCAB_FILES_NAMES
		UpperCAmelCase__      :				Tuple				    =						PRETRAINED_INIT_CONFIGURATION
		UpperCAmelCase__      :				List[str]				    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCAmelCase__      :				List[str]				    =						PRETRAINED_VOCAB_FILES_MAP
		UpperCAmelCase__      :				Tuple				    =						RESOURCE_FILES_NAMES


		def __init__(						self ,    UpperCamelCase__ ,    UpperCamelCase__=None ,    UpperCamelCase__=False ,    UpperCamelCase__="utf8" ,    UpperCamelCase__="[UNK]" ,    UpperCamelCase__="[SEP]" ,    UpperCamelCase__="[PAD]" ,    UpperCamelCase__="[CLS]" ,    UpperCamelCase__="[MASK]" ,    UpperCamelCase__ = None ,    **UpperCamelCase__ ,    ):
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.

						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   {} if sp_model_kwargs is None else sp_model_kwargs
						super().__init__(
						    do_lower_case=UpperCamelCase__ ,    unk_token=UpperCamelCase__ ,    sep_token=UpperCamelCase__ ,    pad_token=UpperCamelCase__ ,    cls_token=UpperCamelCase__ ,    mask_token=UpperCamelCase__ ,    vocab_file=UpperCamelCase__ ,    encoding=UpperCamelCase__ ,    sp_model_kwargs=self.sp_model_kwargs ,    **UpperCamelCase__ ,    )
						SCREAMING_SNAKE_CASE_   :	str													=   do_lower_case
						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   sentencepiece_model_ckpt
						SCREAMING_SNAKE_CASE_   :	List[Any]													=   spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.Load(UpperCamelCase__       )

						# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
						if vocab_file is not None:
										SCREAMING_SNAKE_CASE_   :	Optional[int]													=   self.load_vocab(filepath=UpperCamelCase__       )
						else:
										SCREAMING_SNAKE_CASE_   :	Dict													=   {self.sp_model.id_to_piece(UpperCamelCase__       ): id for id in range(self.sp_model.get_piece_size()       )}
						SCREAMING_SNAKE_CASE_   :	Tuple													=   {v: k for k, v in self.vocab.items()}


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						if text is None:
										return None

						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   self.tokenize(UpperCamelCase__       )
						SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_   :	Dict													=   '', []

						for i, ch in enumerate(UpperCamelCase__       ):
										if ch in self.SP_CHAR_MAPPING:
														SCREAMING_SNAKE_CASE_   :	Any													=   self.SP_CHAR_MAPPING.get(UpperCamelCase__       )
										else:
														SCREAMING_SNAKE_CASE_   :	Tuple													=   unicodedata.normalize('NFKC' ,    UpperCamelCase__       )
										if self.is_whitespace(UpperCamelCase__       ):
														continue
										normalized_text += ch
										char_mapping.extend([i] * len(UpperCamelCase__       )       )

						SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_   :	Tuple													=   normalized_text, [], 0

						if self.do_lower_case:
										SCREAMING_SNAKE_CASE_   :	int													=   text.lower()

						for token in split_tokens:
										if token[:1] == "▁":
														SCREAMING_SNAKE_CASE_   :	Dict													=   token[1:]
										SCREAMING_SNAKE_CASE_   :	List[str]													=   text[offset:].index(UpperCamelCase__       ) + offset
										SCREAMING_SNAKE_CASE_   :	Tuple													=   start + len(UpperCamelCase__       )

										token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)       )
										SCREAMING_SNAKE_CASE_   :	List[Any]													=   end
						return token_mapping


		@property
		def   __snake_case			(						self       ):
						return len(self.vocab       )


		def   __snake_case			(						self       ):
						return dict(self.vocab ,    **self.added_tokens_encoder       )


		def __getstate__(						self       ):
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   self.__dict__.copy()
						SCREAMING_SNAKE_CASE_   :	int													=   None
						return state


		def __setstate__(						self ,    UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	List[Any]													=   d

						# for backward compatibility
						if not hasattr(self ,    'sp_model_kwargs'       ):
										SCREAMING_SNAKE_CASE_   :	Tuple													=   {}

						SCREAMING_SNAKE_CASE_   :	List[str]													=   spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.Load(self.sentencepiece_model_ckpt       )


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase__ ,    UpperCamelCase__       ) for c in text)       )


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__=False ,    UpperCamelCase__=64 ,    UpperCamelCase__=0.1       ):

						if self.sp_model_kwargs.get('enable_sampling'       ) is True:
										SCREAMING_SNAKE_CASE_   :	int													=   True
						if self.sp_model_kwargs.get('alpha'       ) is not None:
										SCREAMING_SNAKE_CASE_   :	Tuple													=   self.sp_model_kwargs.get('alpha'       )
						if self.sp_model_kwargs.get('nbest_size'       ) is not None:
										SCREAMING_SNAKE_CASE_   :	Dict													=   self.sp_model_kwargs.get('nbest_size'       )

						if not enable_sampling:
										SCREAMING_SNAKE_CASE_   :	Optional[int]													=   self.sp_model.EncodeAsPieces(UpperCamelCase__       )
						else:
										SCREAMING_SNAKE_CASE_   :	Dict													=   self.sp_model.SampleEncodeAsPieces(UpperCamelCase__ ,    UpperCamelCase__ ,    UpperCamelCase__       )
						SCREAMING_SNAKE_CASE_   :	Any													=   []
						for pi, piece in enumerate(UpperCamelCase__       ):
										if piece == SPIECE_UNDERLINE:
														if not pieces[pi + 1].startswith(UpperCamelCase__       ) and pi != 0:
																		new_pieces.append(UpperCamelCase__       )
																		continue
														else:
																		continue
										SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   0
										for i, chunk in enumerate(UpperCamelCase__       ):
														if chunk == SPIECE_UNDERLINE:
																		continue
														if self.is_ch_char(UpperCamelCase__       ) or self.is_punct(UpperCamelCase__       ):
																		if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																						new_pieces.append(piece[lst_i:i]       )
																		new_pieces.append(UpperCamelCase__       )
																		SCREAMING_SNAKE_CASE_   :	List[Any]													=   i + 1
														elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																		if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																						new_pieces.append(piece[lst_i:i]       )
																		SCREAMING_SNAKE_CASE_   :	str													=   i
														elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																		if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																						new_pieces.append(piece[lst_i:i]       )
																		SCREAMING_SNAKE_CASE_   :	List[Any]													=   i
										if len(UpperCamelCase__       ) > lst_i:
														new_pieces.append(piece[lst_i:]       )
						return new_pieces


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   ''.join(UpperCamelCase__       ).replace(UpperCamelCase__ ,    ' '       ).strip()
						return out_string


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	Dict													=   self.convert_ids_to_tokens(UpperCamelCase__       )
						SCREAMING_SNAKE_CASE_   :	int													=   ''.join(UpperCamelCase__       ).replace(UpperCamelCase__ ,    ' '       ).strip()
						return out_string


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						return self.vocab.get(UpperCamelCase__ ,    self.vocab.get(self.unk_token       )       )


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						return self.reverse_vocab.get(UpperCamelCase__ ,    self.unk_token       )


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__=None       ):
						if token_ids_a is None:
										return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						SCREAMING_SNAKE_CASE_   :	Optional[int]													=   [self.cls_token_id]
						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   [self.sep_token_id]
						return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__=None       ):
						if offset_mapping_a is None:
										return [(0, 0)] + offset_mapping_a + [(0, 0)]

						return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__=None ,    UpperCamelCase__=False       ):

						if already_has_special_tokens:
										if token_ids_a is not None:
														raise ValueError(
														    'You should not supply a second sequence if the provided sequence of '
														    'ids is already formatted with special tokens for the model.'       )
										return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
										return [1] + ([0] * len(UpperCamelCase__       )) + [1, 1] + ([0] * len(UpperCamelCase__       )) + [1]
						return [1] + ([0] * len(UpperCamelCase__       )) + [1]


		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ = None       ):
						# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
						if token_ids_a is None:
										# [CLS] X [SEP]
										return (len(UpperCamelCase__       ) + 2) * [0]

						# [CLS] A [SEP] [SEP] B [SEP]
						return [0] * (len(UpperCamelCase__       ) + 1) + [1] * (len(UpperCamelCase__       ) + 3)


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						if "\u4e00" <= char <= "\u9fff":
										return True
						return False


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						if ("a" <= char <= "z") or ("A" <= char <= "Z"):
										return True
						return False


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						if char in ",;:.?!~，；：。？！《》【】":
										return True
						return False


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						if char == " " or char == "\t" or char == "\n" or char == "\r":
										return True
						if len(UpperCamelCase__       ) == 1:
										SCREAMING_SNAKE_CASE_   :	int													=   unicodedata.category(UpperCamelCase__       )
										if cat == "Zs":
														return True
						return False


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   {}
						with io.open(UpperCamelCase__ ,    'r' ,    encoding='utf-8'       ) as f:
										for index, line in enumerate(UpperCamelCase__       ):
														SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   line.rstrip('\n'       )
														SCREAMING_SNAKE_CASE_   :	Tuple													=   int(UpperCamelCase__       )

						return token_to_idx






		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ = None       ):
						SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   0
						if os.path.isdir(UpperCamelCase__       ):
										SCREAMING_SNAKE_CASE_   :	int													=   os.path.join(
										    UpperCamelCase__ ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
						else:
										SCREAMING_SNAKE_CASE_   :	List[Any]													=   (filename_prefix + '-' if filename_prefix else '') + save_directory
						with open(UpperCamelCase__ ,    'w' ,    encoding='utf-8'       ) as writer:
										for token, token_index in sorted(self.vocab.items() ,    key=lambda UpperCamelCase__       : kv[1]       ):
														if index != token_index:
																		logger.warning(
																		    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																		    ' Please check that the vocabulary is not corrupted!'       )
																		SCREAMING_SNAKE_CASE_   :	str													=   token_index
														writer.write(token + '\n'       )
														index += 1

						SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   os.path.join(UpperCamelCase__ ,    'sentencepiece.bpe.model'       )
						with open(UpperCamelCase__ ,    'wb'       ) as fi:
										SCREAMING_SNAKE_CASE_   :	int													=   self.sp_model.serialized_model_proto()
										fi.write(UpperCamelCase__       )

						return (vocab_file,)







import os

from bleurt import score  # From: git+https://github.com/google-research/bleurt.git

import datasets


A        =			datasets.logging.get_logger(__name__)


A        =			'\\n@inproceedings{bleurt,\n  title={BLEURT: Learning Robust Metrics for Text Generation},\n  author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh},\n  booktitle={ACL},\n  year={2020},\n  url={https://arxiv.org/abs/2004.04696}\n}\n'

A        =			'\\nBLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018)\nand then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune\nit for your specific application (the latter is expected to perform better).\n\nSee the project\'s README at https://github.com/google-research/bleurt#readme for more information.\n'

A        =			'\nBLEURT score.\n\nArgs:\n    `predictions` (list of str): prediction/candidate sentences\n    `references` (list of str): reference sentences\n    `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None.\n\nReturns:\n    \'scores\': List of scores.\nExamples:\n\n    >>> predictions = ["hello there", "general kenobi"]\n    >>> references = ["hello there", "general kenobi"]\n    >>> bleurt = datasets.load_metric("bleurt")\n    >>> results = bleurt.compute(predictions=predictions, references=references)\n    >>> print([round(v, 2) for v in results["scores"]])\n    [1.03, 1.04]\n'

A        =			{
    'bleurt-tiny-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip',
    'bleurt-tiny-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip',
    'bleurt-base-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip',
    'bleurt-base-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip',
    'bleurt-large-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip',
    'bleurt-large-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip',
    'BLEURT-20-D3': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip',
    'BLEURT-20-D6': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip',
    'BLEURT-20-D12': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip',
    'BLEURT-20': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip',
}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,	_KWARGS_DESCRIPTION						)
class 							__a							(  datasets.Metric						):





		'''simple docstring'''



		def   __snake_case			(						self       ):
						return datasets.MetricInfo(
						    description=_DESCRIPTION ,    citation=_CITATION ,    homepage='https://github.com/google-research/bleurt' ,    inputs_description=_KWARGS_DESCRIPTION ,    features=datasets.Features(
						        {
						            'predictions': datasets.Value('string' ,    id='sequence'       ),
						            'references': datasets.Value('string' ,    id='sequence'       ),
						        }       ) ,    codebase_urls=['https://github.com/google-research/bleurt'] ,    reference_urls=['https://github.com/google-research/bleurt', 'https://arxiv.org/abs/2004.04696'] ,    )


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						# check that config name specifies a valid BLEURT model
						if self.config_name == "default":
										logger.warning(
										    'Using default BLEURT-Base checkpoint for sequence maximum length 128. '
										    'You can use a bigger model for better results with e.g.: datasets.load_metric(\'bleurt\', \'bleurt-large-512\').'       )
										SCREAMING_SNAKE_CASE_   :	List[Any]													=   'bleurt-base-128'

						if self.config_name.lower() in CHECKPOINT_URLS:
										SCREAMING_SNAKE_CASE_   :	Tuple													=   self.config_name.lower()

						elif self.config_name.upper() in CHECKPOINT_URLS:
										SCREAMING_SNAKE_CASE_   :	Any													=   self.config_name.upper()

						else:
										raise KeyError(
										    F'''{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}'''       )

						# download the model checkpoint specified by self.config_name and set up the scorer
						SCREAMING_SNAKE_CASE_   :	List[Any]													=   dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]       )
						SCREAMING_SNAKE_CASE_   :	int													=   score.BleurtScorer(os.path.join(UpperCamelCase__ ,    UpperCamelCase__       )       )






		def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__       ):
						SCREAMING_SNAKE_CASE_   :	Tuple													=   self.scorer.score(references=UpperCamelCase__ ,    candidates=UpperCamelCase__       )
						return {"scores": scores}