



"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase     =							logging.get_logger(__name__)

UpperCamelCase     =							"""▁"""

UpperCamelCase     =							{"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

UpperCamelCase     =							{
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

UpperCamelCase     =							{"""vinai/bartpho-syllable""": 1024}






class       UpperCamelCase__			(   _lowerCAmelCase      ):

			"""simple docstring"""


			A__	:					List[str]							  =     VOCAB_FILES_NAMES
			A__	:					Tuple							  =     PRETRAINED_VOCAB_FILES_MAP
			A__	:					Union[str, Any]							  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__	:					str							  =     ["input_ids", "attention_mask"]


			def __init__(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__="<s>"						,	SCREAMING_SNAKE_CASE__="</s>"						,	SCREAMING_SNAKE_CASE__="</s>"						,	SCREAMING_SNAKE_CASE__="<s>"						,	SCREAMING_SNAKE_CASE__="<unk>"						,	SCREAMING_SNAKE_CASE__="<pad>"						,	SCREAMING_SNAKE_CASE__="<mask>"						,	SCREAMING_SNAKE_CASE__ = None						,	**SCREAMING_SNAKE_CASE__						,	)      ->	None:
				# Mask token behave like a normal word, i.e. include the space before it
				A__											=					AddedToken(SCREAMING_SNAKE_CASE__						,	lstrip=SCREAMING_SNAKE_CASE__						,	rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		) else mask_token

				A__											=					{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=SCREAMING_SNAKE_CASE__						,	eos_token=SCREAMING_SNAKE_CASE__						,	unk_token=SCREAMING_SNAKE_CASE__						,	sep_token=SCREAMING_SNAKE_CASE__						,	cls_token=SCREAMING_SNAKE_CASE__						,	pad_token=SCREAMING_SNAKE_CASE__						,	mask_token=SCREAMING_SNAKE_CASE__						,	sp_model_kwargs=self.sp_model_kwargs						,	**SCREAMING_SNAKE_CASE__						,	)

				A__											=					vocab_file
				A__											=					monolingual_vocab_file
				A__											=					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(str(SCREAMING_SNAKE_CASE__		)		)

				# Load the reduced vocab

				# Keep order of special tokens for backward compatibility
				A__											=					{}
				A__											=					0
				for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
					if str(SCREAMING_SNAKE_CASE__		) not in self.fairseq_tokens_to_ids:
						A__											=					cnt
						cnt += 1
				with open(SCREAMING_SNAKE_CASE__						,	"r"						,	encoding="utf-8"		) as f:
					for line in f.readlines():
						A__											=					line.strip().split()[0]
						A__											=					len(self.fairseq_tokens_to_ids		)
				if str(SCREAMING_SNAKE_CASE__		) not in self.fairseq_tokens_to_ids:
					A__											=					len(self.fairseq_tokens_to_ids		)

				A__											=					{v: k for k, v in self.fairseq_tokens_to_ids.items()}


			def __getstate__(      self		)      ->	Dict:
				A__											=					self.__dict__.copy()
				A__											=					None
				A__											=					self.sp_model.serialized_model_proto()
				return state


			def __setstate__(      self						,	SCREAMING_SNAKE_CASE__		)      ->	List[Any]:
				A__											=					d

				# for backward compatibility
				if not hasattr(self						,	"sp_model_kwargs"		):
					A__											=					{}

				A__											=					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None		)      ->	List[int]:

				if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				A__											=					[self.cls_token_id]
				A__											=					[self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = False		)      ->	List[int]:

				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=SCREAMING_SNAKE_CASE__						,	token_ids_a=SCREAMING_SNAKE_CASE__						,	already_has_special_tokens=SCREAMING_SNAKE_CASE__		)

				if token_ids_a is None:
					return [1] + ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1]
				return [1] + ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1]


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None		)      ->	List[int]:
				A__											=					[self.sep_token_id]
				A__											=					[self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]


			@property
			def   snake_case__	(      self		)      ->	str:
				return len(self.fairseq_ids_to_tokens		)


			def   snake_case__	(      self		)      ->	Dict:
				A__											=					{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__		): i for i in range(self.vocab_size		)}
				vocab.update(self.added_tokens_encoder		)
				return vocab


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__		)      ->	List[str]:
				return self.sp_model.encode(SCREAMING_SNAKE_CASE__						,	out_type=SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__		)      ->	List[Any]:
				if token in self.fairseq_tokens_to_ids:
					return self.fairseq_tokens_to_ids[token]
				else:
					return self.unk_token_id


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__		)      ->	Optional[Any]:
				return self.fairseq_ids_to_tokens[index]


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__		)      ->	str:
				A__											=					"".join(SCREAMING_SNAKE_CASE__		).replace(SCREAMING_SNAKE_CASE__						,	" "		).strip()
				return out_string







			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None		)      ->	Tuple[str]:
				if not os.path.isdir(SCREAMING_SNAKE_CASE__		):
					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""		)
					return
				A__											=					os.path.join(
				    SCREAMING_SNAKE_CASE__						,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
				A__											=					os.path.join(
				    SCREAMING_SNAKE_CASE__						,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["monolingual_vocab_file"]						,	)

				if os.path.abspath(self.vocab_file		) != os.path.abspath(SCREAMING_SNAKE_CASE__		) and os.path.isfile(self.vocab_file		):
					copyfile(self.vocab_file						,	SCREAMING_SNAKE_CASE__		)
				elif not os.path.isfile(self.vocab_file		):
					with open(SCREAMING_SNAKE_CASE__						,	"wb"		) as fi:
						A__											=					self.sp_model.serialized_model_proto()
						fi.write(SCREAMING_SNAKE_CASE__		)

				if os.path.abspath(self.monolingual_vocab_file		) != os.path.abspath(
				    SCREAMING_SNAKE_CASE__		) and os.path.isfile(self.monolingual_vocab_file		):
					copyfile(self.monolingual_vocab_file						,	SCREAMING_SNAKE_CASE__		)
				elif not os.path.isfile(self.monolingual_vocab_file		):
					with open(SCREAMING_SNAKE_CASE__						,	"w"						,	encoding="utf-8"		) as fp:
						for token in self.fairseq_tokens_to_ids:
							if token not in self.all_special_tokens:
								fp.write(f"""{str(SCREAMING_SNAKE_CASE__		)} \n"""		)

				return out_vocab_file, out_monolingual_vocab_file












import datasets

from .evaluate import evaluate


snake_case__   :						int													=   """\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
"""

snake_case__   :						Union[str, Any]													=   """
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
"""

snake_case__   :						Any													=   """
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair as given in the references (see below)
        - 'prediction_text': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair (see above),
        - 'answers': a Dict in the CUAD dataset format
            {
                'text': list of possible texts for the answer, as a list of strings
                'answer_start': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    'exact_match': Exact match (the normalized answer exactly match the gold answer)
    'f1': The F-score of predicted tokens versus the gold answer
    'aupr': Area Under the Precision-Recall curve
    'prec_at_80_recall': Precision at 80% recall
    'prec_at_90_recall': Precision at 90% recall
Examples:
    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> cuad_metric = datasets.load_metric(\"cuad\")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}
"""


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,							_KWARGS_DESCRIPTION     )
class     _a							(	datasets.Metric     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			Optional[int]:
     return datasets.MetricInfo(
         description=_DESCRIPTION		,		citation=_CITATION		,		inputs_description=_KWARGS_DESCRIPTION		,		features=datasets.Features(
             {
                 'predictions': {
                     'id': datasets.Value('string'			),
                     'prediction_text': datasets.features.Sequence(datasets.Value('string'			)			),
                 },
                 'references': {
                     'id': datasets.Value('string'			),
                     'answers': datasets.features.Sequence(
                         {
                             'text': datasets.Value('string'			),
                             'answer_start': datasets.Value('int32'			),
                         }			),
                 },
             }			)		,		codebase_urls=['https://www.atticusprojectai.org/cuad']		,		reference_urls=['https://www.atticusprojectai.org/cuad']		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Dict:
     UpperCamelCase_						=  {prediction['id']: prediction['prediction_text'] for prediction in predictions}
     UpperCamelCase_						=  [
         {
             'paragraphs': [
                 {
                     'qas': [
                         {
                             'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
                             'id': ref['id'],
                         }
                         for ref in references
                     ]
                 }
             ]
         }
     ]
     UpperCamelCase_						=  evaluate(dataset=_UpperCAmelCase		,		predictions=_UpperCAmelCase			)
     return score





