






'''simple docstring'''



import datasets

from .evaluate import evaluate


__UpperCamelCase      			=      """\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
"""

__UpperCamelCase      			=      """
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
"""

__UpperCamelCase      			=      """
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair as given in the references (see below)
        - 'prediction_text': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair (see above),
        - 'answers': a Dict in the CUAD dataset format
            {
                'text': list of possible texts for the answer, as a list of strings
                'answer_start': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    'exact_match': Exact match (the normalized answer exactly match the gold answer)
    'f1': The F-score of predicted tokens versus the gold answer
    'aupr': Area Under the Precision-Recall curve
    'prec_at_80_recall': Precision at 80% recall
    'prec_at_90_recall': Precision at 90% recall
Examples:
    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> cuad_metric = datasets.load_metric(\"cuad\")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}
"""
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,      _KWARGS_DESCRIPTION     )
class 							__SCREAMING_SNAKE_CASE       (							datasets.Metric     ):




      def  __UpperCamelCase							(  self      )							->Optional[Any]:



         '''simple docstring'''
         return datasets.MetricInfo(
             description=_DESCRIPTION ,   citation=_CITATION ,   inputs_description=_KWARGS_DESCRIPTION ,   features=datasets.Features(
                 {
                     'predictions': {
                         'id': datasets.Value('string'      ),
                         'prediction_text': datasets.features.Sequence(datasets.Value('string'      )      ),
                     },
                     'references': {
                         'id': datasets.Value('string'      ),
                         'answers': datasets.features.Sequence(
                             {
                                 'text': datasets.Value('string'      ),
                                 'answer_start': datasets.Value('int32'      ),
                             }      ),
                     },
                 }      ) ,   codebase_urls=['https://www.atticusprojectai.org/cuad'] ,   reference_urls=['https://www.atticusprojectai.org/cuad'] ,   )






      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase      )							->Union[str, Any]:



         '''simple docstring'''
         __a				     =    {prediction['id']: prediction['prediction_text'] for prediction in predictions}
         __a				     =    [
             {
                 'paragraphs': [
                     {
                         'qas': [
                             {
                                 'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
                                 'id': ref['id'],
                             }
                             for ref in references
                         ]
                     }
                 ]
             }
         ]
         __a				     =    evaluate(dataset=lowerCamelCase ,   predictions=lowerCamelCase      )
         return score







'''simple docstring'''



import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase      :       Tuple      			=      get_tests_dir("""fixtures/test_sentencepiece.model""")
@require_sentencepiece
@require_tokenizers
class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase      ,      unittest.TestCase     ):
      __a  					=XLMRobertaTokenizer
      __a  					=XLMRobertaTokenizerFast
      __a  					=True
      __a  					=True




      def  __UpperCamelCase							(  self      )							->Tuple:



         '''simple docstring'''
         super().setUp()

         # We have a SentencePiece fixture for testing
         __a				     =    XLMRobertaTokenizer(lowerCamelCase ,   keep_accents=lowerCamelCase      )
         tokenizer.save_pretrained(self.tmpdirname      )




      def  __UpperCamelCase							(  self      )							->List[Any]:



         '''simple docstring'''
         __a				     =    '<pad>'
         __a				     =    1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase      ) ,   lowerCamelCase      )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase      ) ,   lowerCamelCase      )




      def  __UpperCamelCase							(  self      )							->int:



         '''simple docstring'''
         __a				     =    list(self.get_tokenizer().get_vocab().keys()      )

         self.assertEqual(vocab_keys[0] ,   '<s>'      )
         self.assertEqual(vocab_keys[1] ,   '<pad>'      )
         self.assertEqual(vocab_keys[-1] ,   '<mask>'      )
         self.assertEqual(len(lowerCamelCase      ) ,   1002      )




      def  __UpperCamelCase							(  self      )							->Tuple:



         '''simple docstring'''
         self.assertEqual(self.get_tokenizer().vocab_size ,   1002      )




      def  __UpperCamelCase							(  self      )							->int:



         '''simple docstring'''
         __a				     =    XLMRobertaTokenizer(lowerCamelCase ,   keep_accents=lowerCamelCase      )

         __a				     =    tokenizer.tokenize('This is a test'      )
         self.assertListEqual(lowerCamelCase ,   ['▁This', '▁is', '▁a', '▁t', 'est']      )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(lowerCamelCase      ) ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]] ,   )

         __a				     =    tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
         self.assertListEqual(
             lowerCamelCase ,   [
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '9',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 'é',
                 '.',
             ] ,   )
         __a				     =    tokenizer.convert_tokens_to_ids(lowerCamelCase      )
         self.assertListEqual(
             lowerCamelCase ,   [
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                 #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
             ] ,   )

         __a				     =    tokenizer.convert_ids_to_tokens(lowerCamelCase      )
         self.assertListEqual(
             lowerCamelCase ,   [
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '<unk>',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 '<unk>',
                 '.',
             ] ,   )




      def  __UpperCamelCase							(  self      )							->Dict:



         '''simple docstring'''
         if not self.test_slow_tokenizer:
            # as we don't have a slow version, we can't compare the outputs between slow and fast versions
            return

         __a				     =    (self.rust_tokenizer_class, 'hf-internal-testing/tiny-xlm-roberta', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
               __a				     =    self.rust_tokenizer_class.from_pretrained(lowerCamelCase ,   **lowerCamelCase      )
               __a				     =    self.tokenizer_class.from_pretrained(lowerCamelCase ,   **lowerCamelCase      )

               __a				     =    tempfile.mkdtemp()

               __a				     =    tokenizer_r.save_pretrained(lowerCamelCase      )
               __a				     =    tokenizer_p.save_pretrained(lowerCamelCase      )

               # Checks it save with the same files + the tokenizer.json file for the fast one
               self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )
               __a				     =    tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f      )
               self.assertSequenceEqual(lowerCamelCase ,   lowerCamelCase      )

               # Checks everything loads correctly in the same way
               __a				     =    tokenizer_r.from_pretrained(lowerCamelCase      )
               __a				     =    tokenizer_p.from_pretrained(lowerCamelCase      )

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(lowerCamelCase ,   lowerCamelCase      )      )
                  # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                  # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

               shutil.rmtree(lowerCamelCase      )

               # Save tokenizer rust, legacy_format=True
               __a				     =    tempfile.mkdtemp()

               __a				     =    tokenizer_r.save_pretrained(lowerCamelCase ,   legacy_format=lowerCamelCase      )
               __a				     =    tokenizer_p.save_pretrained(lowerCamelCase      )

               # Checks it save with the same files
               self.assertSequenceEqual(lowerCamelCase ,   lowerCamelCase      )

               # Checks everything loads correctly in the same way
               __a				     =    tokenizer_r.from_pretrained(lowerCamelCase      )
               __a				     =    tokenizer_p.from_pretrained(lowerCamelCase      )

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(lowerCamelCase ,   lowerCamelCase      )      )

               shutil.rmtree(lowerCamelCase      )

               # Save tokenizer rust, legacy_format=False
               __a				     =    tempfile.mkdtemp()

               __a				     =    tokenizer_r.save_pretrained(lowerCamelCase ,   legacy_format=lowerCamelCase      )
               __a				     =    tokenizer_p.save_pretrained(lowerCamelCase      )

               # Checks it saved the tokenizer.json file
               self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )

               # Checks everything loads correctly in the same way
               __a				     =    tokenizer_r.from_pretrained(lowerCamelCase      )
               __a				     =    tokenizer_p.from_pretrained(lowerCamelCase      )

               # Check special tokens are set accordingly on Rust and Python
               for key in tokenizer_pp.special_tokens_map:
                  self.assertTrue(hasattr(lowerCamelCase ,   lowerCamelCase      )      )

               shutil.rmtree(lowerCamelCase      )




      @cached_property
      def  __UpperCamelCase							(  self      )							->List[str]:



         '''simple docstring'''
         return XLMRobertaTokenizer.from_pretrained('xlm-roberta-base'      )




      def  __UpperCamelCase							(  self      )							->Union[str, Any]:



         '''simple docstring'''
         with tempfile.NamedTemporaryFile() as f:
            shutil.copyfile(lowerCamelCase ,   f.name      )
            __a				     =    XLMRobertaTokenizer(f.name ,   keep_accents=lowerCamelCase      )
            __a				     =    pickle.dumps(lowerCamelCase      )
         pickle.loads(lowerCamelCase      )




      def  __UpperCamelCase							(  self      )							->Tuple:



         '''simple docstring'''
         if not self.test_rust_tokenizer:
            return

         __a				     =    self.get_tokenizer()
         __a				     =    self.get_rust_tokenizer()

         __a				     =    'I was born in 92000, and this is falsé.'

         __a				     =    tokenizer.tokenize(lowerCamelCase      )
         __a				     =    rust_tokenizer.tokenize(lowerCamelCase      )
         self.assertListEqual(lowerCamelCase ,   lowerCamelCase      )

         __a				     =    tokenizer.encode(lowerCamelCase ,   add_special_tokens=lowerCamelCase      )
         __a				     =    rust_tokenizer.encode(lowerCamelCase ,   add_special_tokens=lowerCamelCase      )
         self.assertListEqual(lowerCamelCase ,   lowerCamelCase      )

         __a				     =    self.get_rust_tokenizer()
         __a				     =    tokenizer.encode(lowerCamelCase      )
         __a				     =    rust_tokenizer.encode(lowerCamelCase      )
         self.assertListEqual(lowerCamelCase ,   lowerCamelCase      )




      @slow
      def  __UpperCamelCase							(  self      )							->Optional[Any]:



         '''simple docstring'''
         __a				     =    'Hello World!'
         __a				     =    [0, 3_5378, 6661, 38, 2]
         # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
         # xlmr.eval()
         # xlmr.encode(symbols)

         self.assertListEqual(lowerCamelCase ,   self.big_tokenizer.encode(lowerCamelCase      )      )




      @slow
      def  __UpperCamelCase							(  self      )							->Optional[int]:



         '''simple docstring'''
         __a				     =    (
             'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
             ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
         )
         __a				     =    [
             0,
             3293,
             83,
             10,
             4552,
             4989,
             7986,
             678,
             10,
             5915,
             111,
             17_9459,
             12_4850,
             4,
             6044,
             237,
             12,
             6,
             5,
             6,
             4,
             6780,
             705,
             15,
             1388,
             44,
             378,
             1_0114,
             711,
             152,
             20,
             6,
             5,
             2_2376,
             642,
             1221,
             1_5190,
             3_4153,
             450,
             5608,
             959,
             1119,
             5_7702,
             136,
             186,
             47,
             1098,
             2_9367,
             47,
             # 4426, # What fairseq tokenizes from "<unk>": "_<"
             # 3678, # What fairseq tokenizes from "<unk>": "unk"
             # 2740, # What fairseq tokenizes from "<unk>": ">"
             3,  # What we tokenize from "<unk>": "<unk>"
             6,  # Residue from the tokenization: an extra sentencepiece underline
             4,
             6044,
             237,
             6284,
             5_0901,
             528,
             31,
             90,
             34,
             927,
             2,
         ]
         # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
         # xlmr.eval()
         # xlmr.encode(symbols)

         self.assertListEqual(lowerCamelCase ,   self.big_tokenizer.encode(lowerCamelCase      )      )






      @slow
      def  __UpperCamelCase							(  self      )							->Tuple:



         '''simple docstring'''
         # fmt: off
         __a				     =    {'input_ids': [[0, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 5428, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 1_2399, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 12_2009, 11_5774, 23, 805, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [0, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=lowerCamelCase ,   model_name='xlm-roberta-base' ,   revision='d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3' ,   )