'''simple docstring'''





import datasets

from .evaluate import evaluate


_lowerCAmelCase  :int  				=    """\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
"""

_lowerCAmelCase  :Dict  				=    """
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
"""

_lowerCAmelCase  :List[str]  				=    """
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair as given in the references (see below)
        - 'prediction_text': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair (see above),
        - 'answers': a Dict in the CUAD dataset format
            {
                'text': list of possible texts for the answer, as a list of strings
                'answer_start': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    'exact_match': Exact match (the normalized answer exactly match the gold answer)
    'f1': The F-score of predicted tokens versus the gold answer
    'aupr': Area Under the Precision-Recall curve
    'prec_at_80_recall': Precision at 80% recall
    'prec_at_90_recall': Precision at 90% recall
Examples:
    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> cuad_metric = datasets.load_metric(\"cuad\")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}
"""






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,    _KWARGS_DESCRIPTION )
class 				UpperCAmelCase (       datasets.Metric ):




    '''simple docstring'''







    def       _UpperCamelCase    (       self )							-> Optional[int]:
         return datasets.MetricInfo(
             description=_DESCRIPTION    ,							citation=_CITATION    ,							inputs_description=_KWARGS_DESCRIPTION    ,							features=datasets.Features(
                 {
                     'predictions': {
                         'id': datasets.Value('string' ),
                         'prediction_text': datasets.features.Sequence(datasets.Value('string' ) ),
                     },
                     'references': {
                         'id': datasets.Value('string' ),
                         'answers': datasets.features.Sequence(
                             {
                                 'text': datasets.Value('string' ),
                                 'answer_start': datasets.Value('int32' ),
                             } ),
                     },
                 } )    ,							codebase_urls=['https://www.atticusprojectai.org/cuad']    ,							reference_urls=['https://www.atticusprojectai.org/cuad']    ,							)






    def       _UpperCamelCase    (       self    ,							lowercase__    ,							lowercase__ )							-> List[Any]:
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       {prediction['id']: prediction['prediction_text'] for prediction in predictions}
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       [
             {
                 'paragraphs': [
                     {
                         'qas': [
                             {
                                 'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
                                 'id': ref['id'],
                             }
                             for ref in references
                         ]
                     }
                 ]
             }
         ]
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       evaluate(dataset=lowercase__    ,							predictions=lowercase__ )
         return score







'''simple docstring'''





from __future__ import annotations

import unittest

from transformers import LEDConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import TFLEDForConditionalGeneration, TFLEDModel






@require_tf
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Dict							=						LEDConfig
    snake_case__							:   Dict							=						{}
    snake_case__							:   int							=						"gelu"






    def __init__(       self    ,							lowercase__    ,							lowercase__=13    ,							lowercase__=7    ,							lowercase__=True    ,							lowercase__=False    ,							lowercase__=99    ,							lowercase__=32    ,							lowercase__=2    ,							lowercase__=4    ,							lowercase__=37    ,							lowercase__=0.1    ,							lowercase__=0.1    ,							lowercase__=20    ,							lowercase__=2    ,							lowercase__=1    ,							lowercase__=0    ,							lowercase__=4    ,							)							-> Union[str, Any]:
         SCREAMING_SNAKE_CASE					:			int    	=       parent
         SCREAMING_SNAKE_CASE					:			str    	=       batch_size
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       seq_length
         SCREAMING_SNAKE_CASE					:			List[str]    	=       is_training
         SCREAMING_SNAKE_CASE					:			Tuple    	=       use_labels
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       vocab_size
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       hidden_size
         SCREAMING_SNAKE_CASE					:			Tuple    	=       num_hidden_layers
         SCREAMING_SNAKE_CASE					:			Dict    	=       num_attention_heads
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       intermediate_size

         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       hidden_dropout_prob
         SCREAMING_SNAKE_CASE					:			Dict    	=       attention_probs_dropout_prob
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       max_position_embeddings
         SCREAMING_SNAKE_CASE					:			Any    	=       eos_token_id
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       pad_token_id
         SCREAMING_SNAKE_CASE					:			Tuple    	=       bos_token_id
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       attention_window

         # `ModelTesterMixin.test_attention_outputs` is expecting attention tensors to be of size
         # [num_attention_heads, encoder_seq_length, encoder_key_length], but TFLongformerSelfAttention
         # returns attention of shape [num_attention_heads, encoder_seq_length, self.attention_window + 1]
         # because its local attention only attends to `self.attention_window` and one before and one after
         SCREAMING_SNAKE_CASE					:			List[str]    	=       self.attention_window + 2

         # because of padding `encoder_seq_length`, is different from `seq_length`. Relevant for
         # the `test_attention_outputs` and `test_hidden_states_output` tests
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       (
             self.seq_length + (self.attention_window - self.seq_length % self.attention_window) % self.attention_window
         )






    def       _UpperCamelCase    (       self )							-> Dict:
         SCREAMING_SNAKE_CASE					:			Tuple    	=       ids_tensor([self.batch_size, self.seq_length - 1]    ,							self.vocab_size )
         SCREAMING_SNAKE_CASE					:			Tuple    	=       tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size )    ,							1 )
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       tf.concat([input_ids, eos_tensor]    ,							axis=1 )

         SCREAMING_SNAKE_CASE					:			List[str]    	=       ids_tensor([self.batch_size, self.seq_length]    ,							self.vocab_size )

         SCREAMING_SNAKE_CASE					:			Tuple    	=       self.config_cls(
             vocab_size=self.vocab_size    ,							d_model=self.hidden_size    ,							encoder_layers=self.num_hidden_layers    ,							decoder_layers=self.num_hidden_layers    ,							encoder_attention_heads=self.num_attention_heads    ,							decoder_attention_heads=self.num_attention_heads    ,							encoder_ffn_dim=self.intermediate_size    ,							decoder_ffn_dim=self.intermediate_size    ,							dropout=self.hidden_dropout_prob    ,							attention_dropout=self.attention_probs_dropout_prob    ,							max_position_embeddings=self.max_position_embeddings    ,							eos_token_ids=[2]    ,							bos_token_id=self.bos_token_id    ,							pad_token_id=self.pad_token_id    ,							decoder_start_token_id=self.pad_token_id    ,							attention_window=self.attention_window    ,							**self.config_updates    ,							)
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       prepare_led_inputs_dict(lowercase__    ,							lowercase__    ,							lowercase__ )
         SCREAMING_SNAKE_CASE					:			str    	=       tf.concat(
             [tf.zeros_like(lowercase__ )[:, :-1], tf.ones_like(lowercase__ )[:, -1:]]    ,							axis=-1    ,							)
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       global_attention_mask
         return config, inputs_dict






    def       _UpperCamelCase    (       self    ,							lowercase__    ,							lowercase__ )							-> Dict:
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       TFLEDModel(config=lowercase__ ).get_decoder()
         SCREAMING_SNAKE_CASE					:			int    	=       inputs_dict['input_ids']

         SCREAMING_SNAKE_CASE					:			Tuple    	=       input_ids[:1, :]
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       inputs_dict['attention_mask'][:1, :]
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       1

         # first forward pass
         SCREAMING_SNAKE_CASE					:			Any    	=       model(lowercase__    ,							attention_mask=lowercase__    ,							use_cache=lowercase__ )

         SCREAMING_SNAKE_CASE	,   SCREAMING_SNAKE_CASE					:			Tuple    	=       outputs.to_tuple()

         # create hypothetical next token and extent to next_input_ids
         SCREAMING_SNAKE_CASE					:			str    	=       ids_tensor((self.batch_size, 3)    ,							config.vocab_size )
         SCREAMING_SNAKE_CASE					:			List[str]    	=       tf.cast(ids_tensor((self.batch_size, 3)    ,							2 )    ,							tf.inta )

         # append to next input_ids and
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       tf.concat([input_ids, next_tokens]    ,							axis=-1 )
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       tf.concat([attention_mask, next_attn_mask]    ,							axis=-1 )

         SCREAMING_SNAKE_CASE					:			int    	=       model(lowercase__    ,							attention_mask=lowercase__ )[0]
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       model(lowercase__    ,							attention_mask=lowercase__    ,							past_key_values=lowercase__ )[0]

         self.parent.assertEqual(next_tokens.shape[1]    ,							output_from_past.shape[1] )

         # select random slice
         SCREAMING_SNAKE_CASE					:			str    	=       int(ids_tensor((1,)    ,							output_from_past.shape[-1] ) )
         SCREAMING_SNAKE_CASE					:			Tuple    	=       output_from_no_past[:, -3:, random_slice_idx]
         SCREAMING_SNAKE_CASE					:			Tuple    	=       output_from_past[:, :, random_slice_idx]

         # test that outputs are equal for slice
         tf.debugging.assert_near(lowercase__    ,							lowercase__    ,							rtol=1E-3 )






def     __lowerCAmelCase						(       a_       ,      a_       ,      a_       ,      a_=None       ,      a_=None       ,      a_=None       ,      a_=None       ,      )      ->	Any:





     '''simple docstring'''





     if attention_mask is None:
          SCREAMING_SNAKE_CASE					:			int    	=       tf.cast(tf.math.not_equal(a_       ,      config.pad_token_id		)       ,      tf.inta		)
     if decoder_attention_mask is None:
          SCREAMING_SNAKE_CASE					:			List[str]    	=       tf.concat(
              [
                  tf.ones(decoder_input_ids[:, :1].shape       ,      dtype=tf.inta		),
                  tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]       ,      config.pad_token_id		)       ,      tf.inta		),
              ]       ,      axis=-1       ,      )
     if head_mask is None:
          SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       tf.ones((config.encoder_layers, config.encoder_attention_heads)		)
     if decoder_head_mask is None:
          SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       tf.ones((config.decoder_layers, config.decoder_attention_heads)		)
     return {
         "input_ids": input_ids,
         "attention_mask": attention_mask,
         "decoder_input_ids": decoder_input_ids,
         "decoder_attention_mask": decoder_attention_mask,
         "head_mask": head_mask,
         "decoder_head_mask": decoder_head_mask,
     }






@require_tf
class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE   ,    _SCREAMING_SNAKE_CASE   ,    unittest.TestCase ):




    '''simple docstring'''







    snake_case__							:   str							=						(TFLEDForConditionalGeneration, TFLEDModel) if is_tf_available() else ()
    snake_case__							:   List[str]							=						(TFLEDForConditionalGeneration,) if is_tf_available() else ()
    snake_case__							:   Union[str, Any]							=						(
        {
            "conversational": TFLEDForConditionalGeneration,
            "feature-extraction": TFLEDModel,
            "summarization": TFLEDForConditionalGeneration,
            "text2text-generation": TFLEDForConditionalGeneration,
            "translation": TFLEDForConditionalGeneration,
        }
        if is_tf_available()
        else {}
    )
    snake_case__							:   List[Any]							=						True
    snake_case__							:   Tuple							=						False
    snake_case__							:   Dict							=						False
    snake_case__							:   int							=						False






    def       _UpperCamelCase    (       self )							-> Optional[int]:
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       TFLEDModelTester(self )
         SCREAMING_SNAKE_CASE					:			Dict    	=       ConfigTester(self    ,							config_class=lowercase__ )






    def       _UpperCamelCase    (       self )							-> Optional[Any]:
         self.config_tester.run_common_tests()






    def       _UpperCamelCase    (       self )							-> Optional[int]:
         SCREAMING_SNAKE_CASE					:			Any    	=       self.model_tester.prepare_config_and_inputs_for_common()
         self.model_tester.check_decoder_model_past_large_inputs(*lowercase__ )






    def       _UpperCamelCase    (       self )							-> int:
         SCREAMING_SNAKE_CASE	,   SCREAMING_SNAKE_CASE					:			int    	=       self.model_tester.prepare_config_and_inputs_for_common()
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       tf.zeros_like(inputs_dict['attention_mask'] )
         SCREAMING_SNAKE_CASE					:			str    	=       2
         SCREAMING_SNAKE_CASE					:			List[str]    	=       tf.where(
             tf.range(self.model_tester.seq_length )[None, :] < num_global_attn_indices    ,							1    ,							inputs_dict['global_attention_mask']    ,							)

         SCREAMING_SNAKE_CASE					:			Dict    	=       True
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       self.model_tester.seq_length
         SCREAMING_SNAKE_CASE					:			str    	=       self.model_tester.encoder_seq_length

         def check_decoder_attentions_output(lowercase__ ):
              SCREAMING_SNAKE_CASE					:			List[Any]    	=       outputs.decoder_attentions
              self.assertEqual(len(lowercase__ )    ,							self.model_tester.num_hidden_layers )
              self.assertListEqual(
                  list(decoder_attentions[0].shape[-3:] )    ,							[self.model_tester.num_attention_heads, seq_length, seq_length]    ,							)

         def check_encoder_attentions_output(lowercase__ ):
              SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       [t.numpy() for t in outputs.encoder_attentions]
              SCREAMING_SNAKE_CASE					:			Tuple    	=       [t.numpy() for t in outputs.encoder_global_attentions]
              self.assertEqual(len(lowercase__ )    ,							self.model_tester.num_hidden_layers )
              self.assertEqual(len(lowercase__ )    ,							self.model_tester.num_hidden_layers )
              self.assertListEqual(
                  list(attentions[0].shape[-3:] )    ,							[self.model_tester.num_attention_heads, seq_length, seq_length]    ,							)
              self.assertListEqual(
                  list(global_attentions[0].shape[-3:] )    ,							[self.model_tester.num_attention_heads, encoder_seq_length, num_global_attn_indices]    ,							)

         for model_class in self.all_model_classes:
              SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       True
              SCREAMING_SNAKE_CASE					:			Tuple    	=       False
              SCREAMING_SNAKE_CASE					:			str    	=       False
              SCREAMING_SNAKE_CASE					:			Any    	=       model_class(lowercase__ )
              SCREAMING_SNAKE_CASE					:			str    	=       model(self._prepare_for_class(lowercase__    ,							lowercase__ ) )
              SCREAMING_SNAKE_CASE					:			List[str]    	=       len(lowercase__ )
              self.assertEqual(config.output_hidden_states    ,							lowercase__ )
              check_encoder_attentions_output(lowercase__ )

              if self.is_encoder_decoder:
                   SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       model_class(lowercase__ )
                   SCREAMING_SNAKE_CASE					:			Dict    	=       model(self._prepare_for_class(lowercase__    ,							lowercase__ ) )
                   self.assertEqual(config.output_hidden_states    ,							lowercase__ )
                   check_decoder_attentions_output(lowercase__ )

              # Check that output attentions can also be changed via the config
              del inputs_dict["output_attentions"]
              SCREAMING_SNAKE_CASE					:			Dict    	=       True
              SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       model_class(lowercase__ )
              SCREAMING_SNAKE_CASE					:			List[str]    	=       model(self._prepare_for_class(lowercase__    ,							lowercase__ ) )
              self.assertEqual(config.output_hidden_states    ,							lowercase__ )
              check_encoder_attentions_output(lowercase__ )

              # Check attention is always last and order is fine
              SCREAMING_SNAKE_CASE					:			str    	=       True
              SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       True
              SCREAMING_SNAKE_CASE					:			Tuple    	=       model_class(lowercase__ )
              SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       model(self._prepare_for_class(lowercase__    ,							lowercase__ ) )

              self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)    ,							len(lowercase__ ) )
              self.assertEqual(model.config.output_hidden_states    ,							lowercase__ )
              check_encoder_attentions_output(lowercase__ )






    @unittest.skip('LED keeps using potentially symbolic tensors in conditionals and breaks tracing.' )
    def       _UpperCamelCase    (       self )							-> Tuple:
         pass






    def       _UpperCamelCase    (       self )							-> Tuple:
         # TODO: Head-masking not yet implement
         pass






def     __lowerCAmelCase						(       a_		)      ->	Any:





     '''simple docstring'''





     return tf.constant(a_       ,      dtype=tf.intaa		)


_lowerCAmelCase  :Union[str, Any]  				=    1E-4






@slow
@require_tf
class 				UpperCAmelCase (       unittest.TestCase ):




    '''simple docstring'''







    def       _UpperCamelCase    (       self )							-> Optional[Any]:
         SCREAMING_SNAKE_CASE					:			Any    	=       TFLEDForConditionalGeneration.from_pretrained('allenai/led-base-16384' ).led

         # change to intended input here
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       _long_tensor([512 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]] )
         SCREAMING_SNAKE_CASE					:			List[str]    	=       _long_tensor([128 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]] )
         SCREAMING_SNAKE_CASE					:			Any    	=       prepare_led_inputs_dict(model.config    ,							lowercase__    ,							lowercase__ )
         SCREAMING_SNAKE_CASE					:			Tuple    	=       model(**lowercase__ )[0]
         SCREAMING_SNAKE_CASE					:			str    	=       (1, 1_024, 768)
         self.assertEqual(output.shape    ,							lowercase__ )
         # change to expected output here
         SCREAMING_SNAKE_CASE					:			str    	=       tf.convert_to_tensor(
             [[2.3_0_5_0, 2.8_2_7_9, 0.6_5_3_1], [-1.8_4_5_7, -0.1_4_5_5, -3.5_6_6_1], [-1.0_1_8_6, 0.4_5_8_6, -2.2_0_4_3]]    ,							)
         tf.debugging.assert_near(output[:, :3, :3]    ,							lowercase__    ,							atol=1E-3 )






    def       _UpperCamelCase    (       self )							-> str:
         SCREAMING_SNAKE_CASE					:			str    	=       TFLEDForConditionalGeneration.from_pretrained('allenai/led-base-16384' )

         # change to intended input here
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       _long_tensor([512 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]] )
         SCREAMING_SNAKE_CASE					:			str    	=       _long_tensor([128 * [0, 31_414, 232, 328, 740, 1_140, 12_695, 69]] )
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       prepare_led_inputs_dict(model.config    ,							lowercase__    ,							lowercase__ )
         SCREAMING_SNAKE_CASE					:			str    	=       model(**lowercase__ )[0]
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       (1, 1_024, model.config.vocab_size)
         self.assertEqual(output.shape    ,							lowercase__ )
         # change to expected output here
         SCREAMING_SNAKE_CASE					:			str    	=       tf.convert_to_tensor(
             [[3_3.6_5_0_7, 6.4_5_7_2, 1_6.8_0_8_9], [5.8_7_3_9, -2.4_2_3_8, 1_1.2_9_0_2], [-3.2_1_3_9, -4.3_1_4_9, 4.2_7_8_3]]    ,							)
         tf.debugging.assert_near(output[:, :3, :3]    ,							lowercase__    ,							atol=1E-3    ,							rtol=1E-3 )



