






import datasets

from .evaluate import evaluate


lowerCamelCase      :       Optional[Any]    =						'''\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
'''

lowerCamelCase      :       str    =						'''
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
'''

lowerCamelCase      :       Tuple    =						'''
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair as given in the references (see below)
        - \'prediction_text\': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair (see above),
        - \'answers\': a Dict in the CUAD dataset format
            {
                \'text\': list of possible texts for the answer, as a list of strings
                \'answer_start\': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
    \'aupr\': Area Under the Precision-Recall curve
    \'prec_at_80_recall\': Precision at 80% recall
    \'prec_at_90_recall\': Precision at 90% recall
Examples:
    >>> predictions = [{\'prediction_text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\'], \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]
    >>> references = [{\'answers\': {\'answer_start\': [143, 49], \'text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\']}, \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]
    >>> cuad_metric = datasets.load_metric("cuad")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 100.0, \'f1\': 100.0, \'aupr\': 0.0, \'prec_at_80_recall\': 1.0, \'prec_at_90_recall\': 1.0}
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,		_KWARGS_DESCRIPTION      )
class 				_UpperCamelCase      (datasets.Metric      ):


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           return datasets.MetricInfo(
               description=_DESCRIPTION       ,    citation=_CITATION       ,    inputs_description=_KWARGS_DESCRIPTION       ,    features=datasets.Features(
                   {
                       "predictions": {
                           "id": datasets.Value("string"						),
                           "prediction_text": datasets.features.Sequence(datasets.Value("string"						)						),
                       },
                       "references": {
                           "id": datasets.Value("string"						),
                           "answers": datasets.features.Sequence(
                               {
                                   "text": datasets.Value("string"						),
                                   "answer_start": datasets.Value("int32"						),
                               }						),
                       },
                   }						)       ,    codebase_urls=["https://www.atticusprojectai.org/cuad"]       ,    reference_urls=["https://www.atticusprojectai.org/cuad"]       ,    )


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						int:
           __lowerCAmelCase             =       {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
           __lowerCAmelCase             =       [
               {
                   "paragraphs": [
                       {
                           "qas": [
                               {
                                   "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
                                   "id": ref["id"],
                               }
                               for ref in references
                           ]
                       }
                   ]
               }
           ]
           __lowerCAmelCase             =       evaluate(dataset=__UpperCamelCase       ,    predictions=__UpperCamelCase						)
           return score














import tempfile

import torch

from diffusers import PNDMScheduler

from .test_schedulers import SchedulerCommonTest



class 				_UpperCamelCase      (a_      ):
      snake_case_	=				(PNDMScheduler,)
      snake_case_	=				(("""num_inference_steps""", 50),)


      def 			__UpperCAmelCase		(						self       ,    **__UpperCamelCase						)->						int:
           __lowerCAmelCase             =       {
               "num_train_timesteps": 1_0_0_0,
               "beta_start": 0.0_0_0_1,
               "beta_end": 0.0_2,
               "beta_schedule": "linear",
           }

           config.update(**__UpperCamelCase						)
           return config


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase=0       ,    **__UpperCamelCase						)->						Optional[int]:
           __lowerCAmelCase             =       dict(self.forward_default_kwargs						)
           __lowerCAmelCase             =       kwargs.pop("num_inference_steps"       ,    __UpperCamelCase						)
           __lowerCAmelCase             =       self.dummy_sample
           __lowerCAmelCase             =       0.1 * sample
           __lowerCAmelCase             =       [residual + 0.2, residual + 0.1_5, residual + 0.1, residual + 0.0_5]

           for scheduler_class in self.scheduler_classes:
                __lowerCAmelCase             =       self.get_scheduler_config(**__UpperCamelCase						)
                __lowerCAmelCase             =       scheduler_class(**__UpperCamelCase						)
                scheduler.set_timesteps(__UpperCamelCase						)
                # copy over dummy past residuals
                __lowerCAmelCase             =       dummy_past_residuals[:]

                with tempfile.TemporaryDirectory() as tmpdirname:
                     scheduler.save_config(__UpperCamelCase						)
                     __lowerCAmelCase             =       scheduler_class.from_pretrained(__UpperCamelCase						)
                     new_scheduler.set_timesteps(__UpperCamelCase						)
                     # copy over dummy past residuals
                     __lowerCAmelCase             =       dummy_past_residuals[:]

                __lowerCAmelCase             =       scheduler.step_prk(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample
                __lowerCAmelCase             =       new_scheduler.step_prk(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample

                assert torch.sum(torch.abs(output - new_output						)						) < 1e-5, "Scheduler outputs are not identical"

                __lowerCAmelCase             =       scheduler.step_plms(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample
                __lowerCAmelCase             =       new_scheduler.step_plms(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample

                assert torch.sum(torch.abs(output - new_output						)						) < 1e-5, "Scheduler outputs are not identical"


      def 			__UpperCAmelCase		(						self						)->						Dict:
           pass


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase=0       ,    **__UpperCamelCase						)->						Dict:
           __lowerCAmelCase             =       dict(self.forward_default_kwargs						)
           __lowerCAmelCase             =       kwargs.pop("num_inference_steps"       ,    __UpperCamelCase						)
           __lowerCAmelCase             =       self.dummy_sample
           __lowerCAmelCase             =       0.1 * sample
           __lowerCAmelCase             =       [residual + 0.2, residual + 0.1_5, residual + 0.1, residual + 0.0_5]

           for scheduler_class in self.scheduler_classes:
                __lowerCAmelCase             =       self.get_scheduler_config()
                __lowerCAmelCase             =       scheduler_class(**__UpperCamelCase						)
                scheduler.set_timesteps(__UpperCamelCase						)

                # copy over dummy past residuals (must be after setting timesteps)
                __lowerCAmelCase             =       dummy_past_residuals[:]

                with tempfile.TemporaryDirectory() as tmpdirname:
                     scheduler.save_config(__UpperCamelCase						)
                     __lowerCAmelCase             =       scheduler_class.from_pretrained(__UpperCamelCase						)
                     # copy over dummy past residuals
                     new_scheduler.set_timesteps(__UpperCamelCase						)

                     # copy over dummy past residual (must be after setting timesteps)
                     __lowerCAmelCase             =       dummy_past_residuals[:]

                __lowerCAmelCase             =       scheduler.step_prk(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample
                __lowerCAmelCase             =       new_scheduler.step_prk(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample

                assert torch.sum(torch.abs(output - new_output						)						) < 1e-5, "Scheduler outputs are not identical"

                __lowerCAmelCase             =       scheduler.step_plms(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample
                __lowerCAmelCase             =       new_scheduler.step_plms(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample

                assert torch.sum(torch.abs(output - new_output						)						) < 1e-5, "Scheduler outputs are not identical"


      def 			__UpperCAmelCase		(						self       ,    **__UpperCamelCase						)->						Tuple:
           __lowerCAmelCase             =       self.scheduler_classes[0]
           __lowerCAmelCase             =       self.get_scheduler_config(**__UpperCamelCase						)
           __lowerCAmelCase             =       scheduler_class(**__UpperCamelCase						)

           __lowerCAmelCase             =       1_0
           __lowerCAmelCase             =       self.dummy_model()
           __lowerCAmelCase             =       self.dummy_sample_deter
           scheduler.set_timesteps(__UpperCamelCase						)

           for i, t in enumerate(scheduler.prk_timesteps						):
                __lowerCAmelCase             =       model(__UpperCamelCase       ,    __UpperCamelCase						)
                __lowerCAmelCase             =       scheduler.step_prk(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						).prev_sample

           for i, t in enumerate(scheduler.plms_timesteps						):
                __lowerCAmelCase             =       model(__UpperCamelCase       ,    __UpperCamelCase						)
                __lowerCAmelCase             =       scheduler.step_plms(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						).prev_sample

           return sample


      def 			__UpperCAmelCase		(						self						)->						int:
           __lowerCAmelCase             =       dict(self.forward_default_kwargs						)

           __lowerCAmelCase             =       kwargs.pop("num_inference_steps"       ,    __UpperCamelCase						)

           for scheduler_class in self.scheduler_classes:
                __lowerCAmelCase             =       self.get_scheduler_config()
                __lowerCAmelCase             =       scheduler_class(**__UpperCamelCase						)

                __lowerCAmelCase             =       self.dummy_sample
                __lowerCAmelCase             =       0.1 * sample

                if num_inference_steps is not None and hasattr(__UpperCamelCase       ,    "set_timesteps"						):
                     scheduler.set_timesteps(__UpperCamelCase						)
                elif num_inference_steps is not None and not hasattr(__UpperCamelCase       ,    "set_timesteps"						):
                     __lowerCAmelCase             =       num_inference_steps

                # copy over dummy past residuals (must be done after set_timesteps)
                __lowerCAmelCase             =       [residual + 0.2, residual + 0.1_5, residual + 0.1, residual + 0.0_5]
                __lowerCAmelCase             =       dummy_past_residuals[:]

                __lowerCAmelCase             =       scheduler.step_prk(__UpperCamelCase       ,    0       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample
                __lowerCAmelCase             =       scheduler.step_prk(__UpperCamelCase       ,    1       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample

                self.assertEqual(output_a.shape       ,    sample.shape						)
                self.assertEqual(output_a.shape       ,    output_a.shape						)

                __lowerCAmelCase             =       scheduler.step_plms(__UpperCamelCase       ,    0       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample
                __lowerCAmelCase             =       scheduler.step_plms(__UpperCamelCase       ,    1       ,    __UpperCamelCase       ,    **__UpperCamelCase						).prev_sample

                self.assertEqual(output_a.shape       ,    sample.shape						)
                self.assertEqual(output_a.shape       ,    output_a.shape						)


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           for timesteps in [1_0_0, 1_0_0_0]:
                self.check_over_configs(num_train_timesteps=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						Any:
           for steps_offset in [0, 1]:
                self.check_over_configs(steps_offset=__UpperCamelCase						)

           __lowerCAmelCase             =       self.scheduler_classes[0]
           __lowerCAmelCase             =       self.get_scheduler_config(steps_offset=1						)
           __lowerCAmelCase             =       scheduler_class(**__UpperCamelCase						)
           scheduler.set_timesteps(1_0						)
           assert torch.equal(
               scheduler.timesteps       ,    torch.LongTensor(
                   [9_0_1, 8_5_1, 8_5_1, 8_0_1, 8_0_1, 7_5_1, 7_5_1, 7_0_1, 7_0_1, 6_5_1, 6_5_1, 6_0_1, 6_0_1, 5_0_1, 4_0_1, 3_0_1, 2_0_1, 1_0_1, 1]						)       ,    )


      def 			__UpperCAmelCase		(						self						)->						int:
           for beta_start, beta_end in zip([0.0_0_0_1, 0.0_0_1]       ,    [0.0_0_2, 0.0_2]						):
                self.check_over_configs(beta_start=__UpperCamelCase       ,    beta_end=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						Optional[int]:
           for schedule in ["linear", "squaredcos_cap_v2"]:
                self.check_over_configs(beta_schedule=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						Dict:
           for prediction_type in ["epsilon", "v_prediction"]:
                self.check_over_configs(prediction_type=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						List[Any]:
           for t in [1, 5, 1_0]:
                self.check_over_forward(time_step=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						Optional[Any]:
           for t, num_inference_steps in zip([1, 5, 1_0]       ,    [1_0, 5_0, 1_0_0]						):
                self.check_over_forward(num_inference_steps=__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						int:
           # earlier version of set_timesteps() caused an error indexing alpha's with inference steps as power of 3
           __lowerCAmelCase             =       2_7

           for scheduler_class in self.scheduler_classes:
                __lowerCAmelCase             =       self.dummy_sample
                __lowerCAmelCase             =       0.1 * sample

                __lowerCAmelCase             =       self.get_scheduler_config()
                __lowerCAmelCase             =       scheduler_class(**__UpperCamelCase						)

                scheduler.set_timesteps(__UpperCamelCase						)

                # before power of 3 fix, would error on first step, so we only need to do two
                for i, t in enumerate(scheduler.prk_timesteps[:2]						):
                     __lowerCAmelCase             =       scheduler.step_prk(__UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						).prev_sample


      def 			__UpperCAmelCase		(						self						)->						Dict:
           with self.assertRaises(__UpperCamelCase						):
                __lowerCAmelCase             =       self.scheduler_classes[0]
                __lowerCAmelCase             =       self.get_scheduler_config()
                __lowerCAmelCase             =       scheduler_class(**__UpperCamelCase						)

                scheduler.step_plms(self.dummy_sample       ,    1       ,    self.dummy_sample						).prev_sample


      def 			__UpperCAmelCase		(						self						)->						Any:
           __lowerCAmelCase             =       self.full_loop()
           __lowerCAmelCase             =       torch.sum(torch.abs(__UpperCamelCase						)						)
           __lowerCAmelCase             =       torch.mean(torch.abs(__UpperCamelCase						)						)

           assert abs(result_sum.item() - 1_9_8.1_3_1_8						) < 1e-2
           assert abs(result_mean.item() - 0.2_5_8_0						) < 1e-3


      def 			__UpperCAmelCase		(						self						)->						str:
           __lowerCAmelCase             =       self.full_loop(prediction_type="v_prediction"						)
           __lowerCAmelCase             =       torch.sum(torch.abs(__UpperCamelCase						)						)
           __lowerCAmelCase             =       torch.mean(torch.abs(__UpperCamelCase						)						)

           assert abs(result_sum.item() - 6_7.3_9_8_6						) < 1e-2
           assert abs(result_mean.item() - 0.0_8_7_8						) < 1e-3


      def 			__UpperCAmelCase		(						self						)->						Any:
           # We specify different beta, so that the first alpha is 0.99
           __lowerCAmelCase             =       self.full_loop(set_alpha_to_one=__UpperCamelCase       ,    beta_start=0.0_1						)
           __lowerCAmelCase             =       torch.sum(torch.abs(__UpperCamelCase						)						)
           __lowerCAmelCase             =       torch.mean(torch.abs(__UpperCamelCase						)						)

           assert abs(result_sum.item() - 2_3_0.0_3_9_9						) < 1e-2
           assert abs(result_mean.item() - 0.2_9_9_5						) < 1e-3


      def 			__UpperCAmelCase		(						self						)->						Optional[Any]:
           # We specify different beta, so that the first alpha is 0.99
           __lowerCAmelCase             =       self.full_loop(set_alpha_to_one=__UpperCamelCase       ,    beta_start=0.0_1						)
           __lowerCAmelCase             =       torch.sum(torch.abs(__UpperCamelCase						)						)
           __lowerCAmelCase             =       torch.mean(torch.abs(__UpperCamelCase						)						)

           assert abs(result_sum.item() - 1_8_6.9_4_8_2						) < 1e-2
           assert abs(result_mean.item() - 0.2_4_3_4						) < 1e-3






