



"""simple docstring"""





import datasets

from .evaluate import evaluate


__lowerCAmelCase							:       Tuple          =       '''\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
'''

__lowerCAmelCase							:       Union[str, Any]          =       '''
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
'''

__lowerCAmelCase							:       Optional[Any]          =       '''
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair as given in the references (see below)
        - \'prediction_text\': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - \'id\': id of the question-answer pair (see above),
        - \'answers\': a Dict in the CUAD dataset format
            {
                \'text\': list of possible texts for the answer, as a list of strings
                \'answer_start\': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)
    \'f1\': The F-score of predicted tokens versus the gold answer
    \'aupr\': Area Under the Precision-Recall curve
    \'prec_at_80_recall\': Precision at 80% recall
    \'prec_at_90_recall\': Precision at 90% recall
Examples:
    >>> predictions = [{\'prediction_text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\'], \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]
    >>> references = [{\'answers\': {\'answer_start\': [143, 49], \'text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\']}, \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]
    >>> cuad_metric = datasets.load_metric("cuad")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 100.0, \'f1\': 100.0, \'aupr\': 0.0, \'prec_at_80_recall\': 1.0, \'prec_at_90_recall\': 1.0}
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,					_KWARGS_DESCRIPTION		)
class  _lowerCAmelCase     (					datasets.Metric		):





		"""simple docstring"""






		def  UpperCAmelCase__				(       self )			->    int:







								'''simple docstring'''

								return datasets.MetricInfo(
								    description=_DESCRIPTION				,     citation=_CITATION				,     inputs_description=_KWARGS_DESCRIPTION				,     features=datasets.Features(
								        {
								            """predictions""": {
								                """id""": datasets.Value("""string""" ),
								                """prediction_text""": datasets.features.Sequence(datasets.Value("""string""" ) ),
								            },
								            """references""": {
								                """id""": datasets.Value("""string""" ),
								                """answers""": datasets.features.Sequence(
								                    {
								                        """text""": datasets.Value("""string""" ),
								                        """answer_start""": datasets.Value("""int32""" ),
								                    } ),
								            },
								        } )				,     codebase_urls=["""https://www.atticusprojectai.org/cuad"""]				,     reference_urls=["""https://www.atticusprojectai.org/cuad"""]				,     )





		def  UpperCAmelCase__				(       self				,     _lowercase				,     _lowercase )			->    Optional[int]:







								'''simple docstring'''

								snake_case_ :			Union[str, Any]            =      {prediction["""id"""]: prediction["""prediction_text"""] for prediction in predictions}
								snake_case_ :			Union[str, Any]            =      [
								    {
								        """paragraphs""": [
								            {
								                """qas""": [
								                    {
								                        """answers""": [{"""text""": answer_text} for answer_text in ref["""answers"""]["""text"""]],
								                        """id""": ref["""id"""],
								                    }
								                    for ref in references
								                ]
								            }
								        ]
								    }
								]
								snake_case_ :			int            =      evaluate(dataset=_lowercase				,     predictions=_lowercase )
								return score









import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

    from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
    _SCREAMING_SNAKE_CASE   =  False

if is_vision_available():
    from PIL import Image

    from transformers import PixaStructImageProcessor






class 					SCREAMING_SNAKE_CASE_  (       unittest.TestCase ):
       """simple docstring"""


       def __init__(				self	:str,      snake_case	:int,      snake_case	:Dict=7,      snake_case	:List[Any]=3,      snake_case	:int=18,      snake_case	:List[str]=30,      snake_case	:str=400,      snake_case	:int=None,      snake_case	:List[Any]=True,      snake_case	:Optional[int]=True,      snake_case	:Union[str, Any]=None,      ):



          """simple docstring"""

          _lowercase              =size if size is not None else {'height': 20, 'width': 20}
          _lowercase              =parent
          _lowercase              =batch_size
          _lowercase              =num_channels
          _lowercase              =image_size
          _lowercase              =min_resolution
          _lowercase              =max_resolution
          _lowercase              =size
          _lowercase              =do_normalize
          _lowercase              =do_convert_rgb
          _lowercase              =[512, 1024, 2048, 4096]
          _lowercase              =patch_size if patch_size is not None else {'height': 16, 'width': 16}





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              ='https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg'
          _lowercase              =Image.open(requests.get(snake_case,      stream=snake_case).raw).convert('RGB')
          return raw_image






@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11  ,						reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''  ,						)
@require_torch
@require_vision
class 					SCREAMING_SNAKE_CASE_  (       _a  ,						unittest.TestCase ):
       """simple docstring"""


       __lowerCAmelCase    :		Tuple					=PixaStructImageProcessor if is_vision_available() else None





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          _lowercase              =PixaStructImageProcessingTester(self)





       @property
       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          return self.image_processor_tester.prepare_image_processor_dict()





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          self.assertTrue(hasattr(snake_case,      'do_normalize'))
          self.assertTrue(hasattr(snake_case,      'do_convert_rgb'))





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          _lowercase              =self.image_processor_tester.prepare_dummy_image()

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          _lowercase              =2048

          _lowercase              =image_processor(snake_case,      return_tensors='pt',      max_patches=snake_case)
          self.assertTrue(torch.allclose(inputs.flattened_patches.mean(),      torch.tensor(0.0_6_0_6),      atol=1e-3,      rtol=1e-3))





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PIL images
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      Image.Image)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PIL images
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      Image.Image)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          _lowercase              =True

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             with self.assertRaises(snake_case):
                _lowercase              =image_processor(
                    image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches

             _lowercase              ='Hello'

             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case,      header_text=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case,      header_text=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random numpy tensors
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case,      numpify=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      np.ndarray)

          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PyTorch tensors
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case,      torchify=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      torch.Tensor)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )






@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11  ,						reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''  ,						)
@require_torch
@require_vision
class 					SCREAMING_SNAKE_CASE_  (       _a  ,						unittest.TestCase ):
       """simple docstring"""


       __lowerCAmelCase    :		Optional[Any]					=PixaStructImageProcessor if is_vision_available() else None





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =PixaStructImageProcessingTester(self,      num_channels=4)
          _lowercase              =3





       @property
       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          return self.image_processor_tester.prepare_image_processor_dict()





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          self.assertTrue(hasattr(snake_case,      'do_normalize'))
          self.assertTrue(hasattr(snake_case,      'do_convert_rgb'))





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PIL images
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      Image.Image)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * (self.image_processor_tester.num_channels - 1)
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )


