
"""simple docstring"""



import datasets

from .evaluate import evaluate


__lowercase						:     Tuple							      =		"""\\n@article{hendrycks2021cuad,\n      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},\n      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},\n      journal={arXiv preprint arXiv:2103.06268},\n      year={2021}\n}\n"""

__lowercase						:     str							      =		"""\nThis metric wrap the official scoring script for version 1 of the Contract\nUnderstanding Atticus Dataset (CUAD).\nContract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510\ncommercial legal contracts that have been manually labeled to identify 41 categories of important\nclauses that lawyers look for when reviewing contracts in connection with corporate transactions.\n"""

__lowercase						:     Optional[Any]							      =		"""\nComputes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair as given in the references (see below)\n        - 'prediction_text': list of possible texts for the answer, as a list of strings\n        depending on a threshold on the confidence probability of each prediction.\n    references: List of question-answers dictionaries with the following key-values:\n        - 'id': id of the question-answer pair (see above),\n        - 'answers': a Dict in the CUAD dataset format\n            {\n                'text': list of possible texts for the answer, as a list of strings\n                'answer_start': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    'exact_match': Exact match (the normalized answer exactly match the gold answer)\n    'f1': The F-score of predicted tokens versus the gold answer\n    'aupr': Area Under the Precision-Recall curve\n    'prec_at_80_recall': Precision at 80% recall\n    'prec_at_90_recall': Precision at 90% recall\nExamples:\n    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]\n    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]\n    >>> cuad_metric = datasets.load_metric(\"cuad\")\n    >>> results = cuad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}\n"""





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,						_KWARGS_DESCRIPTION					)
class 					lowerCAmelCase			(				datasets.Metric					):
     """simple docstring"""







     def    _lowerCAmelCase		(      self							)					->  Optional[Any]:







            '''simple docstring'''





            return datasets.MetricInfo(
                description=_DESCRIPTION ,		citation=_CITATION ,		inputs_description=_KWARGS_DESCRIPTION ,		features=datasets.Features(
                    {
                        '''predictions''': {
                            '''id''': datasets.Value('''string'''							),
                            '''prediction_text''': datasets.features.Sequence(datasets.Value('''string'''							)							),
                        },
                        '''references''': {
                            '''id''': datasets.Value('''string'''							),
                            '''answers''': datasets.features.Sequence(
                                {
                                    '''text''': datasets.Value('''string'''							),
                                    '''answer_start''': datasets.Value('''int32'''							),
                                }							),
                        },
                    }							) ,		codebase_urls=['''https://www.atticusprojectai.org/cuad'''] ,		reference_urls=['''https://www.atticusprojectai.org/cuad'''] ,		)




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__							)					->  List[Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		{prediction['id']: prediction['prediction_text'] for prediction in predictions}
            lowerCamelCase_       							=		[
                {
                    'paragraphs': [
                        {
                            'qas': [
                                {
                                    'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
                                    'id': ref['id'],
                                }
                                for ref in references
                            ]
                        }
                    ]
                }
            ]
            lowerCamelCase_       							=		evaluate(dataset=__lowerCamelCase ,		predictions=__lowerCamelCase							)
            return score





'''simple docstring'''






# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py

import contextlib
import faulthandler
import io
import multiprocessing
import os
import platform
import signal
import tempfile






def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		)    ->   Dict:
      snake_case__       :   int	      =    multiprocessing.Manager()
      snake_case__       :   Any	      =    manager.list()

      snake_case__       :   List[str]	      =    multiprocessing.Process(target=__SCREAMING_SNAKE_CASE				,      args=(check_program, result, timeout)		)
      p.start()
      p.join(timeout=timeout + 1		)
      if p.is_alive():
            p.kill()

      if not result:
            result.append('timed out'		)

      return {
          "task_id": task_id,
          "passed": result[0] == "passed",
          "result": result[0],
          "completion_id": completion_id,
      }






def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		)    ->   Tuple:
      with create_tempdir():
            # These system calls are needed when cleaning up tempdir.
            import os
            import shutil

            snake_case__       :   List[Any]	      =    shutil.rmtree
            snake_case__       :   Tuple	      =    os.rmdir
            snake_case__       :   Any	      =    os.chdir

            # Disable functionalities that can make destructive changes to the test.
            reliability_guard()

            # Run program.
            try:
                  snake_case__       :   Union[str, Any]	      =    {}
                  with swallow_io():
                        with time_limit(__SCREAMING_SNAKE_CASE		):
                              exec(__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		)
                  result.append('passed'		)
            except TimeoutException:
                  result.append('timed out'		)
            except BaseException as e:
                  result.append(f"failed: {e}"		)

            # Needed for cleaning up.
            snake_case__       :   List[Any]	      =    rmtree
            snake_case__       :   Optional[int]	      =    rmdir
            snake_case__       :   Dict	      =    chdir






@contextlib.contextmanager
def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE		)    ->   List[str]:
      def signal_handler(__SCREAMING_SNAKE_CASE				,      __SCREAMING_SNAKE_CASE		):
            raise TimeoutException('Timed out!'		)

      signal.setitimer(signal.ITIMER_REAL				,      __SCREAMING_SNAKE_CASE		)
      signal.signal(signal.SIGALRM				,      __SCREAMING_SNAKE_CASE		)
      try:
            yield
      finally:
            signal.setitimer(signal.ITIMER_REAL				,      0		)






@contextlib.contextmanager
def 		UpperCamelCase__      (		)    ->   List[str]:
      snake_case__       :   Dict	      =    WriteOnlyStringIO()
      with contextlib.redirect_stdout(__SCREAMING_SNAKE_CASE		):
            with contextlib.redirect_stderr(__SCREAMING_SNAKE_CASE		):
                  with redirect_stdin(__SCREAMING_SNAKE_CASE		):
                        yield






@contextlib.contextmanager
def 		UpperCamelCase__      (		)    ->   Optional[int]:
      with tempfile.TemporaryDirectory() as dirname:
            with chdir(__SCREAMING_SNAKE_CASE		):
                  yield dirname






class        lowercase_      (						lowerCAmelCase_							):
     pass






class        lowercase_      (						io.StringIO							):





     def 			_lowerCAmelCase      (     self     :					Optional[Any]     ,							*__lowerCamelCase     :					int     ,							**__lowerCamelCase     :					Optional[int]			):
           raise OSError





     def 			_lowerCAmelCase      (     self     :					int     ,							*__lowerCamelCase     :					List[Any]     ,							**__lowerCamelCase     :					Tuple			):
           raise OSError





     def 			_lowerCAmelCase      (     self     :					Any     ,							*__lowerCamelCase     :					int     ,							**__lowerCamelCase     :					Union[str, Any]			):
           raise OSError





     def 			_lowerCAmelCase      (     self     :					Tuple     ,							*__lowerCamelCase     :					List[str]     ,							**__lowerCamelCase     :					Optional[Any]			):
           return False






class        lowercase_      (						contextlib._RedirectStream							):  # type: ignore
     A_											=			"stdin"






@contextlib.contextmanager
def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE		)    ->   List[str]:
      if root == ".":
            yield
            return
      snake_case__       :   str	      =    os.getcwd()
      os.chdir(__SCREAMING_SNAKE_CASE		)
      try:
            yield
      except BaseException as exc:
            raise exc
      finally:
            os.chdir(__SCREAMING_SNAKE_CASE		)






def 		UpperCamelCase__      (		__SCREAMING_SNAKE_CASE=None		)    ->   Any:

      if maximum_memory_bytes is not None:
            import resource

            resource.setrlimit(resource.RLIMIT_AS				,      (maximum_memory_bytes, maximum_memory_bytes)		)
            resource.setrlimit(resource.RLIMIT_DATA				,      (maximum_memory_bytes, maximum_memory_bytes)		)
            if not platform.uname().system == "Darwin":
                  resource.setrlimit(resource.RLIMIT_STACK				,      (maximum_memory_bytes, maximum_memory_bytes)		)

      faulthandler.disable()

      import builtins

      snake_case__       :   List[Any]	      =    None
      snake_case__       :   List[str]	      =    None

      import os

      snake_case__       :   List[Any]	      =    '1'

      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   Dict	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   Any	      =    None
      snake_case__       :   Optional[Any]	      =    None
      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   int	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   List[Any]	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   Union[str, Any]	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   int	      =    None
      snake_case__       :   Optional[Any]	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   Dict	      =    None
      snake_case__       :   Optional[Any]	      =    None
      snake_case__       :   List[Any]	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   int	      =    None

      import shutil

      snake_case__       :   List[str]	      =    None
      snake_case__       :   List[Any]	      =    None
      snake_case__       :   List[Any]	      =    None

      import subprocess

      snake_case__       :   int	      =    None  # type: ignore

      snake_case__       :   Dict	      =    None

      import sys

      snake_case__       :   Tuple	      =    None
      snake_case__       :   Tuple	      =    None
      snake_case__       :   List[str]	      =    None
      snake_case__       :   Optional[int]	      =    None
      snake_case__       :   List[Any]	      =    None
