






import argparse
import glob
import logging
import os
import sys
import time
from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Tuple

import numpy as np
import pytorch_lightning as pl
import torch
from callbacks import SeqaSeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
from torch import nn
from torch.utils.data import DataLoader

from transformers import MBartTokenizer, TaForConditionalGeneration
from transformers.models.bart.modeling_bart import shift_tokens_right
from utils import (
    ROUGE_KEYS,
    LegacySeqaSeqDataset,
    SeqaSeqDataset,
    assert_all_frozen,
    calculate_bleu,
    calculate_rouge,
    check_output_dir,
    flatten_list,
    freeze_embeds,
    freeze_params,
    get_git_info,
    label_smoothed_nll_loss,
    lmap,
    pickle_save,
    save_git_info,
    save_json,
    use_task_specific_params,
)


# need the parent dir module
sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
from lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa


snake_case__   :						str													=   logging.getLogger(__name__)


class     _a							(	UpperCAmelCase__     ):






  """simple docstring"""





  A_						       = """summarization"""
  A_						       = ["""loss"""]
  A_						       = ROUGE_KEYS
  A_						       = """rouge2"""




  def __init__(  self		,		_UpperCAmelCase		,		**_UpperCAmelCase			)      ->			Union[str, Any]:
     if hparams.sortish_sampler and hparams.gpus > 1:
        UpperCamelCase_						=  False
     elif hparams.max_tokens_per_batch is not None:
        if hparams.gpus > 1:
           raise NotImplementedError('Dynamic Batch size does not work for multi-gpu training'			)
        if hparams.sortish_sampler:
           raise ValueError('--sortish_sampler and --max_tokens_per_batch may not be used simultaneously'			)

     super().__init__(_UpperCAmelCase		,		num_labels=_UpperCAmelCase		,		mode=self.mode		,		**_UpperCAmelCase			)
     use_task_specific_params(self.model		,		'summarization'			)
     save_git_info(self.hparams.output_dir			)
     UpperCamelCase_						=  Path(self.output_dir			) / 'metrics.json'
     UpperCamelCase_						=  Path(self.output_dir			) / 'hparams.pkl'
     pickle_save(self.hparams		,		self.hparams_save_path			)
     UpperCamelCase_						=  0
     UpperCamelCase_						=  defaultdict(_UpperCAmelCase			)
     UpperCamelCase_						=  self.config.model_type
     UpperCamelCase_						=  self.config.tgt_vocab_size if self.model_type == 'fsmt' else self.config.vocab_size

     UpperCamelCase_						=  {
         "data_dir": self.hparams.data_dir,
         "max_source_length": self.hparams.max_source_length,
         "prefix": self.model.config.prefix or "",
     }
     UpperCamelCase_						=  {
         'train': self.hparams.n_train,
         'val': self.hparams.n_val,
         'test': self.hparams.n_test,
     }
     UpperCamelCase_						=  {k: v if v >= 0 else None for k, v in n_observations_per_split.items()}

     UpperCamelCase_						=  {
         'train': self.hparams.max_target_length,
         'val': self.hparams.val_max_target_length,
         'test': self.hparams.test_max_target_length,
     }
     assert self.target_lens["train"] <= self.target_lens["val"], f"""target_lens: {self.target_lens}"""
     assert self.target_lens["train"] <= self.target_lens["test"], f"""target_lens: {self.target_lens}"""
     if self.hparams.freeze_embeds:
        freeze_embeds(self.model			)
     if self.hparams.freeze_encoder:
        freeze_params(self.model.get_encoder()			)
        assert_all_frozen(self.model.get_encoder()			)

     UpperCamelCase_						=  get_git_info()['repo_sha']
     UpperCamelCase_						=  hparams.num_workers
     UpperCamelCase_						=  None  # default to config
     if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer		,		_UpperCAmelCase			):
        UpperCamelCase_						=  self.tokenizer.lang_code_to_id[hparams.tgt_lang]
        UpperCamelCase_						=  self.decoder_start_token_id
     UpperCamelCase_						=  (
         SeqaSeqDataset if hasattr(self.tokenizer		,		'prepare_seq2seq_batch'			) else LegacySeqaSeqDataset
     )
     UpperCamelCase_						=  False
     UpperCamelCase_						=  self.model.config.num_beams if self.hparams.eval_beams is None else self.hparams.eval_beams
     if self.hparams.eval_max_gen_length is not None:
        UpperCamelCase_						=  self.hparams.eval_max_gen_length
     else:
        UpperCamelCase_						=  self.model.config.max_length
     UpperCamelCase_						=  self.default_val_metric if self.hparams.val_metric is None else self.hparams.val_metric




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			Dict[str, List[str]]:
     UpperCamelCase_						=  {
         k: self.tokenizer.batch_decode(v.tolist()			) if 'mask' not in k else v.shape for k, v in batch.items()
     }
     save_json(_UpperCAmelCase		,		Path(self.output_dir			) / 'text_batch.json'			)
     save_json({k: v.tolist() for k, v in batch.items()}		,		Path(self.output_dir			) / 'tok_batch.json'			)

     UpperCamelCase_						=  True
     return readable_batch




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		**_UpperCAmelCase			)      ->			Union[str, Any]:
     return self.model(_UpperCAmelCase		,		**_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			int:
     UpperCamelCase_						=  self.tokenizer.batch_decode(
         _UpperCAmelCase		,		skip_special_tokens=_UpperCAmelCase		,		clean_up_tokenization_spaces=_UpperCAmelCase			)
     return lmap(str.strip		,		_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			Tuple:
     UpperCamelCase_						=  self.tokenizer.pad_token_id
     UpperCamelCase_		,   UpperCamelCase_						=  batch['input_ids'], batch['attention_mask']
     UpperCamelCase_						=  batch['labels']
     if isinstance(self.model		,		_UpperCAmelCase			):
        UpperCamelCase_						=  self.model._shift_right(_UpperCAmelCase			)
     else:
        UpperCamelCase_						=  shift_tokens_right(_UpperCAmelCase		,		_UpperCAmelCase			)
     if not self.already_saved_batch:  # This would be slightly better if it only happened on rank zero
        UpperCamelCase_						=  decoder_input_ids
        self.save_readable_batch(_UpperCAmelCase			)

     UpperCamelCase_						=  self(_UpperCAmelCase		,		attention_mask=_UpperCAmelCase		,		decoder_input_ids=_UpperCAmelCase		,		use_cache=_UpperCAmelCase			)
     UpperCamelCase_						=  outputs['logits']
     if self.hparams.label_smoothing == 0:
        # Same behavior as modeling_bart.py, besides ignoring pad_token_id
        UpperCamelCase_						=  nn.CrossEntropyLoss(ignore_index=_UpperCAmelCase			)

        assert lm_logits.shape[-1] == self.vocab_size
        UpperCamelCase_						=  ce_loss_fct(lm_logits.view(-1		,		lm_logits.shape[-1]			)		,		tgt_ids.view(-1			)			)
     else:
        UpperCamelCase_						=  nn.functional.log_softmax(_UpperCAmelCase		,		dim=-1			)
        UpperCamelCase_		,   UpperCamelCase_						=  label_smoothed_nll_loss(
            _UpperCAmelCase		,		_UpperCAmelCase		,		self.hparams.label_smoothing		,		ignore_index=_UpperCAmelCase			)
     return (loss,)




  @property
  def  _UpperCAmelCase  (  self			)      ->			int:
     return self.tokenizer.pad_token_id




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Dict:
     UpperCamelCase_						=  self._step(_UpperCAmelCase			)

     UpperCamelCase_						=  dict(zip(self.loss_names		,		_UpperCAmelCase			)			)
     # tokens per batch
     UpperCamelCase_						=  batch['input_ids'].ne(self.pad			).sum() + batch['labels'].ne(self.pad			).sum()
     UpperCamelCase_						=  batch['input_ids'].shape[0]
     UpperCamelCase_						=  batch['input_ids'].eq(self.pad			).sum()
     UpperCamelCase_						=  batch['input_ids'].eq(self.pad			).float().mean()
     # TODO(SS): make a wandb summary metric for this
     return {"loss": loss_tensors[0], "log": logs}




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Dict:
     return self._generative_step(_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase="val"			)      ->			Dict:
     self.step_count += 1
     UpperCamelCase_						=  {k: torch.stack([x[k] for x in outputs]			).mean() for k in self.loss_names}
     UpperCamelCase_						=  losses['loss']
     UpperCamelCase_						=  {
         k: np.array([x[k] for x in outputs]			).mean() for k in self.metric_names + ['gen_time', 'gen_len']
     }
     UpperCamelCase_						=  (
         generative_metrics[self.val_metric] if self.val_metric in generative_metrics else losses[self.val_metric]
     )
     UpperCamelCase_						=  torch.tensor(_UpperCAmelCase			).type_as(_UpperCAmelCase			)
     generative_metrics.update({k: v.item() for k, v in losses.items()}			)
     losses.update(_UpperCAmelCase			)
     UpperCamelCase_						=  {f"""{prefix}_avg_{k}""": x for k, x in losses.items()}
     UpperCamelCase_						=  self.step_count
     self.metrics[prefix].append(_UpperCAmelCase			)  # callback writes this to self.metrics_save_path
     UpperCamelCase_						=  flatten_list([x['preds'] for x in outputs]			)
     return {
         "log": all_metrics,
         "preds": preds,
         f"""{prefix}_loss""": loss,
         f"""{prefix}_{self.val_metric}""": metric_tensor,
     }




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Dict:
     return calculate_rouge(_UpperCAmelCase		,		_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			dict:
     UpperCamelCase_						=  time.time()

     # parser.add_argument('--eval_max_gen_length', type=int, default=None, help='never generate more than n tokens')
     UpperCamelCase_						=  self.model.generate(
         batch['input_ids']		,		attention_mask=batch['attention_mask']		,		use_cache=_UpperCAmelCase		,		decoder_start_token_id=self.decoder_start_token_id		,		num_beams=self.eval_beams		,		max_length=self.eval_max_length		,		)
     UpperCamelCase_						=  (time.time() - ta) / batch['input_ids'].shape[0]
     UpperCamelCase_						=  self.ids_to_clean_text(_UpperCAmelCase			)
     UpperCamelCase_						=  self.ids_to_clean_text(batch['labels']			)
     UpperCamelCase_						=  self._step(_UpperCAmelCase			)
     UpperCamelCase_						=  dict(zip(self.loss_names		,		_UpperCAmelCase			)			)
     UpperCamelCase_						=  self.calc_generative_metrics(_UpperCAmelCase		,		_UpperCAmelCase			)
     UpperCamelCase_						=  np.mean(lmap(_UpperCAmelCase		,		_UpperCAmelCase			)			)
     base_metrics.update(gen_time=_UpperCAmelCase		,		gen_len=_UpperCAmelCase		,		preds=_UpperCAmelCase		,		target=_UpperCAmelCase		,		**_UpperCAmelCase			)
     return base_metrics




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			str:
     return self._generative_step(_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			int:
     return self.validation_epoch_end(_UpperCAmelCase		,		prefix='test'			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			SeqaSeqDataset:
     UpperCamelCase_						=  self.n_obs[type_path]
     UpperCamelCase_						=  self.target_lens[type_path]
     UpperCamelCase_						=  self.dataset_class(
         self.tokenizer		,		type_path=_UpperCAmelCase		,		n_obs=_UpperCAmelCase		,		max_target_length=_UpperCAmelCase		,		**self.dataset_kwargs		,		)
     return dataset




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase		,		_UpperCAmelCase = False			)      ->			DataLoader:
     UpperCamelCase_						=  self.get_dataset(_UpperCAmelCase			)

     if self.hparams.sortish_sampler and type_path != "test" and type_path != "val":
        UpperCamelCase_						=  dataset.make_sortish_sampler(_UpperCAmelCase		,		distributed=self.hparams.gpus > 1			)
        return DataLoader(
            _UpperCAmelCase		,		batch_size=_UpperCAmelCase		,		collate_fn=dataset.collate_fn		,		shuffle=_UpperCAmelCase		,		num_workers=self.num_workers		,		sampler=_UpperCAmelCase		,		)

     elif self.hparams.max_tokens_per_batch is not None and type_path != "test" and type_path != "val":
        UpperCamelCase_						=  dataset.make_dynamic_sampler(
            self.hparams.max_tokens_per_batch		,		distributed=self.hparams.gpus > 1			)
        return DataLoader(
            _UpperCAmelCase		,		batch_sampler=_UpperCAmelCase		,		collate_fn=dataset.collate_fn		,		num_workers=self.num_workers		,		)
     else:
        return DataLoader(
            _UpperCAmelCase		,		batch_size=_UpperCAmelCase		,		collate_fn=dataset.collate_fn		,		shuffle=_UpperCAmelCase		,		num_workers=self.num_workers		,		sampler=_UpperCAmelCase		,		)




  def  _UpperCAmelCase  (  self			)      ->			DataLoader:
     UpperCamelCase_						=  self.get_dataloader('train'		,		batch_size=self.hparams.train_batch_size		,		shuffle=_UpperCAmelCase			)
     return dataloader




  def  _UpperCAmelCase  (  self			)      ->			DataLoader:
     return self.get_dataloader('val'		,		batch_size=self.hparams.eval_batch_size			)




  def  _UpperCAmelCase  (  self			)      ->			DataLoader:
     return self.get_dataloader('test'		,		batch_size=self.hparams.eval_batch_size			)




  @staticmethod
  def  _UpperCAmelCase  (  _UpperCAmelCase		,		_UpperCAmelCase			)      ->			List[Any]:
     BaseTransformer.add_model_specific_args(_UpperCAmelCase		,		_UpperCAmelCase			)
     add_generic_args(_UpperCAmelCase		,		_UpperCAmelCase			)
     parser.add_argument(
         '--max_source_length'		,		default=1024		,		type=_UpperCAmelCase		,		help=(
             'The maximum total input sequence length after tokenization. Sequences longer '
             'than this will be truncated, sequences shorter will be padded.'
         )		,		)
     parser.add_argument(
         '--max_target_length'		,		default=56		,		type=_UpperCAmelCase		,		help=(
             'The maximum total input sequence length after tokenization. Sequences longer '
             'than this will be truncated, sequences shorter will be padded.'
         )		,		)
     parser.add_argument(
         '--val_max_target_length'		,		default=142		,		type=_UpperCAmelCase		,		help=(
             'The maximum total input sequence length after tokenization. Sequences longer '
             'than this will be truncated, sequences shorter will be padded.'
         )		,		)
     parser.add_argument(
         '--test_max_target_length'		,		default=142		,		type=_UpperCAmelCase		,		help=(
             'The maximum total input sequence length after tokenization. Sequences longer '
             'than this will be truncated, sequences shorter will be padded.'
         )		,		)
     parser.add_argument('--freeze_encoder'		,		action='store_true'			)
     parser.add_argument('--freeze_embeds'		,		action='store_true'			)
     parser.add_argument('--sortish_sampler'		,		action='store_true'		,		default=_UpperCAmelCase			)
     parser.add_argument('--overwrite_output_dir'		,		action='store_true'		,		default=_UpperCAmelCase			)
     parser.add_argument('--max_tokens_per_batch'		,		type=_UpperCAmelCase		,		default=_UpperCAmelCase			)
     parser.add_argument('--logger_name'		,		type=_UpperCAmelCase		,		choices=['default', 'wandb', 'wandb_shared']		,		default='default'			)
     parser.add_argument('--n_train'		,		type=_UpperCAmelCase		,		default=-1		,		required=_UpperCAmelCase		,		help='# examples. -1 means use all.'			)
     parser.add_argument('--n_val'		,		type=_UpperCAmelCase		,		default=500		,		required=_UpperCAmelCase		,		help='# examples. -1 means use all.'			)
     parser.add_argument('--n_test'		,		type=_UpperCAmelCase		,		default=-1		,		required=_UpperCAmelCase		,		help='# examples. -1 means use all.'			)
     parser.add_argument(
         '--task'		,		type=_UpperCAmelCase		,		default='summarization'		,		required=_UpperCAmelCase		,		help='# examples. -1 means use all.'			)
     parser.add_argument('--label_smoothing'		,		type=_UpperCAmelCase		,		default=0.0		,		required=_UpperCAmelCase			)
     parser.add_argument('--src_lang'		,		type=_UpperCAmelCase		,		default=''		,		required=_UpperCAmelCase			)
     parser.add_argument('--tgt_lang'		,		type=_UpperCAmelCase		,		default=''		,		required=_UpperCAmelCase			)
     parser.add_argument('--eval_beams'		,		type=_UpperCAmelCase		,		default=_UpperCAmelCase		,		required=_UpperCAmelCase			)
     parser.add_argument(
         '--val_metric'		,		type=_UpperCAmelCase		,		default=_UpperCAmelCase		,		required=_UpperCAmelCase		,		choices=['bleu', 'rouge2', 'loss', None]			)
     parser.add_argument('--eval_max_gen_length'		,		type=_UpperCAmelCase		,		default=_UpperCAmelCase		,		help='never generate more than n tokens'			)
     parser.add_argument('--save_top_k'		,		type=_UpperCAmelCase		,		default=1		,		required=_UpperCAmelCase		,		help='How many checkpoints to save'			)
     parser.add_argument(
         '--early_stopping_patience'		,		type=_UpperCAmelCase		,		default=-1		,		required=_UpperCAmelCase		,		help=(
             '-1 means never early stop. early_stopping_patience is measured in validation checks, not epochs. So'
             ' val_check_interval will effect it.'
         )		,		)
     return parser




class     _a							(	UpperCAmelCase__     ):






  """simple docstring"""





  A_						       = """translation"""
  A_						       = ["""loss"""]
  A_						       = ["""bleu"""]
  A_						       = """bleu"""




  def __init__(  self		,		_UpperCAmelCase		,		**_UpperCAmelCase			)      ->			Optional[int]:
     super().__init__(_UpperCAmelCase		,		**_UpperCAmelCase			)
     UpperCamelCase_						=  hparams.src_lang
     UpperCamelCase_						=  hparams.tgt_lang




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			dict:
     return calculate_bleu(_UpperCAmelCase		,		_UpperCAmelCase			)






def      _snake_case     (__lowercase     ,   __lowercase=None):
   Path(args.output_dir).mkdir(exist_ok=__lowercase)
   check_output_dir(__lowercase     ,   expected_items=3)

   if model is None:
      if "summarization" in args.task:
         UpperCamelCase_						=  SummarizationModule(__lowercase)
      else:
         UpperCamelCase_						=  TranslationModule(__lowercase)
   UpperCamelCase_						=  Path(args.data_dir).name
   if (
       args.logger_name == "default"
       or args.fast_dev_run
       or str(args.output_dir).startswith('/tmp')
       or str(args.output_dir).startswith('/var')
   ):
      UpperCamelCase_						=  True  # don't pollute wandb logs unnecessarily
   elif args.logger_name == "wandb":
      from pytorch_lightning.loggers import WandbLogger

      UpperCamelCase_						=  os.environ.get('WANDB_PROJECT'     ,   __lowercase)
      UpperCamelCase_						=  WandbLogger(name=model.output_dir.name     ,   project=__lowercase)

   elif args.logger_name == "wandb_shared":
      from pytorch_lightning.loggers import WandbLogger

      UpperCamelCase_						=  WandbLogger(name=model.output_dir.name     ,   project=f"""hf_{dataset}""")

   if args.early_stopping_patience >= 0:
      UpperCamelCase_						=  get_early_stopping_callback(model.val_metric     ,   args.early_stopping_patience)
   else:
      UpperCamelCase_						=  False

   UpperCamelCase_						=  args.val_metric == 'loss'
   UpperCamelCase_						=  generic_train(
       __lowercase     ,   __lowercase     ,   logging_callback=SeqaSeqLoggingCallback()     ,   checkpoint_callback=get_checkpoint_callback(
           args.output_dir     ,   model.val_metric     ,   args.save_top_k     ,   __lowercase)     ,   early_stopping_callback=__lowercase     ,   logger=__lowercase     ,   )
   pickle_save(model.hparams     ,   model.output_dir / 'hparams.pkl')
   if not args.do_predict:
      return model

   UpperCamelCase_						=  ''
   UpperCamelCase_						=  sorted(glob.glob(os.path.join(args.output_dir     ,   '*.ckpt')     ,   recursive=__lowercase))
   if checkpoints:
      UpperCamelCase_						=  checkpoints[-1]
      UpperCamelCase_						=  checkpoints[-1]
   trainer.logger.log_hyperparams(model.hparams)

   # test() without a model tests using the best checkpoint automatically
   trainer.test()
   return model


if __name__ == "__main__":
 snake_case__   :						Union[str, Any]													=   argparse.ArgumentParser()
 snake_case__   :						Dict													=   pl.Trainer.add_argparse_args(parser)
 snake_case__   :						List[str]													=   SummarizationModule.add_model_specific_args(parser, os.getcwd())

 snake_case__   :						Optional[Any]													=   parser.parse_args()

 main(args)













import datasets

from .evaluate import evaluate


snake_case__   :						int													=   """\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
"""

snake_case__   :						Union[str, Any]													=   """
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
"""

snake_case__   :						Any													=   """
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair as given in the references (see below)
        - 'prediction_text': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair (see above),
        - 'answers': a Dict in the CUAD dataset format
            {
                'text': list of possible texts for the answer, as a list of strings
                'answer_start': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    'exact_match': Exact match (the normalized answer exactly match the gold answer)
    'f1': The F-score of predicted tokens versus the gold answer
    'aupr': Area Under the Precision-Recall curve
    'prec_at_80_recall': Precision at 80% recall
    'prec_at_90_recall': Precision at 90% recall
Examples:
    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> cuad_metric = datasets.load_metric(\"cuad\")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}
"""


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,							_KWARGS_DESCRIPTION     )
class     _a							(	datasets.Metric     ):






  """simple docstring"""





  def  _UpperCAmelCase  (  self			)      ->			Optional[int]:
     return datasets.MetricInfo(
         description=_DESCRIPTION		,		citation=_CITATION		,		inputs_description=_KWARGS_DESCRIPTION		,		features=datasets.Features(
             {
                 'predictions': {
                     'id': datasets.Value('string'			),
                     'prediction_text': datasets.features.Sequence(datasets.Value('string'			)			),
                 },
                 'references': {
                     'id': datasets.Value('string'			),
                     'answers': datasets.features.Sequence(
                         {
                             'text': datasets.Value('string'			),
                             'answer_start': datasets.Value('int32'			),
                         }			),
                 },
             }			)		,		codebase_urls=['https://www.atticusprojectai.org/cuad']		,		reference_urls=['https://www.atticusprojectai.org/cuad']		,		)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Dict:
     UpperCamelCase_						=  {prediction['id']: prediction['prediction_text'] for prediction in predictions}
     UpperCamelCase_						=  [
         {
             'paragraphs': [
                 {
                     'qas': [
                         {
                             'answers': [{'text': answer_text} for answer_text in ref['answers']['text']],
                             'id': ref['id'],
                         }
                         for ref in references
                     ]
                 }
             ]
         }
     ]
     UpperCamelCase_						=  evaluate(dataset=_UpperCAmelCase		,		predictions=_UpperCAmelCase			)
     return score





