





import argparse
import json
import pickle
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import MaskFormerConfig, MaskFormerForInstanceSegmentation, MaskFormerImageProcessor, SwinConfig
from transformers.utils import logging


logging.set_verbosity_info()
lowercase    : Dict    	=  logging.get_logger(__name__)


def     SCREAMING_SNAKE_CASE			(      lowerCAmelCase					):
       _UpperCamelCase              =						SwinConfig.from_pretrained(
           '''microsoft/swin-tiny-patch4-window7-224'''    ,       out_features=['''stage1''', '''stage2''', '''stage3''', '''stage4''']					)
       _UpperCamelCase              =						MaskFormerConfig(backbone_config=a_					)

       _UpperCamelCase              =						'''huggingface/label-files'''
       if "ade20k-full" in model_name:
              # this should be ok
              _UpperCamelCase              =						847
              _UpperCamelCase              =						'''maskformer-ade20k-full-id2label.json'''
       elif "ade" in model_name:
              # this should be ok
              _UpperCamelCase              =						150
              _UpperCamelCase              =						'''ade20k-id2label.json'''
       elif "coco-stuff" in model_name:
              # this should be ok
              _UpperCamelCase              =						171
              _UpperCamelCase              =						'''maskformer-coco-stuff-id2label.json'''
       elif "coco" in model_name:
              # TODO
              _UpperCamelCase              =						133
              _UpperCamelCase              =						'''coco-panoptic-id2label.json'''
       elif "cityscapes" in model_name:
              # this should be ok
              _UpperCamelCase              =						19
              _UpperCamelCase              =						'''cityscapes-id2label.json'''
       elif "vistas" in model_name:
              # this should be ok
              _UpperCamelCase              =						65
              _UpperCamelCase              =						'''mapillary-vistas-id2label.json'''

       _UpperCamelCase              =						json.load(open(hf_hub_download(a_    ,       a_    ,       repo_type='''dataset'''					)    ,       '''r'''					)					)
       _UpperCamelCase              =						{int(a_					): v for k, v in idalabel.items()}

       return config


def     SCREAMING_SNAKE_CASE			(      lowerCAmelCase					):
       _UpperCamelCase              =						[]
       # stem
       # fmt: off
       rename_keys.append(('''backbone.patch_embed.proj.weight''', '''model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.weight''')					)
       rename_keys.append(('''backbone.patch_embed.proj.bias''', '''model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.bias''')					)
       rename_keys.append(('''backbone.patch_embed.norm.weight''', '''model.pixel_level_module.encoder.model.embeddings.norm.weight''')					)
       rename_keys.append(('''backbone.patch_embed.norm.bias''', '''model.pixel_level_module.encoder.model.embeddings.norm.bias''')					)
       # stages
       for i in range(len(config.backbone_config.depths					)					):
              for j in range(config.backbone_config.depths[i]					):
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.norm1.weight''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.weight''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.norm1.bias''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.bias''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.attn.relative_position_bias_table''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.attn.relative_position_index''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.attn.proj.weight''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.attn.proj.bias''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.norm2.weight''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.weight''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.norm2.bias''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.bias''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.mlp.fc1.weight''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.mlp.fc1.bias''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.mlp.fc2.weight''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.weight''')					)
                     rename_keys.append((f'''backbone.layers.{i}.blocks.{j}.mlp.fc2.bias''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.bias''')					)

              if i < 3:
                     rename_keys.append((f'''backbone.layers.{i}.downsample.reduction.weight''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.reduction.weight''')					)
                     rename_keys.append((f'''backbone.layers.{i}.downsample.norm.weight''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.weight''')					)
                     rename_keys.append((f'''backbone.layers.{i}.downsample.norm.bias''', f'''model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.bias''')					)
              rename_keys.append((f'''backbone.norm{i}.weight''', f'''model.pixel_level_module.encoder.hidden_states_norms.{i}.weight''')					)
              rename_keys.append((f'''backbone.norm{i}.bias''', f'''model.pixel_level_module.encoder.hidden_states_norms.{i}.bias''')					)

       # FPN
       rename_keys.append(('''sem_seg_head.layer_4.weight''', '''model.pixel_level_module.decoder.fpn.stem.0.weight''')					)
       rename_keys.append(('''sem_seg_head.layer_4.norm.weight''', '''model.pixel_level_module.decoder.fpn.stem.1.weight''')					)
       rename_keys.append(('''sem_seg_head.layer_4.norm.bias''', '''model.pixel_level_module.decoder.fpn.stem.1.bias''')					)
       for source_index, target_index in zip(range(3    ,       0    ,       -1					)    ,       range(0    ,       3					)					):
              rename_keys.append((f'''sem_seg_head.adapter_{source_index}.weight''', f'''model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.0.weight''')					)
              rename_keys.append((f'''sem_seg_head.adapter_{source_index}.norm.weight''', f'''model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.weight''')					)
              rename_keys.append((f'''sem_seg_head.adapter_{source_index}.norm.bias''', f'''model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.bias''')					)
              rename_keys.append((f'''sem_seg_head.layer_{source_index}.weight''', f'''model.pixel_level_module.decoder.fpn.layers.{target_index}.block.0.weight''')					)
              rename_keys.append((f'''sem_seg_head.layer_{source_index}.norm.weight''', f'''model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.weight''')					)
              rename_keys.append((f'''sem_seg_head.layer_{source_index}.norm.bias''', f'''model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.bias''')					)
       rename_keys.append(('''sem_seg_head.mask_features.weight''', '''model.pixel_level_module.decoder.mask_projection.weight''')					)
       rename_keys.append(('''sem_seg_head.mask_features.bias''', '''model.pixel_level_module.decoder.mask_projection.bias''')					)

       # Transformer decoder
       for idx in range(config.decoder_config.decoder_layers					):
              # self-attention out projection
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.weight''', f'''model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.bias''', f'''model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.bias''')					)
              # cross-attention out projection
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.weight''', f'''model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.bias''', f'''model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.bias''')					)
              # MLP 1
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.weight''', f'''model.transformer_module.decoder.layers.{idx}.fc1.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.bias''', f'''model.transformer_module.decoder.layers.{idx}.fc1.bias''')					)
              # MLP 2
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.weight''', f'''model.transformer_module.decoder.layers.{idx}.fc2.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.bias''', f'''model.transformer_module.decoder.layers.{idx}.fc2.bias''')					)
              # layernorm 1 (self-attention layernorm)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.weight''', f'''model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.bias''', f'''model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.bias''')					)
              # layernorm 2 (cross-attention layernorm)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.weight''', f'''model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.bias''', f'''model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.bias''')					)
              # layernorm 3 (final layernorm)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.weight''', f'''model.transformer_module.decoder.layers.{idx}.final_layer_norm.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.bias''', f'''model.transformer_module.decoder.layers.{idx}.final_layer_norm.bias''')					)

       rename_keys.append(('''sem_seg_head.predictor.transformer.decoder.norm.weight''', '''model.transformer_module.decoder.layernorm.weight''')					)
       rename_keys.append(('''sem_seg_head.predictor.transformer.decoder.norm.bias''', '''model.transformer_module.decoder.layernorm.bias''')					)

       # heads on top
       rename_keys.append(('''sem_seg_head.predictor.query_embed.weight''', '''model.transformer_module.queries_embedder.weight''')					)

       rename_keys.append(('''sem_seg_head.predictor.input_proj.weight''', '''model.transformer_module.input_projection.weight''')					)
       rename_keys.append(('''sem_seg_head.predictor.input_proj.bias''', '''model.transformer_module.input_projection.bias''')					)

       rename_keys.append(('''sem_seg_head.predictor.class_embed.weight''', '''class_predictor.weight''')					)
       rename_keys.append(('''sem_seg_head.predictor.class_embed.bias''', '''class_predictor.bias''')					)

       for i in range(3					):
              rename_keys.append((f'''sem_seg_head.predictor.mask_embed.layers.{i}.weight''', f'''mask_embedder.{i}.0.weight''')					)
              rename_keys.append((f'''sem_seg_head.predictor.mask_embed.layers.{i}.bias''', f'''mask_embedder.{i}.0.bias''')					)
       # fmt: on

       return rename_keys


def     SCREAMING_SNAKE_CASE			(      lowerCAmelCase    ,       lowerCAmelCase    ,       lowerCAmelCase					):
       _UpperCamelCase              =						dct.pop(a_					)
       _UpperCamelCase              =						val


def     SCREAMING_SNAKE_CASE			(      lowerCAmelCase    ,       lowerCAmelCase					):
       _UpperCamelCase              =						[int(backbone_config.embed_dim * 2**i					) for i in range(len(backbone_config.depths					)					)]
       for i in range(len(backbone_config.depths					)					):
              _UpperCamelCase              =						num_features[i]
              for j in range(backbone_config.depths[i]					):
                     # fmt: off
                     # read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
                     _UpperCamelCase              =						state_dict.pop(f'''backbone.layers.{i}.blocks.{j}.attn.qkv.weight'''					)
                     _UpperCamelCase              =						state_dict.pop(f'''backbone.layers.{i}.blocks.{j}.attn.qkv.bias'''					)
                     # next, add query, keys and values (in that order) to the state dict
                     _UpperCamelCase              =						in_proj_weight[:dim, :]
                     _UpperCamelCase              =						in_proj_bias[: dim]
                     _UpperCamelCase              =						in_proj_weight[
                         dim : dim * 2, :
                     ]
                     _UpperCamelCase              =						in_proj_bias[
                         dim : dim * 2
                     ]
                     _UpperCamelCase              =						in_proj_weight[
                         -dim :, :
                     ]
                     _UpperCamelCase              =						in_proj_bias[-dim :]
                     # fmt: on


def     SCREAMING_SNAKE_CASE			(      lowerCAmelCase    ,       lowerCAmelCase					):
       # fmt: off
       _UpperCamelCase              =						config.decoder_config.hidden_size
       for idx in range(config.decoder_config.decoder_layers					):
              # read in weights + bias of self-attention input projection layer (in the original implementation, this is a single matrix + bias)
              _UpperCamelCase              =						state_dict.pop(f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_weight'''					)
              _UpperCamelCase              =						state_dict.pop(f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_bias'''					)
              # next, add query, keys and values (in that order) to the state dict
              _UpperCamelCase              =						in_proj_weight[: hidden_size, :]
              _UpperCamelCase              =						in_proj_bias[:config.hidden_size]
              _UpperCamelCase              =						in_proj_weight[hidden_size : hidden_size * 2, :]
              _UpperCamelCase              =						in_proj_bias[hidden_size : hidden_size * 2]
              _UpperCamelCase              =						in_proj_weight[-hidden_size :, :]
              _UpperCamelCase              =						in_proj_bias[-hidden_size :]
              # read in weights + bias of cross-attention input projection layer (in the original implementation, this is a single matrix + bias)
              _UpperCamelCase              =						state_dict.pop(f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_weight'''					)
              _UpperCamelCase              =						state_dict.pop(f'''sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_bias'''					)
              # next, add query, keys and values (in that order) to the state dict
              _UpperCamelCase              =						in_proj_weight[: hidden_size, :]
              _UpperCamelCase              =						in_proj_bias[:config.hidden_size]
              _UpperCamelCase              =						in_proj_weight[hidden_size : hidden_size * 2, :]
              _UpperCamelCase              =						in_proj_bias[hidden_size : hidden_size * 2]
              _UpperCamelCase              =						in_proj_weight[-hidden_size :, :]
              _UpperCamelCase              =						in_proj_bias[-hidden_size :]
       # fmt: on


def     SCREAMING_SNAKE_CASE			(      ):
       _UpperCamelCase              =						'''http://images.cocodataset.org/val2017/000000039769.jpg'''
       _UpperCamelCase              =						Image.open(requests.get(a_    ,       stream=a_					).raw					)
       return im


@torch.no_grad()
def     SCREAMING_SNAKE_CASE			(      lowerCAmelCase    ,       lowerCAmelCase    ,       lowerCAmelCase    ,       lowerCAmelCase = False					):
       _UpperCamelCase              =						get_maskformer_config(a_					)

       # load original state_dict
       with open(a_    ,       '''rb'''					) as f:
              _UpperCamelCase              =						pickle.load(a_					)
       _UpperCamelCase              =						data['''model''']

       # for name, param in state_dict.items():
       #     print(name, param.shape)

       # rename keys
       _UpperCamelCase              =						create_rename_keys(a_					)
       for src, dest in rename_keys:
              rename_key(a_    ,       a_    ,       a_					)
       read_in_swin_q_k_v(a_    ,       config.backbone_config					)
       read_in_decoder_q_k_v(a_    ,       a_					)

       # update to torch tensors
       for key, value in state_dict.items():
              _UpperCamelCase              =						torch.from_numpy(a_					)

       # load 🤗 model
       _UpperCamelCase              =						MaskFormerForInstanceSegmentation(a_					)
       model.eval()

       for name, param in model.named_parameters():
              print(a_    ,       param.shape					)

       _UpperCamelCase              =						model.load_state_dict(a_    ,       strict=a_					)
       assert missing_keys == [
           "model.pixel_level_module.encoder.model.layernorm.weight",
           "model.pixel_level_module.encoder.model.layernorm.bias",
       ]
       assert len(a_					) == 0, f'''Unexpected keys: {unexpected_keys}'''

       # verify results
       _UpperCamelCase              =						prepare_img()
       if "vistas" in model_name:
              _UpperCamelCase              =						65
       elif "cityscapes" in model_name:
              _UpperCamelCase              =						65_535
       else:
              _UpperCamelCase              =						255
       _UpperCamelCase              =						True if '''ade''' in model_name else False
       _UpperCamelCase              =						MaskFormerImageProcessor(ignore_index=a_    ,       reduce_labels=a_					)

       _UpperCamelCase              =						image_processor(a_    ,       return_tensors='''pt'''					)

       _UpperCamelCase              =						model(**a_					)

       print('''Logits:'''    ,       outputs.class_queries_logits[0, :3, :3]					)

       if model_name == "maskformer-swin-tiny-ade":
              _UpperCamelCase              =						torch.tensor(
                  [[3.6_3_5_3, -4.4_7_7_0, -2.6_0_6_5], [0.5_0_8_1, -4.2_3_9_4, -3.5_3_4_3], [2.1_9_0_9, -5.0_3_5_3, -1.9_3_2_3]]					)
       assert torch.allclose(outputs.class_queries_logits[0, :3, :3]    ,       a_    ,       atol=1E-4					)
       print('''Looks ok!'''					)

       if pytorch_dump_folder_path is not None:
              print(f'''Saving model and image processor to {pytorch_dump_folder_path}'''					)
              Path(a_					).mkdir(exist_ok=a_					)
              model.save_pretrained(a_					)
              image_processor.save_pretrained(a_					)

       if push_to_hub:
              print('''Pushing model and image processor to the hub...'''					)
              model.push_to_hub(f'''nielsr/{model_name}'''					)
              image_processor.push_to_hub(f'''nielsr/{model_name}'''					)


if __name__ == "__main__":
       lowercase    : int    	=  argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           """--model_name""",
           default="""maskformer-swin-tiny-ade""",
           type=str,
           help=("""Name of the MaskFormer model you'd like to convert""",),
       )
       parser.add_argument(
           """--checkpoint_path""",
           default="""/Users/nielsrogge/Documents/MaskFormer_checkpoints/MaskFormer-Swin-tiny-ADE20k/model.pkl""",
           type=str,
           help="""Path to the original state dict (.pth file).""",
       )
       parser.add_argument(
           """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
       )
       parser.add_argument(
           """--push_to_hub""", action="""store_true""", help="""Whether or not to push the converted model to the 🤗 hub."""
       )

       lowercase    : Optional[Any]    	=  parser.parse_args()
       convert_maskformer_checkpoint(
           args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path, args.push_to_hub
       )













import datasets

from .evaluate import evaluate


lowercase    : str    	=  """\
@article{hendrycks2021cuad,
      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},
      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},
      journal={arXiv preprint arXiv:2103.06268},
      year={2021}
}
"""

lowercase    : Tuple    	=  """
This metric wrap the official scoring script for version 1 of the Contract
Understanding Atticus Dataset (CUAD).
Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510
commercial legal contracts that have been manually labeled to identify 41 categories of important
clauses that lawyers look for when reviewing contracts in connection with corporate transactions.
"""

lowercase    : List[str]    	=  """
Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).
Args:
    predictions: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair as given in the references (see below)
        - 'prediction_text': list of possible texts for the answer, as a list of strings
        depending on a threshold on the confidence probability of each prediction.
    references: List of question-answers dictionaries with the following key-values:
        - 'id': id of the question-answer pair (see above),
        - 'answers': a Dict in the CUAD dataset format
            {
                'text': list of possible texts for the answer, as a list of strings
                'answer_start': list of start positions for the answer, as a list of ints
            }
            Note that answer_start values are not taken into account to compute the metric.
Returns:
    'exact_match': Exact match (the normalized answer exactly match the gold answer)
    'f1': The F-score of predicted tokens versus the gold answer
    'aupr': Area Under the Precision-Recall curve
    'prec_at_80_recall': Precision at 80% recall
    'prec_at_90_recall': Precision at 90% recall
Examples:
    >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}]
    >>> cuad_metric = datasets.load_metric(\"cuad\")
    >>> results = cuad_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0}
"""

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,			_KWARGS_DESCRIPTION )
class  __A(						datasets.Metric ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            return datasets.MetricInfo(
                description=_DESCRIPTION,       citation=_CITATION,       inputs_description=_KWARGS_DESCRIPTION,       features=datasets.Features(
                    {
                        '''predictions''': {
                            '''id''': datasets.Value('''string'''      ),
                            '''prediction_text''': datasets.features.Sequence(datasets.Value('''string'''      )      ),
                        },
                        '''references''': {
                            '''id''': datasets.Value('''string'''      ),
                            '''answers''': datasets.features.Sequence(
                                {
                                    '''text''': datasets.Value('''string'''      ),
                                    '''answer_start''': datasets.Value('''int32'''      ),
                                }      ),
                        },
                    }      ),       codebase_urls=['''https://www.atticusprojectai.org/cuad'''],       reference_urls=['''https://www.atticusprojectai.org/cuad'''],       )



     def 	_UpperCamelCase     (      self,       A,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						{prediction['''id''']: prediction['''prediction_text'''] for prediction in predictions}
            _UpperCamelCase              =						[
                {
                    '''paragraphs''': [
                        {
                            '''qas''': [
                                {
                                    '''answers''': [{'''text''': answer_text} for answer_text in ref['''answers''']['''text''']],
                                    '''id''': ref['''id'''],
                                }
                                for ref in references
                            ]
                        }
                    ]
                }
            ]
            _UpperCamelCase              =						evaluate(dataset=A,       predictions=A      )
            return score






