






import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


SCREAMING_SNAKE_CASE__		:							List[str] 				=				logging.getLogger(__name__)

require_version("""pytorch_lightning>=1.0.4""")

SCREAMING_SNAKE_CASE__		:							Optional[int] 				=				{
    """base""": AutoModel,
    """sequence-classification""": AutoModelForSequenceClassification,
    """question-answering""": AutoModelForQuestionAnswering,
    """pretraining""": AutoModelForPreTraining,
    """token-classification""": AutoModelForTokenClassification,
    """language-modeling""": AutoModelWithLMHead,
    """summarization""": AutoModelForSeqaSeqLM,
    """translation""": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
SCREAMING_SNAKE_CASE__		:							List[str] 				=				{
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
SCREAMING_SNAKE_CASE__		:							Tuple 				=				sorted(arg_to_scheduler.keys())
SCREAMING_SNAKE_CASE__		:							Optional[Any] 				=				"""{""" + """, """.join(arg_to_scheduler_choices) + """}"""







class 				UpperCAmelCase_  (		pl.LightningModule		):




     def __init__(				self	,						_lowerCAmelCase	,						_lowerCAmelCase=None	,						_lowerCAmelCase="base"	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						**_lowerCAmelCase	,						):
          super().__init__()
          # TODO: move to self.save_hyperparameters()
          # self.save_hyperparameters()
          # can also expand arguments into trainer signature for easier reading

          self.save_hyperparameters(_lowerCAmelCase		)
          UpperCAmelCase__      :  int				      =			0
          UpperCAmelCase__      :  int				      =			Path(self.hparams.output_dir		)
          UpperCAmelCase__      :  Dict				      =			self.hparams.cache_dir if self.hparams.cache_dir else None
          if config is None:
               UpperCAmelCase__      :  Optional[int]				      =			AutoConfig.from_pretrained(
                   self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path	,						**({"""num_labels""": num_labels} if num_labels is not None else {})	,						cache_dir=_lowerCAmelCase	,						**_lowerCAmelCase	,						)
          else:
               UpperCAmelCase__      :  PretrainedConfig				      =			config

          UpperCAmelCase__      :  List[str]				      =			("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
          for p in extra_model_params:
               if getattr(self.hparams	,						_lowerCAmelCase	,						_lowerCAmelCase		):
                    assert hasattr(self.config	,						_lowerCAmelCase		), f"model config doesn't have a `{p}` attribute"
                    setattr(self.config	,						_lowerCAmelCase	,						getattr(self.hparams	,						_lowerCAmelCase		)		)

          if tokenizer is None:
               UpperCAmelCase__      :  str				      =			AutoTokenizer.from_pretrained(
                   self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path	,						cache_dir=_lowerCAmelCase	,						)
          else:
               UpperCAmelCase__      :  PreTrainedTokenizer				      =			tokenizer
          UpperCAmelCase__      :  List[str]				      =			MODEL_MODES[mode]
          if model is None:
               UpperCAmelCase__      :  Optional[Any]				      =			self.model_type.from_pretrained(
                   self.hparams.model_name_or_path	,						from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path		)	,						config=self.config	,						cache_dir=_lowerCAmelCase	,						)
          else:
               UpperCAmelCase__      :  Tuple				      =			model




     def   __UpperCAmelCase  (				self	,						*_lowerCAmelCase	,						**_lowerCAmelCase		):
          UpperCAmelCase__      :  Union[str, Any]				      =			self.model_type.from_pretrained(*_lowerCAmelCase	,						**_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Tuple				      =			arg_to_scheduler[self.hparams.lr_scheduler]
          UpperCAmelCase__      :  Optional[int]				      =			get_schedule_func(
              self.opt	,						num_warmup_steps=self.hparams.warmup_steps	,						num_training_steps=self.total_steps()		)
          UpperCAmelCase__      :  Union[str, Any]				      =			{"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
          return scheduler




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Any				      =			self.model
          UpperCAmelCase__      :  int				      =			["""bias""", """LayerNorm.weight"""]
          UpperCAmelCase__      :  int				      =			[
              {
                  """params""": [
                      p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay		)
                  ],  # check this named paramters
                  """weight_decay""": self.hparams.weight_decay,
              },
              {
                  """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay		)],
                  """weight_decay""": 0.0,
              },
          ]
          if self.hparams.adafactor:
               UpperCAmelCase__      :  Dict				      =			Adafactor(
                   _lowerCAmelCase	,						lr=self.hparams.learning_rate	,						scale_parameter=_lowerCAmelCase	,						relative_step=_lowerCAmelCase		)

          else:
               UpperCAmelCase__      :  Union[str, Any]				      =			AdamW(
                   _lowerCAmelCase	,						lr=self.hparams.learning_rate	,						eps=self.hparams.adam_epsilon		)
          UpperCAmelCase__      :  Optional[int]				      =			optimizer

          UpperCAmelCase__      :  int				      =			self.get_lr_scheduler()

          return [optimizer], [scheduler]




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          return self.validation_step(_lowerCAmelCase	,						_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase		):
          return self.validation_end(_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self		):
          UpperCAmelCase__      :  Dict				      =			max(1	,						self.hparams.gpus		)  # TODO: consider num_tpu_cores
          UpperCAmelCase__      :  List[Any]				      =			self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
          return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase		):
          if stage == "test":
               UpperCAmelCase__      :  int				      =			len(self.test_dataloader().dataset		)
          else:
               UpperCAmelCase__      :  Optional[Any]				      =			self.get_dataloader("""train"""	,						self.hparams.train_batch_size	,						shuffle=_lowerCAmelCase		)
               UpperCAmelCase__      :  Optional[int]				      =			len(self.train_dataloader().dataset		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase = False		):
          raise NotImplementedError("""You must implement this for your task"""		)




     def   __UpperCAmelCase  (				self		):
          return self.train_loader




     def   __UpperCAmelCase  (				self		):
          return self.get_dataloader("""dev"""	,						self.hparams.eval_batch_size	,						shuffle=_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self		):
          return self.get_dataloader("""test"""	,						self.hparams.eval_batch_size	,						shuffle=_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase		):
          return os.path.join(
              self.hparams.data_dir	,						"""cached_{}_{}_{}""".format(
                  _lowerCAmelCase	,						list(filter(_lowerCAmelCase	,						self.hparams.model_name_or_path.split("""/"""		)		)		).pop()	,						str(self.hparams.max_seq_length		)	,						)	,						)




     @pl.utilities.rank_zero_only
     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  Optional[int]				      =			self.output_dir.joinpath("""best_tfmr"""		)
          UpperCAmelCase__      :  List[str]				      =			self.step_count
          self.model.save_pretrained(_lowerCAmelCase		)
          self.tokenizer.save_pretrained(_lowerCAmelCase		)




     @staticmethod
     def   __UpperCAmelCase  (				_lowerCAmelCase	,						_lowerCAmelCase		):
          parser.add_argument(
              """--model_name_or_path"""	,						default=_lowerCAmelCase	,						type=_lowerCAmelCase	,						required=_lowerCAmelCase	,						help="""Path to pretrained model or model identifier from huggingface.co/models"""	,						)
          parser.add_argument(
              """--config_name"""	,						default=""""""	,						type=_lowerCAmelCase	,						help="""Pretrained config name or path if not the same as model_name"""		)
          parser.add_argument(
              """--tokenizer_name"""	,						default=_lowerCAmelCase	,						type=_lowerCAmelCase	,						help="""Pretrained tokenizer name or path if not the same as model_name"""	,						)
          parser.add_argument(
              """--cache_dir"""	,						default=str(Path(_lowerCAmelCase		).parent / """test_run""" / """cache"""		)	,						type=_lowerCAmelCase	,						help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""	,						)
          parser.add_argument(
              """--encoder_layerdrop"""	,						type=_lowerCAmelCase	,						help="""Encoder layer dropout probability (Optional). Goes into model.config"""	,						)
          parser.add_argument(
              """--decoder_layerdrop"""	,						type=_lowerCAmelCase	,						help="""Decoder layer dropout probability (Optional). Goes into model.config"""	,						)
          parser.add_argument(
              """--dropout"""	,						type=_lowerCAmelCase	,						help="""Dropout probability (Optional). Goes into model.config"""	,						)
          parser.add_argument(
              """--attention_dropout"""	,						type=_lowerCAmelCase	,						help="""Attention dropout probability (Optional). Goes into model.config"""	,						)
          parser.add_argument("""--learning_rate"""	,						default=5e-5	,						type=_lowerCAmelCase	,						help="""The initial learning rate for Adam."""		)
          parser.add_argument(
              """--lr_scheduler"""	,						default="""linear"""	,						choices=_lowerCAmelCase	,						metavar=_lowerCAmelCase	,						type=_lowerCAmelCase	,						help="""Learning rate scheduler"""	,						)
          parser.add_argument("""--weight_decay"""	,						default=0.0	,						type=_lowerCAmelCase	,						help="""Weight decay if we apply some."""		)
          parser.add_argument("""--adam_epsilon"""	,						default=1e-8	,						type=_lowerCAmelCase	,						help="""Epsilon for Adam optimizer."""		)
          parser.add_argument("""--warmup_steps"""	,						default=0	,						type=_lowerCAmelCase	,						help="""Linear warmup over warmup_steps."""		)
          parser.add_argument("""--num_workers"""	,						default=4	,						type=_lowerCAmelCase	,						help="""kwarg passed to DataLoader"""		)
          parser.add_argument("""--num_train_epochs"""	,						dest="""max_epochs"""	,						default=3	,						type=_lowerCAmelCase		)
          parser.add_argument("""--train_batch_size"""	,						default=32	,						type=_lowerCAmelCase		)
          parser.add_argument("""--eval_batch_size"""	,						default=32	,						type=_lowerCAmelCase		)
          parser.add_argument("""--adafactor"""	,						action="""store_true"""		)







class 				UpperCAmelCase_  (		pl.Callback		):




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          if (
              trainer.is_global_zero and trainer.global_rank == 0
          ):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
               pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.







class 				UpperCAmelCase_  (		pl.Callback		):




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          # print(pl_module.model.rag)
          for name, param in pl_module.model.rag.named_parameters():
               if param.grad is None:
                    print(_lowerCAmelCase		)







class 				UpperCAmelCase_  (		pl.Callback		):




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  Any				      =			trainer.lr_schedulers[0]["""scheduler"""]
          UpperCAmelCase__      :  Optional[int]				      =			{f"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr()		)}
          pl_module.logger.log_metrics(_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          rank_zero_info("""***** Validation results *****"""		)
          UpperCAmelCase__      :  Tuple				      =			trainer.callback_metrics
          # Log results
          for key in sorted(_lowerCAmelCase		):
               if key not in ["log", "progress_bar"]:
                    rank_zero_info("""{} = {}\n""".format(_lowerCAmelCase	,						str(metrics[key]		)		)		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase		):
          rank_zero_info("""***** Test results *****"""		)
          UpperCAmelCase__      :  Any				      =			trainer.callback_metrics
          # Log and save results to file
          UpperCAmelCase__      :  Optional[Any]				      =			os.path.join(pl_module.hparams.output_dir	,						"""test_results.txt"""		)
          with open(_lowerCAmelCase	,						"""w"""		) as writer:
               for key in sorted(_lowerCAmelCase		):
                    if key not in ["log", "progress_bar"]:
                         rank_zero_info("""{} = {}\n""".format(_lowerCAmelCase	,						str(metrics[key]		)		)		)
                         writer.write("""{} = {}\n""".format(_lowerCAmelCase	,						str(metrics[key]		)		)		)





def  _lowerCamelCase						(						__lowerCamelCase ,							__lowerCamelCase       ) ->			None:

     '''simple docstring'''
     #  To allow all pl args uncomment the following line
     #  parser = pl.Trainer.add_argparse_args(parser)
     parser.add_argument(
         """--output_dir""" ,							default=str(Path(__lowerCamelCase       ).parent / """test_run""" / """model_checkpoints"""       ) ,							type=__lowerCamelCase ,							help="""The output directory where the model predictions and checkpoints will be written.""" ,							)
     parser.add_argument(
         """--fp16""" ,							action="""store_true""" ,							help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit""" ,							)

     parser.add_argument(
         """--fp16_opt_level""" ,							type=__lowerCamelCase ,							default="""O2""" ,							help=(
             """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
             """See details at https://nvidia.github.io/apex/amp.html"""
         ) ,							)
     parser.add_argument("""--n_tpu_cores""" ,							dest="""tpu_cores""" ,							type=__lowerCamelCase       )
     parser.add_argument("""--max_grad_norm""" ,							dest="""gradient_clip_val""" ,							default=1.0 ,							type=__lowerCamelCase ,							help="""Max gradient norm"""       )
     parser.add_argument("""--do_train""" ,							action="""store_true""" ,							help="""Whether to run training."""       )
     parser.add_argument("""--do_predict""" ,							action="""store_true""" ,							help="""Whether to run predictions on the test set."""       )
     parser.add_argument(
         """--gradient_accumulation_steps""" ,							dest="""accumulate_grad_batches""" ,							type=__lowerCamelCase ,							default=1 ,							help="""Number of updates steps to accumulate before performing a backward/update pass.""" ,							)
     parser.add_argument("""--seed""" ,							type=__lowerCamelCase ,							default=42 ,							help="""random seed for initialization"""       )
     parser.add_argument(
         """--data_dir""" ,							default=str(Path(__lowerCamelCase       ).parent / """test_run""" / """dummy-train-data"""       ) ,							type=__lowerCamelCase ,							help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task.""" ,							)







def  _lowerCamelCase						(						__lowerCamelCase ,							__lowerCamelCase ,							__lowerCamelCase=None ,							__lowerCamelCase=True ,							__lowerCamelCase=[] ,							__lowerCamelCase=None ,							__lowerCamelCase=None ,							**__lowerCamelCase ,							) ->			Dict:

     '''simple docstring'''
     pl.seed_everything(args.seed       )

     # init model
     UpperCAmelCase__      :  Optional[int]				      =			Path(model.hparams.output_dir       )
     odir.mkdir(exist_ok=__lowerCamelCase       )

     # add custom checkpoints
     if checkpoint_callback is None:
          UpperCAmelCase__      :  List[Any]				      =			pl.callbacks.ModelCheckpoint(
              filepath=args.output_dir ,							prefix="""checkpoint""" ,							monitor="""val_loss""" ,							mode="""min""" ,							save_top_k=1       )
     if early_stopping_callback:
          extra_callbacks.append(__lowerCamelCase       )
     if logging_callback is None:
          UpperCAmelCase__      :  Optional[Any]				      =			LoggingCallback()

     UpperCAmelCase__      :  List[str]				      =			{}

     if args.fpaa:
          UpperCAmelCase__      :  List[Any]				      =			16

     if args.gpus > 1:
          UpperCAmelCase__      :  Dict				      =			"""auto"""
          UpperCAmelCase__      :  str				      =			"""ddp"""

     UpperCAmelCase__      :  Optional[Any]				      =			args.accumulate_grad_batches
     UpperCAmelCase__      :  List[Any]				      =			None
     UpperCAmelCase__      :  List[str]				      =			"""auto"""

     UpperCAmelCase__      :  List[str]				      =			pl.Trainer.from_argparse_args(
         __lowerCamelCase ,							weights_summary=__lowerCamelCase ,							callbacks=[logging_callback] + extra_callbacks + [InitCallback()] + [checkpoint_callback] ,							logger=__lowerCamelCase ,							val_check_interval=1 ,							num_sanity_val_steps=2 ,							**__lowerCamelCase ,							)

     if args.do_train:
          trainer.fit(__lowerCamelCase       )

     else:
          print("""RAG modeling tests with new set functions successfuly executed!"""       )
     return trainer








import absl  # noqa: F401 # Here to have a nice missing dependency error message early on
import nltk  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import six  # noqa: F401 # Here to have a nice missing dependency error message early on
from rouge_score import rouge_scorer, scoring

import datasets


_lowercase											=    """\
@inproceedings{lin-2004-rouge,
    title = \"{ROUGE}: A Package for Automatic Evaluation of Summaries\",
    author = \"Lin, Chin-Yew\",
    booktitle = \"Text Summarization Branches Out\",
    month = jul,
    year = \"2004\",
    address = \"Barcelona, Spain\",
    publisher = \"Association for Computational Linguistics\",
    url = \"https://www.aclweb.org/anthology/W04-1013\",
    pages = \"74--81\",
}
"""

_lowercase											=    """\
ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for
evaluating automatic summarization and machine translation software in natural language processing.
The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.

Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.

This metrics is a wrapper around Google Research reimplementation of ROUGE:
https://github.com/google-research/google-research/tree/master/rouge
"""

_lowercase											=    """
Calculates average rouge scores for a list of hypotheses and references
Args:
    predictions: list of predictions to score. Each prediction
        should be a string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should be a string with tokens separated by spaces.
    rouge_types: A list of rouge types to calculate.
        Valid names:
        `\"rouge{n}\"` (e.g. `\"rouge1\"`, `\"rouge2\"`) where: {n} is the n-gram based scoring,
        `\"rougeL\"`: Longest common subsequence based scoring.
        `\"rougeLSum\"`: rougeLsum splits text using `\"\n\"`.
        See details in https://github.com/huggingface/datasets/issues/617
    use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes.
    use_aggregator: Return aggregates if this is set to True
Returns:
    rouge1: rouge_1 (precision, recall, f1),
    rouge2: rouge_2 (precision, recall, f1),
    rougeL: rouge_l (precision, recall, f1),
    rougeLsum: rouge_lsum (precision, recall, f1)
Examples:

    >>> rouge = datasets.load_metric('rouge')
    >>> predictions = [\"hello there\", \"general kenobi\"]
    >>> references = [\"hello there\", \"general kenobi\"]
    >>> results = rouge.compute(predictions=predictions, references=references)
    >>> print(list(results.keys()))
    ['rouge1', 'rouge2', 'rougeL', 'rougeLsum']
    >>> print(results[\"rouge1\"])
    AggregateScore(low=Score(precision=1.0, recall=1.0, fmeasure=1.0), mid=Score(precision=1.0, recall=1.0, fmeasure=1.0), high=Score(precision=1.0, recall=1.0, fmeasure=1.0))
    >>> print(results[\"rouge1\"].mid.fmeasure)
    1.0
"""


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,      _KWARGS_DESCRIPTION       )
class 			lowercase_		(      datasets.Metric       ):


       def 	_snake_case				(    self				)						->    Union[str, Any]:
              return datasets.MetricInfo(
                  description=_DESCRIPTION						,       citation=_CITATION						,       inputs_description=_KWARGS_DESCRIPTION						,       features=datasets.Features(
                      {
                          '''predictions''': datasets.Value('''string'''						,       id='''sequence'''				),
                          '''references''': datasets.Value('''string'''						,       id='''sequence'''				),
                      }				)						,       codebase_urls=['''https://github.com/google-research/google-research/tree/master/rouge''']						,       reference_urls=[
                      '''https://en.wikipedia.org/wiki/ROUGE_(metric)''',
                      '''https://github.com/google-research/google-research/tree/master/rouge''',
                  ]						,       )





       def 	_snake_case				(    self						,       __A						,       __A						,       __A=None						,       __A=True						,       __A=False				)						->    List[str]:
              if rouge_types is None:
                     SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=['''rouge1''', '''rouge2''', '''rougeL''', '''rougeLsum''']

              SCREAMING_SNAKE_CASE_							:				Tuple		=rouge_scorer.RougeScorer(rouge_types=__A						,       use_stemmer=__A				)
              if use_aggregator:
                     SCREAMING_SNAKE_CASE_							:				List[str]		=scoring.BootstrapAggregator()
              else:
                     SCREAMING_SNAKE_CASE_							:				Tuple		=[]

              for ref, pred in zip(__A						,       __A				):
                     SCREAMING_SNAKE_CASE_							:				int		=scorer.score(__A						,       __A				)
                     if use_aggregator:
                            aggregator.add_scores(__A				)
                     else:
                            scores.append(__A				)

              if use_aggregator:
                     SCREAMING_SNAKE_CASE_							:				Tuple		=aggregator.aggregate()
              else:
                     SCREAMING_SNAKE_CASE_							:				Optional[int]		={}
                     for key in scores[0]:
                            SCREAMING_SNAKE_CASE_							:				Tuple		=[score[key] for score in scores]

              return result
