




"""simple docstring"""







import argparse
import glob
import logging
import os
import sys
import time
from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Tuple

import numpy as np
import pytorch_lightning as pl
import torch
from callbacks import SeqaSeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
from torch import nn
from torch.utils.data import DataLoader

from transformers import MBartTokenizer, TaForConditionalGeneration
from transformers.models.bart.modeling_bart import shift_tokens_right
from utils import (
    ROUGE_KEYS,
    LegacySeqaSeqDataset,
    SeqaSeqDataset,
    assert_all_frozen,
    calculate_bleu,
    calculate_rouge,
    check_output_dir,
    flatten_list,
    freeze_embeds,
    freeze_params,
    get_git_info,
    label_smoothed_nll_loss,
    lmap,
    pickle_save,
    save_git_info,
    save_json,
    use_task_specific_params,
)


# need the parent dir module
sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
from lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa


__A   :  str					=      logging.getLogger(__name__)




class       lowerCAmelCase__       (			lowerCAmelCase_    ):


				"""simple docstring"""





				__UpperCAmelCase :     Optional[Any]       =      "summarization"
				__UpperCAmelCase :     Union[str, Any]       =      ["loss"]
				__UpperCAmelCase :     List[str]       =      ROUGE_KEYS
				__UpperCAmelCase :     Any       =      "rouge2"



				def __init__(     self					:	Dict    ,   lowercase__					:	int    ,   **lowercase__					:	Optional[int]						):
						if hparams.sortish_sampler and hparams.gpus > 1:
								__lowercase				:		Optional[int]												=				False
						elif hparams.max_tokens_per_batch is not None:
								if hparams.gpus > 1:
										raise NotImplementedError("Dynamic Batch size does not work for multi-gpu training"						)
								if hparams.sortish_sampler:
										raise ValueError("--sortish_sampler and --max_tokens_per_batch may not be used simultaneously"						)

						super().__init__(lowercase__    ,   num_labels=lowercase__    ,   mode=self.mode    ,   **lowercase__						)
						use_task_specific_params(self.model    ,   "summarization"						)
						save_git_info(self.hparams.output_dir						)
						__lowercase				:		Union[str, Any]												=				Path(self.output_dir						) / "metrics.json"
						__lowercase				:		Dict												=				Path(self.output_dir						) / "hparams.pkl"
						pickle_save(self.hparams    ,   self.hparams_save_path						)
						__lowercase				:		List[str]												=				0
						__lowercase				:		Optional[Any]												=				defaultdict(lowercase__						)
						__lowercase				:		int												=				self.config.model_type
						__lowercase				:		List[Any]												=				self.config.tgt_vocab_size if self.model_type == "fsmt" else self.config.vocab_size

						__lowercase				:		dict												=				{
						    "data_dir": self.hparams.data_dir,
						    "max_source_length": self.hparams.max_source_length,
						    "prefix": self.model.config.prefix or "",
						}
						__lowercase				:		Dict												=				{
						    "train": self.hparams.n_train,
						    "val": self.hparams.n_val,
						    "test": self.hparams.n_test,
						}
						__lowercase				:		Optional[Any]												=				{k: v if v >= 0 else None for k, v in n_observations_per_split.items()}

						__lowercase				:		Optional[Any]												=				{
						    "train": self.hparams.max_target_length,
						    "val": self.hparams.val_max_target_length,
						    "test": self.hparams.test_max_target_length,
						}
						assert self.target_lens["train"] <= self.target_lens["val"], f'target_lens: {self.target_lens}'
						assert self.target_lens["train"] <= self.target_lens["test"], f'target_lens: {self.target_lens}'
						if self.hparams.freeze_embeds:
								freeze_embeds(self.model						)
						if self.hparams.freeze_encoder:
								freeze_params(self.model.get_encoder()						)
								assert_all_frozen(self.model.get_encoder()						)

						__lowercase				:		Optional[int]												=				get_git_info()["repo_sha"]
						__lowercase				:		Dict												=				hparams.num_workers
						__lowercase				:		List[str]												=				None  # default to config
						if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer    ,   lowercase__						):
								__lowercase				:		Union[str, Any]												=				self.tokenizer.lang_code_to_id[hparams.tgt_lang]
								__lowercase				:		Union[str, Any]												=				self.decoder_start_token_id
						__lowercase				:		Tuple												=				(
						    SeqaSeqDataset if hasattr(self.tokenizer    ,   "prepare_seq2seq_batch"						) else LegacySeqaSeqDataset
						)
						__lowercase				:		Optional[Any]												=				False
						__lowercase				:		str												=				self.model.config.num_beams if self.hparams.eval_beams is None else self.hparams.eval_beams
						if self.hparams.eval_max_gen_length is not None:
								__lowercase				:		Optional[int]												=				self.hparams.eval_max_gen_length
						else:
								__lowercase				:		Dict												=				self.model.config.max_length
						__lowercase				:		Optional[int]												=				self.default_val_metric if self.hparams.val_metric is None else self.hparams.val_metric



				def       snake_case    (     self					:	Optional[int]    ,   lowercase__					:	Dict[str, torch.Tensor]						):
						__lowercase				:		Any												=				{
						    k: self.tokenizer.batch_decode(v.tolist()						) if "mask" not in k else v.shape for k, v in batch.items()
						}
						save_json(lowercase__    ,   Path(self.output_dir						) / "text_batch.json"						)
						save_json({k: v.tolist() for k, v in batch.items()}    ,   Path(self.output_dir						) / "tok_batch.json"						)

						__lowercase				:		Union[str, Any]												=				True
						return readable_batch



				def       snake_case    (     self					:	Dict    ,   lowercase__					:	Any    ,   **lowercase__					:	str						):
						return self.model(lowercase__    ,   **lowercase__						)



				def       snake_case    (     self					:	str    ,   lowercase__					:	List[int]						):
						__lowercase				:		Tuple												=				self.tokenizer.batch_decode(
						    lowercase__    ,   skip_special_tokens=lowercase__    ,   clean_up_tokenization_spaces=lowercase__						)
						return lmap(str.strip    ,   lowercase__						)



				def       snake_case    (     self					:	List[Any]    ,   lowercase__					:	dict						):
						__lowercase				:		Optional[int]												=				self.tokenizer.pad_token_id
						__lowercase				:		str												=				batch["input_ids"], batch["attention_mask"]
						__lowercase				:		List[str]												=				batch["labels"]
						if isinstance(self.model    ,   lowercase__						):
								__lowercase				:		Optional[Any]												=				self.model._shift_right(lowercase__						)
						else:
								__lowercase				:		Any												=				shift_tokens_right(lowercase__    ,   lowercase__						)
						if not self.already_saved_batch:  # This would be slightly better if it only happened on rank zero
								__lowercase				:		int												=				decoder_input_ids
								self.save_readable_batch(lowercase__						)

						__lowercase				:		Optional[int]												=				self(lowercase__    ,   attention_mask=lowercase__    ,   decoder_input_ids=lowercase__    ,   use_cache=lowercase__						)
						__lowercase				:		int												=				outputs["logits"]
						if self.hparams.label_smoothing == 0:
								# Same behavior as modeling_bart.py, besides ignoring pad_token_id
								__lowercase				:		List[str]												=				nn.CrossEntropyLoss(ignore_index=lowercase__						)

								assert lm_logits.shape[-1] == self.vocab_size
								__lowercase				:		Optional[int]												=				ce_loss_fct(lm_logits.view(-1    ,   lm_logits.shape[-1]						)    ,   tgt_ids.view(-1						)						)
						else:
								__lowercase				:		List[Any]												=				nn.functional.log_softmax(lowercase__    ,   dim=-1						)
								__lowercase				:		Optional[Any]												=				label_smoothed_nll_loss(
								    lowercase__    ,   lowercase__    ,   self.hparams.label_smoothing    ,   ignore_index=lowercase__						)
						return (loss,)



				@property
				def       snake_case    (     self					:	Dict						):
						return self.tokenizer.pad_token_id



				def       snake_case    (     self					:	Tuple    ,   lowercase__					:	Dict    ,   lowercase__					:	Any						):
						__lowercase				:		str												=				self._step(lowercase__						)

						__lowercase				:		int												=				dict(zip(self.loss_names    ,   lowercase__						)						)
						# tokens per batch
						__lowercase				:		int												=				batch["input_ids"].ne(self.pad						).sum() + batch["labels"].ne(self.pad						).sum()
						__lowercase				:		str												=				batch["input_ids"].shape[0]
						__lowercase				:		Any												=				batch["input_ids"].eq(self.pad						).sum()
						__lowercase				:		Optional[int]												=				batch["input_ids"].eq(self.pad						).float().mean()
						# TODO(SS): make a wandb summary metric for this
						return {"loss": loss_tensors[0], "log": logs}



				def       snake_case    (     self					:	Optional[int]    ,   lowercase__					:	Optional[int]    ,   lowercase__					:	Optional[int]						):
						return self._generative_step(lowercase__						)



				def       snake_case    (     self					:	Any    ,   lowercase__					:	Dict    ,   lowercase__					:	Any="val"						):
						self.step_count += 1
						__lowercase				:		List[str]												=				{k: torch.stack([x[k] for x in outputs]						).mean() for k in self.loss_names}
						__lowercase				:		List[str]												=				losses["loss"]
						__lowercase				:		Any												=				{
						    k: np.array([x[k] for x in outputs]						).mean() for k in self.metric_names + ["gen_time", "gen_len"]
						}
						__lowercase				:		str												=				(
						    generative_metrics[self.val_metric] if self.val_metric in generative_metrics else losses[self.val_metric]
						)
						__lowercase				:		torch.FloatTensor												=				torch.tensor(lowercase__						).type_as(lowercase__						)
						generative_metrics.update({k: v.item() for k, v in losses.items()}						)
						losses.update(lowercase__						)
						__lowercase				:		Optional[int]												=				{f'{prefix}_avg_{k}': x for k, x in losses.items()}
						__lowercase				:		int												=				self.step_count
						self.metrics[prefix].append(lowercase__						)  # callback writes this to self.metrics_save_path
						__lowercase				:		Optional[Any]												=				flatten_list([x["preds"] for x in outputs]						)
						return {
						    "log": all_metrics,
						    "preds": preds,
						    f'{prefix}_loss': loss,
						    f'{prefix}_{self.val_metric}': metric_tensor,
						}



				def       snake_case    (     self					:	List[Any]    ,   lowercase__					:	str    ,   lowercase__					:	str						):
						return calculate_rouge(lowercase__    ,   lowercase__						)



				def       snake_case    (     self					:	str    ,   lowercase__					:	dict						):
						__lowercase				:		Union[str, Any]												=				time.time()

						# parser.add_argument('--eval_max_gen_length', type=int, default=None, help='never generate more than n tokens')
						__lowercase				:		Optional[int]												=				self.model.generate(
						    batch["input_ids"]    ,   attention_mask=batch["attention_mask"]    ,   use_cache=lowercase__    ,   decoder_start_token_id=self.decoder_start_token_id    ,   num_beams=self.eval_beams    ,   max_length=self.eval_max_length    ,   )
						__lowercase				:		Union[str, Any]												=				(time.time() - ta) / batch["input_ids"].shape[0]
						__lowercase				:		List[str]												=				self.ids_to_clean_text(lowercase__						)
						__lowercase				:		List[str]												=				self.ids_to_clean_text(batch["labels"]						)
						__lowercase				:		int												=				self._step(lowercase__						)
						__lowercase				:		List[str]												=				dict(zip(self.loss_names    ,   lowercase__						)						)
						__lowercase				:		Dict												=				self.calc_generative_metrics(lowercase__    ,   lowercase__						)
						__lowercase				:		int												=				np.mean(lmap(lowercase__    ,   lowercase__						)						)
						base_metrics.update(gen_time=lowercase__    ,   gen_len=lowercase__    ,   preds=lowercase__    ,   target=lowercase__    ,   **lowercase__						)
						return base_metrics



				def       snake_case    (     self					:	int    ,   lowercase__					:	Any    ,   lowercase__					:	List[str]						):
						return self._generative_step(lowercase__						)



				def       snake_case    (     self					:	str    ,   lowercase__					:	str						):
						return self.validation_epoch_end(lowercase__    ,   prefix="test"						)



				def       snake_case    (     self					:	Union[str, Any]    ,   lowercase__					:	Tuple						):
						__lowercase				:		List[str]												=				self.n_obs[type_path]
						__lowercase				:		Optional[int]												=				self.target_lens[type_path]
						__lowercase				:		Tuple												=				self.dataset_class(
						    self.tokenizer    ,   type_path=lowercase__    ,   n_obs=lowercase__    ,   max_target_length=lowercase__    ,   **self.dataset_kwargs    ,   )
						return dataset



				def       snake_case    (     self					:	Tuple    ,   lowercase__					:	str    ,   lowercase__					:	int    ,   lowercase__					:	bool = False						):
						__lowercase				:		Dict												=				self.get_dataset(lowercase__						)

						if self.hparams.sortish_sampler and type_path != "test" and type_path != "val":
								__lowercase				:		Optional[int]												=				dataset.make_sortish_sampler(lowercase__    ,   distributed=self.hparams.gpus > 1						)
								return DataLoader(
								    lowercase__    ,   batch_size=lowercase__    ,   collate_fn=dataset.collate_fn    ,   shuffle=lowercase__    ,   num_workers=self.num_workers    ,   sampler=lowercase__    ,   )

						elif self.hparams.max_tokens_per_batch is not None and type_path != "test" and type_path != "val":
								__lowercase				:		int												=				dataset.make_dynamic_sampler(
								    self.hparams.max_tokens_per_batch    ,   distributed=self.hparams.gpus > 1						)
								return DataLoader(
								    lowercase__    ,   batch_sampler=lowercase__    ,   collate_fn=dataset.collate_fn    ,   num_workers=self.num_workers    ,   )
						else:
								return DataLoader(
								    lowercase__    ,   batch_size=lowercase__    ,   collate_fn=dataset.collate_fn    ,   shuffle=lowercase__    ,   num_workers=self.num_workers    ,   sampler=lowercase__    ,   )



				def       snake_case    (     self					:	List[str]						):
						__lowercase				:		Union[str, Any]												=				self.get_dataloader("train"    ,   batch_size=self.hparams.train_batch_size    ,   shuffle=lowercase__						)
						return dataloader



				def       snake_case    (     self					:	List[Any]						):
						return self.get_dataloader("val"    ,   batch_size=self.hparams.eval_batch_size						)



				def       snake_case    (     self					:	int						):
						return self.get_dataloader("test"    ,   batch_size=self.hparams.eval_batch_size						)



				@staticmethod
				def       snake_case    (     lowercase__					:	Optional[int]    ,   lowercase__					:	List[Any]						):
						BaseTransformer.add_model_specific_args(lowercase__    ,   lowercase__						)
						add_generic_args(lowercase__    ,   lowercase__						)
						parser.add_argument(
						    "--max_source_length"    ,   default=1_0_2_4    ,   type=lowercase__    ,   help=(
						        "The maximum total input sequence length after tokenization. Sequences longer "
						        "than this will be truncated, sequences shorter will be padded."
						    )    ,   )
						parser.add_argument(
						    "--max_target_length"    ,   default=5_6    ,   type=lowercase__    ,   help=(
						        "The maximum total input sequence length after tokenization. Sequences longer "
						        "than this will be truncated, sequences shorter will be padded."
						    )    ,   )
						parser.add_argument(
						    "--val_max_target_length"    ,   default=1_4_2    ,   type=lowercase__    ,   help=(
						        "The maximum total input sequence length after tokenization. Sequences longer "
						        "than this will be truncated, sequences shorter will be padded."
						    )    ,   )
						parser.add_argument(
						    "--test_max_target_length"    ,   default=1_4_2    ,   type=lowercase__    ,   help=(
						        "The maximum total input sequence length after tokenization. Sequences longer "
						        "than this will be truncated, sequences shorter will be padded."
						    )    ,   )
						parser.add_argument("--freeze_encoder"    ,   action="store_true"						)
						parser.add_argument("--freeze_embeds"    ,   action="store_true"						)
						parser.add_argument("--sortish_sampler"    ,   action="store_true"    ,   default=lowercase__						)
						parser.add_argument("--overwrite_output_dir"    ,   action="store_true"    ,   default=lowercase__						)
						parser.add_argument("--max_tokens_per_batch"    ,   type=lowercase__    ,   default=lowercase__						)
						parser.add_argument("--logger_name"    ,   type=lowercase__    ,   choices=["default", "wandb", "wandb_shared"]    ,   default="default"						)
						parser.add_argument("--n_train"    ,   type=lowercase__    ,   default=-1    ,   required=lowercase__    ,   help="# examples. -1 means use all."						)
						parser.add_argument("--n_val"    ,   type=lowercase__    ,   default=5_0_0    ,   required=lowercase__    ,   help="# examples. -1 means use all."						)
						parser.add_argument("--n_test"    ,   type=lowercase__    ,   default=-1    ,   required=lowercase__    ,   help="# examples. -1 means use all."						)
						parser.add_argument(
						    "--task"    ,   type=lowercase__    ,   default="summarization"    ,   required=lowercase__    ,   help="# examples. -1 means use all."						)
						parser.add_argument("--label_smoothing"    ,   type=lowercase__    ,   default=0.0    ,   required=lowercase__						)
						parser.add_argument("--src_lang"    ,   type=lowercase__    ,   default=""    ,   required=lowercase__						)
						parser.add_argument("--tgt_lang"    ,   type=lowercase__    ,   default=""    ,   required=lowercase__						)
						parser.add_argument("--eval_beams"    ,   type=lowercase__    ,   default=lowercase__    ,   required=lowercase__						)
						parser.add_argument(
						    "--val_metric"    ,   type=lowercase__    ,   default=lowercase__    ,   required=lowercase__    ,   choices=["bleu", "rouge2", "loss", None]						)
						parser.add_argument("--eval_max_gen_length"    ,   type=lowercase__    ,   default=lowercase__    ,   help="never generate more than n tokens"						)
						parser.add_argument("--save_top_k"    ,   type=lowercase__    ,   default=1    ,   required=lowercase__    ,   help="How many checkpoints to save"						)
						parser.add_argument(
						    "--early_stopping_patience"    ,   type=lowercase__    ,   default=-1    ,   required=lowercase__    ,   help=(
						        "-1 means never early stop. early_stopping_patience is measured in validation checks, not epochs. So"
						        " val_check_interval will effect it."
						    )    ,   )
						return parser





class       lowerCAmelCase__       (			lowerCAmelCase_    ):


				"""simple docstring"""





				__UpperCAmelCase :     List[Any]       =      "translation"
				__UpperCAmelCase :     Dict       =      ["loss"]
				__UpperCAmelCase :     Tuple       =      ["bleu"]
				__UpperCAmelCase :     Optional[Any]       =      "bleu"



				def __init__(     self					:	List[str]    ,   lowercase__					:	Any    ,   **lowercase__					:	List[Any]						):
						super().__init__(lowercase__    ,   **lowercase__						)
						__lowercase				:		Optional[Any]												=				hparams.src_lang
						__lowercase				:		List[Any]												=				hparams.tgt_lang



				def       snake_case    (     self					:	Optional[Any]    ,   lowercase__					:	int    ,   lowercase__					:	Dict						):
						return calculate_bleu(lowercase__    ,   lowercase__						)





def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase=None				)			->SummarizationModule:
		Path(args.output_dir				).mkdir(exist_ok=_lowerCamelCase				)
		check_output_dir(_lowerCamelCase,  expected_items=3				)

		if model is None:
				if "summarization" in args.task:
						__lowercase				:		SummarizationModule												=				SummarizationModule(_lowerCamelCase				)
				else:
						__lowercase				:		SummarizationModule												=				TranslationModule(_lowerCamelCase				)
		__lowercase				:		Union[str, Any]												=				Path(args.data_dir				).name
		if (
		    args.logger_name == "default"
		    or args.fast_dev_run
		    or str(args.output_dir				).startswith("/tmp"				)
		    or str(args.output_dir				).startswith("/var"				)
		):
				__lowercase				:		Optional[int]												=				True  # don't pollute wandb logs unnecessarily
		elif args.logger_name == "wandb":
				from pytorch_lightning.loggers import WandbLogger

				__lowercase				:		Any												=				os.environ.get("WANDB_PROJECT",  _lowerCamelCase				)
				__lowercase				:		Dict												=				WandbLogger(name=model.output_dir.name,  project=_lowerCamelCase				)

		elif args.logger_name == "wandb_shared":
				from pytorch_lightning.loggers import WandbLogger

				__lowercase				:		int												=				WandbLogger(name=model.output_dir.name,  project=F'hf_{dataset}'				)

		if args.early_stopping_patience >= 0:
				__lowercase				:		List[Any]												=				get_early_stopping_callback(model.val_metric,  args.early_stopping_patience				)
		else:
				__lowercase				:		str												=				False

		__lowercase				:		Dict												=				args.val_metric == "loss"
		__lowercase				:		pl.Trainer												=				generic_train(
		    _lowerCamelCase,  _lowerCamelCase,  logging_callback=SeqaSeqLoggingCallback(),  checkpoint_callback=get_checkpoint_callback(
		        args.output_dir,  model.val_metric,  args.save_top_k,  _lowerCamelCase				),  early_stopping_callback=_lowerCamelCase,  logger=_lowerCamelCase,  )
		pickle_save(model.hparams,  model.output_dir / "hparams.pkl"				)
		if not args.do_predict:
				return model

		__lowercase				:		Any												=				""
		__lowercase				:		Optional[int]												=				sorted(glob.glob(os.path.join(args.output_dir,  "*.ckpt"				),  recursive=_lowerCamelCase				)				)
		if checkpoints:
				__lowercase				:		Optional[int]												=				checkpoints[-1]
				__lowercase				:		int												=				checkpoints[-1]
		trainer.logger.log_hyperparams(model.hparams				)

		# test() without a model tests using the best checkpoint automatically
		trainer.test()
		return model


if __name__ == "__main__":
						__A   :  List[str]					=      argparse.ArgumentParser()
						__A   :  int					=      pl.Trainer.add_argparse_args(parser)
						__A   :  Optional[int]					=      SummarizationModule.add_model_specific_args(parser, os.getcwd())

						__A   :  Optional[Any]					=      parser.parse_args()

						main(args)






"""simple docstring"""







from scipy.stats import spearmanr

import datasets


__A   :  Optional[int]					=      '\nThe Spearman rank-order correlation coefficient is a measure of the\nrelationship between two datasets. Like other correlation coefficients,\nthis one varies between -1 and +1 with 0 implying no correlation.\nPositive correlations imply that as data in dataset x increases, so\ndoes data in dataset y. Negative correlations imply that as x increases,\ny decreases. Correlations of -1 or +1 imply an exact monotonic relationship.\n\nUnlike the Pearson correlation, the Spearman correlation does not\nassume that both datasets are normally distributed.\n\nThe p-value roughly indicates the probability of an uncorrelated system\nproducing datasets that have a Spearman correlation at least as extreme\nas the one computed from these datasets. The p-values are not entirely\nreliable but are probably reasonable for datasets larger than 500 or so.\n'

__A   :  List[Any]					=      '\nArgs:\n    predictions (`List[float]`): Predicted labels, as returned by a model.\n    references (`List[float]`): Ground truth labels.\n    return_pvalue (`bool`): If `True`, returns the p-value. If `False`, returns\n            only the spearmanr score. Defaults to `False`.\nReturns:\n    spearmanr (`float`): Spearman correlation coefficient.\n    p-value (`float`): p-value. **Note**: is only returned if `return_pvalue=True` is input.\nExamples:\n    Example 1:\n        >>> spearmanr_metric = datasets.load_metric("spearmanr")\n        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4])\n        >>> print(results)\n        {\'spearmanr\': -0.7}\n\n    Example 2:\n        >>> spearmanr_metric = datasets.load_metric("spearmanr")\n        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5],\n        ...                                     predictions=[10, 9, 2.5, 6, 4],\n        ...                                     return_pvalue=True)\n        >>> print(results[\'spearmanr\'])\n        -0.7\n        >>> print(round(results[\'spearmanr_pvalue\'], 2))\n        0.19\n'

__A   :  Optional[int]					=      R'\\n@book{kokoska2000crc,\n  title={CRC standard probability and statistics tables and formulae},\n  author={Kokoska, Stephen and Zwillinger, Daniel},\n  year={2000},\n  publisher={Crc Press}\n}\n@article{2020SciPy-NMeth,\n  author  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and\n            Haberland, Matt and Reddy, Tyler and Cournapeau, David and\n            Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and\n            Bright, Jonathan and {van der Walt}, St{\'e}fan J. and\n            Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and\n            Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and\n            Kern, Robert and Larson, Eric and Carey, C J and\n            Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and\n            {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and\n            Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and\n            Harris, Charles R. and Archibald, Anne M. and\n            Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and\n            {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},\n  title   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific\n            Computing in Python}},\n  journal = {Nature Methods},\n  year    = {2020},\n  volume  = {17},\n  pages   = {261--272},\n  adsurl  = {https://rdcu.be/b08Wh},\n  doi     = {10.1038/s41592-019-0686-2},\n}\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,			_KWARGS_DESCRIPTION    )
class       lowerCAmelCase__       (			datasets.Metric    ):


				"""simple docstring"""





				def       snake_case    (     self					:	List[str]						):
						return datasets.MetricInfo(
						    description=_DESCRIPTION    ,   citation=_CITATION    ,   inputs_description=_KWARGS_DESCRIPTION    ,   features=datasets.Features(
						        {
						            "predictions": datasets.Value("float"						),
						            "references": datasets.Value("float"						),
						        }						)    ,   reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html"]    ,   )



				def       snake_case    (     self					:	Optional[int]    ,   lowercase__					:	str    ,   lowercase__					:	Any    ,   lowercase__					:	int=False						):
						__lowercase				:		Optional[int]												=				spearmanr(lowercase__    ,   lowercase__						)
						if return_pvalue:
								return {"spearmanr": results[0], "spearmanr_pvalue": results[1]}
						else:
								return {"spearmanr": results[0]}
