


'''simple docstring'''
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCAmelCase   : Union[str, Any]             =				"""\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
"""

lowerCAmelCase   : str             =				"""\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
"""

lowerCAmelCase   : List[str]             =				"""
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for 'record': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'prediction_text': the predicted answer text
        - for 'multirc': list of question-answer dictionaries with the following keys:
            - 'idx': index of the question-answer pair as specified by the dataset
            - 'prediction': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for 'record': list of question-answers dictionaries with the following keys:
            - 'idx': index of the question as specified by the dataset
            - 'answers': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for 'record':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1': F1 score
    - for 'multirc':
        - 'exact_match': Exact match between answer and gold answer
        - 'f1_m': Per-question macro-F1 score
        - 'f1_a': Average F1 score over all answers
    - for 'axb':
        'matthews_correlation': Matthew Correlation
    - for 'cb':
        - 'accuracy': Accuracy
        - 'f1': F1 score
    - for all others:
        - 'accuracy': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')
    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]
    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')
    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}

    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'matthews_correlation': 1.0}
"""




def 	_A  (       A							,A							)  ->				Union[str, Any]:
	return float((preds == labels).mean()							)




def 	_A  (       A							,A							,A="binary"							)  ->				int:
	lowercase						:		int   					=		simple_accuracy(A							,A							)
	lowercase						:		Union[str, Any]   					=		float(fa_score(y_true=A							,y_pred=A							,average=A							)							)
	return {
	    "accuracy": acc,
	    "f1": fa,
	}




def 	_A  (       A							,A							)  ->				str:
	lowercase						:		Union[str, Any]   					=		{}
	for id_pred, label in zip(A							,A							):
		lowercase						:		Optional[int]   					=		F'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
		lowercase						:		Optional[Any]   					=		id_pred["prediction"]
		if question_id in question_map:
			question_map[question_id].append((pred, label)							)
		else:
			lowercase						:		Tuple   					=		[(pred, label)]
	lowercase					,		lowercase						:		int   					=		[], []
	for question, preds_labels in question_map.items():
		lowercase					,		lowercase						:		Tuple   					=		zip(*A							)
		lowercase						:		Tuple   					=		fa_score(y_true=A							,y_pred=A							,average="macro"							)
		fas.append(A							)
		lowercase						:		List[str]   					=		int(sum(pred == label for pred, label in preds_labels							) == len(A							)							)
		ems.append(A							)
	lowercase						:		Dict   					=		float(sum(A							) / len(A							)							)
	lowercase						:		Any   					=		sum(A							) / len(A							)
	lowercase						:		int   					=		float(fa_score(y_true=A							,y_pred=[id_pred["prediction"] for id_pred in ids_preds]							)							)
	return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,			_KWARGS_DESCRIPTION)
class        _UpperCamelCase	(    datasets.Metric):


				'''simple docstring'''


				def 	a__		(     self							)    ->  Union[str, Any]:
					if self.config_name not in [
					    "boolq",
					    "cb",
					    "copa",
					    "multirc",
					    "record",
					    "rte",
					    "wic",
					    "wsc",
					    "wsc.fixed",
					    "axb",
					    "axg",
					]:
						raise KeyError(
						    "You should supply a configuration name selected in "
						    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"							)
					return datasets.MetricInfo(
					    description=_DESCRIPTION							,      citation=_CITATION							,      inputs_description=_KWARGS_DESCRIPTION							,      features=datasets.Features(self._get_feature_types()							)							,      codebase_urls=[]							,      reference_urls=[]							,      format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None							,      )

				def 	a__		(     self							)    ->  Optional[Any]:
					if self.config_name == "record":
						return {
						    "predictions": {
						        "idx": {
						            "passage": datasets.Value("int64"							),
						            "query": datasets.Value("int64"							),
						        },
						        "prediction_text": datasets.Value("string"							),
						    },
						    "references": {
						        "idx": {
						            "passage": datasets.Value("int64"							),
						            "query": datasets.Value("int64"							),
						        },
						        "answers": datasets.Sequence(datasets.Value("string"							)							),
						    },
						}
					elif self.config_name == "multirc":
						return {
						    "predictions": {
						        "idx": {
						            "answer": datasets.Value("int64"							),
						            "paragraph": datasets.Value("int64"							),
						            "question": datasets.Value("int64"							),
						        },
						        "prediction": datasets.Value("int64"							),
						    },
						    "references": datasets.Value("int64"							),
						}
					else:
						return {
						    "predictions": datasets.Value("int64"							),
						    "references": datasets.Value("int64"							),
						}


				def 	a__		(     self							,      a_							,      a_							)    ->  Any:
					if self.config_name == "axb":
						return {"matthews_correlation": matthews_corrcoef(a_							,      a_							)}
					elif self.config_name == "cb":
						return acc_and_fa(a_							,      a_							,      fa_avg="macro"							)
					elif self.config_name == "record":
						lowercase						:		int   					=		[
						    {
						        "qas": [
						            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
						            for ref in references
						        ]
						    }
						]
						lowercase						:		Optional[int]   					=		{pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
						return evaluate_record(a_							,      a_							)[0]
					elif self.config_name == "multirc":
						return evaluate_multirc(a_							,      a_							)
					elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
						return {"accuracy": simple_accuracy(a_							,      a_							)}
					else:
						raise KeyError(
						    "You should supply a configuration name selected in "
						    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"							)






'''simple docstring'''
# Usage:
# ./gen-card-allenai-wmt16.py

import os
from pathlib import Path




def 	_A  (       A							,A							,A							,A							)  ->				List[Any]:
	lowercase						:		Optional[int]   					=		{
	    "en": "Machine learning is great, isn't it?",
	    "ru": "Машинное обучение - это здорово, не так ли?",
	    "de": "Maschinelles Lernen ist großartig, nicht wahr?",
	}

	# BLUE scores as follows:
	# "pair": [fairseq, transformers]
	lowercase						:		Optional[int]   					=		{
	    "wmt16-en-de-dist-12-1": [28.3, 27.52],
	    "wmt16-en-de-dist-6-1": [27.4, 27.11],
	    "wmt16-en-de-12-1": [26.9, 25.75],
	}
	lowercase						:		Dict   					=		F'''{src_lang}-{tgt_lang}'''

	lowercase						:		Union[str, Any]   					=		F'''
---
language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt16
- allenai
license: apache-2.0
datasets:
- wmt16
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

All 3 models are available:

* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)
* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)
* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "allenai/{model_name}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   | fairseq | transformers
-------|---------|----------
{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}

The score is slightly below the score reported in the paper, as the researchers don\'t use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt16/)
- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

'''
	model_card_dir.mkdir(parents=A							,exist_ok=A							)
	lowercase						:		str   					=		os.path.join(A							,"README.md"							)
	print(F'''Generating {path}'''							)
	with open(A							,"w"							,encoding="utf-8"							) as f:
		f.write(A							)


# make sure we are under the root of the project
lowerCAmelCase   : Union[str, Any]             =				Path(__file__).resolve().parent.parent.parent
lowerCAmelCase   : Union[str, Any]             =				repo_dir / """model_cards"""

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
					lowerCAmelCase   : Optional[Any]             =				model_cards_dir / """allenai""" / model_name
					write_model_card(model_card_dir, src_lang="""en""", tgt_lang="""de""", model_name=model_name)


