



# Usage:
# ./gen-card-allenai-wmt16.py

import os
from pathlib import Path






def        __lowerCAmelCase		(	__lowerCamelCase						:     Union[str, Any]					, __lowerCamelCase						:     str					, __lowerCamelCase						:     List[Any]					, __lowerCamelCase						:     Any						)	->    List[Any]:
					__lowerCAmelCase						 ={
					    """en""": """Machine learning is great, isn't it?""",
					    """ru""": """Машинное обучение - это здорово, не так ли?""",
					    """de""": """Maschinelles Lernen ist großartig, nicht wahr?""",
					}

					# BLUE scores as follows:
					# "pair": [fairseq, transformers]
					__lowerCAmelCase						 ={
					    """wmt16-en-de-dist-12-1""": [28.3, 27.52],
					    """wmt16-en-de-dist-6-1""": [27.4, 27.11],
					    """wmt16-en-de-12-1""": [26.9, 25.75],
					}
					__lowerCAmelCase						 =f"""{src_lang}-{tgt_lang}"""

					__lowerCAmelCase						 =f"""
---
language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt16
- allenai
license: apache-2.0
datasets:
- wmt16
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

All 3 models are available:

* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)
* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)
* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = \"allenai/{model_name}\"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = \"{texts[src_lang]}\"
input_ids = tokenizer.encode(input, return_tensors=\"pt\")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   | fairseq | transformers
-------|---------|----------
{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}

The score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt16/)
- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

"""
					model_card_dir.mkdir(parents=__lowerCamelCase					, exist_ok=__lowerCamelCase						)
					__lowerCAmelCase						 =os.path.join(__lowerCamelCase					, """README.md"""						)
					print(f"""Generating {path}"""						)
					with open(__lowerCamelCase					, """w"""					, encoding="""utf-8"""						) as f:
										f.write(__lowerCamelCase						)


# make sure we are under the root of the project
lowercase_													=      Path(__file__).resolve().parent.parent.parent
lowercase_													=      repo_dir / '''model_cards'''

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
				lowercase_													=      model_cards_dir / '''allenai''' / model_name
				write_model_card(model_card_dir, src_lang='''en''', tgt_lang='''de''', model_name=model_name)









import unittest

from transformers import is_vision_available
from transformers.pipelines import pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
				from PIL import Image
else:






				class     __a			:



					@staticmethod
					def 		UpperCamelCase				(    *snake_case_				:     Any					,						**snake_case_				:     str)->				int:
										pass






@is_pipeline_test
@require_vision
class     __a			(  unittest.TestCase					):



	@require_torch
	def 		UpperCamelCase				(    self				:     Dict)->				List[str]:
						__lowerCAmelCase						 =pipeline(
						    model="""hf-internal-testing/tiny-random-clip-zero-shot-image-classification"""					,						)
						__lowerCAmelCase						 =Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""")
						__lowerCAmelCase						 =image_classifier(snake_case_					,						candidate_labels=["""a""", """b""", """c"""])

						# The floating scores are so close, we enter floating error approximation and the order is not guaranteed across
						# python and torch versions.
						self.assertIn(
						    nested_simplify(snake_case_)					,						[
						        [{"""score""": 0.3_3_3, """label""": """a"""}, {"""score""": 0.3_3_3, """label""": """b"""}, {"""score""": 0.3_3_3, """label""": """c"""}],
						        [{"""score""": 0.3_3_3, """label""": """a"""}, {"""score""": 0.3_3_3, """label""": """c"""}, {"""score""": 0.3_3_3, """label""": """b"""}],
						    ]					,						)

						__lowerCAmelCase						 =image_classifier([image] * 5					,						candidate_labels=["""A""", """B""", """C"""]					,						batch_size=2)
						self.assertEqual(
						    nested_simplify(snake_case_)					,						[
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						    ]					,						)



	@require_tf
	def 		UpperCamelCase				(    self				:     Dict)->				Optional[Any]:
						__lowerCAmelCase						 =pipeline(
						    model="""hf-internal-testing/tiny-random-clip-zero-shot-image-classification"""					,						framework="""tf""")
						__lowerCAmelCase						 =Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""")
						__lowerCAmelCase						 =image_classifier(snake_case_					,						candidate_labels=["""a""", """b""", """c"""])

						self.assertEqual(
						    nested_simplify(snake_case_)					,						[{"""score""": 0.3_3_3, """label""": """a"""}, {"""score""": 0.3_3_3, """label""": """b"""}, {"""score""": 0.3_3_3, """label""": """c"""}]					,						)

						__lowerCAmelCase						 =image_classifier([image] * 5					,						candidate_labels=["""A""", """B""", """C"""]					,						batch_size=2)
						self.assertEqual(
						    nested_simplify(snake_case_)					,						[
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						        [
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						            {"""score""": 0.3_3_3, """label""": ANY(snake_case_)},
						        ],
						    ]					,						)



	@slow
	@require_torch
	def 		UpperCamelCase				(    self				:     Any)->				Dict:
						__lowerCAmelCase						 =pipeline(
						    task="""zero-shot-image-classification"""					,						model="""openai/clip-vit-base-patch32"""					,						)
						# This is an image of 2 cats with remotes and no planes
						__lowerCAmelCase						 =Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""")
						__lowerCAmelCase						 =image_classifier(snake_case_					,						candidate_labels=["""cat""", """plane""", """remote"""])

						self.assertEqual(
						    nested_simplify(snake_case_)					,						[
						        {"""score""": 0.5_1_1, """label""": """remote"""},
						        {"""score""": 0.4_8_5, """label""": """cat"""},
						        {"""score""": 0.0_0_4, """label""": """plane"""},
						    ]					,						)

						__lowerCAmelCase						 =image_classifier([image] * 5					,						candidate_labels=["""cat""", """plane""", """remote"""]					,						batch_size=2)
						self.assertEqual(
						    nested_simplify(snake_case_)					,						[
						        [
						            {"""score""": 0.5_1_1, """label""": """remote"""},
						            {"""score""": 0.4_8_5, """label""": """cat"""},
						            {"""score""": 0.0_0_4, """label""": """plane"""},
						        ],
						    ]
						    * 5					,						)



	@slow
	@require_tf
	def 		UpperCamelCase				(    self				:     Optional[int])->				int:
						__lowerCAmelCase						 =pipeline(
						    task="""zero-shot-image-classification"""					,						model="""openai/clip-vit-base-patch32"""					,						framework="""tf""")
						# This is an image of 2 cats with remotes and no planes
						__lowerCAmelCase						 =Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""")
						__lowerCAmelCase						 =image_classifier(snake_case_					,						candidate_labels=["""cat""", """plane""", """remote"""])
						self.assertEqual(
						    nested_simplify(snake_case_)					,						[
						        {"""score""": 0.5_1_1, """label""": """remote"""},
						        {"""score""": 0.4_8_5, """label""": """cat"""},
						        {"""score""": 0.0_0_4, """label""": """plane"""},
						    ]					,						)

						__lowerCAmelCase						 =image_classifier([image] * 5					,						candidate_labels=["""cat""", """plane""", """remote"""]					,						batch_size=2)
						self.assertEqual(
						    nested_simplify(snake_case_)					,						[
						        [
						            {"""score""": 0.5_1_1, """label""": """remote"""},
						            {"""score""": 0.4_8_5, """label""": """cat"""},
						            {"""score""": 0.0_0_4, """label""": """plane"""},
						        ],
						    ]
						    * 5					,						)




