




from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
					import torch


__A								=       logging.get_logger(__name__)







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			['''pixel_values''']


			def __init__( self     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = PILImageResampling.BILINEAR     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = 1 / 255     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> None:



							'''simple docstring'''
							super().__init__(**lowerCamelCase__				)
							__lowerCamelCase									=       size if size is not None else {'shortest_edge': 256}
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							__lowerCamelCase									=       crop_size if crop_size is not None else {'height': 224, 'width': 224}
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		param_name='crop_size'				)
							__lowerCamelCase									=       do_resize
							__lowerCamelCase									=       size
							__lowerCamelCase									=       resample
							__lowerCamelCase									=       do_center_crop
							__lowerCamelCase									=       crop_size
							__lowerCamelCase									=       do_rescale
							__lowerCamelCase									=       rescale_factor
							__lowerCamelCase									=       do_normalize
							__lowerCamelCase									=       image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
							__lowerCamelCase									=       image_std if image_std is not None else IMAGENET_STANDARD_STD


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = PILImageResampling.BICUBIC     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							if "shortest_edge" not in size:
											raise ValueError(f"""The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}"""				)
							__lowerCamelCase									=       get_resize_output_image_size(lowerCamelCase__     ,		size=size['shortest_edge']     ,		default_to_square=lowerCamelCase__				)
							return resize(lowerCamelCase__     ,		size=lowerCamelCase__     ,		resample=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__				)
							if "height" not in size or "width" not in size:
											raise ValueError(f"""The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}"""				)
							return center_crop(lowerCamelCase__     ,		size=(size['height'], size['width'])     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__				)       -> np.ndarray:



							'''simple docstring'''
							return rescale(lowerCamelCase__     ,		scale=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							return normalize(lowerCamelCase__     ,		mean=lowerCamelCase__     ,		std=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = ChannelDimension.FIRST     ,		**lowerCamelCase__     ,		)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       do_resize if do_resize is not None else self.do_resize
							__lowerCamelCase									=       size if size is not None else self.size
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							__lowerCamelCase									=       resample if resample is not None else self.resample
							__lowerCamelCase									=       do_center_crop if do_center_crop is not None else self.do_center_crop
							__lowerCamelCase									=       crop_size if crop_size is not None else self.crop_size
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		param_name='crop_size'				)
							__lowerCamelCase									=       do_rescale if do_rescale is not None else self.do_rescale
							__lowerCamelCase									=       rescale_factor if rescale_factor is not None else self.rescale_factor
							__lowerCamelCase									=       do_normalize if do_normalize is not None else self.do_normalize
							__lowerCamelCase									=       image_mean if image_mean is not None else self.image_mean
							__lowerCamelCase									=       image_std if image_std is not None else self.image_std

							__lowerCamelCase									=       make_list_of_images(lowerCamelCase__				)

							if not valid_images(lowerCamelCase__				):
											raise ValueError(
											    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
											    'torch.Tensor, tf.Tensor or jax.ndarray.'				)

							if do_resize and size is None:
											raise ValueError('Size must be specified if do_resize is True.'				)

							if do_center_crop and crop_size is None:
											raise ValueError('Crop size must be specified if do_center_crop is True.'				)

							if do_rescale and rescale_factor is None:
											raise ValueError('Rescale factor must be specified if do_rescale is True.'				)

							if do_normalize and (image_mean is None or image_std is None):
											raise ValueError('Image mean and std must be specified if do_normalize is True.'				)

							# All transformations expect numpy arrays.
							__lowerCamelCase									=       [to_numpy_array(lowerCamelCase__				) for image in images]

							if do_resize:
											__lowerCamelCase									=       [self.resize(image=lowerCamelCase__     ,		size=lowerCamelCase__     ,		resample=lowerCamelCase__				) for image in images]

							if do_center_crop:
											__lowerCamelCase									=       [self.center_crop(image=lowerCamelCase__     ,		size=lowerCamelCase__				) for image in images]

							if do_rescale:
											__lowerCamelCase									=       [self.rescale(image=lowerCamelCase__     ,		scale=lowerCamelCase__				) for image in images]

							if do_normalize:
											__lowerCamelCase									=       [self.normalize(image=lowerCamelCase__     ,		mean=lowerCamelCase__     ,		std=lowerCamelCase__				) for image in images]

							__lowerCamelCase									=       [to_channel_dimension_format(lowerCamelCase__     ,		lowerCamelCase__				) for image in images]

							__lowerCamelCase									=       {'pixel_values': images}
							return BatchFeature(data=lowerCamelCase__     ,		tensor_type=lowerCamelCase__				)






			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None				)       -> Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       outputs.logits

							# Resize logits and compute semantic segmentation maps
							if target_sizes is not None:
											if len(lowerCamelCase__				) != len(lowerCamelCase__				):
															raise ValueError(
															    'Make sure that you pass in as many target sizes as the batch dimension of the logits'				)

											if is_torch_tensor(lowerCamelCase__				):
															__lowerCamelCase									=       target_sizes.numpy()

											__lowerCamelCase									=       []

											for idx in range(len(lowerCamelCase__				)				):
															__lowerCamelCase									=       torch.nn.functional.interpolate(
															    logits[idx].unsqueeze(dim=0				)     ,		size=target_sizes[idx]     ,		mode='bilinear'     ,		align_corners=lowerCamelCase__				)
															__lowerCamelCase									=       resized_logits[0].argmax(dim=0				)
															semantic_segmentation.append(lowerCamelCase__				)
							else:
											__lowerCamelCase									=       logits.argmax(dim=1				)
											__lowerCamelCase									=       [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]				)]

							return semantic_segmentation








# Usage:
# ./gen-card-allenai-wmt16.py

import os
from pathlib import Path







def      lowerCamelCase_						(			UpperCamelCase__							: int					,	UpperCamelCase__							: Optional[int]					,	UpperCamelCase__							: int					,	UpperCamelCase__							: str )	->			List[str]:





				"""simple docstring"""


				__lowerCamelCase									=       {
				    'en': 'Machine learning is great, isn\'t it?',
				    'ru': 'Машинное обучение - это здорово, не так ли?',
				    'de': 'Maschinelles Lernen ist großartig, nicht wahr?',
				}

				# BLUE scores as follows:
				# "pair": [fairseq, transformers]
				__lowerCamelCase									=       {
				    'wmt16-en-de-dist-12-1': [28.3, 27.52],
				    'wmt16-en-de-dist-6-1': [27.4, 27.11],
				    'wmt16-en-de-12-1': [26.9, 25.75],
				}
				__lowerCamelCase									=       F"""{src_lang}-{tgt_lang}"""

				__lowerCamelCase									=       F"""
---
language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt16
- allenai
license: apache-2.0
datasets:
- wmt16
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

All 3 models are available:

* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)
* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)
* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = \"allenai/{model_name}\"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = \"{texts[src_lang]}\"
input_ids = tokenizer.encode(input, return_tensors=\"pt\")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   | fairseq | transformers
-------|---------|----------
{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}

The score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt16/)
- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

"""
				model_card_dir.mkdir(parents=UpperCamelCase__					,	exist_ok=UpperCamelCase__ )
				__lowerCamelCase									=       os.path.join(UpperCamelCase__					,	'README.md' )
				print(F"""Generating {path}""" )
				with open(UpperCamelCase__					,	'w'					,	encoding='utf-8' ) as f:
								f.write(UpperCamelCase__ )


# make sure we are under the root of the project
__A								=       Path(__file__).resolve().parent.parent.parent
__A								=       repo_dir / "model_cards"

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
					__A								=       model_cards_dir / "allenai" / model_name
					write_model_card(model_card_dir, src_lang="en", tgt_lang="de", model_name=model_name)


