



# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path







def snake_case__	(      __SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE			)		->				int:
		UpperCAmelCase_						       =					{
		    '''en''': '''Machine learning is great, isn\'t it?''',
		    '''ru''': '''Машинное обучение - это здорово, не так ли?''',
		    '''de''': '''Maschinelles Lernen ist großartig, oder?''',
		}

		# BLUE scores as follows:
		# "pair": [fairseq, transformers]
		UpperCAmelCase_						       =					{
		    '''ru-en''': ['''[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)''', '''39.20'''],
		    '''en-ru''': ['''[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)''', '''33.47'''],
		    '''en-de''': ['''[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)''', '''42.83'''],
		    '''de-en''': ['''[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)''', '''41.35'''],
		}
		UpperCAmelCase_						       =					f'''{src_lang}-{tgt_lang}'''

		UpperCAmelCase_						       =					f'''
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR\'s WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = \"facebook/wmt19-{src_lang}-{tgt_lang}\"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = \"{texts[src_lang]}\"
input_ids = tokenizer.encode(input, return_tensors=\"pt\")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn\'t seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn\'t support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR\'s WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

'''
		os.makedirs(__lowercase      ,  exist_ok=__lowercase			)
		UpperCAmelCase_						       =					os.path.join(__lowercase      ,  "README.md"			)
		print(f'''Generating {path}'''			)
		with open(__lowercase      ,  "w"      ,  encoding="utf-8"			) as f:
				f.write(__lowercase			)


# make sure we are under the root of the project
SCREAMING_SNAKE_CASE  					=    Path(__file__).resolve().parent.parent.parent
SCREAMING_SNAKE_CASE  					=    repo_dir / 'model_cards'

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
			SCREAMING_SNAKE_CASE  					=    model_name.split("-")
			SCREAMING_SNAKE_CASE  					=    model_cards_dir / 'facebook' / model_name
			write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)









import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase			:	Dict								=get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_UpperCamelCase			:	Optional[Any]								=256047
_UpperCamelCase			:	int								=256145






@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase__ (							__snake_case     ,				unittest.TestCase							):
		__snake_case      :       Any     =   NllbTokenizer
		__snake_case      :       Dict     =   NllbTokenizerFast
		__snake_case      :       List[Any]     =   True
		__snake_case      :       Any     =   True
		__snake_case      :       List[str]     =   {}







		def  A__      (					self		):
						super().setUp()

						# We have a SentencePiece fixture for testing
						_A	:				Any   =      NllbTokenizer(A__       ,keep_accents=A__		)
						tokenizer.save_pretrained(self.tmpdirname		)







		def  A__      (					self		):
						_A	:				Dict   =      NllbTokenizer(A__       ,keep_accents=A__		)

						_A	:				Any   =      tokenizer.tokenize('''This is a test'''		)
						self.assertListEqual(A__       ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A__		)       ,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,)

						_A	:				Optional[Any]   =      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
						self.assertListEqual(
						    A__       ,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]       ,)
						_A	:				int   =      tokenizer.convert_tokens_to_ids(A__		)
						self.assertListEqual(
						    A__       ,[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]       ,)

						_A	:				Optional[int]   =      tokenizer.convert_ids_to_tokens(A__		)
						self.assertListEqual(
						    A__       ,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''<unk>''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''<unk>''',
						        '''.''',
						    ]       ,)







		def  A__      (					self		):
						_A	:				Any   =      (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				List[str]   =      self.rust_tokenizer_class.from_pretrained(A__       ,**A__		)
														_A	:				Tuple   =      self.tokenizer_class.from_pretrained(A__       ,**A__		)

														_A	:				Tuple   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__		)
														_A	:				Tuple   =      tokenizer_p.save_pretrained(A__		)

														# Checks it save with the same files + the tokenizer.json file for the fast one
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)
														_A	:				Dict   =      tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f		)
														self.assertSequenceEqual(A__       ,A__		)

														# Checks everything loads correctly in the same way
														_A	:				Optional[int]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				Optional[Any]   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)

														# Save tokenizer rust, legacy_format=True
														_A	:				List[Any]   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__       ,legacy_format=A__		)
														_A	:				List[Any]   =      tokenizer_p.save_pretrained(A__		)

														# Checks it save with the same files
														self.assertSequenceEqual(A__       ,A__		)

														# Checks everything loads correctly in the same way
														_A	:				List[Any]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				int   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)

														# Save tokenizer rust, legacy_format=False
														_A	:				int   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__       ,legacy_format=A__		)
														_A	:				Dict   =      tokenizer_p.save_pretrained(A__		)

														# Checks it saved the tokenizer.json file
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)

														# Checks everything loads correctly in the same way
														_A	:				Optional[int]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				str   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)







		@require_torch
		def  A__      (					self		):
						if not self.test_seqaseq:
										return

						_A	:				Union[str, Any]   =      self.get_tokenizers()
						for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""		):
														# Longer text that will definitely require truncation.
														_A	:				Union[str, Any]   =      [
														    ''' UN Chief Says There Is No Military Solution in Syria''',
														    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
														    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
														    ''' will only worsen the violence and misery for millions of people.''',
														]
														_A	:				Tuple   =      [
														    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
														    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
														    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
														    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
														]
														try:
																		_A	:				List[Any]   =      tokenizer.prepare_seqaseq_batch(
																		    src_texts=A__       ,tgt_texts=A__       ,max_length=3       ,max_target_length=10       ,return_tensors='''pt'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''ron_Latn'''       ,)
														except NotImplementedError:
																		return
														self.assertEqual(batch.input_ids.shape[1]       ,3		)
														self.assertEqual(batch.labels.shape[1]       ,10		)
														# max_target_length will default to max_length if not specified
														_A	:				int   =      tokenizer.prepare_seqaseq_batch(
														    A__       ,tgt_texts=A__       ,max_length=3       ,return_tensors='''pt'''		)
														self.assertEqual(batch.input_ids.shape[1]       ,3		)
														self.assertEqual(batch.labels.shape[1]       ,3		)

														_A	:				Union[str, Any]   =      tokenizer.prepare_seqaseq_batch(
														    src_texts=A__       ,max_length=3       ,max_target_length=10       ,return_tensors='''pt'''		)
														self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,3		)
														self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,3		)
														self.assertNotIn('''decoder_input_ids'''       ,A__		)







		@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''		)
		def  A__      (					self		):
						pass







		def  A__      (					self		):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				List[str]   =      [AddedToken('''<special>'''       ,lstrip=A__		)]

														_A	:				List[Any]   =      self.rust_tokenizer_class.from_pretrained(
														    A__       ,additional_special_tokens=A__       ,**A__		)
														_A	:				List[str]   =      tokenizer_r.encode('''Hey this is a <special> token'''		)

														_A	:				List[Any]   =      tokenizer_r.encode('''<special>'''       ,add_special_tokens=A__		)[0]

														self.assertTrue(special_token_id in r_output		)

														if self.test_slow_tokenizer:
																		_A	:				Dict   =      self.rust_tokenizer_class.from_pretrained(
																		    A__       ,additional_special_tokens=A__       ,**A__       ,)
																		_A	:				List[Any]   =      self.tokenizer_class.from_pretrained(
																		    A__       ,additional_special_tokens=A__       ,**A__		)

																		_A	:				int   =      tokenizer_p.encode('''Hey this is a <special> token'''		)

																		_A	:				Optional[int]   =      tokenizer_cr.encode('''Hey this is a <special> token'''		)

																		self.assertEqual(A__       ,A__		)
																		self.assertEqual(A__       ,A__		)
																		self.assertTrue(special_token_id in p_output		)
																		self.assertTrue(special_token_id in cr_output		)






@require_torch
@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase__ (							unittest.TestCase							):
		__snake_case      :       Union[str, Any]     =   "facebook/nllb-200-distilled-600M"
		__snake_case      :       List[Any]     =   [
		    " UN Chief Says There Is No Military Solution in Syria",
		    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
		]
		__snake_case      :       List[str]     =   [
		    "Şeful ONU declară că nu există o soluţie militară în Siria",
		    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
		    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
		    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
		]
		__snake_case      :       int     =   [
		    2_5_6_0_4_7,
		    1_6_2_9_7,
		    1_3_4_4_0_8,
		    8_1_6_5,
		    2_4_8_0_6_6,
		    1_4_7_3_4,
		    9_5_0,
		    1_1_3_5,
		    1_0_5_7_2_1,
		    3_5_7_3,
		    8_3,
		    2_7_3_5_2,
		    1_0_8,
		    4_9_4_8_6,
		    2,
		]







		@classmethod
		def  A__      (					cls		):
						_A	:				NllbTokenizer   =      NllbTokenizer.from_pretrained(
						    cls.checkpoint_name       ,src_lang='''eng_Latn'''       ,tgt_lang='''ron_Latn'''		)
						_A	:				List[str]   =      1
						return cls







		def  A__      (					self		):
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']       ,256001		)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']       ,256002		)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']       ,256057		)







		def  A__      (					self		):
						_A	:				Dict   =      self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
						self.assertListEqual(self.expected_src_tokens       ,A__		)







		def  A__      (					self		):
						self.assertIn(A__       ,self.tokenizer.all_special_ids		)
						# fmt: off
						_A	:				Dict   =      [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
						# fmt: on

						_A	:				Any   =      self.tokenizer.decode(A__       ,skip_special_tokens=A__		)
						_A	:				Any   =      self.tokenizer.decode(generated_ids[1:]       ,skip_special_tokens=A__		)
						self.assertEqual(A__       ,A__		)
						self.assertNotIn(self.tokenizer.eos_token       ,A__		)







		def  A__      (					self		):
						_A	:				int   =      ['''this is gunna be a long sentence ''' * 20]
						assert isinstance(src_text[0]       ,A__		)
						_A	:				Tuple   =      10
						_A	:				Optional[Any]   =      self.tokenizer(A__       ,max_length=A__       ,truncation=A__		).input_ids[0]
						self.assertEqual(ids[-1]       ,2		)
						self.assertEqual(ids[0]       ,A__		)
						self.assertEqual(len(A__		)       ,A__		)







		def  A__      (					self		):
						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']		)       ,[256203, 3]		)







		def  A__      (					self		):
						_A	:				str   =      tempfile.mkdtemp()
						_A	:				str   =      self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(A__		)
						_A	:				Any   =      NllbTokenizer.from_pretrained(A__		)
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,A__		)







		@require_torch
		def  A__      (					self		):
						_A	:				Optional[int]   =      self.tokenizer(
						    self.src_text       ,text_target=self.tgt_text       ,padding=A__       ,truncation=A__       ,max_length=len(self.expected_src_tokens		)       ,return_tensors='''pt'''       ,)
						_A	:				str   =      shift_tokens_right(
						    batch['''labels''']       ,self.tokenizer.pad_token_id       ,self.tokenizer.lang_code_to_id['''ron_Latn''']		)

						self.assertIsInstance(A__       ,A__		)

						self.assertEqual((2, 15)       ,batch.input_ids.shape		)
						self.assertEqual((2, 15)       ,batch.attention_mask.shape		)
						_A	:				Dict   =      batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens       ,A__		)
						self.assertEqual(A__       ,batch.decoder_input_ids[0, 0]		)  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens       ,[EN_CODE]		)
						self.assertEqual(self.tokenizer.suffix_tokens       ,[self.tokenizer.eos_token_id]		)







		def  A__      (					self		):
						_A	:				str   =      self.tokenizer(self.src_text       ,padding=A__       ,truncation=A__       ,max_length=3       ,return_tensors='''pt'''		)
						_A	:				Tuple   =      self.tokenizer(
						    text_target=self.tgt_text       ,padding=A__       ,truncation=A__       ,max_length=10       ,return_tensors='''pt'''		)
						_A	:				int   =      targets['''input_ids''']
						_A	:				Dict   =      shift_tokens_right(
						    A__       ,self.tokenizer.pad_token_id       ,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,)

						self.assertEqual(batch.input_ids.shape[1]       ,3		)
						self.assertEqual(batch.decoder_input_ids.shape[1]       ,10		)







		@require_torch
		def  A__      (					self		):
						_A	:				str   =      self.tokenizer._build_translation_inputs(
						    '''A test'''       ,return_tensors='''pt'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)

						self.assertEqual(
						    nested_simplify(A__		)       ,{
						        # A, test, EOS, en_XX
						        '''input_ids''': [[256047, 70, 7356, 2]],
						        '''attention_mask''': [[1, 1, 1, 1]],
						        # ar_AR
						        '''forced_bos_token_id''': 256057,
						    }       ,)







		@require_torch
		def  A__      (					self		):
						_A	:				int   =      True
						_A	:				Optional[Any]   =      self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)
						self.assertEqual(
						    inputs.input_ids       ,[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]		)

						_A	:				List[str]   =      False
						_A	:				Union[str, Any]   =      self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)
						self.assertEqual(
						    inputs.input_ids       ,[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]		)



