





'''simple docstring'''






import argparse

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


snake_case_     =	16
snake_case_     =	32
def 							_lowerCamelCase(  UpperCamelCase__    : Accelerator    ,       UpperCamelCase__    : int = 16		)		->  Union[str, Any]:
    A       :       Tuple							=						AutoTokenizer.from_pretrained('''bert-base-cased'''		)
    A       :       List[Any]							=						load_dataset('''glue'''    ,       '''mrpc'''		)

    def tokenize_function(UpperCamelCase__    : int		):
        # max_length=None => use the model max length (it's actually the default)
        A       :       List[Any]							=						tokenizer(examples['''sentence1''']    ,       examples['''sentence2''']    ,       truncation=UpperCamelCase__    ,       max_length=UpperCamelCase__		)
        return outputs

    # Apply the method we just defined to all the examples in all the splits of the dataset
    # starting with the main process first:
    with accelerator.main_process_first():
        A       :       Tuple							=						datasets.map(
            UpperCamelCase__    ,       batched=UpperCamelCase__    ,       remove_columns=['''idx''', '''sentence1''', '''sentence2''']    ,       )

    # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
    # transformers library
    A       :       Optional[int]							=						tokenized_datasets.rename_column('''label'''    ,       '''labels'''		)

    def collate_fn(UpperCamelCase__    : List[Any]		):
        # On TPU it's best to pad everything to the same length or training will be very slow.
        A       :       Optional[int]							=						128 if accelerator.distributed_type == DistributedType.TPU else None
        # When using mixed precision we want round multiples of 8/16
        if accelerator.mixed_precision == "fp8":
            A       :       List[Any]							=						16
        elif accelerator.mixed_precision != "no":
            A       :       List[str]							=						8
        else:
            A       :       Dict							=						None

        return tokenizer.pad(
            UpperCamelCase__    ,       padding='''longest'''    ,       max_length=UpperCamelCase__    ,       pad_to_multiple_of=UpperCamelCase__    ,       return_tensors='''pt'''    ,       )

    # Instantiate dataloaders.
    A       :       Dict							=						DataLoader(
        tokenized_datasets['''train''']    ,       shuffle=UpperCamelCase__    ,       collate_fn=UpperCamelCase__    ,       batch_size=UpperCamelCase__    ,       drop_last=UpperCamelCase__		)
    A       :       List[str]							=						DataLoader(
        tokenized_datasets['''validation''']    ,       shuffle=UpperCamelCase__    ,       collate_fn=UpperCamelCase__    ,       batch_size=UpperCamelCase__    ,       drop_last=(accelerator.mixed_precision == '''fp8''')    ,       )

    return train_dataloader, eval_dataloader
def 							_lowerCamelCase(  UpperCamelCase__    : Any    ,       UpperCamelCase__    : Any		)		->  Optional[int]:
    # Initialize accelerator
    A       :       Optional[Any]							=						Accelerator(cpu=args.cpu    ,       mixed_precision=args.mixed_precision		)
    # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
    A       :       int							=						config['''lr''']
    A       :       List[Any]							=						int(config['''num_epochs''']		)
    A       :       Union[str, Any]							=						int(config['''seed''']		)
    A       :       Optional[Any]							=						int(config['''batch_size''']		)

    A       :       int							=						evaluate.load('''glue'''    ,       '''mrpc'''		)

    # If the batch size is too big we use gradient accumulation
    A       :       List[str]							=						1
    if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
        A       :       List[Any]							=						batch_size // MAX_GPU_BATCH_SIZE
        A       :       int							=						MAX_GPU_BATCH_SIZE

    set_seed(UpperCamelCase__		)
    A,	A       :       Tuple							=						get_dataloaders(UpperCamelCase__    ,       UpperCamelCase__		)
    # Instantiate the model (we build the model here so that the seed also control new weights initialization)
    A       :       Optional[int]							=						AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''    ,       return_dict=UpperCamelCase__		)

    # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
    # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
    # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
    A       :       Optional[Any]							=						model.to(accelerator.device		)
    # Instantiate optimizer
    A       :       Dict							=						AdamW(params=model.parameters()    ,       lr=UpperCamelCase__		)

    # Instantiate scheduler
    A       :       Optional[int]							=						get_linear_schedule_with_warmup(
        optimizer=UpperCamelCase__    ,       num_warmup_steps=100    ,       num_training_steps=(len(UpperCamelCase__		) * num_epochs) // gradient_accumulation_steps    ,       )

    # Prepare everything
    # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
    # prepare method.

    A,	A,	A,	A,	A       :       Union[str, Any]							=						accelerator.prepare(
        UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__		)

    # Now we train the model
    for epoch in range(UpperCamelCase__		):
        model.train()
        for step, batch in enumerate(UpperCamelCase__		):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device		)
            A       :       Tuple							=						model(**UpperCamelCase__		)
            A       :       int							=						outputs.loss
            A       :       Any							=						loss / gradient_accumulation_steps
            accelerator.backward(UpperCamelCase__		)
            if step % gradient_accumulation_steps == 0:
                optimizer.step()
                lr_scheduler.step()
                optimizer.zero_grad()

        model.eval()
        for step, batch in enumerate(UpperCamelCase__		):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device		)
            with torch.no_grad():
                A       :       Tuple							=						model(**UpperCamelCase__		)
            A       :       str							=						outputs.logits.argmax(dim=-1		)
            A,	A       :       Any							=						accelerator.gather_for_metrics((predictions, batch['''labels'''])		)
            metric.add_batch(
                predictions=UpperCamelCase__    ,       references=UpperCamelCase__    ,       )

        A       :       int							=						metric.compute()
        # Use accelerator.print to print only on the main process.
        accelerator.print(F'''epoch {epoch}:'''    ,       UpperCamelCase__		)
def 							_lowerCamelCase(  )		->  Dict:
    A       :       List[Any]							=						argparse.ArgumentParser(description='''Simple example of training script.'''		)
    parser.add_argument(
        '''--mixed_precision'''    ,       type=UpperCamelCase__    ,       default=UpperCamelCase__    ,       choices=['''no''', '''fp16''', '''bf16''', '''fp8''']    ,       help='''Whether to use mixed precision. Choose'''
        '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
        '''and an Nvidia Ampere GPU.'''    ,       )
    parser.add_argument('''--cpu'''    ,       action='''store_true'''    ,       help='''If passed, will train on the CPU.'''		)
    A       :       List[str]							=						parser.parse_args()
    A       :       Dict							=						{'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
    training_function(UpperCamelCase__    ,       UpperCamelCase__		)


if __name__ == "__main__":
     main()







'''simple docstring'''






# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path
def 							_lowerCamelCase(  UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : Optional[Any]    ,       UpperCamelCase__    : List[Any]		)		->  List[Any]:
    A       :       Any							=						{
        '''en''': '''Machine learning is great, isn\'t it?''',
        '''ru''': '''Машинное обучение - это здорово, не так ли?''',
        '''de''': '''Maschinelles Lernen ist großartig, oder?''',
    }

    # BLUE scores as follows:
    # "pair": [fairseq, transformers]
    A       :       List[Any]							=						{
        '''ru-en''': ['''[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)''', '''39.20'''],
        '''en-ru''': ['''[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)''', '''33.47'''],
        '''en-de''': ['''[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)''', '''42.83'''],
        '''de-en''': ['''[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)''', '''41.35'''],
    }
    A       :       List[Any]							=						F'''{src_lang}-{tgt_lang}'''

    A       :       Any							=						F'''
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR\'s WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "facebook/wmt19-{src_lang}-{tgt_lang}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn\'t seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn\'t support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR\'s WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

'''
    os.makedirs(UpperCamelCase__    ,       exist_ok=UpperCamelCase__		)
    A       :       Optional[int]							=						os.path.join(UpperCamelCase__    ,       '''README.md'''		)
    print(F'''Generating {path}'''		)
    with open(UpperCamelCase__    ,       '''w'''    ,       encoding='''utf-8'''		) as f:
        f.write(UpperCamelCase__		)


# make sure we are under the root of the project
snake_case_     =	Path(__file__).resolve().parent.parent.parent
snake_case_     =	repo_dir / """model_cards"""

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
     snake_case_ ,  snake_case_ ,  snake_case_     =	model_name.split("""-""")
     snake_case_     =	model_cards_dir / """facebook""" / model_name
     write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)
