





'''simple docstring'''






import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


snake_case_     =	logging.getLogger(__name__)

snake_case_     =	tf.data.AUTOTUNE
def 							_lowerCamelCase(  )		->  Dict:
    A       :       Any							=						argparse.ArgumentParser(description='''Train a masked language model on TPU.'''		)
    parser.add_argument(
        '''--pretrained_model_config'''    ,       type=UpperCamelCase__    ,       default='''roberta-base'''    ,       help='''The model config to use. Note that we don\'t copy the model\'s weights, only the config!'''    ,       )
    parser.add_argument(
        '''--tokenizer'''    ,       type=UpperCamelCase__    ,       default='''unigram-tokenizer-wikitext'''    ,       help='''The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'''    ,       )

    parser.add_argument(
        '''--per_replica_batch_size'''    ,       type=UpperCamelCase__    ,       default=8    ,       help='''Batch size per TPU core.'''    ,       )

    parser.add_argument(
        '''--no_tpu'''    ,       action='''store_true'''    ,       help='''If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'''    ,       )

    parser.add_argument(
        '''--tpu_name'''    ,       type=UpperCamelCase__    ,       help='''Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'''    ,       default='''local'''    ,       )

    parser.add_argument(
        '''--tpu_zone'''    ,       type=UpperCamelCase__    ,       help='''Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'''    ,       )

    parser.add_argument(
        '''--gcp_project'''    ,       type=UpperCamelCase__    ,       help='''Google cloud project name. Only used for non-Colab TPU nodes.'''		)

    parser.add_argument(
        '''--bfloat16'''    ,       action='''store_true'''    ,       help='''Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'''    ,       )

    parser.add_argument(
        '''--train_dataset'''    ,       type=UpperCamelCase__    ,       help='''Path to training dataset to load. If the path begins with `gs://`'''
        ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''    ,       )

    parser.add_argument(
        '''--shuffle_buffer_size'''    ,       type=UpperCamelCase__    ,       default=2**18    ,       help='''Size of the shuffle buffer (in samples)'''    ,       )

    parser.add_argument(
        '''--eval_dataset'''    ,       type=UpperCamelCase__    ,       help='''Path to evaluation dataset to load. If the path begins with `gs://`'''
        ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''    ,       )

    parser.add_argument(
        '''--num_epochs'''    ,       type=UpperCamelCase__    ,       default=1    ,       help='''Number of epochs to train for.'''    ,       )

    parser.add_argument(
        '''--learning_rate'''    ,       type=UpperCamelCase__    ,       default=1e-4    ,       help='''Learning rate to use for training.'''    ,       )

    parser.add_argument(
        '''--weight_decay_rate'''    ,       type=UpperCamelCase__    ,       default=1e-3    ,       help='''Weight decay rate to use for training.'''    ,       )

    parser.add_argument(
        '''--max_length'''    ,       type=UpperCamelCase__    ,       default=512    ,       help='''Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'''    ,       )

    parser.add_argument(
        '''--mlm_probability'''    ,       type=UpperCamelCase__    ,       default=0.1_5    ,       help='''Fraction of tokens to mask during training.'''    ,       )

    parser.add_argument('''--output_dir'''    ,       type=UpperCamelCase__    ,       required=UpperCamelCase__    ,       help='''Path to save model checkpoints to.'''		)
    parser.add_argument('''--hub_model_id'''    ,       type=UpperCamelCase__    ,       help='''Model ID to upload to on the Hugging Face Hub.'''		)

    A       :       Tuple							=						parser.parse_args()
    return args
def 							_lowerCamelCase(  UpperCamelCase__    : Any		)		->  str:
    try:
        if args.tpu_name:
            A       :       str							=						tf.distribute.cluster_resolver.TPUClusterResolver(
                args.tpu_name    ,       zone=args.tpu_zone    ,       project=args.gcp_project		)
        else:
            A       :       Dict							=						tf.distribute.cluster_resolver.TPUClusterResolver()
    except ValueError:
        raise RuntimeError(
            '''Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '''
            '''--gcp_project. When running on a TPU VM, use --tpu_name local.'''		)

    tf.config.experimental_connect_to_cluster(UpperCamelCase__		)
    tf.tpu.experimental.initialize_tpu_system(UpperCamelCase__		)

    return tpu
def 							_lowerCamelCase(  UpperCamelCase__    : List[Any]		)		->  Optional[int]:
    A       :       List[Any]							=						0
    for file in file_list:
        A       :       List[Any]							=						file.split('''/'''		)[-1]
        A       :       Dict							=						re.search(R'''-\d+-(\d+)\.tfrecord'''    ,       UpperCamelCase__		).group(1		)
        A       :       Dict							=						int(UpperCamelCase__		)
        num_samples += sample_count

    return num_samples
def 							_lowerCamelCase(  UpperCamelCase__    : List[str]    ,       UpperCamelCase__    : Tuple    ,       UpperCamelCase__    : Any    ,       UpperCamelCase__    : Tuple    ,       UpperCamelCase__    : Dict    ,       UpperCamelCase__    : Any=None		)		->  Optional[int]:
    A       :       str							=						count_samples(UpperCamelCase__		)
    A       :       Any							=						tf.data.Dataset.from_tensor_slices(UpperCamelCase__		)
    if shuffle:
        A       :       int							=						dataset.shuffle(len(UpperCamelCase__		)		)
    A       :       List[str]							=						tf.data.TFRecordDataset(UpperCamelCase__    ,       num_parallel_reads=UpperCamelCase__		)
    # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
    A       :       Tuple							=						dataset.apply(tf.data.experimental.assert_cardinality(UpperCamelCase__		)		)
    A       :       List[str]							=						dataset.map(UpperCamelCase__    ,       num_parallel_calls=UpperCamelCase__		)
    if shuffle:
        assert shuffle_buffer_size is not None
        A       :       List[Any]							=						dataset.shuffle(args.shuffle_buffer_size		)
    A       :       Optional[int]							=						dataset.batch(UpperCamelCase__    ,       drop_remainder=UpperCamelCase__		)
    A       :       List[str]							=						dataset.map(UpperCamelCase__    ,       num_parallel_calls=UpperCamelCase__		)
    A       :       Dict							=						dataset.prefetch(UpperCamelCase__		)
    return dataset
def 							_lowerCamelCase(  UpperCamelCase__    : str		)		->  List[Any]:
    if not args.no_tpu:
        A       :       Dict							=						initialize_tpu(UpperCamelCase__		)
        A       :       List[str]							=						tf.distribute.TPUStrategy(UpperCamelCase__		)
    else:
        A       :       List[str]							=						tf.distribute.OneDeviceStrategy(device='''/gpu:0'''		)

    if args.bfloataa:
        tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''		)

    A       :       str							=						AutoTokenizer.from_pretrained(args.tokenizer		)
    A       :       Optional[Any]							=						AutoConfig.from_pretrained(args.pretrained_model_config		)
    A       :       Tuple							=						tokenizer.vocab_size

    A       :       Tuple							=						tf.io.gfile.glob(os.path.join(args.train_dataset    ,       '''*.tfrecord'''		)		)
    if not training_records:
        raise ValueError(F'''No .tfrecord files found in {args.train_dataset}.'''		)
    A       :       List[str]							=						tf.io.gfile.glob(os.path.join(args.eval_dataset    ,       '''*.tfrecord'''		)		)
    if not eval_records:
        raise ValueError(F'''No .tfrecord files found in {args.eval_dataset}.'''		)

    A       :       Optional[Any]							=						count_samples(UpperCamelCase__		)

    A       :       List[Any]							=						num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
    A       :       Tuple							=						steps_per_epoch * args.num_epochs

    with strategy.scope():
        A       :       List[str]							=						TFAutoModelForMaskedLM.from_config(UpperCamelCase__		)
        model(model.dummy_inputs		)  # Pass some dummy inputs through the model to ensure all the weights are built
        A,	A       :       Any							=						create_optimizer(
            num_train_steps=UpperCamelCase__    ,       num_warmup_steps=total_train_steps // 20    ,       init_lr=args.learning_rate    ,       weight_decay_rate=args.weight_decay_rate    ,       )

        # Transformers models compute the right loss for their task by default when labels are passed, and will
        # use this for training unless you specify your own loss function in compile().
        model.compile(optimizer=UpperCamelCase__    ,       metrics=['''accuracy''']		)

    def decode_fn(UpperCamelCase__    : Tuple		):
        A       :       Optional[Any]							=						{
            '''input_ids''': tf.io.FixedLenFeature(dtype=tf.intaa    ,       shape=(args.max_length,)		),
            '''attention_mask''': tf.io.FixedLenFeature(dtype=tf.intaa    ,       shape=(args.max_length,)		),
        }
        return tf.io.parse_single_example(UpperCamelCase__    ,       UpperCamelCase__		)

    # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
    # use their methods in our data pipeline.
    A       :       Optional[int]							=						DataCollatorForLanguageModeling(
        tokenizer=UpperCamelCase__    ,       mlm_probability=args.mlm_probability    ,       mlm=UpperCamelCase__    ,       return_tensors='''tf'''		)

    def mask_with_collator(UpperCamelCase__    : List[Any]		):
        # TF really needs an isin() function
        A       :       Any							=						(
            ~tf.cast(batch['''attention_mask''']    ,       tf.bool		)
            | (batch['''input_ids'''] == tokenizer.cls_token_id)
            | (batch['''input_ids'''] == tokenizer.sep_token_id)
        )
        A,	A       :       Tuple							=						data_collator.tf_mask_tokens(
            batch['''input_ids''']    ,       vocab_size=len(UpperCamelCase__		)    ,       mask_token_id=tokenizer.mask_token_id    ,       special_tokens_mask=UpperCamelCase__    ,       )
        return batch

    A       :       Dict							=						args.per_replica_batch_size * strategy.num_replicas_in_sync

    A       :       Optional[int]							=						prepare_dataset(
        UpperCamelCase__    ,       decode_fn=UpperCamelCase__    ,       mask_fn=UpperCamelCase__    ,       batch_size=UpperCamelCase__    ,       shuffle=UpperCamelCase__    ,       shuffle_buffer_size=args.shuffle_buffer_size    ,       )

    A       :       List[Any]							=						prepare_dataset(
        UpperCamelCase__    ,       decode_fn=UpperCamelCase__    ,       mask_fn=UpperCamelCase__    ,       batch_size=UpperCamelCase__    ,       shuffle=UpperCamelCase__    ,       )

    A       :       str							=						[]
    if args.hub_model_id:
        callbacks.append(
            PushToHubCallback(output_dir=args.output_dir    ,       hub_model_id=args.hub_model_id    ,       tokenizer=UpperCamelCase__		)		)

    model.fit(
        UpperCamelCase__    ,       validation_data=UpperCamelCase__    ,       epochs=args.num_epochs    ,       callbacks=UpperCamelCase__    ,       )

    model.save_pretrained(args.output_dir		)


if __name__ == "__main__":
     snake_case_     =	parse_args()
     main(args)







'''simple docstring'''






# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path
def 							_lowerCamelCase(  UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : Optional[Any]    ,       UpperCamelCase__    : List[Any]		)		->  List[Any]:
    A       :       Any							=						{
        '''en''': '''Machine learning is great, isn\'t it?''',
        '''ru''': '''Машинное обучение - это здорово, не так ли?''',
        '''de''': '''Maschinelles Lernen ist großartig, oder?''',
    }

    # BLUE scores as follows:
    # "pair": [fairseq, transformers]
    A       :       List[Any]							=						{
        '''ru-en''': ['''[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)''', '''39.20'''],
        '''en-ru''': ['''[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)''', '''33.47'''],
        '''en-de''': ['''[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)''', '''42.83'''],
        '''de-en''': ['''[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)''', '''41.35'''],
    }
    A       :       List[Any]							=						F'''{src_lang}-{tgt_lang}'''

    A       :       Any							=						F'''
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR\'s WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "facebook/wmt19-{src_lang}-{tgt_lang}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn\'t seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn\'t support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR\'s WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

'''
    os.makedirs(UpperCamelCase__    ,       exist_ok=UpperCamelCase__		)
    A       :       Optional[int]							=						os.path.join(UpperCamelCase__    ,       '''README.md'''		)
    print(F'''Generating {path}'''		)
    with open(UpperCamelCase__    ,       '''w'''    ,       encoding='''utf-8'''		) as f:
        f.write(UpperCamelCase__		)


# make sure we are under the root of the project
snake_case_     =	Path(__file__).resolve().parent.parent.parent
snake_case_     =	repo_dir / """model_cards"""

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
     snake_case_ ,  snake_case_ ,  snake_case_     =	model_name.split("""-""")
     snake_case_     =	model_cards_dir / """facebook""" / model_name
     write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)
