





'''simple docstring'''






import argparse
import json
import os

import fairseq
import torch
from torch import nn

from transformers import (
    SpeechaTextaConfig,
    SpeechaTextaForCausalLM,
    SpeechaTextaTokenizer,
    SpeechEncoderDecoderConfig,
    SpeechEncoderDecoderModel,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaModel,
    logging,
)


logging.set_verbosity_info()
snake_case_     =	logging.get_logger(__name__)

snake_case_     =	{
    """post_extract_proj""": """feature_projection.projection""",
    """encoder.pos_conv.0""": """encoder.pos_conv_embed.conv""",
    """self_attn.k_proj""": """encoder.layers.*.attention.k_proj""",
    """self_attn.v_proj""": """encoder.layers.*.attention.v_proj""",
    """self_attn.q_proj""": """encoder.layers.*.attention.q_proj""",
    """self_attn.out_proj""": """encoder.layers.*.attention.out_proj""",
    """self_attn_layer_norm""": """encoder.layers.*.layer_norm""",
    """fc1""": """encoder.layers.*.feed_forward.intermediate_dense""",
    """fc2""": """encoder.layers.*.feed_forward.output_dense""",
    """final_layer_norm""": """encoder.layers.*.final_layer_norm""",
    """encoder.layer_norm""": """encoder.layer_norm""",
    """w2v_model.layer_norm""": """feature_projection.layer_norm""",
    """quantizer.weight_proj""": """quantizer.weight_proj""",
    """quantizer.vars""": """quantizer.codevectors""",
    """project_q""": """project_q""",
    """final_proj""": """project_hid""",
    """w2v_encoder.proj""": """lm_head""",
    """mask_emb""": """masked_spec_embed""",
}
snake_case_     =	[
    """lm_head""",
    """quantizer.weight_proj""",
    """quantizer.codevectors""",
    """project_q""",
    """project_hid""",
]
def 							_lowerCamelCase(  UpperCamelCase__    : List[str]    ,       UpperCamelCase__    : Any    ,       UpperCamelCase__    : Union[str, Any]    ,       UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : Any		)		->  Tuple:
    for attribute in key.split('''.'''		):
        A       :       str							=						getattr(UpperCamelCase__    ,       UpperCamelCase__		)

    if weight_type is not None:
        A       :       int							=						getattr(UpperCamelCase__    ,       UpperCamelCase__		).shape
    else:
        A       :       str							=						hf_pointer.shape

    assert hf_shape == value.shape, (
        F'''Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be'''
        F''' {value.shape} for {full_name}'''
    )

    if weight_type == "weight":
        A       :       Optional[Any]							=						value
    elif weight_type == "weight_g":
        A       :       Union[str, Any]							=						value
    elif weight_type == "weight_v":
        A       :       Dict							=						value
    elif weight_type == "bias":
        A       :       Union[str, Any]							=						value
    else:
        A       :       Dict							=						value

    logger.info(F'''{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.'''		)
def 							_lowerCamelCase(  UpperCamelCase__    : List[str]    ,       UpperCamelCase__    : Dict		)		->  Optional[Any]:
    A       :       str							=						[]
    A       :       str							=						fairseq_model.state_dict()

    A       :       int							=						hf_model.feature_extractor

    # if encoder has different dim to decoder -> use proj_weight
    A       :       List[Any]							=						None

    for name, value in fairseq_dict.items():
        A       :       Optional[int]							=						False
        if "conv_layers" in name:
            load_conv_layer(
                UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       hf_model.config.feat_extract_norm == '''group'''    ,       )
            A       :       List[str]							=						True
        elif name.split('''.'''		)[0] == "proj":
            A       :       Optional[int]							=						fairseq_model.proj
            A       :       Any							=						True
        else:
            for key, mapped_key in MAPPING.items():
                if key in name or key.split('''w2v_model.'''		)[-1] == name.split('''.'''		)[0]:
                    A       :       List[Any]							=						True
                    if "*" in mapped_key:
                        A       :       Optional[Any]							=						name.split(UpperCamelCase__		)[0].split('''.'''		)[-2]
                        A       :       Optional[Any]							=						mapped_key.replace('''*'''    ,       UpperCamelCase__		)
                    if "weight_g" in name:
                        A       :       Dict							=						'''weight_g'''
                    elif "weight_v" in name:
                        A       :       str							=						'''weight_v'''
                    elif "bias" in name:
                        A       :       Dict							=						'''bias'''
                    elif "weight" in name:
                        A       :       Dict							=						'''weight'''
                    else:
                        A       :       str							=						None
                    set_recursively(UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__    ,       UpperCamelCase__		)
                continue
        if not is_used:
            unused_weights.append(UpperCamelCase__		)

    logger.warning(F'''Unused weights: {unused_weights}'''		)

    return proj_weight
def 							_lowerCamelCase(  UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : Any    ,       UpperCamelCase__    : Union[str, Any]    ,       UpperCamelCase__    : Tuple    ,       UpperCamelCase__    : Optional[int]		)		->  List[str]:
    A       :       Optional[Any]							=						full_name.split('''conv_layers.'''		)[-1]
    A       :       Optional[Any]							=						name.split('''.'''		)
    A       :       List[str]							=						int(items[0]		)
    A       :       str							=						int(items[1]		)

    if type_id == 0:
        if "bias" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
                F'''{full_name} has size {value.shape}, but'''
                F''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''
            )
            A       :       int							=						value
            logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''		)
        elif "weight" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
                F'''{full_name} has size {value.shape}, but'''
                F''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''
            )
            A       :       Union[str, Any]							=						value
            logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''		)
    elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
        if "bias" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
                F'''{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'''
                " found."
            )
            A       :       Optional[Any]							=						value
            logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''		)
        elif "weight" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
                F'''{full_name} has size {value.shape}, but'''
                F''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''
            )
            A       :       List[str]							=						value
            logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''		)
    else:
        unused_weights.append(UpperCamelCase__		)
def 							_lowerCamelCase(  UpperCamelCase__    : Any		)		->  Dict:
    A,	A       :       Tuple							=						emb.weight.shape
    A       :       List[Any]							=						nn.Linear(UpperCamelCase__    ,       UpperCamelCase__    ,       bias=UpperCamelCase__		)
    A       :       int							=						emb.weight.data
    return lin_layer
def 							_lowerCamelCase(  UpperCamelCase__    : Dict		)		->  Tuple:
    with open(UpperCamelCase__    ,       '''r'''    ,       encoding='''utf-8'''		) as f:
        A       :       List[Any]							=						f.readlines()
        A       :       Any							=						[line.split(''' '''		)[0] for line in lines]

    A       :       int							=						len(UpperCamelCase__		)

    A       :       Optional[int]							=						{
        '''<s>''': 0,
        '''<pad>''': 1,
        '''</s>''': 2,
        '''<unk>''': 3,
    }

    vocab_dict.update(dict(zip(UpperCamelCase__    ,       range(4    ,       num_words + 4		)		)		)		)
    return vocab_dict
@torch.no_grad()
def 							_lowerCamelCase(  UpperCamelCase__    : int    ,       UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : List[str]    ,       UpperCamelCase__    : str    ,       UpperCamelCase__    : str    ,       UpperCamelCase__    : Optional[int]    ,       UpperCamelCase__    : int    ,       )		->  Union[str, Any]:
    A       :       Tuple							=						WavaVecaConfig.from_pretrained(UpperCamelCase__		)
    A       :       Any							=						SpeechaTextaConfig.from_pretrained(
        UpperCamelCase__    ,       vocab_size=UpperCamelCase__    ,       decoder_layers=UpperCamelCase__    ,       do_stable_layer_norm=UpperCamelCase__		)

    A       :       Dict							=						WavaVecaFeatureExtractor(
        feature_size=1    ,       sampling_rate=16_000    ,       padding_value=0    ,       do_normalize=UpperCamelCase__    ,       return_attention_mask=UpperCamelCase__    ,       )

    A,	A,	A       :       Tuple							=						fairseq.checkpoint_utils.load_model_ensemble_and_task(
        [checkpoint_path]    ,       arg_overrides={'''data''': '''/'''.join(dict_path.split('''/'''		)[:-1]		)}		)
    A       :       Union[str, Any]							=						model[0].eval()

    # set weights for wav2vec2 encoder
    A       :       Dict							=						WavaVecaModel(UpperCamelCase__		)
    A       :       Optional[Any]							=						recursively_load_weights_wavaveca(model.encoder    ,       UpperCamelCase__		)

    A       :       str							=						SpeechaTextaForCausalLM(UpperCamelCase__		)
    A,	A       :       Any							=						hf_decoder.model.decoder.load_state_dict(model.decoder.state_dict()    ,       strict=UpperCamelCase__		)

    # set output linear layer
    unexpected_keys.remove('''embed_out'''		)
    A       :       Optional[int]							=						nn.Parameter(model.decoder.embed_out.detach()		)

    # layer norm is init to identity matrix so leaving it is fine
    logger.warning(F'''The following keys are missing when loading the decoder weights: {missing_keys}'''		)
    logger.warning(F'''The following keys are unexpected when loading the decoder weights: {unexpected_keys}'''		)

    A       :       List[str]							=						SpeechEncoderDecoderModel(encoder=UpperCamelCase__    ,       decoder=UpperCamelCase__		)
    A       :       Optional[int]							=						False

    # add projection layer
    A       :       List[Any]							=						nn.Parameter(projection_layer.weight		)
    A       :       Tuple							=						nn.Parameter(projection_layer.bias		)

    A       :       Dict							=						create_vocab_dict(UpperCamelCase__		)

    with open(os.path.join(UpperCamelCase__    ,       '''vocab.json'''		)    ,       '''w'''		) as fp:
        json.dump(UpperCamelCase__    ,       UpperCamelCase__		)

    A       :       Any							=						SpeechaTextaTokenizer(os.path.join(UpperCamelCase__    ,       '''vocab.json'''		)		)
    tokenizer.save_pretrained(UpperCamelCase__		)

    A       :       Dict							=						hf_wavavec.config.to_dict()
    A       :       Optional[Any]							=						tokenizer.pad_token_id
    A       :       int							=						tokenizer.bos_token_id
    A       :       Union[str, Any]							=						tokenizer.eos_token_id
    A       :       Dict							=						'''speech_to_text_2'''
    A       :       Tuple							=						'''wav2vec2'''

    A       :       Optional[Any]							=						SpeechEncoderDecoderConfig.from_dict(UpperCamelCase__		)

    hf_wavavec.save_pretrained(UpperCamelCase__		)
    feature_extractor.save_pretrained(UpperCamelCase__		)


if __name__ == "__main__":
     snake_case_     =	argparse.ArgumentParser()
     parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
     parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
     parser.add_argument("""--dict_path""", default=None, type=str, help="""Path to dict of fine-tuned model""")
     parser.add_argument(
         """--encoder_config_path""",
         default="""facebook/wav2vec2-large-lv60""",
         type=str,
         help="""Path to hf encoder wav2vec2 checkpoint config""",
     )
     parser.add_argument(
         """--decoder_config_path""",
         default="""facebook/s2t-small-mustc-en-fr-st""",
         type=str,
         help="""Path to hf decoder s2t checkpoint config""",
     )
     parser.add_argument("""--vocab_size""", default=1_02_24, type=int, help="""Vocab size of decoder""")
     parser.add_argument("""--num_decoder_layers""", default=7, type=int, help="""Number of decoder layers""")

     snake_case_     =	parser.parse_args()
     convert_wavaveca_checkpoint(
         args.checkpoint_path,
         args.pytorch_dump_folder_path,
         args.dict_path,
         encoder_config_path=args.encoder_config_path,
         decoder_config_path=args.decoder_config_path,
         vocab_size=args.vocab_size,
         num_decoder_layers=args.num_decoder_layers,
     )







'''simple docstring'''






# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path
def 							_lowerCamelCase(  UpperCamelCase__    : List[Any]    ,       UpperCamelCase__    : Optional[Any]    ,       UpperCamelCase__    : List[Any]		)		->  List[Any]:
    A       :       Any							=						{
        '''en''': '''Machine learning is great, isn\'t it?''',
        '''ru''': '''Машинное обучение - это здорово, не так ли?''',
        '''de''': '''Maschinelles Lernen ist großartig, oder?''',
    }

    # BLUE scores as follows:
    # "pair": [fairseq, transformers]
    A       :       List[Any]							=						{
        '''ru-en''': ['''[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)''', '''39.20'''],
        '''en-ru''': ['''[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)''', '''33.47'''],
        '''en-de''': ['''[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)''', '''42.83'''],
        '''de-en''': ['''[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)''', '''41.35'''],
    }
    A       :       List[Any]							=						F'''{src_lang}-{tgt_lang}'''

    A       :       Any							=						F'''
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR\'s WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "facebook/wmt19-{src_lang}-{tgt_lang}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn\'t seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn\'t support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR\'s WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

'''
    os.makedirs(UpperCamelCase__    ,       exist_ok=UpperCamelCase__		)
    A       :       Optional[int]							=						os.path.join(UpperCamelCase__    ,       '''README.md'''		)
    print(F'''Generating {path}'''		)
    with open(UpperCamelCase__    ,       '''w'''    ,       encoding='''utf-8'''		) as f:
        f.write(UpperCamelCase__		)


# make sure we are under the root of the project
snake_case_     =	Path(__file__).resolve().parent.parent.parent
snake_case_     =	repo_dir / """model_cards"""

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
     snake_case_ ,  snake_case_ ,  snake_case_     =	model_name.split("""-""")
     snake_case_     =	model_cards_dir / """facebook""" / model_name
     write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)
