





import argparse
import json

import numpy
import torch

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()






def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			)		->						Any:






    """simple docstring"""


    snake_case_						:   Optional[Any]           =  torch.load(_lowerCamelCase			,    map_location='''cpu'''			)

    snake_case_						:   Optional[Any]           =  chkpt["""model"""]

    # We have the base model one level deeper than the original XLM repository
    snake_case_						:   Optional[Any]           =  {}
    for k, v in state_dict.items():
        if "pred_layer" in k:
            snake_case_						:   int           =  v
        else:
            snake_case_						:   Any           =  v

    snake_case_						:   Dict           =  chkpt["""params"""]
    snake_case_						:   str           =  {n: v for n, v in config.items() if not isinstance(_lowerCamelCase			,    (torch.FloatTensor, numpy.ndarray)			)}

    snake_case_						:   List[str]           =  chkpt["""dico_word2id"""]
    snake_case_						:   Union[str, Any]           =  {s + """</w>""" if s.find('''@@'''			) == -1 and i > 13 else s.replace('''@@'''			,    ''''''			): i for s, i in vocab.items()}

    # Save pytorch-model
    snake_case_						:   Tuple           =  pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
    snake_case_						:   Optional[int]           =  pytorch_dump_folder_path + """/""" + CONFIG_NAME
    snake_case_						:   List[Any]           =  pytorch_dump_folder_path + """/""" + VOCAB_FILES_NAMES["""vocab_file"""]

    print(f'''Save PyTorch model to {pytorch_weights_dump_path}'''			)
    torch.save(_lowerCamelCase			,    _lowerCamelCase			)

    print(f'''Save configuration file to {pytorch_config_dump_path}'''			)
    with open(_lowerCamelCase			,    '''w'''			,    encoding='''utf-8'''			) as f:
        f.write(json.dumps(_lowerCamelCase			,    indent=2			) + '''\n'''			)

    print(f'''Save vocab file to {pytorch_config_dump_path}'''			)
    with open(_lowerCamelCase			,    '''w'''			,    encoding='''utf-8'''			) as f:
        f.write(json.dumps(_lowerCamelCase			,    indent=2			) + '''\n'''			)


if __name__ == "__main__":
 lowerCAmelCase_    =    argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '''--xlm_checkpoint_path''', default=None, type=str, required=True, help='''Path the official PyTorch dump.'''
 )
 parser.add_argument(
     '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
 )
 lowerCAmelCase_    =    parser.parse_args()
 convert_xlm_checkpoint_to_pytorch(args.xlm_checkpoint_path, args.pytorch_dump_folder_path)







import time
from contextlib import contextmanager
from pathlib import Path

import pytest
import requests
from huggingface_hub.hf_api import HfApi, HfFolder


lowerCAmelCase_    =    '''__DUMMY_TRANSFORMERS_USER__'''
lowerCAmelCase_    =    '''Dummy User'''
lowerCAmelCase_    =    '''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'''

lowerCAmelCase_    =    '''https://hub-ci.huggingface.co'''
lowerCAmelCase_    =    CI_HUB_ENDPOINT + '''/datasets/{repo_id}/resolve/{revision}/{path}'''
lowerCAmelCase_    =    CI_HUB_ENDPOINT + '''/{repo_id}/resolve/{revision}/{filename}'''
lowerCAmelCase_    =    Path('''~/.huggingface/hub_ci_token''').expanduser()






@pytest.fixture
def 					lowerCamelCase_						(    _UpperCamelCase			)		->						Optional[int]:






				"""simple docstring"""


				monkeypatch.setattr(
				    '''huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE'''			,    _UpperCamelCase			)






@pytest.fixture
def 					lowerCamelCase_						(    _UpperCamelCase			)		->						Optional[int]:






				"""simple docstring"""


				monkeypatch.setattr('''datasets.config.HF_ENDPOINT'''			,    _UpperCamelCase			)
				monkeypatch.setattr('''datasets.config.HUB_DATASETS_URL'''			,    _UpperCamelCase			)






@pytest.fixture
def 					lowerCamelCase_						(    _UpperCamelCase			)		->						Any:






				"""simple docstring"""


				monkeypatch.setattr('''huggingface_hub.hf_api.HfFolder.path_token'''			,    _UpperCamelCase			)






@pytest.fixture
def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			)		->						Optional[Any]:






				"""simple docstring"""


				HfFolder.save_token(_UpperCamelCase			)
				yield
				HfFolder.delete_token()






@pytest.fixture(scope='''session'''			)
def 					lowerCamelCase_						(    )		->						List[str]:






				"""simple docstring"""


				return HfApi(endpoint=_UpperCamelCase			)






@pytest.fixture(scope='''session'''			)
def 					lowerCamelCase_						(    _UpperCamelCase			)		->						Dict:






				"""simple docstring"""


				snake_case_						:   Union[str, Any]           =  HfFolder.get_token()
				HfFolder.save_token(_UpperCamelCase			)
				yield CI_HUB_USER_TOKEN
				if previous_token is not None:
								HfFolder.save_token(_UpperCamelCase			)






@pytest.fixture
def 					lowerCamelCase_						(    _UpperCamelCase			)		->						List[str]:






				"""simple docstring"""


				def _cleanup_repo(_UpperCamelCase			):
								hf_api.delete_repo(_UpperCamelCase			,    token=_UpperCamelCase			,    repo_type='''dataset'''			)

				return _cleanup_repo






@pytest.fixture
def 					lowerCamelCase_						(    _UpperCamelCase			)		->						List[str]:






				"""simple docstring"""


				@contextmanager
				def _temporary_repo(_UpperCamelCase			):
								try:
												yield repo_id
								finally:
												cleanup_repo(_UpperCamelCase			)

				return _temporary_repo






@pytest.fixture(scope='''session'''			)
def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			)		->						Dict:






				"""simple docstring"""


				snake_case_						:   Optional[int]           =  f'''repo_txt_data-{int(time.time() * 10E3			)}'''
				snake_case_						:   Any           =  f'''{CI_HUB_USER}/{repo_name}'''
				hf_api.create_repo(_UpperCamelCase			,    token=_UpperCamelCase			,    repo_type='''dataset'''			,    private=_UpperCamelCase			)
				hf_api.upload_file(
				    token=_UpperCamelCase			,    path_or_fileobj=str(_UpperCamelCase			)			,    path_in_repo='''data/text_data.txt'''			,    repo_id=_UpperCamelCase			,    repo_type='''dataset'''			,    )
				yield repo_id
				try:
								hf_api.delete_repo(_UpperCamelCase			,    token=_UpperCamelCase			,    repo_type='''dataset'''			)
				except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
								pass






@pytest.fixture()
def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			)		->						Dict:






				"""simple docstring"""


				return hf_private_dataset_repo_txt_data_






@pytest.fixture(scope='''session'''			)
def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			)		->						Optional[Any]:






				"""simple docstring"""


				snake_case_						:   int           =  f'''repo_zipped_txt_data-{int(time.time() * 10E3			)}'''
				snake_case_						:   Tuple           =  f'''{CI_HUB_USER}/{repo_name}'''
				hf_api.create_repo(_UpperCamelCase			,    token=_UpperCamelCase			,    repo_type='''dataset'''			,    private=_UpperCamelCase			)
				hf_api.upload_file(
				    token=_UpperCamelCase			,    path_or_fileobj=str(_UpperCamelCase			)			,    path_in_repo='''data.zip'''			,    repo_id=_UpperCamelCase			,    repo_type='''dataset'''			,    )
				yield repo_id
				try:
								hf_api.delete_repo(_UpperCamelCase			,    token=_UpperCamelCase			,    repo_type='''dataset'''			)
				except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
								pass






@pytest.fixture()
def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			)		->						Dict:






				"""simple docstring"""


				return hf_private_dataset_repo_zipped_txt_data_






@pytest.fixture(scope='''session'''			)
def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			)		->						Optional[Any]:






				"""simple docstring"""


				snake_case_						:   List[str]           =  f'''repo_zipped_img_data-{int(time.time() * 10E3			)}'''
				snake_case_						:   str           =  f'''{CI_HUB_USER}/{repo_name}'''
				hf_api.create_repo(_UpperCamelCase			,    token=_UpperCamelCase			,    repo_type='''dataset'''			,    private=_UpperCamelCase			)
				hf_api.upload_file(
				    token=_UpperCamelCase			,    path_or_fileobj=str(_UpperCamelCase			)			,    path_in_repo='''data.zip'''			,    repo_id=_UpperCamelCase			,    repo_type='''dataset'''			,    )
				yield repo_id
				try:
								hf_api.delete_repo(_UpperCamelCase			,    token=_UpperCamelCase			,    repo_type='''dataset'''			)
				except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
								pass






@pytest.fixture()
def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			)		->						int:






				"""simple docstring"""


				return hf_private_dataset_repo_zipped_img_data_
