






"""simple docstring"""







import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse('1.0.0a'):
	raise Exception('requires fairseq >= 1.0.0a')

logging.set_verbosity_info()
__a       :	Any             = logging.get_logger(__name__)

__a       :	List[str]             = 'Hello world! cécé herlolip'





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_):
					a__          =  FairseqRobertaModel.from_pretrained(lowerCamelCase_)
					roberta.eval()  # disable dropout
					a__          =  roberta.model.encoder.sentence_encoder
					a__          =  XLMRobertaConfig(
					    vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings ,     hidden_size=roberta.cfg.model.encoder_embed_dim ,     num_hidden_layers=roberta.cfg.model.encoder_layers ,     num_attention_heads=roberta.cfg.model.encoder_attention_heads ,     intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim ,     max_position_embeddings=514 ,     type_vocab_size=1 ,     layer_norm_eps=1E-5 ,     )
					if classification_head:
										a__          =  roberta.model.classification_heads['''mnli'''].out_proj.weight.shape[0]

					print('''Our RoBERTa config:''' ,     lowerCamelCase_)

					a__          =  XLMRobertaXLForSequenceClassification(lowerCamelCase_) if classification_head else XLMRobertaXLForMaskedLM(lowerCamelCase_)
					model.eval()

					# Now let's copy all the weights.
					# Embeddings
					a__          =  roberta_sent_encoder.embed_tokens.weight
					a__          =  roberta_sent_encoder.embed_positions.weight
					a__          =  torch.zeros_like(
					    model.roberta.embeddings.token_type_embeddings.weight)  # just zero them out b/c RoBERTa doesn't use them.

					a__          =  roberta_sent_encoder.layer_norm.weight
					a__          =  roberta_sent_encoder.layer_norm.bias

					for i in range(config.num_hidden_layers):
										# Encoder: start of layer
										a__          =  model.roberta.encoder.layer[i]
										a__          =  roberta_sent_encoder.layers[i]

										a__          =  layer.attention
										a__          =  roberta_layer.self_attn_layer_norm.weight
										a__          =  roberta_layer.self_attn_layer_norm.bias

										# self attention
										a__          =  layer.attention.self
										assert (
										    roberta_layer.self_attn.k_proj.weight.data.shape
										    == roberta_layer.self_attn.q_proj.weight.data.shape
										    == roberta_layer.self_attn.v_proj.weight.data.shape
										    == torch.Size((config.hidden_size, config.hidden_size))
										)

										a__          =  roberta_layer.self_attn.q_proj.weight
										a__          =  roberta_layer.self_attn.q_proj.bias
										a__          =  roberta_layer.self_attn.k_proj.weight
										a__          =  roberta_layer.self_attn.k_proj.bias
										a__          =  roberta_layer.self_attn.v_proj.weight
										a__          =  roberta_layer.self_attn.v_proj.bias

										# self-attention output
										a__          =  layer.attention.output
										assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
										a__          =  roberta_layer.self_attn.out_proj.weight
										a__          =  roberta_layer.self_attn.out_proj.bias

										# this one is final layer norm
										a__          =  roberta_layer.final_layer_norm.weight
										a__          =  roberta_layer.final_layer_norm.bias

										# intermediate
										a__          =  layer.intermediate
										assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
										a__          =  roberta_layer.fca.weight
										a__          =  roberta_layer.fca.bias

										# output
										a__          =  layer.output
										assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
										a__          =  roberta_layer.fca.weight
										a__          =  roberta_layer.fca.bias
										# end of layer

					if classification_head:
										a__          =  roberta.model.classification_heads['''mnli'''].dense.weight
										a__          =  roberta.model.classification_heads['''mnli'''].dense.bias
										a__          =  roberta.model.classification_heads['''mnli'''].out_proj.weight
										a__          =  roberta.model.classification_heads['''mnli'''].out_proj.bias
					else:
										# LM Head
										a__          =  roberta.model.encoder.lm_head.dense.weight
										a__          =  roberta.model.encoder.lm_head.dense.bias
										a__          =  roberta.model.encoder.lm_head.layer_norm.weight
										a__          =  roberta.model.encoder.lm_head.layer_norm.bias
										a__          =  roberta.model.encoder.lm_head.weight
										a__          =  roberta.model.encoder.lm_head.bias

					# Let's check that we get the same results.
					a__          =  roberta.encode(lowerCamelCase_).unsqueeze(0)  # batch of size 1

					a__          =  model(lowerCamelCase_)[0]
					if classification_head:
										a__          =  roberta.model.classification_heads['''mnli'''](roberta.extract_features(lowerCamelCase_))
					else:
										a__          =  roberta.model(lowerCamelCase_)[0]
					print(our_output.shape ,     their_output.shape)
					a__          =  torch.max(torch.abs(our_output - their_output)).item()
					print(f'max_absolute_diff = {max_absolute_diff}')  # ~ 1e-7
					a__          =  torch.allclose(lowerCamelCase_ ,     lowerCamelCase_ ,     atol=1E-3)
					print('''Do both models output the same tensors?''' ,     '''🔥''' if success else '''💩''')
					if not success:
										raise Exception('''Something went wRoNg''')

					pathlib.Path(lowerCamelCase_).mkdir(parents=lowerCamelCase_ ,     exist_ok=lowerCamelCase_)
					print(f'Saving model to {pytorch_dump_folder_path}')
					model.save_pretrained(lowerCamelCase_)


if __name__ == "__main__":
	__a       :	Any             = argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--roberta_checkpoint_path', default=None, type=str, required=True, help='Path the official PyTorch dump.'
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
	)
	parser.add_argument(
	    '--classification_head', action='store_true', help='Whether to convert a final classification head.'
	)
	__a       :	int             = parser.parse_args()
	convert_xlm_roberta_xl_checkpoint_to_pytorch(
	    args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
	)










"""simple docstring"""







import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


__a       :	Dict             = {
    'return_dict': False,
    'output_hidden_states': True,
    'output_attentions': True,
    'torchscript': True,
    'torch_dtype': 'float16',
    'use_bfloat16': True,
    'tf_legacy_loss': True,
    'pruned_heads': {'a': 1},
    'tie_word_embeddings': False,
    'is_decoder': True,
    'cross_attention_hidden_size': 128,
    'add_cross_attention': True,
    'tie_encoder_decoder': True,
    'max_length': 50,
    'min_length': 3,
    'do_sample': True,
    'early_stopping': True,
    'num_beams': 3,
    'num_beam_groups': 3,
    'diversity_penalty': 0.5,
    'temperature': 2.0,
    'top_k': 10,
    'top_p': 0.7,
    'typical_p': 0.2,
    'repetition_penalty': 0.8,
    'length_penalty': 0.8,
    'no_repeat_ngram_size': 5,
    'encoder_no_repeat_ngram_size': 5,
    'bad_words_ids': [1, 2, 3],
    'num_return_sequences': 3,
    'chunk_size_feed_forward': 5,
    'output_scores': True,
    'return_dict_in_generate': True,
    'forced_bos_token_id': 2,
    'forced_eos_token_id': 3,
    'remove_invalid_values': True,
    'architectures': ['BertModel'],
    'finetuning_task': 'translation',
    'id2label': {0: 'label'},
    'label2id': {'label': '0'},
    'tokenizer_class': 'BertTokenizerFast',
    'prefix': 'prefix',
    'bos_token_id': 6,
    'pad_token_id': 7,
    'eos_token_id': 8,
    'sep_token_id': 9,
    'decoder_start_token_id': 10,
    'exponential_decay_length_penalty': (5, 1.01),
    'suppress_tokens': [0, 1],
    'begin_suppress_tokens': 2,
    'task_specific_params': {'translation': 'some_params'},
    'problem_type': 'regression',
}







@is_staging_test
class        _SCREAMING_SNAKE_CASE						(  unittest.TestCase							):





							"""simple docstring"""





							@classmethod
							def  lowercase		(   cls:     Union[str, Any]    ):

												'''simple docstring'''

												a__          =  TOKEN
												HfFolder.save_token(__A    )




							@classmethod
							def  lowercase		(   cls:     Any    ):

												'''simple docstring'''

												try:
																	delete_repo(token=cls._token				,		repo_id='''test-config'''    )
												except HTTPError:
																	pass

												try:
																	delete_repo(token=cls._token				,		repo_id='''valid_org/test-config-org'''    )
												except HTTPError:
																	pass

												try:
																	delete_repo(token=cls._token				,		repo_id='''test-dynamic-config'''    )
												except HTTPError:
																	pass




							def  lowercase		(   self:     Union[str, Any]    ):

												'''simple docstring'''

												a__          =  BertConfig(
												    vocab_size=99				,		hidden_size=32				,		num_hidden_layers=5				,		num_attention_heads=4				,		intermediate_size=37    )
												config.push_to_hub('''test-config'''				,		use_auth_token=self._token    )

												a__          =  BertConfig.from_pretrained(F'{USER}/test-config'    )
												for k, v in config.to_dict().items():
																	if k != "transformers_version":
																						self.assertEqual(__A				,		getattr(__A				,		__A    )    )

        # Reset repo
												delete_repo(token=self._token				,		repo_id='''test-config'''    )

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																	config.save_pretrained(__A				,		repo_id='''test-config'''				,		push_to_hub=__A				,		use_auth_token=self._token    )

												a__          =  BertConfig.from_pretrained(F'{USER}/test-config'    )
												for k, v in config.to_dict().items():
																	if k != "transformers_version":
																						self.assertEqual(__A				,		getattr(__A				,		__A    )    )




							def  lowercase		(   self:     Optional[Any]    ):

												'''simple docstring'''

												a__          =  BertConfig(
												    vocab_size=99				,		hidden_size=32				,		num_hidden_layers=5				,		num_attention_heads=4				,		intermediate_size=37    )
												config.push_to_hub('''valid_org/test-config-org'''				,		use_auth_token=self._token    )

												a__          =  BertConfig.from_pretrained('''valid_org/test-config-org'''    )
												for k, v in config.to_dict().items():
																	if k != "transformers_version":
																						self.assertEqual(__A				,		getattr(__A				,		__A    )    )

        # Reset repo
												delete_repo(token=self._token				,		repo_id='''valid_org/test-config-org'''    )

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																	config.save_pretrained(
																	    __A				,		repo_id='''valid_org/test-config-org'''				,		push_to_hub=__A				,		use_auth_token=self._token    )

												a__          =  BertConfig.from_pretrained('''valid_org/test-config-org'''    )
												for k, v in config.to_dict().items():
																	if k != "transformers_version":
																						self.assertEqual(__A				,		getattr(__A				,		__A    )    )





							def  lowercase		(   self:     List[str]    ):

												'''simple docstring'''

												CustomConfig.register_for_auto_class()
												a__          =  CustomConfig(attribute=42    )

												config.push_to_hub('''test-dynamic-config'''				,		use_auth_token=self._token    )

												# This has added the proper auto_map field to the config
												self.assertDictEqual(config.auto_map				,		{'''AutoConfig''': '''custom_configuration.CustomConfig'''}    )

												a__          =  AutoConfig.from_pretrained(F'{USER}/test-dynamic-config'				,		trust_remote_code=__A    )
												# Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
												self.assertEqual(new_config.__class__.__name__				,		'''CustomConfig'''    )
												self.assertEqual(new_config.attribute				,		42    )







class        _SCREAMING_SNAKE_CASE						(  unittest.TestCase							):





							"""simple docstring"""





							def  lowercase		(   self:     int    ):

												'''simple docstring'''

												a__          =  GPTaConfig()

												# attempt to modify each of int/float/bool/str config records and verify they were updated
												a__          =  c.n_embd + 1  # int
												a__          =  c.resid_pdrop + 1.0  # float
												a__          =  not c.scale_attn_weights  # bool
												a__          =  c.summary_type + '''foo'''  # str
												c.update_from_string(
												    F'n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}'    )
												self.assertEqual(__A				,		c.n_embd				,		'''mismatch for key: n_embd'''    )
												self.assertEqual(__A				,		c.resid_pdrop				,		'''mismatch for key: resid_pdrop'''    )
												self.assertEqual(__A				,		c.scale_attn_weights				,		'''mismatch for key: scale_attn_weights'''    )
												self.assertEqual(__A				,		c.summary_type				,		'''mismatch for key: summary_type'''    )




							def  lowercase		(   self:     List[str]    ):

												'''simple docstring'''

												a__          =  PretrainedConfig()
												a__          =  [key for key in base_config.__dict__ if key not in config_common_kwargs]
												# If this part of the test fails, you have arguments to addin config_common_kwargs above.
												self.assertListEqual(
												    __A				,		['''is_encoder_decoder''', '''_name_or_path''', '''_commit_hash''', '''transformers_version''']    )
												a__          =  [key for key, value in config_common_kwargs.items() if value == getattr(__A				,		__A    )]
												if len(__A    ) > 0:
																	raise ValueError(
																	    '''The following keys are set with the default values in'''
																	    ''' `test_configuration_common.config_common_kwargs` pick another value for them:'''
																	    F' {", ".join(__A    )}.'    )




							def  lowercase		(   self:     str    ):

												'''simple docstring'''

												with self.assertRaises(__A    ):
																	# config is in subfolder, the following should not work without specifying the subfolder
																	a__          =  BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert-subfolder'''    )

												a__          =  BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert-subfolder'''				,		subfolder='''bert'''    )

												self.assertIsNotNone(__A    )




							def  lowercase		(   self:     Dict    ):

												'''simple docstring'''

												a__          =  mock.Mock()
												a__          =  500
												a__          =  {}
												a__          =  HTTPError
												a__          =  {}

												# Download this model to make sure it's in the cache.
												a__          =  BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )

												# Under the mock environment we get a 500 error when trying to reach the model.
												with mock.patch('''requests.Session.request'''				,		return_value=__A    ) as mock_head:
																	a__          =  BertConfig.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
																	# This check we did call the fake head request
																	mock_head.assert_called()




							def  lowercase		(   self:     Union[str, Any]    ):

												'''simple docstring'''

												a__          =  BertConfig.from_pretrained(
												    '''https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json'''    )




							def  lowercase		(   self:     Optional[Any]    ):

												'''simple docstring'''

												a__          =  AutoConfig.from_pretrained('''bert-base-cased'''    )
												a__          =  ['''config.4.0.0.json''']

												with tempfile.TemporaryDirectory() as tmp_dir:
																	configuration.save_pretrained(__A    )
																	a__          =  2
																	json.dump(configuration.to_dict()				,		open(os.path.join(__A				,		'''config.4.0.0.json'''    )				,		'''w'''    )    )

																	# This should pick the new configuration file as the version of Transformers is > 4.0.0
																	a__          =  AutoConfig.from_pretrained(__A    )
																	self.assertEqual(new_configuration.hidden_size				,		2    )

																	# Will need to be adjusted if we reach v42 and this test is still here.
																	# Should pick the old configuration file as the version of Transformers is < 4.42.0
																	a__          =  ['''config.42.0.0.json''']
																	a__          =  768
																	configuration.save_pretrained(__A    )
																	shutil.move(os.path.join(__A				,		'''config.4.0.0.json'''    )				,		os.path.join(__A				,		'''config.42.0.0.json'''    )    )
																	a__          =  AutoConfig.from_pretrained(__A    )
																	self.assertEqual(new_configuration.hidden_size				,		768    )





							def  lowercase		(   self:     Optional[Any]    ):

												'''simple docstring'''

												a__          =  '''hf-internal-testing/test-two-configs'''

												import transformers as new_transformers

												a__          =  '''v4.0.0'''
												a__       ,a__          =  new_transformers.models.auto.AutoConfig.from_pretrained(
												    __A				,		return_unused_kwargs=__A    )
												self.assertEqual(new_configuration.hidden_size				,		2    )
												# This checks `_configuration_file` ia not kept in the kwargs by mistake.
												self.assertDictEqual(__A				,		{}    )

												# Testing an older version by monkey-patching the version in the module it's used.
												import transformers as old_transformers

												a__          =  '''v3.0.0'''
												a__          =  old_transformers.models.auto.AutoConfig.from_pretrained(__A    )
												self.assertEqual(old_configuration.hidden_size				,		768    )


