



import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse('''1.0.0a'''):
				raise Exception('''requires fairseq >= 1.0.0a''')

logging.set_verbosity_info()
_lowercase     							=						logging.get_logger(__name__)

_lowercase     							=						'''Hello world! cécé herlolip'''






def      _A				(UpperCamelCase		:      str      ,			UpperCamelCase		:      str      ,			UpperCamelCase		:      bool ):



			'''simple docstring'''







			lowerCamelCase__ :      List[Any]								=		FairseqRobertaModel.from_pretrained(A__ )
			roberta.eval()  # disable dropout
			lowerCamelCase__ :      Any								=		roberta.model.encoder.sentence_encoder
			lowerCamelCase__ :      Optional[Any]								=		XLMRobertaConfig(
			    vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings      ,			hidden_size=roberta.cfg.model.encoder_embed_dim      ,			num_hidden_layers=roberta.cfg.model.encoder_layers      ,			num_attention_heads=roberta.cfg.model.encoder_attention_heads      ,			intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim      ,			max_position_embeddings=514      ,			type_vocab_size=1      ,			layer_norm_eps=1E-5      ,			)
			if classification_head:
						lowerCamelCase__ :      List[Any]								=		roberta.model.classification_heads["""mnli"""].out_proj.weight.shape[0]

			print("""Our RoBERTa config:"""      ,			A__ )

			lowerCamelCase__ :      List[str]								=		XLMRobertaXLForSequenceClassification(A__ ) if classification_head else XLMRobertaXLForMaskedLM(A__ )
			model.eval()

			# Now let's copy all the weights.
			# Embeddings
			lowerCamelCase__ :      int								=		roberta_sent_encoder.embed_tokens.weight
			lowerCamelCase__ :      str								=		roberta_sent_encoder.embed_positions.weight
			lowerCamelCase__ :      Optional[int]								=		torch.zeros_like(
			    model.roberta.embeddings.token_type_embeddings.weight )  # just zero them out b/c RoBERTa doesn't use them.

			lowerCamelCase__ :      Union[str, Any]								=		roberta_sent_encoder.layer_norm.weight
			lowerCamelCase__ :      Optional[int]								=		roberta_sent_encoder.layer_norm.bias

			for i in range(config.num_hidden_layers ):
						# Encoder: start of layer
						lowerCamelCase__ :      Optional[Any]								=		model.roberta.encoder.layer[i]
						lowerCamelCase__ :      List[str]								=		roberta_sent_encoder.layers[i]

						lowerCamelCase__ :      Dict								=		layer.attention
						lowerCamelCase__ :      Optional[Any]								=		roberta_layer.self_attn_layer_norm.weight
						lowerCamelCase__ :      Optional[Any]								=		roberta_layer.self_attn_layer_norm.bias

						# self attention
						lowerCamelCase__ :      Optional[int]								=		layer.attention.self
						assert (
						    roberta_layer.self_attn.k_proj.weight.data.shape
						    == roberta_layer.self_attn.q_proj.weight.data.shape
						    == roberta_layer.self_attn.v_proj.weight.data.shape
						    == torch.Size((config.hidden_size, config.hidden_size) )
						)

						lowerCamelCase__ :      List[str]								=		roberta_layer.self_attn.q_proj.weight
						lowerCamelCase__ :      List[str]								=		roberta_layer.self_attn.q_proj.bias
						lowerCamelCase__ :      Optional[int]								=		roberta_layer.self_attn.k_proj.weight
						lowerCamelCase__ :      Optional[Any]								=		roberta_layer.self_attn.k_proj.bias
						lowerCamelCase__ :      List[str]								=		roberta_layer.self_attn.v_proj.weight
						lowerCamelCase__ :      Any								=		roberta_layer.self_attn.v_proj.bias

						# self-attention output
						lowerCamelCase__ :      str								=		layer.attention.output
						assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
						lowerCamelCase__ :      Any								=		roberta_layer.self_attn.out_proj.weight
						lowerCamelCase__ :      Any								=		roberta_layer.self_attn.out_proj.bias

						# this one is final layer norm
						lowerCamelCase__ :      Union[str, Any]								=		roberta_layer.final_layer_norm.weight
						lowerCamelCase__ :      Dict								=		roberta_layer.final_layer_norm.bias

						# intermediate
						lowerCamelCase__ :      Optional[int]								=		layer.intermediate
						assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
						lowerCamelCase__ :      List[str]								=		roberta_layer.fca.weight
						lowerCamelCase__ :      List[str]								=		roberta_layer.fca.bias

						# output
						lowerCamelCase__ :      int								=		layer.output
						assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
						lowerCamelCase__ :      Union[str, Any]								=		roberta_layer.fca.weight
						lowerCamelCase__ :      Optional[int]								=		roberta_layer.fca.bias
						# end of layer

			if classification_head:
						lowerCamelCase__ :      str								=		roberta.model.classification_heads["""mnli"""].dense.weight
						lowerCamelCase__ :      str								=		roberta.model.classification_heads["""mnli"""].dense.bias
						lowerCamelCase__ :      int								=		roberta.model.classification_heads["""mnli"""].out_proj.weight
						lowerCamelCase__ :      Union[str, Any]								=		roberta.model.classification_heads["""mnli"""].out_proj.bias
			else:
						# LM Head
						lowerCamelCase__ :      List[str]								=		roberta.model.encoder.lm_head.dense.weight
						lowerCamelCase__ :      List[str]								=		roberta.model.encoder.lm_head.dense.bias
						lowerCamelCase__ :      Union[str, Any]								=		roberta.model.encoder.lm_head.layer_norm.weight
						lowerCamelCase__ :      Dict								=		roberta.model.encoder.lm_head.layer_norm.bias
						lowerCamelCase__ :      Tuple								=		roberta.model.encoder.lm_head.weight
						lowerCamelCase__ :      Optional[Any]								=		roberta.model.encoder.lm_head.bias

			# Let's check that we get the same results.
			lowerCamelCase__ :      Optional[Any]								=		roberta.encode(A__ ).unsqueeze(0 )  # batch of size 1

			lowerCamelCase__ :      Optional[int]								=		model(A__ )[0]
			if classification_head:
						lowerCamelCase__ :      Dict								=		roberta.model.classification_heads["""mnli"""](roberta.extract_features(A__ ) )
			else:
						lowerCamelCase__ :      str								=		roberta.model(A__ )[0]
			print(our_output.shape      ,			their_output.shape )
			lowerCamelCase__ :      int								=		torch.max(torch.abs(our_output - their_output ) ).item()
			print(f"max_absolute_diff = {max_absolute_diff}" )  # ~ 1e-7
			lowerCamelCase__ :      Dict								=		torch.allclose(A__      ,			A__      ,			atol=1E-3 )
			print("""Do both models output the same tensors?"""      ,			"""🔥""" if success else """💩""" )
			if not success:
						raise Exception("""Something went wRoNg""" )

			pathlib.Path(A__ ).mkdir(parents=A__      ,			exist_ok=A__ )
			print(f"Saving model to {pytorch_dump_folder_path}" )
			model.save_pretrained(A__ )


if __name__ == "__main__":
				_lowercase     							=						argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '''--roberta_checkpoint_path''', default=None, type=str, required=True, help='''Path the official PyTorch dump.'''
				)
				parser.add_argument(
				    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
				)
				parser.add_argument(
				    '''--classification_head''', action='''store_true''', help='''Whether to convert a final classification head.'''
				)
				_lowercase     							=						parser.parse_args()
				convert_xlm_roberta_xl_checkpoint_to_pytorch(
				    args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
				)








import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
				import torch

				from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
				_lowercase     							=						False

if is_vision_available():
				from PIL import Image

				from transformers import PixaStructImageProcessor
class 				__A					( unittest.TestCase      ):





			def __init__(self							,    __magic_name__							,    __magic_name__=7							,    __magic_name__=3							,    __magic_name__=18							,    __magic_name__=30							,    __magic_name__=400							,    __magic_name__=None							,    __magic_name__=True							,    __magic_name__=True							,    __magic_name__=None							,    ):
						lowerCamelCase__ :      Optional[Any]								=		size if size is not None else {"""height""": 20, """width""": 20}
						lowerCamelCase__ :      List[str]								=		parent
						lowerCamelCase__ :      str								=		batch_size
						lowerCamelCase__ :      List[str]								=		num_channels
						lowerCamelCase__ :      Optional[int]								=		image_size
						lowerCamelCase__ :      Union[str, Any]								=		min_resolution
						lowerCamelCase__ :      List[Any]								=		max_resolution
						lowerCamelCase__ :      Optional[int]								=		size
						lowerCamelCase__ :      List[str]								=		do_normalize
						lowerCamelCase__ :      str								=		do_convert_rgb
						lowerCamelCase__ :      str								=		[512, 1024, 2048, 4096]
						lowerCamelCase__ :      Any								=		patch_size if patch_size is not None else {"""height""": 16, """width""": 16}





			def        _snake_case    (self				):
						return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}





			def        _snake_case    (self				):
						lowerCamelCase__ :      List[str]								=		"""https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg"""
						lowerCamelCase__ :      int								=		Image.open(requests.get(__magic_name__							,    stream=__magic_name__				).raw				).convert("""RGB"""				)
						return raw_image







@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11					, reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''					, )
@require_torch
@require_vision
class 				__A					( A_					, unittest.TestCase      ):
			UpperCamelCase					:Union[str, Any]     =			PixaStructImageProcessor if is_vision_available() else None





			def        _snake_case    (self				):
						lowerCamelCase__ :      List[str]								=		PixaStructImageProcessingTester(self				)





			@property
			def        _snake_case    (self				):
						return self.image_processor_tester.prepare_image_processor_dict()





			def        _snake_case    (self				):
						lowerCamelCase__ :      int								=		self.image_processing_class(**self.image_processor_dict				)
						self.assertTrue(hasattr(__magic_name__							,    """do_normalize"""				)				)
						self.assertTrue(hasattr(__magic_name__							,    """do_convert_rgb"""				)				)





			def        _snake_case    (self				):
						lowerCamelCase__ :      Tuple								=		self.image_processor_tester.prepare_dummy_image()

						lowerCamelCase__ :      Tuple								=		self.image_processing_class(**self.image_processor_dict				)
						lowerCamelCase__ :      List[str]								=		2048

						lowerCamelCase__ :      Any								=		image_processor(__magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				)
						self.assertTrue(torch.allclose(inputs.flattened_patches.mean()							,    torch.tensor(0.06_06				)							,    atol=1E-3							,    rtol=1E-3				)				)





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      List[Any]								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PIL images
						lowerCamelCase__ :      Optional[Any]								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    Image.Image				)

						# Test not batched input
						lowerCamelCase__ :      Optional[Any]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      List[str]								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PIL images
						lowerCamelCase__ :      Optional[Any]								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    Image.Image				)

						# Test not batched input
						lowerCamelCase__ :      List[Any]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						lowerCamelCase__ :      List[str]								=		True

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									with self.assertRaises(__magic_name__				):
												lowerCamelCase__ :      Tuple								=		image_processor(
												    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches

									lowerCamelCase__ :      Optional[Any]								=		"""Hello"""

									lowerCamelCase__ :      Dict								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__							,    header_text=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      Tuple								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__							,    header_text=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      Any								=		self.image_processing_class(**self.image_processor_dict				)
						# create random numpy tensors
						lowerCamelCase__ :      Dict								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__							,    numpify=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    np.ndarray				)

						lowerCamelCase__ :      Optional[int]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      List[str]								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      List[str]								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      Optional[int]								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PyTorch tensors
						lowerCamelCase__ :      Dict								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__							,    torchify=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    torch.Tensor				)

						# Test not batched input
						lowerCamelCase__ :      List[str]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      Tuple								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )







@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11					, reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''					, )
@require_torch
@require_vision
class 				__A					( A_					, unittest.TestCase      ):
			UpperCamelCase					:Optional[int]     =			PixaStructImageProcessor if is_vision_available() else None





			def        _snake_case    (self				):
						lowerCamelCase__ :      Optional[Any]								=		PixaStructImageProcessingTester(self							,    num_channels=4				)
						lowerCamelCase__ :      Optional[int]								=		3





			@property
			def        _snake_case    (self				):
						return self.image_processor_tester.prepare_image_processor_dict()





			def        _snake_case    (self				):
						lowerCamelCase__ :      Dict								=		self.image_processing_class(**self.image_processor_dict				)
						self.assertTrue(hasattr(__magic_name__							,    """do_normalize"""				)				)
						self.assertTrue(hasattr(__magic_name__							,    """do_convert_rgb"""				)				)





			def        _snake_case    (self				):
						# Initialize image_processor
						lowerCamelCase__ :      str								=		self.image_processing_class(**self.image_processor_dict				)
						# create random PIL images
						lowerCamelCase__ :      List[Any]								=		prepare_image_inputs(self.image_processor_tester							,    equal_resolution=__magic_name__				)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__							,    Image.Image				)

						# Test not batched input
						lowerCamelCase__ :      List[str]								=		(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * (self.image_processor_tester.num_channels - 1)
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									lowerCamelCase__ :      Optional[Any]								=		image_processor(
									    image_inputs[0]							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (1, max_patch, expected_hidden_dim)							,    )

									# Test batched
									lowerCamelCase__ :      str								=		image_processor(
									    __magic_name__							,    return_tensors="""pt"""							,    max_patches=__magic_name__				).flattened_patches
									self.assertEqual(
									    encoded_images.shape							,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)							,    )



