






import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse('''1.0.0a'''):
 raise Exception('''requires fairseq >= 1.0.0a''')

logging.set_verbosity_info()
lowerCamelCase			:					List[str]         =	logging.get_logger(__name__)

lowerCamelCase			:					str         =	'''Hello world! cécé herlolip'''





def       __lowerCAmelCase      (						__snake_case					, __snake_case					, __snake_case       ):
     __lowerCAmelCase								=					FairseqRobertaModel.from_pretrained(__snake_case       )
     roberta.eval()  # disable dropout
     __lowerCAmelCase								=					roberta.model.encoder.sentence_encoder
     __lowerCAmelCase								=					XLMRobertaConfig(
         vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings					, hidden_size=roberta.cfg.model.encoder_embed_dim					, num_hidden_layers=roberta.cfg.model.encoder_layers					, num_attention_heads=roberta.cfg.model.encoder_attention_heads					, intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim					, max_position_embeddings=514					, type_vocab_size=1					, layer_norm_eps=1E-5					, )
     if classification_head:
          __lowerCAmelCase								=					roberta.model.classification_heads["mnli"].out_proj.weight.shape[0]

     print("Our RoBERTa config:"					, __snake_case       )

     __lowerCAmelCase								=					XLMRobertaXLForSequenceClassification(__snake_case       ) if classification_head else XLMRobertaXLForMaskedLM(__snake_case       )
     model.eval()

     # Now let's copy all the weights.
     # Embeddings
     __lowerCAmelCase								=					roberta_sent_encoder.embed_tokens.weight
     __lowerCAmelCase								=					roberta_sent_encoder.embed_positions.weight
     __lowerCAmelCase								=					torch.zeros_like(
         model.roberta.embeddings.token_type_embeddings.weight       )  # just zero them out b/c RoBERTa doesn't use them.

     __lowerCAmelCase								=					roberta_sent_encoder.layer_norm.weight
     __lowerCAmelCase								=					roberta_sent_encoder.layer_norm.bias

     for i in range(config.num_hidden_layers       ):
          # Encoder: start of layer
          __lowerCAmelCase								=					model.roberta.encoder.layer[i]
          __lowerCAmelCase								=					roberta_sent_encoder.layers[i]

          __lowerCAmelCase								=					layer.attention
          __lowerCAmelCase								=					roberta_layer.self_attn_layer_norm.weight
          __lowerCAmelCase								=					roberta_layer.self_attn_layer_norm.bias

          # self attention
          __lowerCAmelCase								=					layer.attention.self
          assert (
              roberta_layer.self_attn.k_proj.weight.data.shape
              == roberta_layer.self_attn.q_proj.weight.data.shape
              == roberta_layer.self_attn.v_proj.weight.data.shape
              == torch.Size((config.hidden_size, config.hidden_size)       )
          )

          __lowerCAmelCase								=					roberta_layer.self_attn.q_proj.weight
          __lowerCAmelCase								=					roberta_layer.self_attn.q_proj.bias
          __lowerCAmelCase								=					roberta_layer.self_attn.k_proj.weight
          __lowerCAmelCase								=					roberta_layer.self_attn.k_proj.bias
          __lowerCAmelCase								=					roberta_layer.self_attn.v_proj.weight
          __lowerCAmelCase								=					roberta_layer.self_attn.v_proj.bias

          # self-attention output
          __lowerCAmelCase								=					layer.attention.output
          assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
          __lowerCAmelCase								=					roberta_layer.self_attn.out_proj.weight
          __lowerCAmelCase								=					roberta_layer.self_attn.out_proj.bias

          # this one is final layer norm
          __lowerCAmelCase								=					roberta_layer.final_layer_norm.weight
          __lowerCAmelCase								=					roberta_layer.final_layer_norm.bias

          # intermediate
          __lowerCAmelCase								=					layer.intermediate
          assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
          __lowerCAmelCase								=					roberta_layer.fca.weight
          __lowerCAmelCase								=					roberta_layer.fca.bias

          # output
          __lowerCAmelCase								=					layer.output
          assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
          __lowerCAmelCase								=					roberta_layer.fca.weight
          __lowerCAmelCase								=					roberta_layer.fca.bias
          # end of layer

     if classification_head:
          __lowerCAmelCase								=					roberta.model.classification_heads["mnli"].dense.weight
          __lowerCAmelCase								=					roberta.model.classification_heads["mnli"].dense.bias
          __lowerCAmelCase								=					roberta.model.classification_heads["mnli"].out_proj.weight
          __lowerCAmelCase								=					roberta.model.classification_heads["mnli"].out_proj.bias
     else:
          # LM Head
          __lowerCAmelCase								=					roberta.model.encoder.lm_head.dense.weight
          __lowerCAmelCase								=					roberta.model.encoder.lm_head.dense.bias
          __lowerCAmelCase								=					roberta.model.encoder.lm_head.layer_norm.weight
          __lowerCAmelCase								=					roberta.model.encoder.lm_head.layer_norm.bias
          __lowerCAmelCase								=					roberta.model.encoder.lm_head.weight
          __lowerCAmelCase								=					roberta.model.encoder.lm_head.bias

     # Let's check that we get the same results.
     __lowerCAmelCase								=					roberta.encode(__snake_case       ).unsqueeze(0       )  # batch of size 1

     __lowerCAmelCase								=					model(__snake_case       )[0]
     if classification_head:
          __lowerCAmelCase								=					roberta.model.classification_heads["mnli"](roberta.extract_features(__snake_case       )       )
     else:
          __lowerCAmelCase								=					roberta.model(__snake_case       )[0]
     print(our_output.shape					, their_output.shape       )
     __lowerCAmelCase								=					torch.max(torch.abs(our_output - their_output       )       ).item()
     print(F"""max_absolute_diff = {max_absolute_diff}"""       )  # ~ 1e-7
     __lowerCAmelCase								=					torch.allclose(__snake_case					, __snake_case					, atol=1E-3       )
     print("Do both models output the same tensors?"					, "🔥" if success else "💩"       )
     if not success:
          raise Exception("Something went wRoNg"       )

     pathlib.Path(__snake_case       ).mkdir(parents=__snake_case					, exist_ok=__snake_case       )
     print(F"""Saving model to {pytorch_dump_folder_path}"""       )
     model.save_pretrained(__snake_case       )


if __name__ == "__main__":
 lowerCamelCase			:					Any         =	argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '''--roberta_checkpoint_path''', default=None, type=str, required=True, help='''Path the official PyTorch dump.'''
 )
 parser.add_argument(
     '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
 )
 parser.add_argument(
     '''--classification_head''', action='''store_true''', help='''Whether to convert a final classification head.'''
 )
 lowerCamelCase			:					Tuple         =	parser.parse_args()
 convert_xlm_roberta_xl_checkpoint_to_pytorch(
     args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
 )














import json
import os
import shutil
import tempfile
import unittest

import numpy as np

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
 from PIL import Image

 from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor



@require_tokenizers
@require_vision
class 							_UpperCamelCase			(unittest.TestCase			):


      def  __UpperCAmelCase		(	self	)->	Optional[Any]:
           __lowerCAmelCase								=					tempfile.mkdtemp()

           # fmt: off
           __lowerCAmelCase								=					["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "want", "##want", "##ed", "wa", "un", "runn", "##ing", ",", "low", "lowest"]
           # fmt: on
           __lowerCAmelCase								=					os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["vocab_file"]	)
           with open(self.vocab_file					,						"w"					,						encoding="utf-8"	) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)

           __lowerCAmelCase								=					{
               "do_resize": True,
               "size": {"height": 1_8, "width": 1_8},
               "do_normalize": True,
               "image_mean": [0.5, 0.5, 0.5],
               "image_std": [0.5, 0.5, 0.5],
           }
           __lowerCAmelCase								=					os.path.join(self.tmpdirname					,						__UpperCamelCase	)
           with open(self.image_processor_file					,						"w"					,						encoding="utf-8"	) as fp:
                json.dump(__UpperCamelCase					,						__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						**__UpperCamelCase	)->	Optional[Any]:
           return BertTokenizer.from_pretrained(self.tmpdirname					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						**__UpperCamelCase	)->	Any:
           return ViTImageProcessor.from_pretrained(self.tmpdirname					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	Any:
           shutil.rmtree(self.tmpdirname	)


      def  __UpperCAmelCase		(	self	)->	int:
           __lowerCAmelCase								=					[np.random.randint(2_5_5					,						size=(3, 3_0, 4_0_0)					,						dtype=np.uinta	)]

           __lowerCAmelCase								=					[Image.fromarray(np.moveaxis(__UpperCamelCase					,						0					,						-1	)	) for x in image_inputs]

           return image_inputs


      def  __UpperCAmelCase		(	self	)->	Optional[Any]:
           __lowerCAmelCase								=					self.get_tokenizer()
           __lowerCAmelCase								=					self.get_image_processor()

           __lowerCAmelCase								=					VisionTextDualEncoderProcessor(tokenizer=__UpperCamelCase					,						image_processor=__UpperCamelCase	)

           processor.save_pretrained(self.tmpdirname	)
           __lowerCAmelCase								=					VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname	)

           self.assertEqual(processor.tokenizer.get_vocab()					,						tokenizer.get_vocab()	)
           self.assertIsInstance(processor.tokenizer					,						(BertTokenizer, BertTokenizerFast)	)

           self.assertEqual(processor.image_processor.to_json_string()					,						image_processor.to_json_string()	)
           self.assertIsInstance(processor.image_processor					,						__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	str:
           __lowerCAmelCase								=					VisionTextDualEncoderProcessor(
               tokenizer=self.get_tokenizer()					,						image_processor=self.get_image_processor()	)
           processor.save_pretrained(self.tmpdirname	)

           __lowerCAmelCase								=					self.get_tokenizer(bos_token="(BOS)"					,						eos_token="(EOS)"	)
           __lowerCAmelCase								=					self.get_image_processor(do_normalize=__UpperCamelCase					,						padding_value=1.0	)

           __lowerCAmelCase								=					VisionTextDualEncoderProcessor.from_pretrained(
               self.tmpdirname					,						bos_token="(BOS)"					,						eos_token="(EOS)"					,						do_normalize=__UpperCamelCase					,						padding_value=1.0	)

           self.assertEqual(processor.tokenizer.get_vocab()					,						tokenizer_add_kwargs.get_vocab()	)
           self.assertIsInstance(processor.tokenizer					,						(BertTokenizer, BertTokenizerFast)	)

           self.assertEqual(processor.image_processor.to_json_string()					,						image_processor_add_kwargs.to_json_string()	)
           self.assertIsInstance(processor.image_processor					,						__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	Union[str, Any]:
           __lowerCAmelCase								=					self.get_image_processor()
           __lowerCAmelCase								=					self.get_tokenizer()

           __lowerCAmelCase								=					VisionTextDualEncoderProcessor(tokenizer=__UpperCamelCase					,						image_processor=__UpperCamelCase	)

           __lowerCAmelCase								=					self.prepare_image_inputs()

           __lowerCAmelCase								=					image_processor(__UpperCamelCase					,						return_tensors="np"	)
           __lowerCAmelCase								=					processor(images=__UpperCamelCase					,						return_tensors="np"	)

           for key in input_feat_extract.keys():
                self.assertAlmostEqual(input_feat_extract[key].sum()					,						input_processor[key].sum()					,						delta=1e-2	)


      def  __UpperCAmelCase		(	self	)->	Dict:
           __lowerCAmelCase								=					self.get_image_processor()
           __lowerCAmelCase								=					self.get_tokenizer()

           __lowerCAmelCase								=					VisionTextDualEncoderProcessor(tokenizer=__UpperCamelCase					,						image_processor=__UpperCamelCase	)

           __lowerCAmelCase								=					"lower newer"

           __lowerCAmelCase								=					processor(text=__UpperCamelCase	)

           __lowerCAmelCase								=					tokenizer(__UpperCamelCase	)

           for key in encoded_tok.keys():
                self.assertListEqual(encoded_tok[key]					,						encoded_processor[key]	)


      def  __UpperCAmelCase		(	self	)->	List[str]:
           __lowerCAmelCase								=					self.get_image_processor()
           __lowerCAmelCase								=					self.get_tokenizer()

           __lowerCAmelCase								=					VisionTextDualEncoderProcessor(tokenizer=__UpperCamelCase					,						image_processor=__UpperCamelCase	)

           __lowerCAmelCase								=					"lower newer"
           __lowerCAmelCase								=					self.prepare_image_inputs()

           __lowerCAmelCase								=					processor(text=__UpperCamelCase					,						images=__UpperCamelCase	)

           self.assertListEqual(list(inputs.keys()	)					,						["input_ids", "token_type_ids", "attention_mask", "pixel_values"]	)

           # test if it raises when no input is passed
           with self.assertRaises(__UpperCamelCase	):
                processor()


      def  __UpperCAmelCase		(	self	)->	Optional[int]:
           __lowerCAmelCase								=					self.get_image_processor()
           __lowerCAmelCase								=					self.get_tokenizer()

           __lowerCAmelCase								=					VisionTextDualEncoderProcessor(tokenizer=__UpperCamelCase					,						image_processor=__UpperCamelCase	)

           __lowerCAmelCase								=					[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

           __lowerCAmelCase								=					processor.batch_decode(__UpperCamelCase	)
           __lowerCAmelCase								=					tokenizer.batch_decode(__UpperCamelCase	)

           self.assertListEqual(__UpperCamelCase					,						__UpperCamelCase	)


      def  __UpperCAmelCase		(	self	)->	Any:
           __lowerCAmelCase								=					self.get_image_processor()
           __lowerCAmelCase								=					self.get_tokenizer()

           __lowerCAmelCase								=					VisionTextDualEncoderProcessor(tokenizer=__UpperCamelCase					,						image_processor=__UpperCamelCase	)

           __lowerCAmelCase								=					"lower newer"
           __lowerCAmelCase								=					self.prepare_image_inputs()

           __lowerCAmelCase								=					processor(text=__UpperCamelCase					,						images=__UpperCamelCase	)

           self.assertListEqual(list(inputs.keys()	)					,						processor.model_input_names	)






