

import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("""1.0.0a"""):
      raise Exception("""requires fairseq >= 1.0.0a""")

logging.set_verbosity_info()
__lowercase				      =						logging.get_logger(__name__)

__lowercase				      =						"""Hello world! cécé herlolip"""







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		FairseqRobertaModel.from_pretrained(SCREAMING_SNAKE_CASE   )
      roberta.eval()  # disable dropout
      A_							       =		roberta.model.encoder.sentence_encoder
      A_							       =		XLMRobertaConfig(
          vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings		,				hidden_size=roberta.cfg.model.encoder_embed_dim		,				num_hidden_layers=roberta.cfg.model.encoder_layers		,				num_attention_heads=roberta.cfg.model.encoder_attention_heads		,				intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim		,				max_position_embeddings=514		,				type_vocab_size=1		,				layer_norm_eps=1E-5		,				)
      if classification_head:
            A_							       =		roberta.model.classification_heads['''mnli'''].out_proj.weight.shape[0]

      print('''Our RoBERTa config:'''		,				SCREAMING_SNAKE_CASE   )

      A_							       =		XLMRobertaXLForSequenceClassification(SCREAMING_SNAKE_CASE   ) if classification_head else XLMRobertaXLForMaskedLM(SCREAMING_SNAKE_CASE   )
      model.eval()

      # Now let's copy all the weights.
      # Embeddings
      A_							       =		roberta_sent_encoder.embed_tokens.weight
      A_							       =		roberta_sent_encoder.embed_positions.weight
      A_							       =		torch.zeros_like(
          model.roberta.embeddings.token_type_embeddings.weight   )  # just zero them out b/c RoBERTa doesn't use them.

      A_							       =		roberta_sent_encoder.layer_norm.weight
      A_							       =		roberta_sent_encoder.layer_norm.bias

      for i in range(config.num_hidden_layers   ):
            # Encoder: start of layer
            A_							       =		model.roberta.encoder.layer[i]
            A_							       =		roberta_sent_encoder.layers[i]

            A_							       =		layer.attention
            A_							       =		roberta_layer.self_attn_layer_norm.weight
            A_							       =		roberta_layer.self_attn_layer_norm.bias

            # self attention
            A_							       =		layer.attention.self
            assert (
                roberta_layer.self_attn.k_proj.weight.data.shape
                == roberta_layer.self_attn.q_proj.weight.data.shape
                == roberta_layer.self_attn.v_proj.weight.data.shape
                == torch.Size((config.hidden_size, config.hidden_size)   )
            )

            A_							       =		roberta_layer.self_attn.q_proj.weight
            A_							       =		roberta_layer.self_attn.q_proj.bias
            A_							       =		roberta_layer.self_attn.k_proj.weight
            A_							       =		roberta_layer.self_attn.k_proj.bias
            A_							       =		roberta_layer.self_attn.v_proj.weight
            A_							       =		roberta_layer.self_attn.v_proj.bias

            # self-attention output
            A_							       =		layer.attention.output
            assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
            A_							       =		roberta_layer.self_attn.out_proj.weight
            A_							       =		roberta_layer.self_attn.out_proj.bias

            # this one is final layer norm
            A_							       =		roberta_layer.final_layer_norm.weight
            A_							       =		roberta_layer.final_layer_norm.bias

            # intermediate
            A_							       =		layer.intermediate
            assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
            A_							       =		roberta_layer.fca.weight
            A_							       =		roberta_layer.fca.bias

            # output
            A_							       =		layer.output
            assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
            A_							       =		roberta_layer.fca.weight
            A_							       =		roberta_layer.fca.bias
            # end of layer

      if classification_head:
            A_							       =		roberta.model.classification_heads['''mnli'''].dense.weight
            A_							       =		roberta.model.classification_heads['''mnli'''].dense.bias
            A_							       =		roberta.model.classification_heads['''mnli'''].out_proj.weight
            A_							       =		roberta.model.classification_heads['''mnli'''].out_proj.bias
      else:
            # LM Head
            A_							       =		roberta.model.encoder.lm_head.dense.weight
            A_							       =		roberta.model.encoder.lm_head.dense.bias
            A_							       =		roberta.model.encoder.lm_head.layer_norm.weight
            A_							       =		roberta.model.encoder.lm_head.layer_norm.bias
            A_							       =		roberta.model.encoder.lm_head.weight
            A_							       =		roberta.model.encoder.lm_head.bias

      # Let's check that we get the same results.
      A_							       =		roberta.encode(SCREAMING_SNAKE_CASE   ).unsqueeze(0   )  # batch of size 1

      A_							       =		model(SCREAMING_SNAKE_CASE   )[0]
      if classification_head:
            A_							       =		roberta.model.classification_heads['''mnli'''](roberta.extract_features(SCREAMING_SNAKE_CASE   )   )
      else:
            A_							       =		roberta.model(SCREAMING_SNAKE_CASE   )[0]
      print(our_output.shape		,				their_output.shape   )
      A_							       =		torch.max(torch.abs(our_output - their_output   )   ).item()
      print(f"max_absolute_diff = {max_absolute_diff}"   )  # ~ 1e-7
      A_							       =		torch.allclose(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				atol=1E-3   )
      print('''Do both models output the same tensors?'''		,				'''🔥''' if success else '''💩'''   )
      if not success:
            raise Exception('''Something went wRoNg'''   )

      pathlib.Path(SCREAMING_SNAKE_CASE   ).mkdir(parents=SCREAMING_SNAKE_CASE		,				exist_ok=SCREAMING_SNAKE_CASE   )
      print(f"Saving model to {pytorch_dump_folder_path}"   )
      model.save_pretrained(SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
      __lowercase				      =						argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--roberta_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
      )
      parser.add_argument(
          """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
      )
      parser.add_argument(
          """--classification_head""", action="""store_true""", help="""Whether to convert a final classification head."""
      )
      __lowercase				      =						parser.parse_args()
      convert_xlm_roberta_xl_checkpoint_to_pytorch(
          args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
      )








import inspect
import os
import re

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
__lowercase				      =						"""src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
__lowercase				      =						direct_transformers_import(PATH_TO_TRANSFORMERS)

__lowercase				      =						transformers.models.auto.configuration_auto.CONFIG_MAPPING

__lowercase				      =						{
    # used to compute the property `self.chunk_length`
    """EncodecConfig""": ["""overlap"""],
    # used as `self.bert_model = BertModel(config, ...)`
    """DPRConfig""": True,
    # not used in modeling files, but it's an important information
    """FSMTConfig""": ["""langs"""],
    # used internally in the configuration class file
    """GPTNeoConfig""": ["""attention_types"""],
    # used internally in the configuration class file
    """EsmConfig""": ["""is_folding_model"""],
    # used during training (despite we don't have training script for these models yet)
    """Mask2FormerConfig""": ["""ignore_value"""],
    # `ignore_value` used during training (despite we don't have training script for these models yet)
    # `norm` used in conversion script (despite not using in the modeling file)
    """OneFormerConfig""": ["""ignore_value""", """norm"""],
    # used during preprocessing and collation, see `collating_graphormer.py`
    """GraphormerConfig""": ["""spatial_pos_max"""],
    # used internally in the configuration class file
    """T5Config""": ["""feed_forward_proj"""],
    # used internally in the configuration class file
    # `tokenizer_class` get default value `T5Tokenizer` intentionally
    """MT5Config""": ["""feed_forward_proj""", """tokenizer_class"""],
    """UMT5Config""": ["""feed_forward_proj""", """tokenizer_class"""],
    # used internally in the configuration class file
    """LongT5Config""": ["""feed_forward_proj"""],
    # used internally in the configuration class file
    """SwitchTransformersConfig""": ["""feed_forward_proj"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """BioGptConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """GLPNConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """SegformerConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """CvtConfig""": ["""layer_norm_eps"""],
    # having default values other than `1e-5` - we can't fix them without breaking
    """PerceiverConfig""": ["""layer_norm_eps"""],
    # used internally to calculate the feature size
    """InformerConfig""": ["""num_static_real_features""", """num_time_features"""],
    # used internally to calculate the feature size
    """TimeSeriesTransformerConfig""": ["""num_static_real_features""", """num_time_features"""],
    # used internally to calculate the feature size
    """AutoformerConfig""": ["""num_static_real_features""", """num_time_features"""],
    # used internally to calculate `mlp_dim`
    """SamVisionConfig""": ["""mlp_ratio"""],
    # For (head) training, but so far not implemented
    """ClapAudioConfig""": ["""num_classes"""],
    # Not used, but providing useful information to users
    """SpeechT5HifiGanConfig""": ["""sampling_rate"""],
}


# TODO (ydshieh): Check the failing cases, try to fix them or move some cases to the above block once we are sure
SPECIAL_CASES_TO_ALLOW.update(
    {
        """CLIPSegConfig""": True,
        """DeformableDetrConfig""": True,
        """DetaConfig""": True,
        """DinatConfig""": True,
        """DonutSwinConfig""": True,
        """EfficientFormerConfig""": True,
        """FSMTConfig""": True,
        """JukeboxConfig""": True,
        """LayoutLMv2Config""": True,
        """MaskFormerSwinConfig""": True,
        """MT5Config""": True,
        """NatConfig""": True,
        """OneFormerConfig""": True,
        """PerceiverConfig""": True,
        """RagConfig""": True,
        """SpeechT5Config""": True,
        """SwinConfig""": True,
        """Swin2SRConfig""": True,
        """Swinv2Config""": True,
        """SwitchTransformersConfig""": True,
        """TableTransformerConfig""": True,
        """TapasConfig""": True,
        """TransfoXLConfig""": True,
        """UniSpeechConfig""": True,
        """UniSpeechSatConfig""": True,
        """WavLMConfig""": True,
        """WhisperConfig""": True,
        # TODO: @Arthur (for `alignment_head` and `alignment_layer`)
        """JukeboxPriorConfig""": True,
        # TODO: @Younes (for `is_decoder`)
        """Pix2StructTextConfig""": True,
    }
)







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		False
      for attribute in attributes:
            for modeling_source in source_strings:
                  # check if we can find `config.xxx`, `getattr(config, "xxx", ...)` or `getattr(self.config, "xxx", ...)`
                  if (
                      f"config.{attribute}" in modeling_source
                      or f"getattr(config, \"{attribute}\"" in modeling_source
                      or f"getattr(self.config, \"{attribute}\"" in modeling_source
                  ):
                        A_							       =		True
                  # Deal with multi-line cases
                  elif (
                      re.search(
                          Rf"getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*\"{attribute}\""		,				SCREAMING_SNAKE_CASE		,				)
                      is not None
                  ):
                        A_							       =		True
                  # `SequenceSummary` is called with `SequenceSummary(config)`
                  elif attribute in [
                      "summary_type",
                      "summary_use_proj",
                      "summary_activation",
                      "summary_last_dropout",
                      "summary_proj_to_labels",
                      "summary_first_dropout",
                  ]:
                        if "SequenceSummary" in modeling_source:
                              A_							       =		True
                  if attribute_used:
                        break
            if attribute_used:
                  break

    # common and important attributes, even if they do not always appear in the modeling files
      A_							       =		[
          '''bos_index''',
          '''eos_index''',
          '''pad_index''',
          '''unk_index''',
          '''mask_index''',
          '''image_size''',
          '''use_cache''',
          '''out_features''',
          '''out_indices''',
      ]
      A_							       =		['''encoder_no_repeat_ngram_size''']

      # Special cases to be allowed
      A_							       =		True
      if not attribute_used:
            A_							       =		False
            for attribute in attributes:
                  # Allow if the default value in the configuration class is different from the one in `PretrainedConfig`
                  if attribute in ["is_encoder_decoder"] and default_value is True:
                        A_							       =		True
                  elif attribute in ["tie_word_embeddings"] and default_value is False:
                        A_							       =		True

                  # Allow cases without checking the default value in the configuration class
                  elif attribute in attributes_to_allow + attributes_used_in_generation:
                        A_							       =		True
                  elif attribute.endswith('''_token_id'''   ):
                        A_							       =		True

                  # configuration class specific cases
                  if not case_allowed:
                        A_							       =		SPECIAL_CASES_TO_ALLOW.get(config_class.__name__		,				[]   )
                        A_							       =		allowed_cases is True or attribute in allowed_cases

      return attribute_used or case_allowed







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      A_							       =		dict(inspect.signature(config_class.__init__   ).parameters   )
      A_							       =		[x for x in list(signature.keys()   ) if x not in ['''self''', '''kwargs''']]
      A_							       =		[signature[param].default for param in parameter_names]

      # If `attribute_map` exists, an attribute can have different names to be used in the modeling files, and as long
      # as one variant is used, the test should pass
      A_							       =		{}
      if len(config_class.attribute_map   ) > 0:
            A_							       =		{v: k for k, v in config_class.attribute_map.items()}

      # Get the path to modeling source files
      A_							       =		inspect.getsourcefile(SCREAMING_SNAKE_CASE   )
      A_							       =		os.path.dirname(SCREAMING_SNAKE_CASE   )
      # Let's check against all frameworks: as long as one framework uses an attribute, we are good.
      A_							       =		[os.path.join(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ) for fn in os.listdir(SCREAMING_SNAKE_CASE   ) if fn.startswith('''modeling_'''   )]

      # Get the source code strings
      A_							       =		[]
      for path in modeling_paths:
            if os.path.isfile(SCREAMING_SNAKE_CASE   ):
                  with open(SCREAMING_SNAKE_CASE   ) as fp:
                        modeling_sources.append(fp.read()   )

      A_							       =		[]
      for config_param, default_value in zip(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
            # `attributes` here is all the variant names for `config_param`
            A_							       =		[config_param]
            # some configuration classes have non-empty `attribute_map`, and both names could be used in the
            # corresponding modeling files. As long as one of them appears, it is fine.
            if config_param in reversed_attribute_map:
                  attributes.append(reversed_attribute_map[config_param]   )

            if not check_attribute_being_used(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ):
                  unused_attributes.append(attributes[0]   )

      return sorted(SCREAMING_SNAKE_CASE   )







def 							_lowerCamelCase   (			):
      '''simple docstring'''







      A_							       =		{}
      for _config_class in list(CONFIG_MAPPING.values()   ):
            # Skip deprecated models
            if "models.deprecated" in _config_class.__module__:
                  continue
            # Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.)
            A_							       =		[
                cls
                for name, cls in inspect.getmembers(
                    inspect.getmodule(_config_class   )		,				lambda SCREAMING_SNAKE_CASE   : inspect.isclass(SCREAMING_SNAKE_CASE   )
                    and issubclass(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
                    and inspect.getmodule(SCREAMING_SNAKE_CASE   ) == inspect.getmodule(_config_class   )		,				)
            ]
            for config_class in config_classes_in_module:
                  A_							       =		check_config_attributes_being_used(SCREAMING_SNAKE_CASE   )
                  if len(SCREAMING_SNAKE_CASE   ) > 0:
                        A_							       =		unused_attributes

      if len(SCREAMING_SNAKE_CASE   ) > 0:
            A_							       =		'''The following configuration classes contain unused attributes in the corresponding modeling files:\n'''
            for name, attributes in configs_with_unused_attributes.items():
                  error += f"{name}: {attributes}\n"

            raise ValueError(SCREAMING_SNAKE_CASE   )


if __name__ == "__main__":
      check_config_attributes()





