




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
       OpenAIGPTConfig,
       OpenAIGPTDoubleHeadsModel,
       OpenAIGPTForSequenceClassification,
       OpenAIGPTLMHeadModel,
       OpenAIGPTModel,
   )




class      a__			:







     def __init__(      self    ,						A    ,						A=13    ,						A=7    ,						A=True    ,						A=True    ,						A=True    ,						A=99    ,						A=32    ,						A=5    ,						A=4    ,						A=37    ,						A="gelu"    ,						A=0.1    ,						A=0.1    ,						A=512    ,						A=16    ,						A=2    ,						A=0.0_2    ,						A=3    ,						A=4    ,						A=None    ,						)    ->       Union[str, Any]:



          '''simple docstring'''



          a           = parent
          a           = batch_size
          a           = seq_length
          a           = is_training
          a           = use_token_type_ids
          a           = use_labels
          a           = vocab_size
          a           = hidden_size
          a           = num_hidden_layers
          a           = num_attention_heads
          a           = intermediate_size
          a           = hidden_act
          a           = hidden_dropout_prob
          a           = attention_probs_dropout_prob
          a           = max_position_embeddings
          a           = type_vocab_size
          a           = type_sequence_label_size
          a           = initializer_range
          a           = num_labels
          a           = num_choices
          a           = scope
          a           = self.vocab_size - 1







     def      lowerCAmelCase_							(      self					)    ->       int:



          '''simple docstring'''



          a           = ids_tensor([self.batch_size, self.seq_length]    ,						self.vocab_size					)

          a           = None
          if self.use_token_type_ids:
               a           = ids_tensor([self.batch_size, self.seq_length]    ,						self.type_vocab_size					)

          a           = None
          a           = None
          a           = None
          if self.use_labels:
               a           = ids_tensor([self.batch_size]    ,						self.type_sequence_label_size					)
               a           = ids_tensor([self.batch_size, self.seq_length]    ,						self.num_labels					)
               a           = ids_tensor([self.batch_size]    ,						self.num_choices					)

          a           = OpenAIGPTConfig(
              vocab_size=self.vocab_size    ,						n_embd=self.hidden_size    ,						n_layer=self.num_hidden_layers    ,						n_head=self.num_attention_heads    ,						n_positions=self.max_position_embeddings    ,						pad_token_id=self.pad_token_id    ,						)

          a           = ids_tensor([self.num_hidden_layers, self.num_attention_heads]    ,						2					)

          return (
              config,
              input_ids,
              head_mask,
              token_type_ids,
              sequence_labels,
              token_labels,
              choice_labels,
          )







     def      lowerCAmelCase_							(      self    ,						A    ,						A    ,						A    ,						A    ,						*A					)    ->       int:



          '''simple docstring'''



          a           = OpenAIGPTModel(config=A					)
          model.to(A					)
          model.eval()

          a           = model(A    ,						token_type_ids=A    ,						head_mask=A					)
          a           = model(A    ,						token_type_ids=A					)
          a           = model(A					)

          self.parent.assertEqual(result.last_hidden_state.shape    ,						(self.batch_size, self.seq_length, self.hidden_size)					)







     def      lowerCAmelCase_							(      self    ,						A    ,						A    ,						A    ,						A    ,						*A					)    ->       Tuple:



          '''simple docstring'''



          a           = OpenAIGPTLMHeadModel(A					)
          model.to(A					)
          model.eval()

          a           = model(A    ,						token_type_ids=A    ,						labels=A					)
          self.parent.assertEqual(result.loss.shape    ,						()					)
          self.parent.assertEqual(result.logits.shape    ,						(self.batch_size, self.seq_length, self.vocab_size)					)







     def      lowerCAmelCase_							(      self    ,						A    ,						A    ,						A    ,						A    ,						*A					)    ->       Dict:



          '''simple docstring'''



          a           = OpenAIGPTDoubleHeadsModel(A					)
          model.to(A					)
          model.eval()

          a           = model(A    ,						token_type_ids=A    ,						labels=A					)
          self.parent.assertEqual(result.loss.shape    ,						()					)
          self.parent.assertEqual(result.logits.shape    ,						(self.batch_size, self.seq_length, self.vocab_size)					)







     def      lowerCAmelCase_							(      self    ,						A    ,						A    ,						A    ,						A    ,						*A					)    ->       Optional[Any]:



          '''simple docstring'''



          a           = self.num_labels
          a           = OpenAIGPTForSequenceClassification(A					)
          model.to(A					)
          model.eval()

          a           = ids_tensor([self.batch_size]    ,						self.type_sequence_label_size					)
          a           = model(A    ,						token_type_ids=A    ,						labels=A					)
          self.parent.assertEqual(result.logits.shape    ,						(self.batch_size, self.num_labels)					)







     def      lowerCAmelCase_							(      self					)    ->       str:



          '''simple docstring'''



          a           = self.prepare_config_and_inputs()
          (
              (
              a			
          )    , (
              a			
          )    , (
              a			
          )    , (
              a			
          )    , (
              a			
          )    , (
              a			
          )    , (
              a			
          )    , 			
          )           = config_and_inputs
          a           = {
              "input_ids": input_ids,
              "token_type_ids": token_type_ids,
              "head_mask": head_mask,
          }

          return config, inputs_dict







@require_torch
class      a__			(   _lowercase  ,   _lowercase  ,   _lowercase  ,   unittest.TestCase ):
     a						:     Dict						   =      (
         (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
         if is_torch_available()
         else ()
     )
     a						:     Optional[Any]						   =      (
         (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
     )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
     a						:     Dict						   =      (
         {
             '''feature-extraction''': OpenAIGPTModel,
             '''text-classification''': OpenAIGPTForSequenceClassification,
             '''text-generation''': OpenAIGPTLMHeadModel,
             '''zero-shot''': OpenAIGPTForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )







     def      lowerCAmelCase_							(      self    ,						A    ,						A    ,						A    ,						A    ,						A					)    ->       Optional[Any]:



          '''simple docstring'''



          if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
               # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
               # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
               # tiny config could not be created.
               return True

          return False







     def      lowerCAmelCase_							(      self    ,						A    ,						A    ,						A=False					)    ->       List[str]:



          '''simple docstring'''



          a           = super()._prepare_for_class(A    ,						A    ,						return_labels=A					)

          if return_labels:
               if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                    a           = torch.zeros(
                        (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)    ,						dtype=torch.long    ,						device=A    ,						)
                    a           = inputs_dict["labels"]
                    a           = inputs_dict["labels"]
                    a           = torch.zeros(
                        (self.model_tester.batch_size, self.model_tester.num_choices)    ,						dtype=torch.long    ,						device=A    ,						)
                    a           = torch.zeros(
                        self.model_tester.batch_size    ,						dtype=torch.long    ,						device=A					)
          return inputs_dict







     def      lowerCAmelCase_							(      self					)    ->       List[str]:



          '''simple docstring'''



          a           = OpenAIGPTModelTester(self					)
          a           = ConfigTester(self    ,						config_class=A    ,						n_embd=37					)







     def      lowerCAmelCase_							(      self					)    ->       Optional[Any]:



          '''simple docstring'''



          self.config_tester.run_common_tests()







     def      lowerCAmelCase_							(      self					)    ->       Optional[int]:



          '''simple docstring'''



          a           = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_openai_gpt_model(*A					)







     def      lowerCAmelCase_							(      self					)    ->       int:



          '''simple docstring'''



          a           = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_lm_head_model(*A					)







     def      lowerCAmelCase_							(      self					)    ->       Optional[int]:



          '''simple docstring'''



          a           = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_double_lm_head_model(*A					)







     def      lowerCAmelCase_							(      self					)    ->       Optional[int]:



          '''simple docstring'''



          a           = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*A					)







     @slow
     def      lowerCAmelCase_							(      self					)    ->       Union[str, Any]:



          '''simple docstring'''



          for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               a           = OpenAIGPTModel.from_pretrained(A					)
               self.assertIsNotNone(A					)







@require_torch
class      a__			(   unittest.TestCase ):







     @slow
     def      lowerCAmelCase_							(      self					)    ->       Optional[int]:



          '''simple docstring'''



          a           = OpenAIGPTLMHeadModel.from_pretrained("openai-gpt"					)
          model.to(A					)
          a           = torch.tensor([[481, 4735, 544]]    ,						dtype=torch.long    ,						device=A					)  # the president is
          a           = [
              481,
              4735,
              544,
              246,
              963,
              870,
              762,
              239,
              244,
              40477,
              244,
              249,
              719,
              881,
              487,
              544,
              240,
              244,
              603,
              481,
          ]  # the president is a very good man. " \n " i\'m sure he is, " said the

          a           = model.generate(A    ,						do_sample=A					)
          self.assertListEqual(output_ids[0].tolist()    ,						A					)






import argparse

import torch

from transformers import YosoConfig, YosoForMaskedLM






def     __a					(		__UpperCAmelCase       ):
 if "model" in orig_key:
  a__												= orig_key.replace('''model.'''							,  ''''''       )
 if "norm1" in orig_key:
  a__												= orig_key.replace('''norm1'''							,  '''attention.output.LayerNorm'''       )
 if "norm2" in orig_key:
  a__												= orig_key.replace('''norm2'''							,  '''output.LayerNorm'''       )
 if "norm" in orig_key:
  a__												= orig_key.replace('''norm'''							,  '''LayerNorm'''       )
 if "transformer" in orig_key:
  a__												= orig_key.split('''.'''       )[0].split('''_'''       )[-1]
  a__												= orig_key.replace(f"transformer_{layer_num}"							,  f"encoder.layer.{layer_num}"       )
 if "mha.attn" in orig_key:
  a__												= orig_key.replace('''mha.attn'''							,  '''attention.self'''       )
 if "mha" in orig_key:
  a__												= orig_key.replace('''mha'''							,  '''attention'''       )
 if "W_q" in orig_key:
  a__												= orig_key.replace('''W_q'''							,  '''self.query'''       )
 if "W_k" in orig_key:
  a__												= orig_key.replace('''W_k'''							,  '''self.key'''       )
 if "W_v" in orig_key:
  a__												= orig_key.replace('''W_v'''							,  '''self.value'''       )
 if "ff1" in orig_key:
  a__												= orig_key.replace('''ff1'''							,  '''intermediate.dense'''       )
 if "ff2" in orig_key:
  a__												= orig_key.replace('''ff2'''							,  '''output.dense'''       )
 if "ff" in orig_key:
  a__												= orig_key.replace('''ff'''							,  '''output.dense'''       )
 if "mlm_class" in orig_key:
  a__												= orig_key.replace('''mlm.mlm_class'''							,  '''cls.predictions.decoder'''       )
 if "mlm" in orig_key:
  a__												= orig_key.replace('''mlm'''							,  '''cls.predictions.transform'''       )
 if "cls" not in orig_key:
  a__												= '''yoso.''' + orig_key

 return orig_key






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase       ):
 for key in orig_state_dict.copy().keys():
  a__												= orig_state_dict.pop(__UpperCAmelCase       )

  if ("pooler" in key) or ("sen_class" in key):
   continue
  else:
   a__												= val

 a__												= orig_state_dict['''cls.predictions.decoder.bias''']
 a__												= torch.arange(__UpperCAmelCase       ).expand((1, -1)       ) + 2

 return orig_state_dict






def     __a					(		__UpperCAmelCase							,  __UpperCAmelCase							,  __UpperCAmelCase       ):
 a__												= torch.load(__UpperCAmelCase							,  map_location='''cpu'''       )['''model_state_dict''']
 a__												= YosoConfig.from_json_file(__UpperCAmelCase       )
 a__												= YosoForMaskedLM(__UpperCAmelCase       )

 a__												= convert_checkpoint_helper(config.max_position_embeddings							,  __UpperCAmelCase       )

 print(model.load_state_dict(__UpperCAmelCase       )       )
 model.eval()
 model.save_pretrained(__UpperCAmelCase       )

 print(f"Checkpoint successfuly converted. Model saved at {pytorch_dump_path}"       )


if __name__ == "__main__":
     a_      :      int    =     argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '--pytorch_model_path', default=None, type=str, required=True, help='Path to YOSO pytorch checkpoint.'
     )
     parser.add_argument(
         '--config_file',
         default=None,
         type=str,
         required=True,
         help='The json file for YOSO model config.',
     )
     parser.add_argument(
         '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
     )
     a_      :      Any    =     parser.parse_args()
     convert_yoso_checkpoint(args.pytorch_model_path, args.config_file, args.pytorch_dump_path)





