






import argparse

import torch

from transformers import OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()



def 	__a					(   lowerCAmelCase_       : Union[str, Any]      ,lowerCAmelCase_       : Dict      ,lowerCAmelCase_       : str    )				->			str:






						'''simple docstring'''



						if openai_config_file == "":
												UpperCAmelCase_=       OpenAIGPTConfig()
						else:
												UpperCAmelCase_=       OpenAIGPTConfig.from_json_file(lowerCAmelCase_    )
						UpperCAmelCase_=       OpenAIGPTModel(lowerCAmelCase_    )

						# Load weights from numpy
						load_tf_weights_in_openai_gpt(lowerCAmelCase_      ,lowerCAmelCase_      ,lowerCAmelCase_    )

						# Save pytorch-model
						UpperCAmelCase_=       pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
						UpperCAmelCase_=       pytorch_dump_folder_path + """/""" + CONFIG_NAME
						print(F"""Save PyTorch model to {pytorch_weights_dump_path}"""    )
						torch.save(model.state_dict()      ,lowerCAmelCase_    )
						print(F"""Save configuration file to {pytorch_config_dump_path}"""    )
						with open(lowerCAmelCase_      ,"""w"""      ,encoding="""utf-8"""    ) as f:
												f.write(config.to_json_string()    )


if __name__ == "__main__":
	__A       =						argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '''--openai_checkpoint_folder_path''',
	    default=None,
	    type=str,
	    required=True,
	    help='''Path to the TensorFlow checkpoint path.''',
	)
	parser.add_argument(
	    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
	)
	parser.add_argument(
	    '''--openai_config_file''',
	    default='''''',
	    type=str,
	    help=(
	        '''An optional config json file corresponding to the pre-trained OpenAI model. \n'''
	        '''This specifies the model architecture.'''
	    ),
	)
	__A       =						parser.parse_args()
	convert_openai_checkpoint_to_pytorch(
	    args.openai_checkpoint_folder_path, args.openai_config_file, args.pytorch_dump_folder_path
	)









import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment



def 	__a					(   )				->			str:






						'''simple docstring'''



						UpperCAmelCase_=       torch.nn.Linear(2      ,4    )
						UpperCAmelCase_=       torch.optim.AdamW(model.parameters()      ,lr=1.0    )
						UpperCAmelCase_=       torch.optim.lr_scheduler.OneCycleLR(lowerCAmelCase_      ,max_lr=0.01      ,steps_per_epoch=2      ,epochs=1    )
						UpperCAmelCase_=       DataLoader(TensorDataset(torch.tensor([1, 2, 3]    )    )    )
						UpperCAmelCase_=       DataLoader(TensorDataset(torch.tensor([4, 5, 6]    )    )    )

						return model, optimizer, scheduler, train_dl, valid_dl



def 	__a					(   lowerCAmelCase_       : Any    )				->			Union[str, Any]:






						'''simple docstring'''



						return (model.weight.abs().sum() + model.bias.abs().sum()).item()



def 	__a					(   lowerCAmelCase_       : Tuple    )				->			Tuple:






						'''simple docstring'''



						UpperCAmelCase_=       torch.nn.Linear(*tuple(model.weight.T.shape    )    ).state_dict()
						model.load_state_dict(lowerCAmelCase_    )






class 		lowercase						(			snake_case__):






	"""simple docstring"""


	@require_cuda
	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Optional[Any]:
							UpperCAmelCase_=       Accelerator()
							assert PartialState._shared_state["_cpu"] is False
							assert PartialState._shared_state["device"].type == "cuda"
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       Accelerator(cpu=__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Tuple					)      ->			Union[str, Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_=       GradientState()
							assert state.num_steps == 1
							UpperCAmelCase_=       4
							assert state.num_steps == 4

							assert state.sync_gradients is True
							UpperCAmelCase_=       False
							assert state.sync_gradients is False
							GradientState._reset_state()




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			List[str]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()

							(
							    (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       
							)=       accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							self.assertTrue(prepared_model in accelerator._models					)
							self.assertTrue(prepared_optimizer in accelerator._optimizers					)
							self.assertTrue(prepared_scheduler in accelerator._schedulers					)
							self.assertTrue(prepared_train_dl in accelerator._dataloaders					)
							self.assertTrue(prepared_valid_dl in accelerator._dataloaders					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Dict:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							accelerator.free_memory()

							self.assertTrue(len(accelerator._models					) == 0					)
							self.assertTrue(len(accelerator._optimizers					) == 0					)
							self.assertTrue(len(accelerator._schedulers					) == 0					)
							self.assertTrue(len(accelerator._dataloaders					) == 0					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Optional[Any]:
							PartialState._reset_state()

							# Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
							def noop(*__UpperCAmelCase     :						Dict						,  **__UpperCAmelCase     :						Tuple					):
													pass

							with patch("""torch.cuda.set_device"""						,  __UpperCAmelCase					), patch_environment(ACCELERATE_TORCH_DEVICE="""cuda:64"""					):
													UpperCAmelCase_=       Accelerator()
													self.assertEqual(str(accelerator.state.device					)						,  """cuda:64"""					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							UpperCAmelCase_=       get_signature(__UpperCAmelCase					)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# make sure loaded weights match
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							UpperCAmelCase_=       get_signature(__UpperCAmelCase					)

							# saving hook
							def save_config(__UpperCAmelCase     :						Tuple						,  __UpperCAmelCase     :						List[str]						,  __UpperCAmelCase     :						Tuple					):
													UpperCAmelCase_=       {"""class_name""": models[0].__class__.__name__}

													with open(os.path.join(__UpperCAmelCase						,  """data.json"""					)						,  """w"""					) as f:
																			json.dump(__UpperCAmelCase						,  __UpperCAmelCase					)

        # loading hook
							def load_config(__UpperCAmelCase     :						Tuple						,  __UpperCAmelCase     :						Union[str, Any]					):
													with open(os.path.join(__UpperCAmelCase						,  """data.json"""					)						,  """r"""					) as f:
																			UpperCAmelCase_=       json.load(__UpperCAmelCase					)

													UpperCAmelCase_=       config["""class_name"""]

							UpperCAmelCase_=       accelerator.register_save_state_pre_hook(__UpperCAmelCase					)
							UpperCAmelCase_=       accelerator.register_load_state_pre_hook(__UpperCAmelCase					)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match with hooks
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# random class name to verify correct one is loaded
													UpperCAmelCase_=       """random"""

													# make sure loaded weights match with hooks
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)

													# mode.class_name is loaded from config
													self.assertTrue(model.class_name == model.__class__.__name__					)

							# remove hooks
							save_hook.remove()
							load_hook.remove()

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match with hooks removed
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# random class name to verify correct one is loaded
													UpperCAmelCase_=       """random"""

													# make sure loaded weights match with hooks removed
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)

													# mode.class_name is NOT loaded from config
													self.assertTrue(model.class_name != model.__class__.__name__					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Union[str, Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							UpperCAmelCase_=       None

							# This should work
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       accelerator.prepare(
							    __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							self.assertTrue(dummy_obj is None					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Any:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							UpperCAmelCase_=       [1, 2, 3]

							# This should work
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       accelerator.prepare(
							    __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Dummy object should have `_is_accelerate_prepared` set to `True`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Model is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Optimizer is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Scheduler is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`"""						,  )




	@slow
	@require_bnb
	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Optional[Any]:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map={"""""": 0}						,  )
							UpperCAmelCase_=       Accelerator()

							# This should work
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@slow
	@require_bnb
	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Tuple:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       Accelerator()

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													model.tie_weights()
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       """cpu"""

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  device_map=__UpperCAmelCase						,  load_in_abit=__UpperCAmelCase						,  llm_inta_enable_fpaa_cpu_offload=__UpperCAmelCase					)

							# This should not work and get value error
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@slow
	@require_bnb
	@require_multi_gpu
	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Tuple:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       {"""distributed_type""": DistributedType.MULTI_GPU}

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													model.tie_weights()
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       1

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map=__UpperCAmelCase						,  )
							UpperCAmelCase_=       Accelerator()

							# This should not work and get value error
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)

							PartialState._reset_state()




	@slow
	@require_bnb
	@require_multi_gpu
	def   _SCREAMING_SNAKE_CASE		(     self     :						Any					)      ->			Dict:
							from transformers import AutoModelForCausalLM

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       1

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map=__UpperCAmelCase						,  )
							UpperCAmelCase_=       Accelerator()

							# This should work
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@require_cuda
	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			Optional[int]:
							UpperCAmelCase_=       torch.nn.Linear(10						,  10					)
							UpperCAmelCase_=       torch.optim.SGD(model.parameters()						,  lr=0.01					)
							UpperCAmelCase_=       Accelerator(cpu=__UpperCAmelCase					)
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




