
"""simple docstring"""






import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)



class 			lowercase					( nn.Module		):






			def __init__(self  :	Optional[int]			)       ->	List[str]:





				"""simple docstring"""


				super().__init__()
				lowerCAmelCase												=  nn.Linear(3      ,4			)
				lowerCAmelCase												=  nn.BatchNormad(4			)
				lowerCAmelCase												=  nn.Linear(4      ,5			)






			def 					UpperCAmelCase							(self  :	int      ,SCREAMING_SNAKE_CASE_  :	List[Any]			)       ->	Any:





				"""simple docstring"""


				return self.lineara(self.batchnorm(self.lineara(SCREAMING_SNAKE_CASE_			)			)			)





class 			lowercase					( unittest.TestCase		):






			def 					UpperCAmelCase							(self  :	Dict			)       ->	Any:





				"""simple docstring"""


				lowerCAmelCase												=  ModelForTest()
				with TemporaryDirectory() as tmp_dir:
					offload_state_dict(SCREAMING_SNAKE_CASE_      ,model.state_dict()			)
					lowerCAmelCase												=  os.path.join(SCREAMING_SNAKE_CASE_      ,'''index.json'''			)
					self.assertTrue(os.path.isfile(SCREAMING_SNAKE_CASE_			)			)
					# TODO: add tests on what is inside the index

					for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
						lowerCAmelCase												=  os.path.join(SCREAMING_SNAKE_CASE_      ,F"""{key}.dat"""			)
						self.assertTrue(os.path.isfile(SCREAMING_SNAKE_CASE_			)			)
						# TODO: add tests on the fact weights are properly loaded






			def 					UpperCAmelCase							(self  :	Dict			)       ->	Optional[int]:





				"""simple docstring"""


				lowerCAmelCase												=  [torch.floataa, torch.floataa, torch.bfloataa]

				for dtype in dtypes:
					lowerCAmelCase												=  torch.randn(2      ,3      ,dtype=SCREAMING_SNAKE_CASE_			)
					with TemporaryDirectory() as tmp_dir:
						lowerCAmelCase												=  offload_weight(SCREAMING_SNAKE_CASE_      ,'''weight'''      ,SCREAMING_SNAKE_CASE_      ,{}			)
						lowerCAmelCase												=  os.path.join(SCREAMING_SNAKE_CASE_      ,'''weight.dat'''			)
						self.assertTrue(os.path.isfile(SCREAMING_SNAKE_CASE_			)			)
						self.assertDictEqual(SCREAMING_SNAKE_CASE_      ,{'''weight''': {'''shape''': [2, 3], '''dtype''': str(SCREAMING_SNAKE_CASE_			).split('''.'''			)[1]}}			)

						lowerCAmelCase												=  load_offloaded_weight(SCREAMING_SNAKE_CASE_      ,index['''weight''']			)
						self.assertTrue(torch.equal(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)			)






			def 					UpperCAmelCase							(self  :	Dict			)       ->	Optional[Any]:





				"""simple docstring"""


				lowerCAmelCase												=  ModelForTest()
				lowerCAmelCase												=  model.state_dict()
				lowerCAmelCase												=  {k: v for k, v in state_dict.items() if '''linear2''' not in k}
				lowerCAmelCase												=  {k: v for k, v in state_dict.items() if '''linear2''' in k}

				with TemporaryDirectory() as tmp_dir:
					offload_state_dict(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)
					lowerCAmelCase												=  OffloadedWeightsLoader(state_dict=SCREAMING_SNAKE_CASE_      ,save_folder=SCREAMING_SNAKE_CASE_			)

					# Every key is there with the right value
					self.assertEqual(sorted(SCREAMING_SNAKE_CASE_			)      ,sorted(state_dict.keys()			)			)
					for key, param in state_dict.items():
						self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_      ,weight_map[key]			)			)

				lowerCAmelCase												=  {k: v for k, v in state_dict.items() if '''weight''' in k}
				lowerCAmelCase												=  {k: v for k, v in state_dict.items() if '''weight''' not in k}

				with TemporaryDirectory() as tmp_dir:
					offload_state_dict(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)
					lowerCAmelCase												=  OffloadedWeightsLoader(state_dict=SCREAMING_SNAKE_CASE_      ,save_folder=SCREAMING_SNAKE_CASE_			)

					# Every key is there with the right value
					self.assertEqual(sorted(SCREAMING_SNAKE_CASE_			)      ,sorted(state_dict.keys()			)			)
					for key, param in state_dict.items():
						self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_      ,weight_map[key]			)			)

				with TemporaryDirectory() as tmp_dir:
					offload_state_dict(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)
					# Duplicates are removed
					lowerCAmelCase												=  OffloadedWeightsLoader(state_dict=SCREAMING_SNAKE_CASE_      ,save_folder=SCREAMING_SNAKE_CASE_			)

					# Every key is there with the right value
					self.assertEqual(sorted(SCREAMING_SNAKE_CASE_			)      ,sorted(state_dict.keys()			)			)
					for key, param in state_dict.items():
						self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_      ,weight_map[key]			)			)






			def 					UpperCAmelCase							(self  :	Tuple			)       ->	Optional[int]:





				"""simple docstring"""


				lowerCAmelCase												=  {'''a.1''': 0, '''a.10''': 1, '''a.2''': 2}
				lowerCAmelCase												=  extract_submodules_state_dict(SCREAMING_SNAKE_CASE_      ,['''a.1''', '''a.2''']			)
				self.assertDictEqual(SCREAMING_SNAKE_CASE_      ,{'''a.1''': 0, '''a.2''': 2}			)

				lowerCAmelCase												=  {'''a.1.a''': 0, '''a.10.a''': 1, '''a.2.a''': 2}
				lowerCAmelCase												=  extract_submodules_state_dict(SCREAMING_SNAKE_CASE_      ,['''a.1''', '''a.2''']			)
				self.assertDictEqual(SCREAMING_SNAKE_CASE_      ,{'''a.1.a''': 0, '''a.2.a''': 2}			)




"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase        =				logging.get_logger(__name__)

UpperCAmelCase        =				{"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/config.json"""}



class 			lowercase					( lowercase__		):
			lowercase        =     '''openai-gpt'''
			lowercase        =     {
			    '''max_position_embeddings''': '''n_positions''',
			    '''hidden_size''': '''n_embd''',
			    '''num_attention_heads''': '''n_head''',
			    '''num_hidden_layers''': '''n_layer''',
			}






			def __init__(self  :	List[Any]      ,SCREAMING_SNAKE_CASE_  :	str=40_478      ,SCREAMING_SNAKE_CASE_  :	Optional[int]=512      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]=768      ,SCREAMING_SNAKE_CASE_  :	Dict=12      ,SCREAMING_SNAKE_CASE_  :	Dict=12      ,SCREAMING_SNAKE_CASE_  :	int="gelu"      ,SCREAMING_SNAKE_CASE_  :	Dict=0.1      ,SCREAMING_SNAKE_CASE_  :	List[Any]=0.1      ,SCREAMING_SNAKE_CASE_  :	Dict=0.1      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]=1e-5      ,SCREAMING_SNAKE_CASE_  :	List[str]=0.02      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]="cls_index"      ,SCREAMING_SNAKE_CASE_  :	str=True      ,SCREAMING_SNAKE_CASE_  :	Any=None      ,SCREAMING_SNAKE_CASE_  :	List[Any]=True      ,SCREAMING_SNAKE_CASE_  :	List[str]=0.1      ,**SCREAMING_SNAKE_CASE_  :	int      ,)       ->	int:





				"""simple docstring"""


				lowerCAmelCase												=  vocab_size
				lowerCAmelCase												=  n_positions
				lowerCAmelCase												=  n_embd
				lowerCAmelCase												=  n_layer
				lowerCAmelCase												=  n_head
				lowerCAmelCase												=  afn
				lowerCAmelCase												=  resid_pdrop
				lowerCAmelCase												=  embd_pdrop
				lowerCAmelCase												=  attn_pdrop
				lowerCAmelCase												=  layer_norm_epsilon
				lowerCAmelCase												=  initializer_range
				lowerCAmelCase												=  summary_type
				lowerCAmelCase												=  summary_use_proj
				lowerCAmelCase												=  summary_activation
				lowerCAmelCase												=  summary_first_dropout
				lowerCAmelCase												=  summary_proj_to_labels
				super().__init__(**SCREAMING_SNAKE_CASE_			)


