






import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


__A       =						1E-4

if is_torch_available():
	import torch

	from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
	from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder






@require_torch
class 		lowercase						:






	"""simple docstring"""


	def __init__(     self     :						Tuple						,  __UpperCAmelCase     :						List[str]						,  __UpperCAmelCase     :						List[Any]=16						,  __UpperCAmelCase     :						Union[str, Any]=13						,  __UpperCAmelCase     :						Optional[int]=7						,  __UpperCAmelCase     :						List[Any]=14						,  __UpperCAmelCase     :						List[Any]=10						,  __UpperCAmelCase     :						Union[str, Any]=19						,  __UpperCAmelCase     :						str=5						,  __UpperCAmelCase     :						int=4						,  __UpperCAmelCase     :						Union[str, Any]=True						,  __UpperCAmelCase     :						List[Any]=16						,  __UpperCAmelCase     :						Union[str, Any]=2						,  __UpperCAmelCase     :						Union[str, Any]=4						,  __UpperCAmelCase     :						Optional[Any]=4						,  __UpperCAmelCase     :						Optional[Any]="gelu"						,  __UpperCAmelCase     :						Dict=0.1						,  __UpperCAmelCase     :						Tuple=0.1						,  __UpperCAmelCase     :						Tuple=[1, 2, 3, 4, 5]						,  __UpperCAmelCase     :						int=25						,  __UpperCAmelCase     :						Dict=5						,  )      ->			List[Any]:
							UpperCAmelCase_=       d_model
							UpperCAmelCase_=       parent
							UpperCAmelCase_=       batch_size
							UpperCAmelCase_=       prediction_length
							UpperCAmelCase_=       context_length
							UpperCAmelCase_=       cardinality
							UpperCAmelCase_=       num_time_features
							UpperCAmelCase_=       lags_sequence
							UpperCAmelCase_=       embedding_dimension
							UpperCAmelCase_=       is_training
							UpperCAmelCase_=       hidden_size
							UpperCAmelCase_=       num_hidden_layers
							UpperCAmelCase_=       num_attention_heads
							UpperCAmelCase_=       intermediate_size
							UpperCAmelCase_=       hidden_act
							UpperCAmelCase_=       hidden_dropout_prob
							UpperCAmelCase_=       attention_probs_dropout_prob

							UpperCAmelCase_=       context_length
							UpperCAmelCase_=       prediction_length + label_length
							UpperCAmelCase_=       label_length

							UpperCAmelCase_=       moving_average
							UpperCAmelCase_=       autocorrelation_factor




	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Tuple:
							return AutoformerConfig(
							    d_model=self.d_model						,  encoder_layers=self.num_hidden_layers						,  decoder_layers=self.num_hidden_layers						,  encoder_attention_heads=self.num_attention_heads						,  decoder_attention_heads=self.num_attention_heads						,  encoder_ffn_dim=self.intermediate_size						,  decoder_ffn_dim=self.intermediate_size						,  dropout=self.hidden_dropout_prob						,  attention_dropout=self.attention_probs_dropout_prob						,  prediction_length=self.prediction_length						,  context_length=self.context_length						,  label_length=self.label_length						,  lags_sequence=self.lags_sequence						,  num_time_features=self.num_time_features						,  num_static_categorical_features=1						,  cardinality=[self.cardinality]						,  embedding_dimension=[self.embedding_dimension]						,  moving_average=self.moving_average						,  )




	def   _SCREAMING_SNAKE_CASE		(     self     :						str						,  __UpperCAmelCase     :						List[str]					)      ->			List[Any]:
							UpperCAmelCase_=       config.context_length + max(config.lags_sequence					)

							UpperCAmelCase_=       ids_tensor([self.batch_size, 1]						,  config.cardinality[0]					)
							UpperCAmelCase_=       floats_tensor([self.batch_size, _past_length, config.num_time_features]					)
							UpperCAmelCase_=       floats_tensor([self.batch_size, _past_length]					)
							UpperCAmelCase_=       floats_tensor([self.batch_size, _past_length]					) > 0.5

							# decoder inputs
							UpperCAmelCase_=       floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]					)
							UpperCAmelCase_=       floats_tensor([self.batch_size, config.prediction_length]					)

							UpperCAmelCase_=       {
							    """past_values""": past_values,
							    """static_categorical_features""": static_categorical_features,
							    """past_time_features""": past_time_features,
							    """past_observed_mask""": past_observed_mask,
							    """future_time_features""": future_time_features,
							    """future_values""": future_values,
							}
							return inputs_dict




	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Dict:
							UpperCAmelCase_=       self.get_config()
							UpperCAmelCase_=       self.prepare_autoformer_inputs_dict(lowerCAmelCase__					)
							return config, inputs_dict




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[str]:
							UpperCAmelCase_,       UpperCAmelCase_=       self.prepare_config_and_inputs()
							return config, inputs_dict




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]						,  __UpperCAmelCase     :						int						,  __UpperCAmelCase     :						str					)      ->			Optional[Any]:
							UpperCAmelCase_=       AutoformerModel(config=lowerCAmelCase__					).to(lowerCAmelCase__					).eval()
							UpperCAmelCase_=       model(**lowerCAmelCase__					)

							UpperCAmelCase_=       outputs.encoder_last_hidden_state
							UpperCAmelCase_=       outputs.last_hidden_state

							with tempfile.TemporaryDirectory() as tmpdirname:
													UpperCAmelCase_=       model.get_encoder()
													encoder.save_pretrained(lowerCAmelCase__					)
													UpperCAmelCase_=       AutoformerEncoder.from_pretrained(lowerCAmelCase__					).to(lowerCAmelCase__					)

							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       model.create_network_inputs(**lowerCAmelCase__					)
							UpperCAmelCase_,       UpperCAmelCase_=       model.decomposition_layer(transformer_inputs[:, : config.context_length, ...]					)

							UpperCAmelCase_=       torch.cat(
							    (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])						,  dim=-1						,  )
							UpperCAmelCase_=       encoder(inputs_embeds=lowerCAmelCase__					)[0]
							self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1E-3					)

							UpperCAmelCase_=       (
							    torch.mean(transformer_inputs[:, : config.context_length, ...]						,  dim=1					)
							    .unsqueeze(1					)
							    .repeat(1						,  config.prediction_length						,  1					)
							)
							UpperCAmelCase_=       torch.zeros(
							    [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]						,  device=enc_input.device						,  )

							UpperCAmelCase_=       torch.cat(
							    (
							        torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)						,  dim=1					),
							        feature[:, config.context_length - config.label_length :, ...],
							    )						,  dim=-1						,  )
							UpperCAmelCase_=       torch.cat(
							    (
							        torch.cat((trend_input[:, -config.label_length :, ...], mean)						,  dim=1					),
							        feature[:, config.context_length - config.label_length :, ...],
							    )						,  dim=-1						,  )

							with tempfile.TemporaryDirectory() as tmpdirname:
													UpperCAmelCase_=       model.get_decoder()
													decoder.save_pretrained(lowerCAmelCase__					)
													UpperCAmelCase_=       AutoformerDecoder.from_pretrained(lowerCAmelCase__					).to(lowerCAmelCase__					)

							UpperCAmelCase_=       decoder(
							    trend=lowerCAmelCase__						,  inputs_embeds=lowerCAmelCase__						,  encoder_hidden_states=lowerCAmelCase__						,  )[0]

							self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1E-3					)






@require_torch
class 		lowercase						(			lowerCamelCase_			,   lowerCamelCase_			,   unittest.TestCase):






	"""simple docstring"""


	a__				: str		       =							(AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
	a__				: str		       =							(AutoformerForPrediction,) if is_torch_available() else ()
	a__				: Tuple		       =							{"""feature-extraction""": AutoformerModel} if is_torch_available() else {}
	a__				: int		       =							False
	a__				: Optional[Any]		       =							False
	a__				: Tuple		       =							False
	a__				: Optional[int]		       =							False
	a__				: int		       =							False
	a__				: str		       =							False




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]					)      ->			List[str]:
							UpperCAmelCase_=       AutoformerModelTester(self					)
							UpperCAmelCase_=       ConfigTester(self						,  config_class=lowerCAmelCase__						,  has_text_modality=lowerCAmelCase__					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]					)      ->			int:
							self.config_tester.run_common_tests()




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Any:
							UpperCAmelCase_,       UpperCAmelCase_=       self.model_tester.prepare_config_and_inputs()
							for model_class in self.all_model_classes:
													UpperCAmelCase_=       model_class(lowerCAmelCase__					)

													with tempfile.TemporaryDirectory() as tmpdirname:
																			model.save_pretrained(lowerCAmelCase__					)
																			UpperCAmelCase_,       UpperCAmelCase_=       model_class.from_pretrained(lowerCAmelCase__						,  output_loading_info=lowerCAmelCase__					)
													self.assertEqual(info["""missing_keys"""]						,  []					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			Optional[int]:
							UpperCAmelCase_=       self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_encoder_decoder_model_standalone(*lowerCAmelCase__					)




	@unittest.skip(reason="""Model has no tokens embeddings"""					)
	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			List[str]:
							pass




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			Optional[Any]:
							UpperCAmelCase_=       inspect.signature(getattr(lowerCAmelCase__						,  """forward"""					)					)
							# The main input is the name of the argument after `self`
							UpperCAmelCase_=       list(model_signature.parameters.keys()					)[1]
							self.assertEqual(AutoformerModel.main_input_name						,  lowerCAmelCase__					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						str					)      ->			List[Any]:
							UpperCAmelCase_,       UpperCAmelCase_=       self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													UpperCAmelCase_=       model_class(lowerCAmelCase__					)
													UpperCAmelCase_=       inspect.signature(model.forward					)
													# signature.parameters is an OrderedDict => so arg_names order is deterministic
													UpperCAmelCase_=       [*signature.parameters.keys()]

													UpperCAmelCase_=       [
													    """past_values""",
													    """past_time_features""",
													    """past_observed_mask""",
													    """static_categorical_features""",
													    """static_real_features""",
													    """future_values""",
													    """future_time_features""",
													]

													if model.__class__.__name__ in ["AutoformerForPrediction"]:
																			expected_arg_names.append("""future_observed_mask"""					)

													expected_arg_names.extend(
													    [
													        """decoder_attention_mask""",
													        """head_mask""",
													        """decoder_head_mask""",
													        """cross_attn_head_mask""",
													        """encoder_outputs""",
													        """past_key_values""",
													        """output_hidden_states""",
													        """output_attentions""",
													        """use_cache""",
													        """return_dict""",
													    ]					)

													self.assertListEqual(arg_names[: len(lowerCAmelCase__					)]						,  lowerCAmelCase__					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Any:
							UpperCAmelCase_,       UpperCAmelCase_=       self.model_tester.prepare_config_and_inputs_for_common()
							UpperCAmelCase_=       True

							UpperCAmelCase_=       getattr(self.model_tester						,  """seq_length"""						,  lowerCAmelCase__					)
							UpperCAmelCase_=       getattr(self.model_tester						,  """decoder_seq_length"""						,  lowerCAmelCase__					)
							UpperCAmelCase_=       getattr(self.model_tester						,  """encoder_seq_length"""						,  lowerCAmelCase__					)
							UpperCAmelCase_=       getattr(self.model_tester						,  """d_model"""						,  lowerCAmelCase__					)
							UpperCAmelCase_=       getattr(self.model_tester						,  """num_attention_heads"""						,  lowerCAmelCase__					)
							UpperCAmelCase_=       d_model // num_attention_heads

							for model_class in self.all_model_classes:
													UpperCAmelCase_=       True
													UpperCAmelCase_=       False
													UpperCAmelCase_=       True
													UpperCAmelCase_=       model_class(lowerCAmelCase__					)
													model.to(lowerCAmelCase__					)
													model.eval()
													with torch.no_grad():
																			UpperCAmelCase_=       model(**self._prepare_for_class(lowerCAmelCase__						,  lowerCAmelCase__					)					)
													UpperCAmelCase_=       outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
													self.assertEqual(len(lowerCAmelCase__					)						,  self.model_tester.num_hidden_layers					)

													# check that output_attentions also work using config
													del inputs_dict["output_attentions"]
													UpperCAmelCase_=       True
													UpperCAmelCase_=       model_class(lowerCAmelCase__					)
													model.to(lowerCAmelCase__					)
													model.eval()
													with torch.no_grad():
																			UpperCAmelCase_=       model(**self._prepare_for_class(lowerCAmelCase__						,  lowerCAmelCase__					)					)
													UpperCAmelCase_=       outputs.encoder_attentions
													self.assertEqual(len(lowerCAmelCase__					)						,  self.model_tester.num_hidden_layers					)

													self.assertListEqual(
													    list(attentions[0].shape[-3:]					)						,  [self.model_tester.num_attention_heads, encoder_seq_length, dim]						,  )
													UpperCAmelCase_=       len(lowerCAmelCase__					)

													UpperCAmelCase_=       7

													if "last_hidden_state" in outputs:
																			correct_outlen += 1

													if "trend" in outputs:
																			correct_outlen += 1

													if "past_key_values" in outputs:
																			correct_outlen += 1  # past_key_values have been returned

													if "loss" in outputs:
																			correct_outlen += 1

													if "params" in outputs:
																			correct_outlen += 1

													self.assertEqual(lowerCAmelCase__						,  lowerCAmelCase__					)

													# decoder attentions
													UpperCAmelCase_=       outputs.decoder_attentions
													self.assertIsInstance(lowerCAmelCase__						,  (list, tuple)					)
													self.assertEqual(len(lowerCAmelCase__					)						,  self.model_tester.num_hidden_layers					)
													self.assertListEqual(
													    list(decoder_attentions[0].shape[-3:]					)						,  [self.model_tester.num_attention_heads, decoder_seq_length, dim]						,  )

													# cross attentions
													UpperCAmelCase_=       outputs.cross_attentions
													self.assertIsInstance(lowerCAmelCase__						,  (list, tuple)					)
													self.assertEqual(len(lowerCAmelCase__					)						,  self.model_tester.num_hidden_layers					)
													self.assertListEqual(
													    list(cross_attentions[0].shape[-3:]					)						,  [self.model_tester.num_attention_heads, decoder_seq_length, dim]						,  )

							# Check attention is always last and order is fine
							UpperCAmelCase_=       True
							UpperCAmelCase_=       True
							UpperCAmelCase_=       model_class(lowerCAmelCase__					)
							model.to(lowerCAmelCase__					)
							model.eval()
							with torch.no_grad():
													UpperCAmelCase_=       model(**self._prepare_for_class(lowerCAmelCase__						,  lowerCAmelCase__					)					)

							self.assertEqual(out_len + 2						,  len(lowerCAmelCase__					)					)

							UpperCAmelCase_=       outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

							self.assertEqual(len(lowerCAmelCase__					)						,  self.model_tester.num_hidden_layers					)
							self.assertListEqual(
							    list(self_attentions[0].shape[-3:]					)						,  [self.model_tester.num_attention_heads, encoder_seq_length, dim]						,  )




	@is_flaky()
	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Any:
							super().test_retain_grad_hidden_states_attentions()



def 	__a					(   lowerCAmelCase_       : List[str]="train-batch.pt"    )				->			Optional[Any]:






						'''simple docstring'''



						UpperCAmelCase_=       hf_hub_download(repo_id="""hf-internal-testing/tourism-monthly-batch"""      ,filename=SCREAMING_SNAKE_CASE_      ,repo_type="""dataset"""    )
						UpperCAmelCase_=       torch.load(SCREAMING_SNAKE_CASE_      ,map_location=SCREAMING_SNAKE_CASE_    )
						return batch






@require_torch
@slow
class 		lowercase						(			unittest.TestCase):






	"""simple docstring"""


	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			Union[str, Any]:
							UpperCAmelCase_=       AutoformerModel.from_pretrained("""huggingface/autoformer-tourism-monthly"""					).to(lowerCAmelCase__					)
							UpperCAmelCase_=       prepare_batch()

							with torch.no_grad():
													UpperCAmelCase_=       model(
													    past_values=batch["""past_values"""]						,  past_time_features=batch["""past_time_features"""]						,  past_observed_mask=batch["""past_observed_mask"""]						,  static_categorical_features=batch["""static_categorical_features"""]						,  future_values=batch["""future_values"""]						,  future_time_features=batch["""future_time_features"""]						,  )[0]

							UpperCAmelCase_=       torch.Size(
							    (64, model.config.prediction_length + model.config.label_length, model.config.feature_size)					)
							self.assertEqual(output.shape						,  lowerCAmelCase__					)

							UpperCAmelCase_=       torch.tensor(
							    [[0.3_593, -1.3_398, 0.6_330], [0.2_279, 1.5_396, -0.1_792], [0.0_450, 1.3_225, -0.2_335]]						,  device=lowerCAmelCase__					)
							self.assertTrue(torch.allclose(output[0, :3, :3]						,  lowerCAmelCase__						,  atol=lowerCAmelCase__					)					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			Dict:
							UpperCAmelCase_=       AutoformerForPrediction.from_pretrained("""huggingface/autoformer-tourism-monthly"""					).to(lowerCAmelCase__					)
							UpperCAmelCase_=       prepare_batch("""val-batch.pt"""					)
							with torch.no_grad():
													UpperCAmelCase_=       model(
													    past_values=batch["""past_values"""]						,  past_time_features=batch["""past_time_features"""]						,  past_observed_mask=batch["""past_observed_mask"""]						,  static_categorical_features=batch["""static_categorical_features"""]						,  ).encoder_last_hidden_state
							UpperCAmelCase_=       torch.Size((64, model.config.context_length, model.config.d_model)					)
							self.assertEqual(output.shape						,  lowerCAmelCase__					)

							UpperCAmelCase_=       torch.tensor(
							    [[-0.0_734, -0.9_036, 0.8_358], [4.7_186, 2.4_113, 1.9_581], [1.7_953, 2.3_558, 1.2_970]]						,  device=lowerCAmelCase__					)
							self.assertTrue(torch.allclose(output[0, :3, :3]						,  lowerCAmelCase__						,  atol=lowerCAmelCase__					)					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			int:
							UpperCAmelCase_=       AutoformerForPrediction.from_pretrained("""huggingface/autoformer-tourism-monthly"""					).to(lowerCAmelCase__					)
							UpperCAmelCase_=       prepare_batch("""val-batch.pt"""					)
							with torch.no_grad():
													UpperCAmelCase_=       model.generate(
													    static_categorical_features=batch["""static_categorical_features"""]						,  past_time_features=batch["""past_time_features"""]						,  past_values=batch["""past_values"""]						,  future_time_features=batch["""future_time_features"""]						,  past_observed_mask=batch["""past_observed_mask"""]						,  )
							UpperCAmelCase_=       torch.Size((64, model.config.num_parallel_samples, model.config.prediction_length)					)
							self.assertEqual(outputs.sequences.shape						,  lowerCAmelCase__					)

							UpperCAmelCase_=       torch.tensor([3_130.6_763, 4_056.5_293, 7_053.0_786]						,  device=lowerCAmelCase__					)
							UpperCAmelCase_=       outputs.sequences.mean(dim=1					)
							self.assertTrue(torch.allclose(mean_prediction[0, -3:]						,  lowerCAmelCase__						,  rtol=1E-1					)					)









import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment



def 	__a					(   )				->			str:






						'''simple docstring'''



						UpperCAmelCase_=       torch.nn.Linear(2      ,4    )
						UpperCAmelCase_=       torch.optim.AdamW(model.parameters()      ,lr=1.0    )
						UpperCAmelCase_=       torch.optim.lr_scheduler.OneCycleLR(lowerCAmelCase_      ,max_lr=0.01      ,steps_per_epoch=2      ,epochs=1    )
						UpperCAmelCase_=       DataLoader(TensorDataset(torch.tensor([1, 2, 3]    )    )    )
						UpperCAmelCase_=       DataLoader(TensorDataset(torch.tensor([4, 5, 6]    )    )    )

						return model, optimizer, scheduler, train_dl, valid_dl



def 	__a					(   lowerCAmelCase_       : Any    )				->			Union[str, Any]:






						'''simple docstring'''



						return (model.weight.abs().sum() + model.bias.abs().sum()).item()



def 	__a					(   lowerCAmelCase_       : Tuple    )				->			Tuple:






						'''simple docstring'''



						UpperCAmelCase_=       torch.nn.Linear(*tuple(model.weight.T.shape    )    ).state_dict()
						model.load_state_dict(lowerCAmelCase_    )






class 		lowercase						(			snake_case__):






	"""simple docstring"""


	@require_cuda
	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Optional[Any]:
							UpperCAmelCase_=       Accelerator()
							assert PartialState._shared_state["_cpu"] is False
							assert PartialState._shared_state["device"].type == "cuda"
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       Accelerator(cpu=__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Tuple					)      ->			Union[str, Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_=       GradientState()
							assert state.num_steps == 1
							UpperCAmelCase_=       4
							assert state.num_steps == 4

							assert state.sync_gradients is True
							UpperCAmelCase_=       False
							assert state.sync_gradients is False
							GradientState._reset_state()




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			List[str]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()

							(
							    (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       
							)=       accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							self.assertTrue(prepared_model in accelerator._models					)
							self.assertTrue(prepared_optimizer in accelerator._optimizers					)
							self.assertTrue(prepared_scheduler in accelerator._schedulers					)
							self.assertTrue(prepared_train_dl in accelerator._dataloaders					)
							self.assertTrue(prepared_valid_dl in accelerator._dataloaders					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Dict:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							accelerator.free_memory()

							self.assertTrue(len(accelerator._models					) == 0					)
							self.assertTrue(len(accelerator._optimizers					) == 0					)
							self.assertTrue(len(accelerator._schedulers					) == 0					)
							self.assertTrue(len(accelerator._dataloaders					) == 0					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Optional[Any]:
							PartialState._reset_state()

							# Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
							def noop(*__UpperCAmelCase     :						Dict						,  **__UpperCAmelCase     :						Tuple					):
													pass

							with patch("""torch.cuda.set_device"""						,  __UpperCAmelCase					), patch_environment(ACCELERATE_TORCH_DEVICE="""cuda:64"""					):
													UpperCAmelCase_=       Accelerator()
													self.assertEqual(str(accelerator.state.device					)						,  """cuda:64"""					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							UpperCAmelCase_=       get_signature(__UpperCAmelCase					)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# make sure loaded weights match
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							UpperCAmelCase_=       get_signature(__UpperCAmelCase					)

							# saving hook
							def save_config(__UpperCAmelCase     :						Tuple						,  __UpperCAmelCase     :						List[str]						,  __UpperCAmelCase     :						Tuple					):
													UpperCAmelCase_=       {"""class_name""": models[0].__class__.__name__}

													with open(os.path.join(__UpperCAmelCase						,  """data.json"""					)						,  """w"""					) as f:
																			json.dump(__UpperCAmelCase						,  __UpperCAmelCase					)

        # loading hook
							def load_config(__UpperCAmelCase     :						Tuple						,  __UpperCAmelCase     :						Union[str, Any]					):
													with open(os.path.join(__UpperCAmelCase						,  """data.json"""					)						,  """r"""					) as f:
																			UpperCAmelCase_=       json.load(__UpperCAmelCase					)

													UpperCAmelCase_=       config["""class_name"""]

							UpperCAmelCase_=       accelerator.register_save_state_pre_hook(__UpperCAmelCase					)
							UpperCAmelCase_=       accelerator.register_load_state_pre_hook(__UpperCAmelCase					)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match with hooks
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# random class name to verify correct one is loaded
													UpperCAmelCase_=       """random"""

													# make sure loaded weights match with hooks
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)

													# mode.class_name is loaded from config
													self.assertTrue(model.class_name == model.__class__.__name__					)

							# remove hooks
							save_hook.remove()
							load_hook.remove()

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match with hooks removed
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# random class name to verify correct one is loaded
													UpperCAmelCase_=       """random"""

													# make sure loaded weights match with hooks removed
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)

													# mode.class_name is NOT loaded from config
													self.assertTrue(model.class_name != model.__class__.__name__					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Union[str, Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							UpperCAmelCase_=       None

							# This should work
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       accelerator.prepare(
							    __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							self.assertTrue(dummy_obj is None					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Any:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							UpperCAmelCase_=       [1, 2, 3]

							# This should work
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       accelerator.prepare(
							    __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Dummy object should have `_is_accelerate_prepared` set to `True`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Model is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Optimizer is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Scheduler is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`"""						,  )




	@slow
	@require_bnb
	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Optional[Any]:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map={"""""": 0}						,  )
							UpperCAmelCase_=       Accelerator()

							# This should work
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@slow
	@require_bnb
	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Tuple:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       Accelerator()

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													model.tie_weights()
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       """cpu"""

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  device_map=__UpperCAmelCase						,  load_in_abit=__UpperCAmelCase						,  llm_inta_enable_fpaa_cpu_offload=__UpperCAmelCase					)

							# This should not work and get value error
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@slow
	@require_bnb
	@require_multi_gpu
	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Tuple:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       {"""distributed_type""": DistributedType.MULTI_GPU}

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													model.tie_weights()
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       1

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map=__UpperCAmelCase						,  )
							UpperCAmelCase_=       Accelerator()

							# This should not work and get value error
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)

							PartialState._reset_state()




	@slow
	@require_bnb
	@require_multi_gpu
	def   _SCREAMING_SNAKE_CASE		(     self     :						Any					)      ->			Dict:
							from transformers import AutoModelForCausalLM

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       1

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map=__UpperCAmelCase						,  )
							UpperCAmelCase_=       Accelerator()

							# This should work
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@require_cuda
	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			Optional[int]:
							UpperCAmelCase_=       torch.nn.Linear(10						,  10					)
							UpperCAmelCase_=       torch.optim.SGD(model.parameters()						,  lr=0.01					)
							UpperCAmelCase_=       Accelerator(cpu=__UpperCAmelCase					)
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




