





import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


_SCREAMING_SNAKE_CASE      	=		1e-4

if is_torch_available():
						import torch

						from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
						from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder
@require_torch
class SCREAMING_SNAKE_CASE_						:






						def __init__(      self      :      Tuple		,   _A      :      Union[str, Any]		,   _A      :      Tuple=16		,   _A      :      Optional[Any]=13		,   _A      :      Union[str, Any]=7		,   _A      :      List[str]=14		,   _A      :      int=10		,   _A      :      str=19		,   _A      :      Optional[int]=5		,   _A      :      List[str]=4		,   _A      :      Tuple=True		,   _A      :      List[str]=16		,   _A      :      List[Any]=2		,   _A      :      Any=4		,   _A      :      List[str]=4		,   _A      :      Any="gelu"		,   _A      :      Tuple=0.1		,   _A      :      Optional[Any]=0.1		,   _A      :      List[Any]=[1, 2, 3, 4, 5]		,   _A      :      Optional[int]=25		,   _A      :      List[str]=5		,   )							->    List[str]:







							"""simple docstring"""


							snake_case_		:			List[str]											=	d_model
							snake_case_		:			List[Any]											=	parent
							snake_case_		:			List[str]											=	batch_size
							snake_case_		:			Any											=	prediction_length
							snake_case_		:			Optional[int]											=	context_length
							snake_case_		:			Optional[int]											=	cardinality
							snake_case_		:			Union[str, Any]											=	num_time_features
							snake_case_		:			Any											=	lags_sequence
							snake_case_		:			Union[str, Any]											=	embedding_dimension
							snake_case_		:			List[str]											=	is_training
							snake_case_		:			int											=	hidden_size
							snake_case_		:			str											=	num_hidden_layers
							snake_case_		:			Dict											=	num_attention_heads
							snake_case_		:			Optional[int]											=	intermediate_size
							snake_case_		:			Any											=	hidden_act
							snake_case_		:			int											=	hidden_dropout_prob
							snake_case_		:			str											=	attention_probs_dropout_prob

							snake_case_		:			Optional[Any]											=	context_length
							snake_case_		:			Dict											=	prediction_length + label_length
							snake_case_		:			Union[str, Any]											=	label_length

							snake_case_		:			str											=	moving_average
							snake_case_		:			Optional[Any]											=	autocorrelation_factor






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    List[str]:







							"""simple docstring"""


							return AutoformerConfig(
							    d_model=self.d_model		,   encoder_layers=self.num_hidden_layers		,   decoder_layers=self.num_hidden_layers		,   encoder_attention_heads=self.num_attention_heads		,   decoder_attention_heads=self.num_attention_heads		,   encoder_ffn_dim=self.intermediate_size		,   decoder_ffn_dim=self.intermediate_size		,   dropout=self.hidden_dropout_prob		,   attention_dropout=self.attention_probs_dropout_prob		,   prediction_length=self.prediction_length		,   context_length=self.context_length		,   label_length=self.label_length		,   lags_sequence=self.lags_sequence		,   num_time_features=self.num_time_features		,   num_static_categorical_features=1		,   cardinality=[self.cardinality]		,   embedding_dimension=[self.embedding_dimension]		,   moving_average=self.moving_average		,   )






						def UpperCAmelCase_				(      self      :      Optional[Any]		,   _A      :      Dict	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			List[str]											=	config.context_length + max(config.lags_sequence	)

							snake_case_		:			Union[str, Any]											=	ids_tensor([self.batch_size, 1]		,   config.cardinality[0]	)
							snake_case_		:			Any											=	floats_tensor([self.batch_size, _past_length, config.num_time_features]	)
							snake_case_		:			int											=	floats_tensor([self.batch_size, _past_length]	)
							snake_case_		:			List[Any]											=	floats_tensor([self.batch_size, _past_length]	) > 0.5

							# decoder inputs
							snake_case_		:			Optional[Any]											=	floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]	)
							snake_case_		:			Tuple											=	floats_tensor([self.batch_size, config.prediction_length]	)

							snake_case_		:			List[str]											=	{
							    'past_values': past_values,
							    'static_categorical_features': static_categorical_features,
							    'past_time_features': past_time_features,
							    'past_observed_mask': past_observed_mask,
							    'future_time_features': future_time_features,
							    'future_values': future_values,
							}
							return inputs_dict






						def UpperCAmelCase_				(      self      :      Any	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			Union[str, Any]											=	self.get_config()
							snake_case_		:			int											=	self.prepare_autoformer_inputs_dict(_A	)
							return config, inputs_dict






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    List[str]:







							"""simple docstring"""


							snake_case_						,snake_case_		:			str											=	self.prepare_config_and_inputs()
							return config, inputs_dict






						def UpperCAmelCase_				(      self      :      Optional[Any]		,   _A      :      Union[str, Any]		,   _A      :      int	)							->    List[str]:







							"""simple docstring"""


							snake_case_		:			Optional[int]											=	AutoformerModel(config=_A	).to(_A	).eval()
							snake_case_		:			int											=	model(**_A	)

							snake_case_		:			Union[str, Any]											=	outputs.encoder_last_hidden_state
							snake_case_		:			Optional[int]											=	outputs.last_hidden_state

							with tempfile.TemporaryDirectory() as tmpdirname:
								snake_case_		:			Union[str, Any]											=	model.get_encoder()
								encoder.save_pretrained(_A	)
								snake_case_		:			str											=	AutoformerEncoder.from_pretrained(_A	).to(_A	)

							snake_case_						,snake_case_						,snake_case_						,snake_case_						,snake_case_		:			Optional[int]											=	model.create_network_inputs(**_A	)
							snake_case_						,snake_case_		:			Dict											=	model.decomposition_layer(transformer_inputs[:, : config.context_length, ...]	)

							snake_case_		:			str											=	torch.cat(
							    (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])		,   dim=-1		,   )
							snake_case_		:			Any											=	encoder(inputs_embeds=_A	)[0]
							self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1E-3	)

							snake_case_		:			str											=	(
							    torch.mean(transformer_inputs[:, : config.context_length, ...]		,   dim=1	)
							    .unsqueeze(1	)
							    .repeat(1		,   config.prediction_length		,   1	)
							)
							snake_case_		:			List[Any]											=	torch.zeros(
							    [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]		,   device=enc_input.device		,   )

							snake_case_		:			Dict											=	torch.cat(
							    (
							        torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)		,   dim=1	),
							        feature[:, config.context_length - config.label_length :, ...],
							    )		,   dim=-1		,   )
							snake_case_		:			str											=	torch.cat(
							    (
							        torch.cat((trend_input[:, -config.label_length :, ...], mean)		,   dim=1	),
							        feature[:, config.context_length - config.label_length :, ...],
							    )		,   dim=-1		,   )

							with tempfile.TemporaryDirectory() as tmpdirname:
								snake_case_		:			List[str]											=	model.get_decoder()
								decoder.save_pretrained(_A	)
								snake_case_		:			Dict											=	AutoformerDecoder.from_pretrained(_A	).to(_A	)

							snake_case_		:			int											=	decoder(
							    trend=_A		,   inputs_embeds=_A		,   encoder_hidden_states=_A		,   )[0]

							self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1E-3	)
@require_torch
class SCREAMING_SNAKE_CASE_						(					snake_case_       ,				snake_case_       ,				unittest.TestCase					):
						__magic_name__:			Tuple       =       (AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
						__magic_name__:			Optional[int]       =       (AutoformerForPrediction,) if is_torch_available() else ()
						__magic_name__:			Any       =       {"feature-extraction": AutoformerModel} if is_torch_available() else {}
						__magic_name__:			Optional[Any]       =       False
						__magic_name__:			Tuple       =       False
						__magic_name__:			Dict       =       False
						__magic_name__:			Dict       =       False
						__magic_name__:			Union[str, Any]       =       False
						__magic_name__:			List[Any]       =       False






						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Union[str, Any]:







							"""simple docstring"""


							snake_case_		:			Dict											=	AutoformerModelTester(self	)
							snake_case_		:			Union[str, Any]											=	ConfigTester(self		,   config_class=_A		,   has_text_modality=_A	)






						def UpperCAmelCase_				(      self      :      List[str]	)							->    int:







							"""simple docstring"""


							self.config_tester.run_common_tests()






						def UpperCAmelCase_				(      self      :      List[Any]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_						,snake_case_		:			str											=	self.model_tester.prepare_config_and_inputs()
							for model_class in self.all_model_classes:
								snake_case_		:			Tuple											=	model_class(_A	)

								with tempfile.TemporaryDirectory() as tmpdirname:
									model.save_pretrained(_A	)
									snake_case_						,snake_case_		:			Tuple											=	model_class.from_pretrained(_A		,   output_loading_info=_A	)
								self.assertEqual(info['missing_keys']		,   []	)






						def UpperCAmelCase_				(      self      :      Dict	)							->    int:







							"""simple docstring"""


							snake_case_		:			Union[str, Any]											=	self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_encoder_decoder_model_standalone(*_A	)






						@unittest.skip(reason='Model has no tokens embeddings'	)
						def UpperCAmelCase_				(      self      :      Optional[Any]	)							->    Optional[int]:







							"""simple docstring"""


							pass






						def UpperCAmelCase_				(      self      :      List[str]	)							->    Tuple:







							"""simple docstring"""


							snake_case_		:			int											=	inspect.signature(getattr(_A		,   'forward'	)	)
							# The main input is the name of the argument after `self`
							snake_case_		:			str											=	list(model_signature.parameters.keys()	)[1]
							self.assertEqual(AutoformerModel.main_input_name		,   _A	)






						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    str:







							"""simple docstring"""


							snake_case_						,snake_case_		:			List[str]											=	self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
								snake_case_		:			Optional[int]											=	model_class(_A	)
								snake_case_		:			Dict											=	inspect.signature(model.forward	)
								# signature.parameters is an OrderedDict => so arg_names order is deterministic
								snake_case_		:			List[str]											=	[*signature.parameters.keys()]

								snake_case_		:			List[Any]											=	[
								    'past_values',
								    'past_time_features',
								    'past_observed_mask',
								    'static_categorical_features',
								    'static_real_features',
								    'future_values',
								    'future_time_features',
								]

								if model.__class__.__name__ in ["AutoformerForPrediction"]:
									expected_arg_names.append('future_observed_mask'	)

								expected_arg_names.extend(
								    [
								        'decoder_attention_mask',
								        'head_mask',
								        'decoder_head_mask',
								        'cross_attn_head_mask',
								        'encoder_outputs',
								        'past_key_values',
								        'output_hidden_states',
								        'output_attentions',
								        'use_cache',
								        'return_dict',
								    ]	)

								self.assertListEqual(arg_names[: len(_A	)]		,   _A	)






						def UpperCAmelCase_				(      self      :      List[Any]	)							->    Optional[int]:







							"""simple docstring"""


							snake_case_						,snake_case_		:			Dict											=	self.model_tester.prepare_config_and_inputs_for_common()
							snake_case_		:			str											=	True

							snake_case_		:			Union[str, Any]											=	getattr(self.model_tester		,   'seq_length'		,   _A	)
							snake_case_		:			str											=	getattr(self.model_tester		,   'decoder_seq_length'		,   _A	)
							snake_case_		:			str											=	getattr(self.model_tester		,   'encoder_seq_length'		,   _A	)
							snake_case_		:			Optional[Any]											=	getattr(self.model_tester		,   'd_model'		,   _A	)
							snake_case_		:			Dict											=	getattr(self.model_tester		,   'num_attention_heads'		,   _A	)
							snake_case_		:			Optional[int]											=	d_model // num_attention_heads

							for model_class in self.all_model_classes:
								snake_case_		:			Tuple											=	True
								snake_case_		:			Tuple											=	False
								snake_case_		:			Optional[Any]											=	True
								snake_case_		:			Any											=	model_class(_A	)
								model.to(_A	)
								model.eval()
								with torch.no_grad():
									snake_case_		:			Dict											=	model(**self._prepare_for_class(_A		,   _A	)	)
								snake_case_		:			Dict											=	outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
								self.assertEqual(len(_A	)		,   self.model_tester.num_hidden_layers	)

								# check that output_attentions also work using config
								del inputs_dict["output_attentions"]
								snake_case_		:			List[Any]											=	True
								snake_case_		:			List[Any]											=	model_class(_A	)
								model.to(_A	)
								model.eval()
								with torch.no_grad():
									snake_case_		:			Dict											=	model(**self._prepare_for_class(_A		,   _A	)	)
								snake_case_		:			Union[str, Any]											=	outputs.encoder_attentions
								self.assertEqual(len(_A	)		,   self.model_tester.num_hidden_layers	)

								self.assertListEqual(
								    list(attentions[0].shape[-3:]	)		,   [self.model_tester.num_attention_heads, encoder_seq_length, dim]		,   )
								snake_case_		:			Optional[int]											=	len(_A	)

								snake_case_		:			Tuple											=	7

								if "last_hidden_state" in outputs:
									correct_outlen += 1

								if "trend" in outputs:
									correct_outlen += 1

								if "past_key_values" in outputs:
									correct_outlen += 1  # past_key_values have been returned

								if "loss" in outputs:
									correct_outlen += 1

								if "params" in outputs:
									correct_outlen += 1

								self.assertEqual(_A		,   _A	)

								# decoder attentions
								snake_case_		:			Any											=	outputs.decoder_attentions
								self.assertIsInstance(_A		,   (list, tuple)	)
								self.assertEqual(len(_A	)		,   self.model_tester.num_hidden_layers	)
								self.assertListEqual(
								    list(decoder_attentions[0].shape[-3:]	)		,   [self.model_tester.num_attention_heads, decoder_seq_length, dim]		,   )

								# cross attentions
								snake_case_		:			Any											=	outputs.cross_attentions
								self.assertIsInstance(_A		,   (list, tuple)	)
								self.assertEqual(len(_A	)		,   self.model_tester.num_hidden_layers	)
								self.assertListEqual(
								    list(cross_attentions[0].shape[-3:]	)		,   [self.model_tester.num_attention_heads, decoder_seq_length, dim]		,   )

							# Check attention is always last and order is fine
							snake_case_		:			List[Any]											=	True
							snake_case_		:			Optional[Any]											=	True
							snake_case_		:			List[Any]											=	model_class(_A	)
							model.to(_A	)
							model.eval()
							with torch.no_grad():
								snake_case_		:			List[str]											=	model(**self._prepare_for_class(_A		,   _A	)	)

							self.assertEqual(out_len + 2		,   len(_A	)	)

							snake_case_		:			int											=	outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

							self.assertEqual(len(_A	)		,   self.model_tester.num_hidden_layers	)
							self.assertListEqual(
							    list(self_attentions[0].shape[-3:]	)		,   [self.model_tester.num_attention_heads, encoder_seq_length, dim]		,   )






						@is_flaky()
						def UpperCAmelCase_				(      self      :      List[str]	)							->    Any:







							"""simple docstring"""


							super().test_retain_grad_hidden_states_attentions()


def 							SCREAMING_SNAKE_CASE__						(    __a="train-batch.pt"   ):
	snake_case_		:			Dict											=	hf_hub_download(repo_id='hf-internal-testing/tourism-monthly-batch'   , filename=__a   , repo_type='dataset'   )
	snake_case_		:			Union[str, Any]											=	torch.load(__a   , map_location=__a   )
	return batch


@require_torch
@slow
class SCREAMING_SNAKE_CASE_						(					unittest.TestCase					):






						def UpperCAmelCase_				(      self      :      Any	)							->    List[str]:







							"""simple docstring"""


							snake_case_		:			Tuple											=	AutoformerModel.from_pretrained('huggingface/autoformer-tourism-monthly'	).to(_A	)
							snake_case_		:			Tuple											=	prepare_batch()

							with torch.no_grad():
								snake_case_		:			Tuple											=	model(
								    past_values=batch['past_values']		,   past_time_features=batch['past_time_features']		,   past_observed_mask=batch['past_observed_mask']		,   static_categorical_features=batch['static_categorical_features']		,   future_values=batch['future_values']		,   future_time_features=batch['future_time_features']		,   )[0]

							snake_case_		:			Union[str, Any]											=	torch.Size(
							    (64, model.config.prediction_length + model.config.label_length, model.config.feature_size)	)
							self.assertEqual(output.shape		,   _A	)

							snake_case_		:			List[Any]											=	torch.tensor(
							    [[0.3_5_9_3, -1.3_3_9_8, 0.6_3_3_0], [0.2_2_7_9, 1.5_3_9_6, -0.1_7_9_2], [0.0_4_5_0, 1.3_2_2_5, -0.2_3_3_5]]		,   device=_A	)
							self.assertTrue(torch.allclose(output[0, :3, :3]		,   _A		,   atol=_A	)	)






						def UpperCAmelCase_				(      self      :      Optional[int]	)							->    str:







							"""simple docstring"""


							snake_case_		:			int											=	AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'	).to(_A	)
							snake_case_		:			Union[str, Any]											=	prepare_batch('val-batch.pt'	)
							with torch.no_grad():
								snake_case_		:			List[Any]											=	model(
								    past_values=batch['past_values']		,   past_time_features=batch['past_time_features']		,   past_observed_mask=batch['past_observed_mask']		,   static_categorical_features=batch['static_categorical_features']		,   ).encoder_last_hidden_state
							snake_case_		:			str											=	torch.Size((64, model.config.context_length, model.config.d_model)	)
							self.assertEqual(output.shape		,   _A	)

							snake_case_		:			int											=	torch.tensor(
							    [[-0.0_7_3_4, -0.9_0_3_6, 0.8_3_5_8], [4.7_1_8_6, 2.4_1_1_3, 1.9_5_8_1], [1.7_9_5_3, 2.3_5_5_8, 1.2_9_7_0]]		,   device=_A	)
							self.assertTrue(torch.allclose(output[0, :3, :3]		,   _A		,   atol=_A	)	)






						def UpperCAmelCase_				(      self      :      Union[str, Any]	)							->    str:







							"""simple docstring"""


							snake_case_		:			Optional[int]											=	AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'	).to(_A	)
							snake_case_		:			str											=	prepare_batch('val-batch.pt'	)
							with torch.no_grad():
								snake_case_		:			Tuple											=	model.generate(
								    static_categorical_features=batch['static_categorical_features']		,   past_time_features=batch['past_time_features']		,   past_values=batch['past_values']		,   future_time_features=batch['future_time_features']		,   past_observed_mask=batch['past_observed_mask']		,   )
							snake_case_		:			Any											=	torch.Size((64, model.config.num_parallel_samples, model.config.prediction_length)	)
							self.assertEqual(outputs.sequences.shape		,   _A	)

							snake_case_		:			str											=	torch.tensor([3_1_3_0.6_7_6_3, 4_0_5_6.5_2_9_3, 7_0_5_3.0_7_8_6]		,   device=_A	)
							snake_case_		:			str											=	outputs.sequences.mean(dim=1	)
							self.assertTrue(torch.allclose(mean_prediction[0, -3:]		,   _A		,   rtol=1E-1	)	)








import argparse
import json

import numpy
import torch

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()


def 							SCREAMING_SNAKE_CASE__						(    __a   , __a   ):
	# Load checkpoint
	snake_case_		:			Union[str, Any]											=	torch.load(__a   , map_location='cpu'   )

	snake_case_		:			Union[str, Any]											=	chkpt['model']

	# We have the base model one level deeper than the original XLM repository
	snake_case_		:			str											=	{}
	for k, v in state_dict.items():
		if "pred_layer" in k:
			snake_case_		:			Tuple											=	v
		else:
			snake_case_		:			Dict											=	v

	snake_case_		:			Tuple											=	chkpt['params']
	snake_case_		:			List[Any]											=	{n: v for n, v in config.items() if not isinstance(__a   , (torch.FloatTensor, numpy.ndarray)   )}

	snake_case_		:			Optional[int]											=	chkpt['dico_word2id']
	snake_case_		:			List[str]											=	{s + '</w>' if s.find('@@'   ) == -1 and i > 13 else s.replace('@@'   , ''   ): i for s, i in vocab.items()}

	# Save pytorch-model
	snake_case_		:			List[str]											=	pytorch_dump_folder_path + '/' + WEIGHTS_NAME
	snake_case_		:			Dict											=	pytorch_dump_folder_path + '/' + CONFIG_NAME
	snake_case_		:			Any											=	pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['vocab_file']

	print(f"""Save PyTorch model to {pytorch_weights_dump_path}"""   )
	torch.save(__a   , __a   )

	print(f"""Save configuration file to {pytorch_config_dump_path}"""   )
	with open(__a   , 'w'   , encoding='utf-8'   ) as f:
		f.write(json.dumps(__a   , indent=2   ) + '\n'   )

	print(f"""Save vocab file to {pytorch_config_dump_path}"""   )
	with open(__a   , 'w'   , encoding='utf-8'   ) as f:
		f.write(json.dumps(__a   , indent=2   ) + '\n'   )


if __name__ == "__main__":
						_SCREAMING_SNAKE_CASE      	=		argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    """--xlm_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
						)
						parser.add_argument(
						    """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
						)
						_SCREAMING_SNAKE_CASE      	=		parser.parse_args()
						convert_xlm_checkpoint_to_pytorch(args.xlm_checkpoint_path, args.pytorch_dump_folder_path)

