





'''simple docstring'''



import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("""1.0.0a"""):
 raise Exception("""requires fairseq >= 1.0.0a""")

logging.set_verbosity_info()
__magic_name__ :    Dict			 =      logging.get_logger(__name__)

__magic_name__ :    Optional[Any]			 =      "Hello world! cécé herlolip"




def 					snake_case_   (		SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__							):







  '''simple docstring'''



  _snake_case  		=					FairseqRobertaModel.from_pretrained(SCREAMING_SNAKE_CASE__							)
  roberta.eval()  # disable dropout
  _snake_case  		=					roberta.model.encoder.sentence_encoder
  _snake_case  		=					XLMRobertaConfig(
      vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings       ,						hidden_size=roberta.cfg.model.encoder_embed_dim       ,						num_hidden_layers=roberta.cfg.model.encoder_layers       ,						num_attention_heads=roberta.cfg.model.encoder_attention_heads       ,						intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim       ,						max_position_embeddings=5_14       ,						type_vocab_size=1       ,						layer_norm_eps=1E-5       ,						)
  if classification_head:
    _snake_case  		=					roberta.model.classification_heads["mnli"].out_proj.weight.shape[0]

  print("Our RoBERTa config:"       ,						SCREAMING_SNAKE_CASE__							)

  _snake_case  		=					XLMRobertaXLForSequenceClassification(SCREAMING_SNAKE_CASE__							) if classification_head else XLMRobertaXLForMaskedLM(SCREAMING_SNAKE_CASE__							)
  model.eval()

  # Now let's copy all the weights.
  # Embeddings
  _snake_case  		=					roberta_sent_encoder.embed_tokens.weight
  _snake_case  		=					roberta_sent_encoder.embed_positions.weight
  _snake_case  		=					torch.zeros_like(
      model.roberta.embeddings.token_type_embeddings.weight							)  # just zero them out b/c RoBERTa doesn't use them.

  _snake_case  		=					roberta_sent_encoder.layer_norm.weight
  _snake_case  		=					roberta_sent_encoder.layer_norm.bias

  for i in range(config.num_hidden_layers							):
    # Encoder: start of layer
    _snake_case  		=					model.roberta.encoder.layer[i]
    _snake_case  		=					roberta_sent_encoder.layers[i]

    _snake_case  		=					layer.attention
    _snake_case  		=					roberta_layer.self_attn_layer_norm.weight
    _snake_case  		=					roberta_layer.self_attn_layer_norm.bias

    # self attention
    _snake_case  		=					layer.attention.self
    assert (
        roberta_layer.self_attn.k_proj.weight.data.shape
        == roberta_layer.self_attn.q_proj.weight.data.shape
        == roberta_layer.self_attn.v_proj.weight.data.shape
        == torch.Size((config.hidden_size, config.hidden_size)							)
    )

    _snake_case  		=					roberta_layer.self_attn.q_proj.weight
    _snake_case  		=					roberta_layer.self_attn.q_proj.bias
    _snake_case  		=					roberta_layer.self_attn.k_proj.weight
    _snake_case  		=					roberta_layer.self_attn.k_proj.bias
    _snake_case  		=					roberta_layer.self_attn.v_proj.weight
    _snake_case  		=					roberta_layer.self_attn.v_proj.bias

    # self-attention output
    _snake_case  		=					layer.attention.output
    assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
    _snake_case  		=					roberta_layer.self_attn.out_proj.weight
    _snake_case  		=					roberta_layer.self_attn.out_proj.bias

    # this one is final layer norm
    _snake_case  		=					roberta_layer.final_layer_norm.weight
    _snake_case  		=					roberta_layer.final_layer_norm.bias

    # intermediate
    _snake_case  		=					layer.intermediate
    assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
    _snake_case  		=					roberta_layer.fca.weight
    _snake_case  		=					roberta_layer.fca.bias

    # output
    _snake_case  		=					layer.output
    assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
    _snake_case  		=					roberta_layer.fca.weight
    _snake_case  		=					roberta_layer.fca.bias
    # end of layer

  if classification_head:
    _snake_case  		=					roberta.model.classification_heads["mnli"].dense.weight
    _snake_case  		=					roberta.model.classification_heads["mnli"].dense.bias
    _snake_case  		=					roberta.model.classification_heads["mnli"].out_proj.weight
    _snake_case  		=					roberta.model.classification_heads["mnli"].out_proj.bias
  else:
    # LM Head
    _snake_case  		=					roberta.model.encoder.lm_head.dense.weight
    _snake_case  		=					roberta.model.encoder.lm_head.dense.bias
    _snake_case  		=					roberta.model.encoder.lm_head.layer_norm.weight
    _snake_case  		=					roberta.model.encoder.lm_head.layer_norm.bias
    _snake_case  		=					roberta.model.encoder.lm_head.weight
    _snake_case  		=					roberta.model.encoder.lm_head.bias

  # Let's check that we get the same results.
  _snake_case  		=					roberta.encode(SCREAMING_SNAKE_CASE__							).unsqueeze(0							)  # batch of size 1

  _snake_case  		=					model(SCREAMING_SNAKE_CASE__							)[0]
  if classification_head:
    _snake_case  		=					roberta.model.classification_heads["mnli"](roberta.extract_features(SCREAMING_SNAKE_CASE__							)							)
  else:
    _snake_case  		=					roberta.model(SCREAMING_SNAKE_CASE__							)[0]
  print(our_output.shape       ,						their_output.shape							)
  _snake_case  		=					torch.max(torch.abs(our_output - their_output							)							).item()
  print(f'''max_absolute_diff = {max_absolute_diff}'''							)  # ~ 1e-7
  _snake_case  		=					torch.allclose(SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       ,						atol=1E-3							)
  print("Do both models output the same tensors?"       ,						"🔥" if success else "💩"							)
  if not success:
    raise Exception("Something went wRoNg"							)

  pathlib.Path(SCREAMING_SNAKE_CASE__							).mkdir(parents=SCREAMING_SNAKE_CASE__       ,						exist_ok=SCREAMING_SNAKE_CASE__							)
  print(f'''Saving model to {pytorch_dump_folder_path}'''							)
  model.save_pretrained(SCREAMING_SNAKE_CASE__							)


if __name__ == "__main__":
 __magic_name__ :    Optional[Any]			 =      argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     """--roberta_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
 )
 parser.add_argument(
     """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
 )
 parser.add_argument(
     """--classification_head""", action="""store_true""", help="""Whether to convert a final classification head."""
 )
 __magic_name__ :    Dict			 =      parser.parse_args()
 convert_xlm_roberta_xl_checkpoint_to_pytorch(
     args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
 )








import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


SCREAMING_SNAKE_CASE__     :     Optional[Any]		    =       1E-4

if is_torch_available():
				import torch

				from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
				from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder



@require_torch
class       a_       :





	def __init__(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE=16      ,     SCREAMING_SNAKE_CASE=13      ,     SCREAMING_SNAKE_CASE=7      ,     SCREAMING_SNAKE_CASE=14      ,     SCREAMING_SNAKE_CASE=10      ,     SCREAMING_SNAKE_CASE=19      ,     SCREAMING_SNAKE_CASE=5      ,     SCREAMING_SNAKE_CASE=4      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE=16      ,     SCREAMING_SNAKE_CASE=2      ,     SCREAMING_SNAKE_CASE=4      ,     SCREAMING_SNAKE_CASE=4      ,     SCREAMING_SNAKE_CASE="gelu"      ,     SCREAMING_SNAKE_CASE=0.1      ,     SCREAMING_SNAKE_CASE=0.1      ,     SCREAMING_SNAKE_CASE=[1, 2, 3, 4, 5]      ,     SCREAMING_SNAKE_CASE=25      ,     SCREAMING_SNAKE_CASE=5      ,     )  ->					Union[str, Any]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						d_model
				SCREAMING_SNAKE_CASE_								=						parent
				SCREAMING_SNAKE_CASE_								=						batch_size
				SCREAMING_SNAKE_CASE_								=						prediction_length
				SCREAMING_SNAKE_CASE_								=						context_length
				SCREAMING_SNAKE_CASE_								=						cardinality
				SCREAMING_SNAKE_CASE_								=						num_time_features
				SCREAMING_SNAKE_CASE_								=						lags_sequence
				SCREAMING_SNAKE_CASE_								=						embedding_dimension
				SCREAMING_SNAKE_CASE_								=						is_training
				SCREAMING_SNAKE_CASE_								=						hidden_size
				SCREAMING_SNAKE_CASE_								=						num_hidden_layers
				SCREAMING_SNAKE_CASE_								=						num_attention_heads
				SCREAMING_SNAKE_CASE_								=						intermediate_size
				SCREAMING_SNAKE_CASE_								=						hidden_act
				SCREAMING_SNAKE_CASE_								=						hidden_dropout_prob
				SCREAMING_SNAKE_CASE_								=						attention_probs_dropout_prob

				SCREAMING_SNAKE_CASE_								=						context_length
				SCREAMING_SNAKE_CASE_								=						prediction_length + label_length
				SCREAMING_SNAKE_CASE_								=						label_length

				SCREAMING_SNAKE_CASE_								=						moving_average
				SCREAMING_SNAKE_CASE_								=						autocorrelation_factor





	def 						A_(							self      )  ->					List[str]:




				"""simple docstring"""



				return AutoformerConfig(
				    d_model=self.d_model      ,     encoder_layers=self.num_hidden_layers      ,     decoder_layers=self.num_hidden_layers      ,     encoder_attention_heads=self.num_attention_heads      ,     decoder_attention_heads=self.num_attention_heads      ,     encoder_ffn_dim=self.intermediate_size      ,     decoder_ffn_dim=self.intermediate_size      ,     dropout=self.hidden_dropout_prob      ,     attention_dropout=self.attention_probs_dropout_prob      ,     prediction_length=self.prediction_length      ,     context_length=self.context_length      ,     label_length=self.label_length      ,     lags_sequence=self.lags_sequence      ,     num_time_features=self.num_time_features      ,     num_static_categorical_features=1      ,     cardinality=[self.cardinality]      ,     embedding_dimension=[self.embedding_dimension]      ,     moving_average=self.moving_average      ,     )





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					Optional[int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						config.context_length + max(config.lags_sequence      )

				SCREAMING_SNAKE_CASE_								=						ids_tensor([self.batch_size, 1]      ,     config.cardinality[0]      )
				SCREAMING_SNAKE_CASE_								=						floats_tensor([self.batch_size, _past_length, config.num_time_features]      )
				SCREAMING_SNAKE_CASE_								=						floats_tensor([self.batch_size, _past_length]      )
				SCREAMING_SNAKE_CASE_								=						floats_tensor([self.batch_size, _past_length]      ) > 0.5

				# decoder inputs
				SCREAMING_SNAKE_CASE_								=						floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]      )
				SCREAMING_SNAKE_CASE_								=						floats_tensor([self.batch_size, config.prediction_length]      )

				SCREAMING_SNAKE_CASE_								=						{
				    'past_values': past_values,
				    'static_categorical_features': static_categorical_features,
				    'past_time_features': past_time_features,
				    'past_observed_mask': past_observed_mask,
				    'future_time_features': future_time_features,
				    'future_values': future_values,
				}
				return inputs_dict





	def 						A_(							self      )  ->					Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self.get_config()
				SCREAMING_SNAKE_CASE_								=						self.prepare_autoformer_inputs_dict(SCREAMING_SNAKE_CASE      )
				return config, inputs_dict





	def 						A_(							self      )  ->					Union[str, Any]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						self.prepare_config_and_inputs()
				return config, inputs_dict





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE      )  ->					Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						AutoformerModel(config=SCREAMING_SNAKE_CASE      ).to(SCREAMING_SNAKE_CASE      ).eval()
				SCREAMING_SNAKE_CASE_								=						model(**SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						outputs.encoder_last_hidden_state
				SCREAMING_SNAKE_CASE_								=						outputs.last_hidden_state

				with tempfile.TemporaryDirectory() as tmpdirname:
							SCREAMING_SNAKE_CASE_								=						model.get_encoder()
							encoder.save_pretrained(SCREAMING_SNAKE_CASE      )
							SCREAMING_SNAKE_CASE_								=						AutoformerEncoder.from_pretrained(SCREAMING_SNAKE_CASE      ).to(SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						model.create_network_inputs(**SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						model.decomposition_layer(transformer_inputs[:, : config.context_length, ...]      )

				SCREAMING_SNAKE_CASE_								=						torch.cat(
				    (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])      ,     dim=-1      ,     )
				SCREAMING_SNAKE_CASE_								=						encoder(inputs_embeds=SCREAMING_SNAKE_CASE      )[0]
				self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1e-3      )

				SCREAMING_SNAKE_CASE_								=						(
				    torch.mean(transformer_inputs[:, : config.context_length, ...]      ,     dim=1      )
				    .unsqueeze(1      )
				    .repeat(1      ,     config.prediction_length      ,     1      )
				)
				SCREAMING_SNAKE_CASE_								=						torch.zeros(
				    [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]      ,     device=enc_input.device      ,     )

				SCREAMING_SNAKE_CASE_								=						torch.cat(
				    (
				        torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)      ,     dim=1      ),
				        feature[:, config.context_length - config.label_length :, ...],
				    )      ,     dim=-1      ,     )
				SCREAMING_SNAKE_CASE_								=						torch.cat(
				    (
				        torch.cat((trend_input[:, -config.label_length :, ...], mean)      ,     dim=1      ),
				        feature[:, config.context_length - config.label_length :, ...],
				    )      ,     dim=-1      ,     )

				with tempfile.TemporaryDirectory() as tmpdirname:
							SCREAMING_SNAKE_CASE_								=						model.get_decoder()
							decoder.save_pretrained(SCREAMING_SNAKE_CASE      )
							SCREAMING_SNAKE_CASE_								=						AutoformerDecoder.from_pretrained(SCREAMING_SNAKE_CASE      ).to(SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						decoder(
				    trend=SCREAMING_SNAKE_CASE      ,     inputs_embeds=SCREAMING_SNAKE_CASE      ,     encoder_hidden_states=SCREAMING_SNAKE_CASE      ,     )[0]

				self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1e-3      )




@require_torch
class       a_       (     SCREAMING_SNAKE_CASE__  ,    SCREAMING_SNAKE_CASE__  ,    unittest.TestCase     ):
	A    							= (AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
	A    							= (AutoformerForPrediction,) if is_torch_available() else ()
	A    							= {'''feature-extraction''': AutoformerModel} if is_torch_available() else {}
	A    							= False
	A    							= False
	A    							= False
	A    							= False
	A    							= False
	A    							= False





	def 						A_(							self      )  ->					Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						AutoformerModelTester(self      )
				SCREAMING_SNAKE_CASE_								=						ConfigTester(self      ,     config_class=SCREAMING_SNAKE_CASE      ,     has_text_modality=SCREAMING_SNAKE_CASE      )





	def 						A_(							self      )  ->					int:




				"""simple docstring"""



				self.config_tester.run_common_tests()





	def 						A_(							self      )  ->					Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						self.model_tester.prepare_config_and_inputs()
				for model_class in self.all_model_classes:
							SCREAMING_SNAKE_CASE_								=						model_class(SCREAMING_SNAKE_CASE      )

							with tempfile.TemporaryDirectory() as tmpdirname:
										model.save_pretrained(SCREAMING_SNAKE_CASE      )
										SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						model_class.from_pretrained(SCREAMING_SNAKE_CASE      ,     output_loading_info=SCREAMING_SNAKE_CASE      )
							self.assertEqual(info['missing_keys']      ,     []      )





	def 						A_(							self      )  ->					Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self.model_tester.prepare_config_and_inputs_for_common()
				self.model_tester.check_encoder_decoder_model_standalone(*SCREAMING_SNAKE_CASE      )





	@unittest.skip(reason='Model has no tokens embeddings'      )
	def 						A_(							self      )  ->					Union[str, Any]:




				"""simple docstring"""



				pass





	def 						A_(							self      )  ->					int:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						inspect.signature(getattr(SCREAMING_SNAKE_CASE      ,     'forward'      )      )
				# The main input is the name of the argument after `self`
				SCREAMING_SNAKE_CASE_								=						list(model_signature.parameters.keys()      )[1]
				self.assertEqual(AutoformerModel.main_input_name      ,     SCREAMING_SNAKE_CASE      )





	def 						A_(							self      )  ->					Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
							SCREAMING_SNAKE_CASE_								=						model_class(SCREAMING_SNAKE_CASE      )
							SCREAMING_SNAKE_CASE_								=						inspect.signature(model.forward      )
							# signature.parameters is an OrderedDict => so arg_names order is deterministic
							SCREAMING_SNAKE_CASE_								=						[*signature.parameters.keys()]

							SCREAMING_SNAKE_CASE_								=						[
							    'past_values',
							    'past_time_features',
							    'past_observed_mask',
							    'static_categorical_features',
							    'static_real_features',
							    'future_values',
							    'future_time_features',
							]

							if model.__class__.__name__ in ["AutoformerForPrediction"]:
										expected_arg_names.append('future_observed_mask'      )

							expected_arg_names.extend(
							    [
							        'decoder_attention_mask',
							        'head_mask',
							        'decoder_head_mask',
							        'cross_attn_head_mask',
							        'encoder_outputs',
							        'past_key_values',
							        'output_hidden_states',
							        'output_attentions',
							        'use_cache',
							        'return_dict',
							    ]      )

							self.assertListEqual(arg_names[: len(SCREAMING_SNAKE_CASE      )]      ,     SCREAMING_SNAKE_CASE      )





	def 						A_(							self      )  ->					str:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						self.model_tester.prepare_config_and_inputs_for_common()
				SCREAMING_SNAKE_CASE_								=						True

				SCREAMING_SNAKE_CASE_								=						getattr(self.model_tester      ,     'seq_length'      ,     SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						getattr(self.model_tester      ,     'decoder_seq_length'      ,     SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						getattr(self.model_tester      ,     'encoder_seq_length'      ,     SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						getattr(self.model_tester      ,     'd_model'      ,     SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						getattr(self.model_tester      ,     'num_attention_heads'      ,     SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						d_model // num_attention_heads

				for model_class in self.all_model_classes:
							SCREAMING_SNAKE_CASE_								=						True
							SCREAMING_SNAKE_CASE_								=						False
							SCREAMING_SNAKE_CASE_								=						True
							SCREAMING_SNAKE_CASE_								=						model_class(SCREAMING_SNAKE_CASE      )
							model.to(SCREAMING_SNAKE_CASE      )
							model.eval()
							with torch.no_grad():
										SCREAMING_SNAKE_CASE_								=						model(**self._prepare_for_class(SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE      )      )
							SCREAMING_SNAKE_CASE_								=						outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
							self.assertEqual(len(SCREAMING_SNAKE_CASE      )      ,     self.model_tester.num_hidden_layers      )

							# check that output_attentions also work using config
							del inputs_dict["output_attentions"]
							SCREAMING_SNAKE_CASE_								=						True
							SCREAMING_SNAKE_CASE_								=						model_class(SCREAMING_SNAKE_CASE      )
							model.to(SCREAMING_SNAKE_CASE      )
							model.eval()
							with torch.no_grad():
										SCREAMING_SNAKE_CASE_								=						model(**self._prepare_for_class(SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE      )      )
							SCREAMING_SNAKE_CASE_								=						outputs.encoder_attentions
							self.assertEqual(len(SCREAMING_SNAKE_CASE      )      ,     self.model_tester.num_hidden_layers      )

							self.assertListEqual(
							    list(attentions[0].shape[-3:]      )      ,     [self.model_tester.num_attention_heads, encoder_seq_length, dim]      ,     )
							SCREAMING_SNAKE_CASE_								=						len(SCREAMING_SNAKE_CASE      )

							SCREAMING_SNAKE_CASE_								=						7

							if "last_hidden_state" in outputs:
										correct_outlen += 1

							if "trend" in outputs:
										correct_outlen += 1

							if "past_key_values" in outputs:
										correct_outlen += 1  # past_key_values have been returned

							if "loss" in outputs:
										correct_outlen += 1

							if "params" in outputs:
										correct_outlen += 1

							self.assertEqual(SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE      )

							# decoder attentions
							SCREAMING_SNAKE_CASE_								=						outputs.decoder_attentions
							self.assertIsInstance(SCREAMING_SNAKE_CASE      ,     (list, tuple)      )
							self.assertEqual(len(SCREAMING_SNAKE_CASE      )      ,     self.model_tester.num_hidden_layers      )
							self.assertListEqual(
							    list(decoder_attentions[0].shape[-3:]      )      ,     [self.model_tester.num_attention_heads, decoder_seq_length, dim]      ,     )

							# cross attentions
							SCREAMING_SNAKE_CASE_								=						outputs.cross_attentions
							self.assertIsInstance(SCREAMING_SNAKE_CASE      ,     (list, tuple)      )
							self.assertEqual(len(SCREAMING_SNAKE_CASE      )      ,     self.model_tester.num_hidden_layers      )
							self.assertListEqual(
							    list(cross_attentions[0].shape[-3:]      )      ,     [self.model_tester.num_attention_heads, decoder_seq_length, dim]      ,     )

				# Check attention is always last and order is fine
				SCREAMING_SNAKE_CASE_								=						True
				SCREAMING_SNAKE_CASE_								=						True
				SCREAMING_SNAKE_CASE_								=						model_class(SCREAMING_SNAKE_CASE      )
				model.to(SCREAMING_SNAKE_CASE      )
				model.eval()
				with torch.no_grad():
							SCREAMING_SNAKE_CASE_								=						model(**self._prepare_for_class(SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE      )      )

				self.assertEqual(out_len + 2      ,     len(SCREAMING_SNAKE_CASE      )      )

				SCREAMING_SNAKE_CASE_								=						outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

				self.assertEqual(len(SCREAMING_SNAKE_CASE      )      ,     self.model_tester.num_hidden_layers      )
				self.assertListEqual(
				    list(self_attentions[0].shape[-3:]      )      ,     [self.model_tester.num_attention_heads, encoder_seq_length, dim]      ,     )





	@is_flaky()
	def 						A_(							self      )  ->					Any:




				"""simple docstring"""



				super().test_retain_grad_hidden_states_attentions()




def lowercase    (  SCREAMING_SNAKE_CASE="train-batch.pt" )			->							str:


			'''simple docstring'''







			SCREAMING_SNAKE_CASE_								=						hf_hub_download(repo_id='hf-internal-testing/tourism-monthly-batch'       ,    filename=SCREAMING_SNAKE_CASE       ,    repo_type='dataset' )
			SCREAMING_SNAKE_CASE_								=						torch.load(SCREAMING_SNAKE_CASE       ,    map_location=SCREAMING_SNAKE_CASE )
			return batch



@require_torch
@slow
class       a_       (     unittest.TestCase     ):





	def 						A_(							self      )  ->					List[str]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						AutoformerModel.from_pretrained('huggingface/autoformer-tourism-monthly'      ).to(SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						prepare_batch()

				with torch.no_grad():
							SCREAMING_SNAKE_CASE_								=						model(
							    past_values=batch['past_values']      ,     past_time_features=batch['past_time_features']      ,     past_observed_mask=batch['past_observed_mask']      ,     static_categorical_features=batch['static_categorical_features']      ,     future_values=batch['future_values']      ,     future_time_features=batch['future_time_features']      ,     )[0]

				SCREAMING_SNAKE_CASE_								=						torch.Size(
				    (64, model.config.prediction_length + model.config.label_length, model.config.feature_size)      )
				self.assertEqual(output.shape      ,     SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						torch.tensor(
				    [[0.3_5_9_3, -1.3_3_9_8, 0.6_3_3_0], [0.2_2_7_9, 1.5_3_9_6, -0.1_7_9_2], [0.0_4_5_0, 1.3_2_2_5, -0.2_3_3_5]]      ,     device=SCREAMING_SNAKE_CASE      )
				self.assertTrue(torch.allclose(output[0, :3, :3]      ,     SCREAMING_SNAKE_CASE      ,     atol=SCREAMING_SNAKE_CASE      )      )





	def 						A_(							self      )  ->					Dict:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'      ).to(SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						prepare_batch('val-batch.pt'      )
				with torch.no_grad():
							SCREAMING_SNAKE_CASE_								=						model(
							    past_values=batch['past_values']      ,     past_time_features=batch['past_time_features']      ,     past_observed_mask=batch['past_observed_mask']      ,     static_categorical_features=batch['static_categorical_features']      ,     ).encoder_last_hidden_state
				SCREAMING_SNAKE_CASE_								=						torch.Size((64, model.config.context_length, model.config.d_model)      )
				self.assertEqual(output.shape      ,     SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						torch.tensor(
				    [[-0.0_7_3_4, -0.9_0_3_6, 0.8_3_5_8], [4.7_1_8_6, 2.4_1_1_3, 1.9_5_8_1], [1.7_9_5_3, 2.3_5_5_8, 1.2_9_7_0]]      ,     device=SCREAMING_SNAKE_CASE      )
				self.assertTrue(torch.allclose(output[0, :3, :3]      ,     SCREAMING_SNAKE_CASE      ,     atol=SCREAMING_SNAKE_CASE      )      )





	def 						A_(							self      )  ->					Any:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'      ).to(SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						prepare_batch('val-batch.pt'      )
				with torch.no_grad():
							SCREAMING_SNAKE_CASE_								=						model.generate(
							    static_categorical_features=batch['static_categorical_features']      ,     past_time_features=batch['past_time_features']      ,     past_values=batch['past_values']      ,     future_time_features=batch['future_time_features']      ,     past_observed_mask=batch['past_observed_mask']      ,     )
				SCREAMING_SNAKE_CASE_								=						torch.Size((64, model.config.num_parallel_samples, model.config.prediction_length)      )
				self.assertEqual(outputs.sequences.shape      ,     SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						torch.tensor([3_1_3_0.6_7_6_3, 4_0_5_6.5_2_9_3, 7_0_5_3.0_7_8_6]      ,     device=SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						outputs.sequences.mean(dim=1      )
				self.assertTrue(torch.allclose(mean_prediction[0, -3:]      ,     SCREAMING_SNAKE_CASE      ,     rtol=1e-1      )      )
