'''simple docstring'''






import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


__a									=   1E-4

if is_torch_available():
     import torch

     from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
     from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder





@require_torch
class        A__				:





   """simple docstring"""





   def __init__(					self	:						List[str]		,	lowerCAmelCase__	:						List[str]		,	lowerCAmelCase__	:						List[Any]=1_6		,	lowerCAmelCase__	:						str=1_3		,	lowerCAmelCase__	:						Tuple=7		,	lowerCAmelCase__	:						Tuple=1_4		,	lowerCAmelCase__	:						List[Any]=1_0		,	lowerCAmelCase__	:						Dict=1_9		,	lowerCAmelCase__	:						Tuple=5		,	lowerCAmelCase__	:						int=4		,	lowerCAmelCase__	:						str=True		,	lowerCAmelCase__	:						Dict=1_6		,	lowerCAmelCase__	:						str=2		,	lowerCAmelCase__	:						Optional[Any]=4		,	lowerCAmelCase__	:						Tuple=4		,	lowerCAmelCase__	:						Tuple="gelu"		,	lowerCAmelCase__	:						List[str]=0.1		,	lowerCAmelCase__	:						List[Any]=0.1		,	lowerCAmelCase__	:						Any=[1, 2, 3, 4, 5]		,	lowerCAmelCase__	:						str=2_5		,	lowerCAmelCase__	:						str=5		,	)			->       Optional[Any]:







      """simple docstring"""




      _UpperCAmelCase				:      Tuple				=  d_model
      _UpperCAmelCase				:      Any				=  parent
      _UpperCAmelCase				:      Tuple				=  batch_size
      _UpperCAmelCase				:      List[Any]				=  prediction_length
      _UpperCAmelCase				:      str				=  context_length
      _UpperCAmelCase				:      Dict				=  cardinality
      _UpperCAmelCase				:      int				=  num_time_features
      _UpperCAmelCase				:      Any				=  lags_sequence
      _UpperCAmelCase				:      Optional[int]				=  embedding_dimension
      _UpperCAmelCase				:      Dict				=  is_training
      _UpperCAmelCase				:      Any				=  hidden_size
      _UpperCAmelCase				:      Tuple				=  num_hidden_layers
      _UpperCAmelCase				:      Any				=  num_attention_heads
      _UpperCAmelCase				:      Dict				=  intermediate_size
      _UpperCAmelCase				:      Union[str, Any]				=  hidden_act
      _UpperCAmelCase				:      Dict				=  hidden_dropout_prob
      _UpperCAmelCase				:      Optional[int]				=  attention_probs_dropout_prob

      _UpperCAmelCase				:      str				=  context_length
      _UpperCAmelCase				:      Dict				=  prediction_length + label_length
      _UpperCAmelCase				:      Optional[int]				=  label_length

      _UpperCAmelCase				:      List[str]				=  moving_average
      _UpperCAmelCase				:      Optional[int]				=  autocorrelation_factor




   def      _lowerCAmelCase						(					self	:						Tuple     )			->       Optional[Any]:







      """simple docstring"""




      return AutoformerConfig(
          d_model=self.d_model		,	encoder_layers=self.num_hidden_layers		,	decoder_layers=self.num_hidden_layers		,	encoder_attention_heads=self.num_attention_heads		,	decoder_attention_heads=self.num_attention_heads		,	encoder_ffn_dim=self.intermediate_size		,	decoder_ffn_dim=self.intermediate_size		,	dropout=self.hidden_dropout_prob		,	attention_dropout=self.attention_probs_dropout_prob		,	prediction_length=self.prediction_length		,	context_length=self.context_length		,	label_length=self.label_length		,	lags_sequence=self.lags_sequence		,	num_time_features=self.num_time_features		,	num_static_categorical_features=1		,	cardinality=[self.cardinality]		,	embedding_dimension=[self.embedding_dimension]		,	moving_average=self.moving_average		,	)




   def      _lowerCAmelCase						(					self	:						List[Any]		,	lowerCAmelCase__	:						Any     )			->       Dict:







      """simple docstring"""




      _UpperCAmelCase				:      Tuple				=  config.context_length + max(config.lags_sequence     )

      _UpperCAmelCase				:      int				=  ids_tensor([self.batch_size, 1]		,	config.cardinality[0]     )
      _UpperCAmelCase				:      str				=  floats_tensor([self.batch_size, _past_length, config.num_time_features]     )
      _UpperCAmelCase				:      int				=  floats_tensor([self.batch_size, _past_length]     )
      _UpperCAmelCase				:      Tuple				=  floats_tensor([self.batch_size, _past_length]     ) > 0.5

      # decoder inputs
      _UpperCAmelCase				:      Tuple				=  floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]     )
      _UpperCAmelCase				:      Optional[Any]				=  floats_tensor([self.batch_size, config.prediction_length]     )

      _UpperCAmelCase				:      List[Any]				=  {
          "past_values": past_values,
          "static_categorical_features": static_categorical_features,
          "past_time_features": past_time_features,
          "past_observed_mask": past_observed_mask,
          "future_time_features": future_time_features,
          "future_values": future_values,
      }
      return inputs_dict




   def      _lowerCAmelCase						(					self	:						int     )			->       Dict:







      """simple docstring"""




      _UpperCAmelCase				:      Optional[int]				=  self.get_config()
      _UpperCAmelCase				:      Optional[int]				=  self.prepare_autoformer_inputs_dict(lowerCAmelCase__     )
      return config, inputs_dict




   def      _lowerCAmelCase						(					self	:						List[str]     )			->       Any:







      """simple docstring"""




      _UpperCAmelCase				,			_UpperCAmelCase				:      Dict				=  self.prepare_config_and_inputs()
      return config, inputs_dict




   def      _lowerCAmelCase						(					self	:						Optional[int]		,	lowerCAmelCase__	:						Dict		,	lowerCAmelCase__	:						Union[str, Any]     )			->       int:







      """simple docstring"""




      _UpperCAmelCase				:      Optional[Any]				=  AutoformerModel(config=lowerCAmelCase__     ).to(lowerCAmelCase__     ).eval()
      _UpperCAmelCase				:      str				=  model(**lowerCAmelCase__     )

      _UpperCAmelCase				:      List[Any]				=  outputs.encoder_last_hidden_state
      _UpperCAmelCase				:      Optional[int]				=  outputs.last_hidden_state

      with tempfile.TemporaryDirectory() as tmpdirname:
         _UpperCAmelCase				:      Tuple				=  model.get_encoder()
         encoder.save_pretrained(lowerCAmelCase__     )
         _UpperCAmelCase				:      Dict				=  AutoformerEncoder.from_pretrained(lowerCAmelCase__     ).to(lowerCAmelCase__     )

      _UpperCAmelCase				,			_UpperCAmelCase				,			_UpperCAmelCase				,			_UpperCAmelCase				,			_UpperCAmelCase				:      Dict				=  model.create_network_inputs(**lowerCAmelCase__     )
      _UpperCAmelCase				,			_UpperCAmelCase				:      Any				=  model.decomposition_layer(transformer_inputs[:, : config.context_length, ...]     )

      _UpperCAmelCase				:      Tuple				=  torch.cat(
          (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])		,	dim=-1		,	)
      _UpperCAmelCase				:      str				=  encoder(inputs_embeds=lowerCAmelCase__     )[0]
      self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1e-3     )

      _UpperCAmelCase				:      List[str]				=  (
          torch.mean(transformer_inputs[:, : config.context_length, ...]		,	dim=1     )
          .unsqueeze(1     )
          .repeat(1		,	config.prediction_length		,	1     )
      )
      _UpperCAmelCase				:      List[str]				=  torch.zeros(
          [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]		,	device=enc_input.device		,	)

      _UpperCAmelCase				:      str				=  torch.cat(
          (
              torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)		,	dim=1     ),
              feature[:, config.context_length - config.label_length :, ...],
          )		,	dim=-1		,	)
      _UpperCAmelCase				:      Optional[int]				=  torch.cat(
          (
              torch.cat((trend_input[:, -config.label_length :, ...], mean)		,	dim=1     ),
              feature[:, config.context_length - config.label_length :, ...],
          )		,	dim=-1		,	)

      with tempfile.TemporaryDirectory() as tmpdirname:
         _UpperCAmelCase				:      Any				=  model.get_decoder()
         decoder.save_pretrained(lowerCAmelCase__     )
         _UpperCAmelCase				:      Dict				=  AutoformerDecoder.from_pretrained(lowerCAmelCase__     ).to(lowerCAmelCase__     )

      _UpperCAmelCase				:      Optional[int]				=  decoder(
          trend=lowerCAmelCase__		,	inputs_embeds=lowerCAmelCase__		,	encoder_hidden_states=lowerCAmelCase__		,	)[0]

      self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1e-3     )





@require_torch
class        A__				(			_a   ,						_a   ,						unittest.TestCase  ):





   """simple docstring"""





   UpperCamelCase_			:		int       							=	(AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
   UpperCamelCase_			:		Tuple       							=	(AutoformerForPrediction,) if is_torch_available() else ()
   UpperCamelCase_			:		Tuple       							=	{'''feature-extraction''': AutoformerModel} if is_torch_available() else {}
   UpperCamelCase_			:		Optional[int]       							=	False
   UpperCamelCase_			:		List[str]       							=	False
   UpperCamelCase_			:		Optional[Any]       							=	False
   UpperCamelCase_			:		Optional[int]       							=	False
   UpperCamelCase_			:		str       							=	False
   UpperCamelCase_			:		str       							=	False




   def      _lowerCAmelCase						(					self	:						Union[str, Any]     )			->       Dict:







      """simple docstring"""




      _UpperCAmelCase				:      Optional[int]				=  AutoformerModelTester(self     )
      _UpperCAmelCase				:      Union[str, Any]				=  ConfigTester(self		,	config_class=lowerCAmelCase__		,	has_text_modality=lowerCAmelCase__     )




   def      _lowerCAmelCase						(					self	:						Optional[int]     )			->       List[Any]:







      """simple docstring"""




      self.config_tester.run_common_tests()




   def      _lowerCAmelCase						(					self	:						List[str]     )			->       Dict:







      """simple docstring"""




      _UpperCAmelCase				,			_UpperCAmelCase				:      Tuple				=  self.model_tester.prepare_config_and_inputs()
      for model_class in self.all_model_classes:
         _UpperCAmelCase				:      Dict				=  model_class(lowerCAmelCase__     )

         with tempfile.TemporaryDirectory() as tmpdirname:
            model.save_pretrained(lowerCAmelCase__     )
            _UpperCAmelCase				,			_UpperCAmelCase				:      str				=  model_class.from_pretrained(lowerCAmelCase__		,	output_loading_info=lowerCAmelCase__     )
         self.assertEqual(info["missing_keys"]		,	[]     )




   def      _lowerCAmelCase						(					self	:						Optional[int]     )			->       Optional[Any]:







      """simple docstring"""




      _UpperCAmelCase				:      List[str]				=  self.model_tester.prepare_config_and_inputs_for_common()
      self.model_tester.check_encoder_decoder_model_standalone(*lowerCAmelCase__     )




   @unittest.skip(reason="Model has no tokens embeddings"     )
   def      _lowerCAmelCase						(					self	:						int     )			->       Union[str, Any]:







      """simple docstring"""




      pass




   def      _lowerCAmelCase						(					self	:						int     )			->       List[Any]:







      """simple docstring"""




      _UpperCAmelCase				:      Any				=  inspect.signature(getattr(lowerCAmelCase__		,	"forward"     )     )
      # The main input is the name of the argument after `self`
      _UpperCAmelCase				:      List[Any]				=  list(model_signature.parameters.keys()     )[1]
      self.assertEqual(AutoformerModel.main_input_name		,	lowerCAmelCase__     )




   def      _lowerCAmelCase						(					self	:						Tuple     )			->       Dict:







      """simple docstring"""




      _UpperCAmelCase				,			_UpperCAmelCase				:      List[str]				=  self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
         _UpperCAmelCase				:      List[str]				=  model_class(lowerCAmelCase__     )
         _UpperCAmelCase				:      Dict				=  inspect.signature(model.forward     )
         # signature.parameters is an OrderedDict => so arg_names order is deterministic
         _UpperCAmelCase				:      Optional[Any]				=  [*signature.parameters.keys()]

         _UpperCAmelCase				:      List[Any]				=  [
             "past_values",
             "past_time_features",
             "past_observed_mask",
             "static_categorical_features",
             "static_real_features",
             "future_values",
             "future_time_features",
         ]

         if model.__class__.__name__ in ["AutoformerForPrediction"]:
            expected_arg_names.append("future_observed_mask"     )

         expected_arg_names.extend(
             [
                 "decoder_attention_mask",
                 "head_mask",
                 "decoder_head_mask",
                 "cross_attn_head_mask",
                 "encoder_outputs",
                 "past_key_values",
                 "output_hidden_states",
                 "output_attentions",
                 "use_cache",
                 "return_dict",
             ]     )

         self.assertListEqual(arg_names[: len(lowerCAmelCase__     )]		,	lowerCAmelCase__     )




   def      _lowerCAmelCase						(					self	:						List[Any]     )			->       Tuple:







      """simple docstring"""




      _UpperCAmelCase				,			_UpperCAmelCase				:      Optional[int]				=  self.model_tester.prepare_config_and_inputs_for_common()
      _UpperCAmelCase				:      Any				=  True

      _UpperCAmelCase				:      Union[str, Any]				=  getattr(self.model_tester		,	"seq_length"		,	lowerCAmelCase__     )
      _UpperCAmelCase				:      Dict				=  getattr(self.model_tester		,	"decoder_seq_length"		,	lowerCAmelCase__     )
      _UpperCAmelCase				:      int				=  getattr(self.model_tester		,	"encoder_seq_length"		,	lowerCAmelCase__     )
      _UpperCAmelCase				:      Union[str, Any]				=  getattr(self.model_tester		,	"d_model"		,	lowerCAmelCase__     )
      _UpperCAmelCase				:      Optional[int]				=  getattr(self.model_tester		,	"num_attention_heads"		,	lowerCAmelCase__     )
      _UpperCAmelCase				:      Union[str, Any]				=  d_model // num_attention_heads

      for model_class in self.all_model_classes:
         _UpperCAmelCase				:      List[Any]				=  True
         _UpperCAmelCase				:      List[str]				=  False
         _UpperCAmelCase				:      Optional[int]				=  True
         _UpperCAmelCase				:      str				=  model_class(lowerCAmelCase__     )
         model.to(lowerCAmelCase__     )
         model.eval()
         with torch.no_grad():
            _UpperCAmelCase				:      Dict				=  model(**self._prepare_for_class(lowerCAmelCase__		,	lowerCAmelCase__     )     )
         _UpperCAmelCase				:      str				=  outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
         self.assertEqual(len(lowerCAmelCase__     )		,	self.model_tester.num_hidden_layers     )

         # check that output_attentions also work using config
         del inputs_dict["output_attentions"]
         _UpperCAmelCase				:      Dict				=  True
         _UpperCAmelCase				:      Any				=  model_class(lowerCAmelCase__     )
         model.to(lowerCAmelCase__     )
         model.eval()
         with torch.no_grad():
            _UpperCAmelCase				:      Optional[Any]				=  model(**self._prepare_for_class(lowerCAmelCase__		,	lowerCAmelCase__     )     )
         _UpperCAmelCase				:      str				=  outputs.encoder_attentions
         self.assertEqual(len(lowerCAmelCase__     )		,	self.model_tester.num_hidden_layers     )

         self.assertListEqual(
             list(attentions[0].shape[-3:]     )		,	[self.model_tester.num_attention_heads, encoder_seq_length, dim]		,	)
         _UpperCAmelCase				:      Optional[Any]				=  len(lowerCAmelCase__     )

         _UpperCAmelCase				:      List[str]				=  7

         if "last_hidden_state" in outputs:
            correct_outlen += 1

         if "trend" in outputs:
            correct_outlen += 1

         if "past_key_values" in outputs:
            correct_outlen += 1  # past_key_values have been returned

         if "loss" in outputs:
            correct_outlen += 1

         if "params" in outputs:
            correct_outlen += 1

         self.assertEqual(lowerCAmelCase__		,	lowerCAmelCase__     )

         # decoder attentions
         _UpperCAmelCase				:      Any				=  outputs.decoder_attentions
         self.assertIsInstance(lowerCAmelCase__		,	(list, tuple)     )
         self.assertEqual(len(lowerCAmelCase__     )		,	self.model_tester.num_hidden_layers     )
         self.assertListEqual(
             list(decoder_attentions[0].shape[-3:]     )		,	[self.model_tester.num_attention_heads, decoder_seq_length, dim]		,	)

         # cross attentions
         _UpperCAmelCase				:      List[Any]				=  outputs.cross_attentions
         self.assertIsInstance(lowerCAmelCase__		,	(list, tuple)     )
         self.assertEqual(len(lowerCAmelCase__     )		,	self.model_tester.num_hidden_layers     )
         self.assertListEqual(
             list(cross_attentions[0].shape[-3:]     )		,	[self.model_tester.num_attention_heads, decoder_seq_length, dim]		,	)

      # Check attention is always last and order is fine
      _UpperCAmelCase				:      Optional[Any]				=  True
      _UpperCAmelCase				:      Union[str, Any]				=  True
      _UpperCAmelCase				:      Tuple				=  model_class(lowerCAmelCase__     )
      model.to(lowerCAmelCase__     )
      model.eval()
      with torch.no_grad():
         _UpperCAmelCase				:      int				=  model(**self._prepare_for_class(lowerCAmelCase__		,	lowerCAmelCase__     )     )

      self.assertEqual(out_len + 2		,	len(lowerCAmelCase__     )     )

      _UpperCAmelCase				:      Union[str, Any]				=  outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

      self.assertEqual(len(lowerCAmelCase__     )		,	self.model_tester.num_hidden_layers     )
      self.assertListEqual(
          list(self_attentions[0].shape[-3:]     )		,	[self.model_tester.num_attention_heads, encoder_seq_length, dim]		,	)




   @is_flaky()
   def      _lowerCAmelCase						(					self	:						Any     )			->       Tuple:







      """simple docstring"""




      super().test_retain_grad_hidden_states_attentions()






def   __UpperCAmelCase     (    a_:			Union[str, Any]="train-batch.pt"				):
   _UpperCAmelCase				:      Optional[Any]				=  hf_hub_download(repo_id="hf-internal-testing/tourism-monthly-batch",   filename=_snake_case,   repo_type="dataset"				)
   _UpperCAmelCase				:      str				=  torch.load(_snake_case,   map_location=_snake_case				)
   return batch






@require_torch
@slow
class        A__				(			unittest.TestCase  ):





   """simple docstring"""





   def      _lowerCAmelCase						(					self	:						str     )			->       Dict:







      """simple docstring"""




      _UpperCAmelCase				:      Optional[Any]				=  AutoformerModel.from_pretrained("huggingface/autoformer-tourism-monthly"     ).to(lowerCAmelCase__     )
      _UpperCAmelCase				:      int				=  prepare_batch()

      with torch.no_grad():
         _UpperCAmelCase				:      List[str]				=  model(
             past_values=batch["past_values"]		,	past_time_features=batch["past_time_features"]		,	past_observed_mask=batch["past_observed_mask"]		,	static_categorical_features=batch["static_categorical_features"]		,	future_values=batch["future_values"]		,	future_time_features=batch["future_time_features"]		,	)[0]

      _UpperCAmelCase				:      List[str]				=  torch.Size(
          (6_4, model.config.prediction_length + model.config.label_length, model.config.feature_size)     )
      self.assertEqual(output.shape		,	lowerCAmelCase__     )

      _UpperCAmelCase				:      str				=  torch.tensor(
          [[0.3593, -1.3398, 0.6330], [0.2279, 1.5396, -0.1792], [0.0450, 1.3225, -0.2335]]		,	device=lowerCAmelCase__     )
      self.assertTrue(torch.allclose(output[0, :3, :3]		,	lowerCAmelCase__		,	atol=lowerCAmelCase__     )     )




   def      _lowerCAmelCase						(					self	:						Optional[Any]     )			->       List[Any]:







      """simple docstring"""




      _UpperCAmelCase				:      Union[str, Any]				=  AutoformerForPrediction.from_pretrained("huggingface/autoformer-tourism-monthly"     ).to(lowerCAmelCase__     )
      _UpperCAmelCase				:      int				=  prepare_batch("val-batch.pt"     )
      with torch.no_grad():
         _UpperCAmelCase				:      Optional[int]				=  model(
             past_values=batch["past_values"]		,	past_time_features=batch["past_time_features"]		,	past_observed_mask=batch["past_observed_mask"]		,	static_categorical_features=batch["static_categorical_features"]		,	).encoder_last_hidden_state
      _UpperCAmelCase				:      List[str]				=  torch.Size((6_4, model.config.context_length, model.config.d_model)     )
      self.assertEqual(output.shape		,	lowerCAmelCase__     )

      _UpperCAmelCase				:      List[Any]				=  torch.tensor(
          [[-0.0734, -0.9036, 0.8358], [4.7186, 2.4113, 1.9581], [1.7953, 2.3558, 1.2970]]		,	device=lowerCAmelCase__     )
      self.assertTrue(torch.allclose(output[0, :3, :3]		,	lowerCAmelCase__		,	atol=lowerCAmelCase__     )     )




   def      _lowerCAmelCase						(					self	:						Optional[Any]     )			->       Any:







      """simple docstring"""




      _UpperCAmelCase				:      Dict				=  AutoformerForPrediction.from_pretrained("huggingface/autoformer-tourism-monthly"     ).to(lowerCAmelCase__     )
      _UpperCAmelCase				:      int				=  prepare_batch("val-batch.pt"     )
      with torch.no_grad():
         _UpperCAmelCase				:      Any				=  model.generate(
             static_categorical_features=batch["static_categorical_features"]		,	past_time_features=batch["past_time_features"]		,	past_values=batch["past_values"]		,	future_time_features=batch["future_time_features"]		,	past_observed_mask=batch["past_observed_mask"]		,	)
      _UpperCAmelCase				:      List[str]				=  torch.Size((6_4, model.config.num_parallel_samples, model.config.prediction_length)     )
      self.assertEqual(outputs.sequences.shape		,	lowerCAmelCase__     )

      _UpperCAmelCase				:      Union[str, Any]				=  torch.tensor([3_1_3_0.6_7_6_3, 4_0_5_6.5_2_9_3, 7_0_5_3.0_7_8_6]		,	device=lowerCAmelCase__     )
      _UpperCAmelCase				:      Any				=  outputs.sequences.mean(dim=1     )
      self.assertTrue(torch.allclose(mean_prediction[0, -3:]		,	lowerCAmelCase__		,	rtol=1e-1     )     )







import torch

from transformers import AutoModel






class 					SCREAMING_SNAKE_CASE_  (       torch.nn.Module ):
       """simple docstring"""


       def __init__(				self	:Dict,      snake_case	:str="sayef/fsner-bert-base-uncased"):



          """simple docstring"""

          super(snake_case,      self).__init__()

          _lowercase              =AutoModel.from_pretrained(snake_case,      return_dict=snake_case)
          _lowercase              =torch.nn.CosineSimilarity(3,      1e-0_8)
          _lowercase              =torch.nn.Softmax(dim=1)





       def 				UpperCamelCase__						(				self	:str,      **snake_case	:int):



          """simple docstring"""

          return self.bert(**snake_case).last_hidden_state





       def 				UpperCamelCase__						(				self	:Union[str, Any],      snake_case	:Optional[Any]):



          """simple docstring"""

          return token_embeddings.sum(2,      keepdim=snake_case)





       def 				UpperCamelCase__						(				self	:List[Any],      snake_case	:int,      snake_case	:Dict,      snake_case	:Dict=1):



          """simple docstring"""

          return self.softmax(T * self.cos(snake_case,      snake_case))





       def 				UpperCamelCase__						(				self	:List[str],      snake_case	:int,      snake_case	:List[str]):



          """simple docstring"""

          _lowercase              =W_supports['sizes'].tolist()
          _lowercase              =W_supports['start_token_id'].item()
          _lowercase              =W_supports['end_token_id'].item()

          del W_supports["sizes"]
          del W_supports["start_token_id"]
          del W_supports["end_token_id"]

          _lowercase              =self.BERT(**snake_case)
          _lowercase              =self.BERT(**snake_case)

          _lowercase              =None
          _lowercase              =None

          _lowercase              =W_supports['input_ids'] == start_token_id
          _lowercase              =W_supports['input_ids'] == end_token_id

          for i, size in enumerate(snake_case):
             if i == 0:
                _lowercase              =0
             else:
                _lowercase              =support_sizes[i - 1]

             _lowercase              =S[s : s + size][start_token_masks[s : s + size]]
             _lowercase              =S[s : s + size][end_token_masks[s : s + size]]

             _lowercase              =torch.matmul(q[i],      s_start.T).sum(1).softmax(0)
             _lowercase              =torch.matmul(q[i],      s_end.T).sum(1).softmax(0)

             if p_starts is not None:
                _lowercase              =torch.vstack((p_starts, p_start))
                _lowercase              =torch.vstack((p_ends, p_end))
             else:
                _lowercase              =p_start
                _lowercase              =p_end

          return p_starts, p_ends


