




"""simple docstring"""

import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           MegatronBertForCausalLM,
           MegatronBertForMaskedLM,
           MegatronBertForMultipleChoice,
           MegatronBertForNextSentencePrediction,
           MegatronBertForPreTraining,
           MegatronBertForQuestionAnswering,
           MegatronBertForSequenceClassification,
           MegatronBertForTokenClassification,
           MegatronBertModel,
       )





class 							lowercase_			:



 '''simple docstring'''

 def __init__(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any=13       ,				_UpperCAmelCase			:			Union[str, Any]=7       ,				_UpperCAmelCase			:			List[Any]=True       ,				_UpperCAmelCase			:			int=True       ,				_UpperCAmelCase			:			List[str]=True       ,				_UpperCAmelCase			:			str=True       ,				_UpperCAmelCase			:			Tuple=99       ,				_UpperCAmelCase			:			Any=64       ,				_UpperCAmelCase			:			Dict=32       ,				_UpperCAmelCase			:			Union[str, Any]=5       ,				_UpperCAmelCase			:			Optional[Any]=4       ,				_UpperCAmelCase			:			int=37       ,				_UpperCAmelCase			:			str="gelu"       ,				_UpperCAmelCase			:			Union[str, Any]=0.1       ,				_UpperCAmelCase			:			str=0.1       ,				_UpperCAmelCase			:			List[Any]=512       ,				_UpperCAmelCase			:			Any=16       ,				_UpperCAmelCase			:			Tuple=2       ,				_UpperCAmelCase			:			Tuple=0.02       ,				_UpperCAmelCase			:			Any=3       ,				_UpperCAmelCase			:			Any=4       ,				_UpperCAmelCase			:			Tuple=None       ,				):
       _A     =  parent
       _A     =  batch_size
       _A     =  seq_length
       _A     =  is_training
       _A     =  use_input_mask
       _A     =  use_token_type_ids
       _A     =  use_labels
       _A     =  vocab_size
       _A     =  hidden_size
       _A     =  embedding_size
       _A     =  num_hidden_layers
       _A     =  num_attention_heads
       _A     =  intermediate_size
       _A     =  hidden_act
       _A     =  hidden_dropout_prob
       _A     =  attention_probs_dropout_prob
       _A     =  max_position_embeddings
       _A     =  type_vocab_size
       _A     =  type_sequence_label_size
       _A     =  initializer_range
       _A     =  num_labels
       _A     =  num_choices
       _A     =  scope

 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       _A     =  ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size						)

       _A     =  None
       if self.use_input_mask:
             _A     =  random_attention_mask([self.batch_size, self.seq_length]						)

       _A     =  None
       if self.use_token_type_ids:
             _A     =  ids_tensor([self.batch_size, self.seq_length]       ,				self.type_vocab_size						)

       _A     =  None
       _A     =  None
       _A     =  None
       if self.use_labels:
             _A     =  ids_tensor([self.batch_size]       ,				self.type_sequence_label_size						)
             _A     =  ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels						)
             _A     =  ids_tensor([self.batch_size]       ,				self.num_choices						)

       _A     =  self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

 def      lowerCAmelCase_				(					self			:			Dict						):
       return MegatronBertConfig(
           vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				embedding_size=self.embedding_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				type_vocab_size=self.type_vocab_size       ,				is_decoder=_UpperCAmelCase       ,				initializer_range=self.initializer_range       ,				)

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[Any]						):
       _A     =  MegatronBertModel(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase						)
       _A     =  model(_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase						)
       _A     =  model(_UpperCAmelCase						)

       self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)						)

 def      lowerCAmelCase_				(					self			:			Any       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Dict       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Optional[int]						):
       _A     =  MegatronBertForMaskedLM(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)						)

 def      lowerCAmelCase_				(					self			:			int       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			int       ,				_UpperCAmelCase			:			int       ,				_UpperCAmelCase			:			int						):
       _A     =  MegatronBertForCausalLM(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)						)

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Any						):
       _A     =  MegatronBertForNextSentencePrediction(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, 2)						)

 def      lowerCAmelCase_				(					self			:			Tuple       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Dict						):
       _A     =  MegatronBertForPreTraining(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase       ,				next_sentence_label=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.prediction_logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)						)
       self.parent.assertEqual(result.seq_relationship_logits.shape       ,				(self.batch_size, 2)						)

 def      lowerCAmelCase_				(					self			:			List[str]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			List[str]						):
       _A     =  MegatronBertForQuestionAnswering(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				start_positions=_UpperCAmelCase       ,				end_positions=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)						)

 def      lowerCAmelCase_				(					self			:			List[Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			str						):
       _A     =  self.num_labels
       _A     =  MegatronBertForSequenceClassification(_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_labels)						)

 def      lowerCAmelCase_				(					self			:			Tuple       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Tuple						):
       _A     =  self.num_labels
       _A     =  MegatronBertForTokenClassification(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)						)

 def      lowerCAmelCase_				(					self			:			List[Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Dict       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any						):
       _A     =  self.num_choices
       _A     =  MegatronBertForMultipleChoice(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  input_ids.unsqueeze(1						).expand(-1       ,				self.num_choices       ,				-1						).contiguous()
       _A     =  token_type_ids.unsqueeze(1						).expand(-1       ,				self.num_choices       ,				-1						).contiguous()
       _A     =  input_mask.unsqueeze(1						).expand(-1       ,				self.num_choices       ,				-1						).contiguous()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_choices)						)



 def      lowerCAmelCase_				(					self			:			Any						):
       _A     =  self.prepare_config_and_inputs()
       (
           (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       
       )     =  config_and_inputs
       _A     =  {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
       return config, inputs_dict





@require_torch
class 							lowercase_			(	__lowerCAmelCase     ,      __lowerCAmelCase     ,      unittest.TestCase				):



 '''simple docstring'''
 UpperCAmelCase       :	Tuple     =      (
     (
         MegatronBertModel,
         MegatronBertForMaskedLM,
         MegatronBertForCausalLM,
         MegatronBertForMultipleChoice,
         MegatronBertForNextSentencePrediction,
         MegatronBertForPreTraining,
         MegatronBertForQuestionAnswering,
         MegatronBertForSequenceClassification,
         MegatronBertForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 UpperCAmelCase       :	Optional[int]     =      (
     {
         '''feature-extraction''': MegatronBertModel,
         '''fill-mask''': MegatronBertForMaskedLM,
         '''question-answering''': MegatronBertForQuestionAnswering,
         '''text-classification''': MegatronBertForSequenceClassification,
         '''text-generation''': MegatronBertForCausalLM,
         '''token-classification''': MegatronBertForTokenClassification,
         '''zero-shot''': MegatronBertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 UpperCAmelCase       :	int     =      True
 # test_resize_embeddings = False
 UpperCAmelCase       :	str     =      False

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[Any]=False						):
       _A     =  super()._prepare_for_class(_UpperCAmelCase       ,				_UpperCAmelCase       ,				return_labels=_UpperCAmelCase						)

       if return_labels:
             if model_class in get_values(_UpperCAmelCase						):
                   _A     =  torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.seq_length)       ,				dtype=torch.long       ,				device=_UpperCAmelCase						)
                   _A     =  torch.zeros(
                       self.model_tester.batch_size       ,				dtype=torch.long       ,				device=_UpperCAmelCase						)
       return inputs_dict

 def      lowerCAmelCase_				(					self			:			Any						):
       _A     =  MegatronBertModelTester(self						)
       _A     =  ConfigTester(self       ,				config_class=_UpperCAmelCase       ,				hidden_size=37						)

 def      lowerCAmelCase_				(					self			:			Optional[int]						):
       self.config_tester.run_common_tests()

 def      lowerCAmelCase_				(					self			:			Dict						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_model(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Dict						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_masked_lm(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Tuple						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[str]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Optional[int]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_pretraining(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[str]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_question_answering(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[Any]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*_UpperCAmelCase						)



 def      lowerCAmelCase_				(					self			:			List[Any]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_token_classification(*_UpperCAmelCase						)






def 		_snake_case	(   _snake_case				:   Any				)       ->   List[Any]:






      '''simple docstring'''


      return torch.tensor(
          _snake_case							,    dtype=torch.long							,    device=_snake_case							,    )


a							      =     1e-4







@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowercase_			(	unittest.TestCase				):



 '''simple docstring'''

 @slow
 @unittest.skip('Model is not available.'						)
 def      lowerCAmelCase_				(					self			:			Optional[int]						):
       _A     =  'nvidia/megatron-bert-uncased-345m'
       if "MYDIR" in os.environ:
             _A     =  os.path.join(os.environ['MYDIR']       ,				_UpperCAmelCase						)
       _A     =  MegatronBertModel.from_pretrained(_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.half()
       _A     =  _long_tensor([[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]						)
       with torch.no_grad():
             _A     =  model(_UpperCAmelCase						)[0]
       _A     =  torch.Size((1, 9, 1_024)						)
       self.assertEqual(output.shape       ,				_UpperCAmelCase						)

       _A     =  [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
       for ii in range(3						):
             for jj in range(3						):
                   _A     =  output[0, ii, jj]
                   _A     =  expected[3 * ii + jj]
                   _A     =  'ii={} jj={} a={} b={}'.format(_UpperCAmelCase       ,				_UpperCAmelCase       ,				_UpperCAmelCase       ,				_UpperCAmelCase						)
                   self.assertTrue(math.isclose(_UpperCAmelCase       ,				_UpperCAmelCase       ,				rel_tol=_UpperCAmelCase       ,				abs_tol=_UpperCAmelCase						)       ,				msg=_UpperCAmelCase						)





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A				:       List[str]      =  {
    'configuration_time_series_transformer': [
        'TIME_SERIES_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'TimeSeriesTransformerConfig',
    ],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	A				:       str      =  [
	    'TIME_SERIES_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'TimeSeriesTransformerForPrediction',
	    'TimeSeriesTransformerModel',
	    'TimeSeriesTransformerPreTrainedModel',
	]


if TYPE_CHECKING:
	from .configuration_time_series_transformer import (
	    TIME_SERIES_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    TimeSeriesTransformerConfig,
	)

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_time_series_transformer import (
		    TIME_SERIES_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TimeSeriesTransformerForPrediction,
		    TimeSeriesTransformerModel,
		    TimeSeriesTransformerPreTrainedModel,
		)

else:
	import sys

	A				:       str      =  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





