


'''simple docstring'''



import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      MegatronBertForCausalLM,
      MegatronBertForMaskedLM,
      MegatronBertForMultipleChoice,
      MegatronBertForNextSentencePrediction,
      MegatronBertForPreTraining,
      MegatronBertForQuestionAnswering,
      MegatronBertForSequenceClassification,
      MegatronBertForTokenClassification,
      MegatronBertModel,
  )


class 						_snake_case   :




     def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase=13							,							_lowerCamelCase=7							,							_lowerCamelCase=True							,							_lowerCamelCase=True							,							_lowerCamelCase=True							,							_lowerCamelCase=True							,							_lowerCamelCase=99							,							_lowerCamelCase=64							,							_lowerCamelCase=32							,							_lowerCamelCase=5							,							_lowerCamelCase=4							,							_lowerCamelCase=37							,							_lowerCamelCase="gelu"							,							_lowerCamelCase=0.1							,							_lowerCamelCase=0.1							,							_lowerCamelCase=512							,							_lowerCamelCase=16							,							_lowerCamelCase=2							,							_lowerCamelCase=0.02							,							_lowerCamelCase=3							,							_lowerCamelCase=4							,							_lowerCamelCase=None							,							):
        UpperCAmelCase__					:   List[str]       				=       parent
        UpperCAmelCase__					:   List[Any]       				=       batch_size
        UpperCAmelCase__					:   Optional[Any]       				=       seq_length
        UpperCAmelCase__					:   Tuple       				=       is_training
        UpperCAmelCase__					:   Union[str, Any]       				=       use_input_mask
        UpperCAmelCase__					:   Dict       				=       use_token_type_ids
        UpperCAmelCase__					:   Dict       				=       use_labels
        UpperCAmelCase__					:   Optional[Any]       				=       vocab_size
        UpperCAmelCase__					:   Any       				=       hidden_size
        UpperCAmelCase__					:   List[str]       				=       embedding_size
        UpperCAmelCase__					:   str       				=       num_hidden_layers
        UpperCAmelCase__					:   Union[str, Any]       				=       num_attention_heads
        UpperCAmelCase__					:   str       				=       intermediate_size
        UpperCAmelCase__					:   Union[str, Any]       				=       hidden_act
        UpperCAmelCase__					:   List[str]       				=       hidden_dropout_prob
        UpperCAmelCase__					:   Dict       				=       attention_probs_dropout_prob
        UpperCAmelCase__					:   str       				=       max_position_embeddings
        UpperCAmelCase__					:   Any       				=       type_vocab_size
        UpperCAmelCase__					:   Union[str, Any]       				=       type_sequence_label_size
        UpperCAmelCase__					:   Dict       				=       initializer_range
        UpperCAmelCase__					:   Dict       				=       num_labels
        UpperCAmelCase__					:   List[str]       				=       num_choices
        UpperCAmelCase__					:   str       				=       scope




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Any       				=       ids_tensor([self.batch_size, self.seq_length]							,							self.vocab_size)

        UpperCAmelCase__					:   Any       				=       None
        if self.use_input_mask:
           UpperCAmelCase__					:   Tuple       				=       random_attention_mask([self.batch_size, self.seq_length])

        UpperCAmelCase__					:   Optional[Any]       				=       None
        if self.use_token_type_ids:
           UpperCAmelCase__					:   int       				=       ids_tensor([self.batch_size, self.seq_length]							,							self.type_vocab_size)

        UpperCAmelCase__					:   Union[str, Any]       				=       None
        UpperCAmelCase__					:   List[str]       				=       None
        UpperCAmelCase__					:   Tuple       				=       None
        if self.use_labels:
           UpperCAmelCase__					:   Union[str, Any]       				=       ids_tensor([self.batch_size]							,							self.type_sequence_label_size)
           UpperCAmelCase__					:   Optional[Any]       				=       ids_tensor([self.batch_size, self.seq_length]							,							self.num_labels)
           UpperCAmelCase__					:   List[str]       				=       ids_tensor([self.batch_size]							,							self.num_choices)

        UpperCAmelCase__					:   Optional[int]       				=       self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




     def 			snake_case__  ( self):
        return MegatronBertConfig(
            vocab_size=self.vocab_size							,							hidden_size=self.hidden_size							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							intermediate_size=self.intermediate_size							,							embedding_size=self.embedding_size							,							hidden_act=self.hidden_act							,							hidden_dropout_prob=self.hidden_dropout_prob							,							attention_probs_dropout_prob=self.attention_probs_dropout_prob							,							max_position_embeddings=self.max_position_embeddings							,							type_vocab_size=self.type_vocab_size							,							is_decoder=__A							,							initializer_range=self.initializer_range							,							)




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   Optional[int]       				=       MegatronBertModel(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   str       				=       model(__A							,							attention_mask=__A							,							token_type_ids=__A)
        UpperCAmelCase__					:   Any       				=       model(__A							,							token_type_ids=__A)
        UpperCAmelCase__					:   Optional[int]       				=       model(__A)

        self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size))
        self.parent.assertEqual(result.pooler_output.shape							,							(self.batch_size, self.hidden_size))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   Tuple       				=       MegatronBertForMaskedLM(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   Optional[int]       				=       model(__A							,							attention_mask=__A							,							token_type_ids=__A							,							labels=__A)
        self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   Tuple       				=       MegatronBertForCausalLM(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   Dict       				=       model(__A							,							attention_mask=__A							,							token_type_ids=__A							,							labels=__A)
        self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   Tuple       				=       MegatronBertForNextSentencePrediction(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   Optional[int]       				=       model(
            __A							,							attention_mask=__A							,							token_type_ids=__A							,							labels=__A							,							)
        self.parent.assertEqual(result.logits.shape							,							(self.batch_size, 2))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   Any       				=       MegatronBertForPreTraining(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   List[Any]       				=       model(
            __A							,							attention_mask=__A							,							token_type_ids=__A							,							labels=__A							,							next_sentence_label=__A							,							)
        self.parent.assertEqual(result.prediction_logits.shape							,							(self.batch_size, self.seq_length, self.vocab_size))
        self.parent.assertEqual(result.seq_relationship_logits.shape							,							(self.batch_size, 2))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   List[Any]       				=       MegatronBertForQuestionAnswering(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   Dict       				=       model(
            __A							,							attention_mask=__A							,							token_type_ids=__A							,							start_positions=__A							,							end_positions=__A							,							)
        self.parent.assertEqual(result.start_logits.shape							,							(self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape							,							(self.batch_size, self.seq_length))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   Tuple       				=       self.num_labels
        UpperCAmelCase__					:   str       				=       MegatronBertForSequenceClassification(__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   Optional[int]       				=       model(__A							,							attention_mask=__A							,							token_type_ids=__A							,							labels=__A)
        self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_labels))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   Optional[int]       				=       self.num_labels
        UpperCAmelCase__					:   Any       				=       MegatronBertForTokenClassification(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   List[str]       				=       model(__A							,							attention_mask=__A							,							token_type_ids=__A							,							labels=__A)
        self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.seq_length, self.num_labels))




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase):
        UpperCAmelCase__					:   str       				=       self.num_choices
        UpperCAmelCase__					:   List[Any]       				=       MegatronBertForMultipleChoice(config=__A)
        model.to(__A)
        model.eval()
        UpperCAmelCase__					:   Optional[Any]       				=       input_ids.unsqueeze(1).expand(-1							,							self.num_choices							,							-1).contiguous()
        UpperCAmelCase__					:   Optional[Any]       				=       token_type_ids.unsqueeze(1).expand(-1							,							self.num_choices							,							-1).contiguous()
        UpperCAmelCase__					:   List[str]       				=       input_mask.unsqueeze(1).expand(-1							,							self.num_choices							,							-1).contiguous()
        UpperCAmelCase__					:   List[Any]       				=       model(
            __A							,							attention_mask=__A							,							token_type_ids=__A							,							labels=__A							,							)
        self.parent.assertEqual(result.logits.shape							,							(self.batch_size, self.num_choices))




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Dict       				=       self.prepare_config_and_inputs()
        (
            UpperCAmelCase__						
        )					:   Optional[Any]       				=       config_and_inputs
        UpperCAmelCase__					:   Any       				=       {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict


@require_torch
class 						_snake_case   (       lowerCamelCase__    ,				lowerCamelCase__    ,				unittest.TestCase		):
     lowerCAmelCase :int						=   (
         (
             MegatronBertModel,
             MegatronBertForMaskedLM,
             MegatronBertForCausalLM,
             MegatronBertForMultipleChoice,
             MegatronBertForNextSentencePrediction,
             MegatronBertForPreTraining,
             MegatronBertForQuestionAnswering,
             MegatronBertForSequenceClassification,
             MegatronBertForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     lowerCAmelCase :List[Any]						=   (
         {
             '''feature-extraction''': MegatronBertModel,
             '''fill-mask''': MegatronBertForMaskedLM,
             '''question-answering''': MegatronBertForQuestionAnswering,
             '''text-classification''': MegatronBertForSequenceClassification,
             '''text-generation''': MegatronBertForCausalLM,
             '''token-classification''': MegatronBertForTokenClassification,
             '''zero-shot''': MegatronBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     lowerCAmelCase :Union[str, Any]						=   True
     # test_resize_embeddings = False
     lowerCAmelCase :Any						=   False




     def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase=False):
        UpperCAmelCase__					:   Optional[Any]       				=       super()._prepare_for_class(__A							,							__A							,							return_labels=__A)

        if return_labels:
           if model_class in get_values(__A):
              UpperCAmelCase__					:   Union[str, Any]       				=       torch.zeros(
                  (self.model_tester.batch_size, self.model_tester.seq_length)							,							dtype=torch.long							,							device=__A)
              UpperCAmelCase__					:   Dict       				=       torch.zeros(
                  self.model_tester.batch_size							,							dtype=torch.long							,							device=__A)
        return inputs_dict




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   int       				=       MegatronBertModelTester(self)
        UpperCAmelCase__					:   List[Any]       				=       ConfigTester(self							,							config_class=__A							,							hidden_size=37)




     def 			snake_case__  ( self):
        self.config_tester.run_common_tests()




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Tuple       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_model(*__A)




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   str       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__A)




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Any       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__A)




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   str       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__A)




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Optional[int]       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_pretraining(*__A)




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Any       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_question_answering(*__A)




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Union[str, Any]       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__A)




     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Union[str, Any]       				=       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_megatron_bert_for_token_classification(*__A)







def       _UpperCamelCase (						UpperCamelCase__   ):
   return torch.tensor(
       UpperCamelCase__	, dtype=torch.long	, device=UpperCamelCase__	, )


__A		=1E-4


@require_torch
@require_sentencepiece
@require_tokenizers
class 						_snake_case   (       unittest.TestCase		):




     @slow
     @unittest.skip("""Model is not available.""")
     def 			snake_case__  ( self):
        UpperCAmelCase__					:   Union[str, Any]       				=       '''nvidia/megatron-bert-uncased-345m'''
        if "MYDIR" in os.environ:
           UpperCAmelCase__					:   int       				=       os.path.join(os.environ["""MYDIR"""]							,							__A)
        UpperCAmelCase__					:   Any       				=       MegatronBertModel.from_pretrained(__A)
        model.to(__A)
        model.half()
        UpperCAmelCase__					:   str       				=       _long_tensor([[101, 7110, 1005, 1056, 2023, 1_1333, 1_7413, 1029, 102]])
        with torch.no_grad():
           UpperCAmelCase__					:   Optional[Any]       				=       model(__A)[0]
        UpperCAmelCase__					:   str       				=       torch.Size((1, 9, 1024))
        self.assertEqual(output.shape							,							__A)

        UpperCAmelCase__					:   Dict       				=       [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
        for ii in range(3):
           for jj in range(3):
              UpperCAmelCase__					:   List[Any]       				=       output[0, ii, jj]
              UpperCAmelCase__					:   List[str]       				=       expected[3 * ii + jj]
              UpperCAmelCase__					:   str       				=       '''ii={} jj={} a={} b={}'''.format(__A							,							__A							,							__A							,							__A)
              self.assertTrue(math.isclose(__A							,							__A							,							rel_tol=__A							,							abs_tol=__A)							,							msg=__A)



'''simple docstring'''



def       _UpperCamelCase (						UpperCamelCase__ = 4_0_0_0_0_0_0   ):
			UpperCAmelCase__					:   List[str]       				=       [0, 1]
			UpperCAmelCase__					:   Any       				=       0
			while fib[i] <= n:
						fib.append(fib[i] + fib[i + 1]   )
						if fib[i + 2] > n:
									break
						i += 1
			UpperCAmelCase__					:   str       				=       0
			for j in range(len(UpperCamelCase__   ) - 1   ):
						if fib[j] % 2 == 0:
									total += fib[j]

			return total


if __name__ == "__main__":
		print(f"""{solution() = }""")