

'''simple docstring'''






import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     MegatronBertForCausalLM,
     MegatronBertForMaskedLM,
     MegatronBertForMultipleChoice,
     MegatronBertForNextSentencePrediction,
     MegatronBertForPreTraining,
     MegatronBertForQuestionAnswering,
     MegatronBertForSequenceClassification,
     MegatronBertForTokenClassification,
     MegatronBertModel,
 )






class      lowercase_      :




  """simple docstring"""




  def __init__( self  :	Optional[int]   ,lowercase__  :	Tuple   ,lowercase__  :	Optional[Any]=1_3   ,lowercase__  :	Optional[Any]=7   ,lowercase__  :	Tuple=True   ,lowercase__  :	Optional[Any]=True   ,lowercase__  :	Tuple=True   ,lowercase__  :	int=True   ,lowercase__  :	Any=9_9   ,lowercase__  :	Optional[Any]=6_4   ,lowercase__  :	Dict=3_2   ,lowercase__  :	Tuple=5   ,lowercase__  :	Optional[Any]=4   ,lowercase__  :	Union[str, Any]=3_7   ,lowercase__  :	List[str]="gelu"   ,lowercase__  :	Optional[Any]=0.1   ,lowercase__  :	str=0.1   ,lowercase__  :	Optional[Any]=5_1_2   ,lowercase__  :	List[str]=1_6   ,lowercase__  :	Optional[int]=2   ,lowercase__  :	int=0.0_2   ,lowercase__  :	Any=3   ,lowercase__  :	Optional[Any]=4   ,lowercase__  :	List[Any]=None   ,):
   __lowercase				    =					parent
   __lowercase				    =					batch_size
   __lowercase				    =					seq_length
   __lowercase				    =					is_training
   __lowercase				    =					use_input_mask
   __lowercase				    =					use_token_type_ids
   __lowercase				    =					use_labels
   __lowercase				    =					vocab_size
   __lowercase				    =					hidden_size
   __lowercase				    =					embedding_size
   __lowercase				    =					num_hidden_layers
   __lowercase				    =					num_attention_heads
   __lowercase				    =					intermediate_size
   __lowercase				    =					hidden_act
   __lowercase				    =					hidden_dropout_prob
   __lowercase				    =					attention_probs_dropout_prob
   __lowercase				    =					max_position_embeddings
   __lowercase				    =					type_vocab_size
   __lowercase				    =					type_sequence_label_size
   __lowercase				    =					initializer_range
   __lowercase				    =					num_labels
   __lowercase				    =					num_choices
   __lowercase				    =					scope






  def    SCREAMING_SNAKE_CASE ( self  :	str     ):
   __lowercase				    =					ids_tensor([self.batch_size, self.seq_length]   ,self.vocab_size     )

   __lowercase				    =					None
   if self.use_input_mask:
    __lowercase				    =					random_attention_mask([self.batch_size, self.seq_length]     )

   __lowercase				    =					None
   if self.use_token_type_ids:
    __lowercase				    =					ids_tensor([self.batch_size, self.seq_length]   ,self.type_vocab_size     )

   __lowercase				    =					None
   __lowercase				    =					None
   __lowercase				    =					None
   if self.use_labels:
    __lowercase				    =					ids_tensor([self.batch_size]   ,self.type_sequence_label_size     )
    __lowercase				    =					ids_tensor([self.batch_size, self.seq_length]   ,self.num_labels     )
    __lowercase				    =					ids_tensor([self.batch_size]   ,self.num_choices     )

   __lowercase				    =					self.get_config()

   return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






  def    SCREAMING_SNAKE_CASE ( self  :	Union[str, Any]     ):
   return MegatronBertConfig(
       vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,intermediate_size=self.intermediate_size   ,embedding_size=self.embedding_size   ,hidden_act=self.hidden_act   ,hidden_dropout_prob=self.hidden_dropout_prob   ,attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,type_vocab_size=self.type_vocab_size   ,is_decoder=lowercase__   ,initializer_range=self.initializer_range   ,)






  def    SCREAMING_SNAKE_CASE ( self  :	Dict   ,lowercase__  :	List[str]   ,lowercase__  :	List[Any]   ,lowercase__  :	Dict   ,lowercase__  :	Optional[Any]   ,lowercase__  :	Tuple   ,lowercase__  :	Any   ,lowercase__  :	Union[str, Any]     ):
   __lowercase				    =					MegatronBertModel(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__     )
   __lowercase				    =					model(lowercase__   ,token_type_ids=lowercase__     )
   __lowercase				    =					model(lowercase__     )

   self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size)     )
   self.parent.assertEqual(result.pooler_output.shape   ,(self.batch_size, self.hidden_size)     )






  def    SCREAMING_SNAKE_CASE ( self  :	Any   ,lowercase__  :	Any   ,lowercase__  :	Optional[int]   ,lowercase__  :	Any   ,lowercase__  :	str   ,lowercase__  :	str   ,lowercase__  :	Tuple   ,lowercase__  :	int     ):
   __lowercase				    =					MegatronBertForMaskedLM(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,labels=lowercase__     )
   self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)     )






  def    SCREAMING_SNAKE_CASE ( self  :	Union[str, Any]   ,lowercase__  :	List[str]   ,lowercase__  :	Dict   ,lowercase__  :	List[str]   ,lowercase__  :	Union[str, Any]   ,lowercase__  :	Any   ,lowercase__  :	Tuple   ,lowercase__  :	str     ):
   __lowercase				    =					MegatronBertForCausalLM(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,labels=lowercase__     )
   self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)     )






  def    SCREAMING_SNAKE_CASE ( self  :	List[str]   ,lowercase__  :	Optional[Any]   ,lowercase__  :	List[str]   ,lowercase__  :	Optional[int]   ,lowercase__  :	Dict   ,lowercase__  :	str   ,lowercase__  :	Any   ,lowercase__  :	Any     ):
   __lowercase				    =					MegatronBertForNextSentencePrediction(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(
       lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,labels=lowercase__   ,)
   self.parent.assertEqual(result.logits.shape   ,(self.batch_size, 2)     )






  def    SCREAMING_SNAKE_CASE ( self  :	int   ,lowercase__  :	Optional[int]   ,lowercase__  :	Optional[Any]   ,lowercase__  :	str   ,lowercase__  :	Tuple   ,lowercase__  :	List[str]   ,lowercase__  :	List[Any]   ,lowercase__  :	Any     ):
   __lowercase				    =					MegatronBertForPreTraining(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(
       lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,labels=lowercase__   ,next_sentence_label=lowercase__   ,)
   self.parent.assertEqual(result.prediction_logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)     )
   self.parent.assertEqual(result.seq_relationship_logits.shape   ,(self.batch_size, 2)     )






  def    SCREAMING_SNAKE_CASE ( self  :	str   ,lowercase__  :	Any   ,lowercase__  :	Dict   ,lowercase__  :	List[Any]   ,lowercase__  :	Any   ,lowercase__  :	int   ,lowercase__  :	Dict   ,lowercase__  :	Any     ):
   __lowercase				    =					MegatronBertForQuestionAnswering(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(
       lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,start_positions=lowercase__   ,end_positions=lowercase__   ,)
   self.parent.assertEqual(result.start_logits.shape   ,(self.batch_size, self.seq_length)     )
   self.parent.assertEqual(result.end_logits.shape   ,(self.batch_size, self.seq_length)     )






  def    SCREAMING_SNAKE_CASE ( self  :	List[str]   ,lowercase__  :	Optional[int]   ,lowercase__  :	str   ,lowercase__  :	Union[str, Any]   ,lowercase__  :	Optional[int]   ,lowercase__  :	Optional[Any]   ,lowercase__  :	List[Any]   ,lowercase__  :	List[Any]     ):
   __lowercase				    =					self.num_labels
   __lowercase				    =					MegatronBertForSequenceClassification(lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,labels=lowercase__     )
   self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_labels)     )






  def    SCREAMING_SNAKE_CASE ( self  :	Union[str, Any]   ,lowercase__  :	Union[str, Any]   ,lowercase__  :	Tuple   ,lowercase__  :	Optional[Any]   ,lowercase__  :	Tuple   ,lowercase__  :	List[Any]   ,lowercase__  :	str   ,lowercase__  :	Dict     ):
   __lowercase				    =					self.num_labels
   __lowercase				    =					MegatronBertForTokenClassification(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					model(lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,labels=lowercase__     )
   self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.num_labels)     )






  def    SCREAMING_SNAKE_CASE ( self  :	str   ,lowercase__  :	Tuple   ,lowercase__  :	Tuple   ,lowercase__  :	List[Any]   ,lowercase__  :	str   ,lowercase__  :	Tuple   ,lowercase__  :	str   ,lowercase__  :	Dict     ):
   __lowercase				    =					self.num_choices
   __lowercase				    =					MegatronBertForMultipleChoice(config=lowercase__     )
   model.to(lowercase__     )
   model.eval()
   __lowercase				    =					input_ids.unsqueeze(1     ).expand(-1   ,self.num_choices   ,-1     ).contiguous()
   __lowercase				    =					token_type_ids.unsqueeze(1     ).expand(-1   ,self.num_choices   ,-1     ).contiguous()
   __lowercase				    =					input_mask.unsqueeze(1     ).expand(-1   ,self.num_choices   ,-1     ).contiguous()
   __lowercase				    =					model(
       lowercase__   ,attention_mask=lowercase__   ,token_type_ids=lowercase__   ,labels=lowercase__   ,)
   self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.num_choices)     )






  def    SCREAMING_SNAKE_CASE ( self  :	Any     ):
   __lowercase				    =					self.prepare_config_and_inputs()
   (
       (
       __lowercase
   )    ,      (
       __lowercase
   )    ,      (
       __lowercase
   )    ,      (
       __lowercase
   )    ,      (
       __lowercase
   )    ,      (
       __lowercase
   )    ,      (
       __lowercase
   )    ,      
   )				    =					config_and_inputs
   __lowercase				    =					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
   return config, inputs_dict






@require_torch
class      lowercase_      (lowerCamelCase__							,			lowerCamelCase__							,			unittest.TestCase ):




  """simple docstring"""




  SCREAMING_SNAKE_CASE       :    Optional[Any]							     =     (
      (
          MegatronBertModel,
          MegatronBertForMaskedLM,
          MegatronBertForCausalLM,
          MegatronBertForMultipleChoice,
          MegatronBertForNextSentencePrediction,
          MegatronBertForPreTraining,
          MegatronBertForQuestionAnswering,
          MegatronBertForSequenceClassification,
          MegatronBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  SCREAMING_SNAKE_CASE       :    List[Any]							     =     (
      {
          'feature-extraction': MegatronBertModel,
          'fill-mask': MegatronBertForMaskedLM,
          'question-answering': MegatronBertForQuestionAnswering,
          'text-classification': MegatronBertForSequenceClassification,
          'text-generation': MegatronBertForCausalLM,
          'token-classification': MegatronBertForTokenClassification,
          'zero-shot': MegatronBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  SCREAMING_SNAKE_CASE       :    Optional[int]							     =     True
  # test_resize_embeddings = False
  SCREAMING_SNAKE_CASE       :    str							     =     False






  def    SCREAMING_SNAKE_CASE ( self  :	str   ,lowercase__  :	Union[str, Any]   ,lowercase__  :	str   ,lowercase__  :	Dict=False     ):
   __lowercase				    =					super()._prepare_for_class(lowercase__   ,lowercase__   ,return_labels=lowercase__     )

   if return_labels:
    if model_class in get_values(lowercase__     ):
     __lowercase				    =					torch.zeros(
         (self.model_tester.batch_size, self.model_tester.seq_length)   ,dtype=torch.long   ,device=lowercase__     )
     __lowercase				    =					torch.zeros(
         self.model_tester.batch_size   ,dtype=torch.long   ,device=lowercase__     )
   return inputs_dict






  def    SCREAMING_SNAKE_CASE ( self  :	Tuple     ):
   __lowercase				    =					MegatronBertModelTester(self     )
   __lowercase				    =					ConfigTester(self   ,config_class=lowercase__   ,hidden_size=3_7     )






  def    SCREAMING_SNAKE_CASE ( self  :	str     ):
   self.config_tester.run_common_tests()






  def    SCREAMING_SNAKE_CASE ( self  :	Optional[Any]     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_model(*lowercase__     )






  def    SCREAMING_SNAKE_CASE ( self  :	Union[str, Any]     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_for_masked_lm(*lowercase__     )






  def    SCREAMING_SNAKE_CASE ( self  :	int     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*lowercase__     )






  def    SCREAMING_SNAKE_CASE ( self  :	Any     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*lowercase__     )






  def    SCREAMING_SNAKE_CASE ( self  :	Tuple     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_for_pretraining(*lowercase__     )






  def    SCREAMING_SNAKE_CASE ( self  :	List[str]     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_for_question_answering(*lowercase__     )






  def    SCREAMING_SNAKE_CASE ( self  :	Tuple     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*lowercase__     )






  def    SCREAMING_SNAKE_CASE ( self  :	List[Any]     ):
   __lowercase				    =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_megatron_bert_for_token_classification(*lowercase__     )


def 						_A	(       A__ ):






 """simple docstring"""



 return torch.tensor(
     A__	,							dtype=torch.long	,							device=A__	,							)


lowerCAmelCase__   		=      1e-4






@require_torch
@require_sentencepiece
@require_tokenizers
class      lowercase_      (unittest.TestCase ):




  """simple docstring"""




  @slow
  @unittest.skip('''Model is not available.'''     )
  def    SCREAMING_SNAKE_CASE ( self  :	Optional[Any]     ):
   __lowercase				    =					'''nvidia/megatron-bert-uncased-345m'''
   if "MYDIR" in os.environ:
    __lowercase				    =					os.path.join(os.environ['''MYDIR''']   ,lowercase__     )
   __lowercase				    =					MegatronBertModel.from_pretrained(lowercase__     )
   model.to(lowercase__     )
   model.half()
   __lowercase				    =					_long_tensor([[1_0_1, 7_1_1_0, 1_0_0_5, 1_0_5_6, 2_0_2_3, 1_1_3_3_3, 1_7_4_1_3, 1_0_2_9, 1_0_2]]     )
   with torch.no_grad():
    __lowercase				    =					model(lowercase__     )[0]
   __lowercase				    =					torch.Size((1, 9, 1_0_2_4)     )
   self.assertEqual(output.shape   ,lowercase__     )

   __lowercase				    =					[-0.6_0_4_0, -0.2_5_1_7, -0.1_0_2_5, 0.3_4_2_0, -0.6_7_5_8, -0.0_0_1_7, -0.1_0_8_9, -0.1_9_9_0, 0.5_7_2_8]
   for ii in range(3     ):
    for jj in range(3     ):
     __lowercase				    =					output[0, ii, jj]
     __lowercase				    =					expected[3 * ii + jj]
     __lowercase				    =					'''ii={} jj={} a={} b={}'''.format(lowercase__   ,lowercase__   ,lowercase__   ,lowercase__     )
     self.assertTrue(math.isclose(lowercase__   ,lowercase__   ,rel_tol=lowercase__   ,abs_tol=lowercase__     )   ,msg=lowercase__     )








'''simple docstring'''

import jax.numpy as jnp

from ...utils import logging
from ..ta.modeling_flax_ta import FlaxTaEncoderModel, FlaxTaForConditionalGeneration, FlaxTaModel
from .configuration_mta import MTaConfig


lowercase__		:     int			=							logging.get_logger(__name__)

lowercase__		:     Any			=							'T5Config'







def    a__							(							lowercase   :	jnp.array,   lowercase   :	int,   lowercase   :	int )						->						jnp.ndarray:





     """simple docstring"""

     _UpperCamelCase							       =			jnp.zeros_like(lowercase )
     _UpperCamelCase							       =			shifted_input_ids.at[:, 1:].set(input_ids[:, :-1] )
     _UpperCamelCase							       =			shifted_input_ids.at[:, 0].set(lowercase )

     _UpperCamelCase							       =			jnp.where(shifted_input_ids == -100,   lowercase,   lowercase )
     return shifted_input_ids
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       _snake_case      :	Optional[Any]									=       'mt5'
       _snake_case      :	Union[str, Any]									=       MTaConfig

class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       _snake_case      :	Tuple									=       'mt5'
       _snake_case      :	int									=       MTaConfig

class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       _snake_case      :	Optional[int]									=       'mt5'
       _snake_case      :	Optional[Any]									=       MTaConfig



