


'''simple docstring'''

import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    MegatronBertForCausalLM,
							    MegatronBertForMaskedLM,
							    MegatronBertForMultipleChoice,
							    MegatronBertForNextSentencePrediction,
							    MegatronBertForPreTraining,
							    MegatronBertForQuestionAnswering,
							    MegatronBertForSequenceClassification,
							    MegatronBertForTokenClassification,
							    MegatronBertModel,
							)
class _lowerCamelCase			:



		'''simple docstring'''







		def __init__(   self						:		Optional[int]    ,	_A						:		int    ,	_A						:		str=13    ,	_A						:		Optional[Any]=7    ,	_A						:		Optional[int]=True    ,	_A						:		List[str]=True    ,	_A						:		Tuple=True    ,	_A						:		List[str]=True    ,	_A						:		Optional[Any]=99    ,	_A						:		str=64    ,	_A						:		Tuple=32    ,	_A						:		int=5    ,	_A						:		str=4    ,	_A						:		Dict=37    ,	_A						:		List[Any]="gelu"    ,	_A						:		Optional[Any]=0.1    ,	_A						:		Tuple=0.1    ,	_A						:		Dict=512    ,	_A						:		Tuple=16    ,	_A						:		Union[str, Any]=2    ,	_A						:		int=0.02    ,	_A						:		List[Any]=3    ,	_A						:		List[str]=4    ,	_A						:		Dict=None    ,	)    ->					Optional[int]:
			__magic_name__			:       List[Any]	      =	parent
			__magic_name__			:       Optional[Any]	      =	batch_size
			__magic_name__			:       Dict	      =	seq_length
			__magic_name__			:       str	      =	is_training
			__magic_name__			:       Optional[int]	      =	use_input_mask
			__magic_name__			:       int	      =	use_token_type_ids
			__magic_name__			:       str	      =	use_labels
			__magic_name__			:       Any	      =	vocab_size
			__magic_name__			:       Optional[Any]	      =	hidden_size
			__magic_name__			:       List[str]	      =	embedding_size
			__magic_name__			:       Any	      =	num_hidden_layers
			__magic_name__			:       List[Any]	      =	num_attention_heads
			__magic_name__			:       Optional[Any]	      =	intermediate_size
			__magic_name__			:       List[Any]	      =	hidden_act
			__magic_name__			:       Optional[Any]	      =	hidden_dropout_prob
			__magic_name__			:       Tuple	      =	attention_probs_dropout_prob
			__magic_name__			:       str	      =	max_position_embeddings
			__magic_name__			:       int	      =	type_vocab_size
			__magic_name__			:       Optional[Any]	      =	type_sequence_label_size
			__magic_name__			:       Tuple	      =	initializer_range
			__magic_name__			:       Optional[Any]	      =	num_labels
			__magic_name__			:       List[Any]	      =	num_choices
			__magic_name__			:       Tuple	      =	scope







		def   __lowerCAmelCase						(   self						:		Union[str, Any]     )    ->					Any:
			__magic_name__			:       List[str]	      =	ids_tensor([self.batch_size, self.seq_length]    ,	self.vocab_size     )

			__magic_name__			:       Dict	      =	None
			if self.use_input_mask:
				__magic_name__			:       Optional[int]	      =	random_attention_mask([self.batch_size, self.seq_length]     )

			__magic_name__			:       Union[str, Any]	      =	None
			if self.use_token_type_ids:
				__magic_name__			:       Union[str, Any]	      =	ids_tensor([self.batch_size, self.seq_length]    ,	self.type_vocab_size     )

			__magic_name__			:       str	      =	None
			__magic_name__			:       Optional[int]	      =	None
			__magic_name__			:       str	      =	None
			if self.use_labels:
				__magic_name__			:       Dict	      =	ids_tensor([self.batch_size]    ,	self.type_sequence_label_size     )
				__magic_name__			:       Optional[int]	      =	ids_tensor([self.batch_size, self.seq_length]    ,	self.num_labels     )
				__magic_name__			:       Dict	      =	ids_tensor([self.batch_size]    ,	self.num_choices     )

			__magic_name__			:       Optional[Any]	      =	self.get_config()

			return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







		def   __lowerCAmelCase						(   self						:		str     )    ->					str:
			return MegatronBertConfig(
			    vocab_size=self.vocab_size    ,	hidden_size=self.hidden_size    ,	num_hidden_layers=self.num_hidden_layers    ,	num_attention_heads=self.num_attention_heads    ,	intermediate_size=self.intermediate_size    ,	embedding_size=self.embedding_size    ,	hidden_act=self.hidden_act    ,	hidden_dropout_prob=self.hidden_dropout_prob    ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,	max_position_embeddings=self.max_position_embeddings    ,	type_vocab_size=self.type_vocab_size    ,	is_decoder=_A    ,	initializer_range=self.initializer_range    ,	)







		def   __lowerCAmelCase						(   self						:		Optional[Any]    ,	_A						:		Tuple    ,	_A						:		Tuple    ,	_A						:		Tuple    ,	_A						:		str    ,	_A						:		Tuple    ,	_A						:		Tuple    ,	_A						:		int     )    ->					str:
			__magic_name__			:       Dict	      =	MegatronBertModel(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       Optional[int]	      =	model(_A    ,	attention_mask=_A    ,	token_type_ids=_A     )
			__magic_name__			:       Union[str, Any]	      =	model(_A    ,	token_type_ids=_A     )
			__magic_name__			:       Union[str, Any]	      =	model(_A     )

			self.parent.assertEqual(result.last_hidden_state.shape    ,	(self.batch_size, self.seq_length, self.hidden_size)     )
			self.parent.assertEqual(result.pooler_output.shape    ,	(self.batch_size, self.hidden_size)     )







		def   __lowerCAmelCase						(   self						:		Optional[Any]    ,	_A						:		Any    ,	_A						:		int    ,	_A						:		Any    ,	_A						:		Dict    ,	_A						:		Dict    ,	_A						:		Optional[Any]    ,	_A						:		Tuple     )    ->					str:
			__magic_name__			:       Optional[Any]	      =	MegatronBertForMaskedLM(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       int	      =	model(_A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	labels=_A     )
			self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.seq_length, self.vocab_size)     )







		def   __lowerCAmelCase						(   self						:		List[Any]    ,	_A						:		Dict    ,	_A						:		str    ,	_A						:		Union[str, Any]    ,	_A						:		str    ,	_A						:		Optional[Any]    ,	_A						:		List[str]    ,	_A						:		Any     )    ->					Dict:
			__magic_name__			:       Optional[Any]	      =	MegatronBertForCausalLM(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       Tuple	      =	model(_A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	labels=_A     )
			self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.seq_length, self.vocab_size)     )







		def   __lowerCAmelCase						(   self						:		Dict    ,	_A						:		Any    ,	_A						:		Dict    ,	_A						:		Any    ,	_A						:		List[Any]    ,	_A						:		Any    ,	_A						:		Dict    ,	_A						:		Optional[Any]     )    ->					List[str]:
			__magic_name__			:       int	      =	MegatronBertForNextSentencePrediction(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       Union[str, Any]	      =	model(
			    _A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	labels=_A    ,	)
			self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, 2)     )







		def   __lowerCAmelCase						(   self						:		Tuple    ,	_A						:		Union[str, Any]    ,	_A						:		List[str]    ,	_A						:		str    ,	_A						:		Dict    ,	_A						:		Union[str, Any]    ,	_A						:		int    ,	_A						:		int     )    ->					Any:
			__magic_name__			:       Tuple	      =	MegatronBertForPreTraining(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       Optional[int]	      =	model(
			    _A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	labels=_A    ,	next_sentence_label=_A    ,	)
			self.parent.assertEqual(result.prediction_logits.shape    ,	(self.batch_size, self.seq_length, self.vocab_size)     )
			self.parent.assertEqual(result.seq_relationship_logits.shape    ,	(self.batch_size, 2)     )







		def   __lowerCAmelCase						(   self						:		Optional[Any]    ,	_A						:		Tuple    ,	_A						:		Dict    ,	_A						:		List[str]    ,	_A						:		Optional[Any]    ,	_A						:		Tuple    ,	_A						:		Optional[int]    ,	_A						:		Any     )    ->					Tuple:
			__magic_name__			:       Tuple	      =	MegatronBertForQuestionAnswering(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       Optional[Any]	      =	model(
			    _A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	start_positions=_A    ,	end_positions=_A    ,	)
			self.parent.assertEqual(result.start_logits.shape    ,	(self.batch_size, self.seq_length)     )
			self.parent.assertEqual(result.end_logits.shape    ,	(self.batch_size, self.seq_length)     )







		def   __lowerCAmelCase						(   self						:		Dict    ,	_A						:		Any    ,	_A						:		Tuple    ,	_A						:		str    ,	_A						:		Union[str, Any]    ,	_A						:		Optional[Any]    ,	_A						:		Tuple    ,	_A						:		str     )    ->					Dict:
			__magic_name__			:       List[str]	      =	self.num_labels
			__magic_name__			:       List[str]	      =	MegatronBertForSequenceClassification(_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       Optional[int]	      =	model(_A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	labels=_A     )
			self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.num_labels)     )







		def   __lowerCAmelCase						(   self						:		int    ,	_A						:		Optional[int]    ,	_A						:		List[Any]    ,	_A						:		str    ,	_A						:		Optional[Any]    ,	_A						:		List[str]    ,	_A						:		Optional[Any]    ,	_A						:		List[Any]     )    ->					List[str]:
			__magic_name__			:       Tuple	      =	self.num_labels
			__magic_name__			:       List[Any]	      =	MegatronBertForTokenClassification(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       Tuple	      =	model(_A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	labels=_A     )
			self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.seq_length, self.num_labels)     )







		def   __lowerCAmelCase						(   self						:		Union[str, Any]    ,	_A						:		List[Any]    ,	_A						:		Any    ,	_A						:		Tuple    ,	_A						:		Any    ,	_A						:		Dict    ,	_A						:		Optional[int]    ,	_A						:		List[Any]     )    ->					List[str]:
			__magic_name__			:       Any	      =	self.num_choices
			__magic_name__			:       Any	      =	MegatronBertForMultipleChoice(config=_A     )
			model.to(_A     )
			model.eval()
			__magic_name__			:       List[Any]	      =	input_ids.unsqueeze(1     ).expand(-1    ,	self.num_choices    ,	-1     ).contiguous()
			__magic_name__			:       List[str]	      =	token_type_ids.unsqueeze(1     ).expand(-1    ,	self.num_choices    ,	-1     ).contiguous()
			__magic_name__			:       List[str]	      =	input_mask.unsqueeze(1     ).expand(-1    ,	self.num_choices    ,	-1     ).contiguous()
			__magic_name__			:       Dict	      =	model(
			    _A    ,	attention_mask=_A    ,	token_type_ids=_A    ,	labels=_A    ,	)
			self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.num_choices)     )







		def   __lowerCAmelCase						(   self						:		List[str]     )    ->					int:
			__magic_name__			:       Tuple	      =	self.prepare_config_and_inputs()
			(
			    (
			    __magic_name__			
			)      ,							(
			    __magic_name__			
			)      ,							(
			    __magic_name__			
			)      ,							(
			    __magic_name__			
			)      ,							(
			    __magic_name__			
			)      ,							(
			    __magic_name__			
			)      ,							(
			    __magic_name__			
			)      ,										
			)			:       List[str]	      =	config_and_inputs
			__magic_name__			:       Optional[Any]	      =	{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
			return config, inputs_dict







@require_torch
class _lowerCamelCase			(		_lowercase						,	_lowercase						,	unittest.TestCase				):



		'''simple docstring'''







		A_      :     Any        =							(
		    (
		        MegatronBertModel,
		        MegatronBertForMaskedLM,
		        MegatronBertForCausalLM,
		        MegatronBertForMultipleChoice,
		        MegatronBertForNextSentencePrediction,
		        MegatronBertForPreTraining,
		        MegatronBertForQuestionAnswering,
		        MegatronBertForSequenceClassification,
		        MegatronBertForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		A_      :     Any        =							(
		    {
		        '''feature-extraction''': MegatronBertModel,
		        '''fill-mask''': MegatronBertForMaskedLM,
		        '''question-answering''': MegatronBertForQuestionAnswering,
		        '''text-classification''': MegatronBertForSequenceClassification,
		        '''text-generation''': MegatronBertForCausalLM,
		        '''token-classification''': MegatronBertForTokenClassification,
		        '''zero-shot''': MegatronBertForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		A_      :     Any        =							True
		# test_resize_embeddings = False
		A_      :     Tuple        =							False







		def   __lowerCAmelCase						(   self						:		List[str]    ,	_A						:		Any    ,	_A						:		Union[str, Any]    ,	_A						:		List[str]=False     )    ->					List[Any]:
			__magic_name__			:       str	      =	super()._prepare_for_class(_A    ,	_A    ,	return_labels=_A     )

			if return_labels:
				if model_class in get_values(_A     ):
					__magic_name__			:       List[Any]	      =	torch.zeros(
					    (self.model_tester.batch_size, self.model_tester.seq_length)    ,	dtype=torch.long    ,	device=_A     )
					__magic_name__			:       Tuple	      =	torch.zeros(
					    self.model_tester.batch_size    ,	dtype=torch.long    ,	device=_A     )
			return inputs_dict







		def   __lowerCAmelCase						(   self						:		List[str]     )    ->					Union[str, Any]:
			__magic_name__			:       str	      =	MegatronBertModelTester(self     )
			__magic_name__			:       int	      =	ConfigTester(self    ,	config_class=_A    ,	hidden_size=37     )







		def   __lowerCAmelCase						(   self						:		Any     )    ->					Tuple:
			self.config_tester.run_common_tests()







		def   __lowerCAmelCase						(   self						:		Optional[int]     )    ->					Optional[int]:
			__magic_name__			:       List[Any]	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_model(*_A     )







		def   __lowerCAmelCase						(   self						:		Optional[Any]     )    ->					Dict:
			__magic_name__			:       str	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_for_masked_lm(*_A     )







		def   __lowerCAmelCase						(   self						:		Tuple     )    ->					Dict:
			__magic_name__			:       int	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*_A     )







		def   __lowerCAmelCase						(   self						:		List[Any]     )    ->					int:
			__magic_name__			:       Optional[Any]	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*_A     )







		def   __lowerCAmelCase						(   self						:		Union[str, Any]     )    ->					Any:
			__magic_name__			:       Optional[Any]	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_for_pretraining(*_A     )







		def   __lowerCAmelCase						(   self						:		Optional[int]     )    ->					List[Any]:
			__magic_name__			:       Tuple	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_for_question_answering(*_A     )







		def   __lowerCAmelCase						(   self						:		Dict     )    ->					Tuple:
			__magic_name__			:       Union[str, Any]	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*_A     )







		def   __lowerCAmelCase						(   self						:		Union[str, Any]     )    ->					Optional[Any]:
			__magic_name__			:       Union[str, Any]	      =	self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_megatron_bert_for_token_classification(*_A     )







def lowerCamelCase     (     lowerCAmelCase							:      List[Any]						):





	"""simple docstring"""







	return torch.tensor(
	    __UpperCAmelCase						,						dtype=torch.long						,						device=__UpperCAmelCase						,						)


lowerCAmelCase :str             =     1E-4





@require_torch
@require_sentencepiece
@require_tokenizers
class _lowerCamelCase			(		unittest.TestCase				):



		'''simple docstring'''







		@slow
		@unittest.skip('Model is not available.'     )
		def   __lowerCAmelCase						(   self						:		Tuple     )    ->					int:
			__magic_name__			:       Optional[Any]	      =	'nvidia/megatron-bert-uncased-345m'
			if "MYDIR" in os.environ:
				__magic_name__			:       int	      =	os.path.join(os.environ['MYDIR']    ,	_A     )
			__magic_name__			:       Tuple	      =	MegatronBertModel.from_pretrained(_A     )
			model.to(_A     )
			model.half()
			__magic_name__			:       Union[str, Any]	      =	_long_tensor([[101, 7110, 1005, 1056, 2023, 11333, 17413, 1029, 102]]     )
			with torch.no_grad():
				__magic_name__			:       Any	      =	model(_A     )[0]
			__magic_name__			:       List[Any]	      =	torch.Size((1, 9, 1024)     )
			self.assertEqual(output.shape    ,	_A     )

			__magic_name__			:       str	      =	[-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
			for ii in range(3     ):
				for jj in range(3     ):
					__magic_name__			:       Union[str, Any]	      =	output[0, ii, jj]
					__magic_name__			:       Union[str, Any]	      =	expected[3 * ii + jj]
					__magic_name__			:       Optional[int]	      =	'ii={} jj={} a={} b={}'.format(_A    ,	_A    ,	_A    ,	_A     )
					self.assertTrue(math.isclose(_A    ,	_A    ,	rel_tol=_A    ,	abs_tol=_A     )    ,	msg=_A     )




import math
from collections.abc import Iterator
from itertools import takewhile






def     __a					(		__UpperCAmelCase       ):

 if 1 < number < 4:
  # 2 and 3 are primes
  return True
 elif number < 2 or number % 2 == 0 or number % 3 == 0:
  # Negatives, 0, 1, all even numbers, all multiples of 3 are not primes
  return False

 # All primes number are in format of 6k +/- 1
 for i in range(5							,  int(math.sqrt(__UpperCAmelCase       ) + 1       )							,  6       ):
  if number % i == 0 or number % (i + 2) == 0:
   return False
 return True






def     __a					(		):
 a__												= 2
 while True:
  if is_prime(__UpperCAmelCase       ):
   yield num
  num += 1






def     __a					(		__UpperCAmelCase = 200_0000       ):
 return sum(takewhile(lambda __UpperCAmelCase       : x < n							,  prime_generator()       )       )


if __name__ == "__main__":
     print(f'{solution() = }')





