




import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    YosoForMaskedLM,
	    YosoForMultipleChoice,
	    YosoForQuestionAnswering,
	    YosoForSequenceClassification,
	    YosoForTokenClassification,
	    YosoModel,
	)
	from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST
class 				__snake_case     :



				def __init__(					self					:						Any						,		_UpperCAmelCase					:						Dict						,		_UpperCAmelCase					:						Dict=13						,		_UpperCAmelCase					:						Union[str, Any]=7						,		_UpperCAmelCase					:						Optional[Any]=True						,		_UpperCAmelCase					:						List[str]=True						,		_UpperCAmelCase					:						Dict=True						,		_UpperCAmelCase					:						str=True						,		_UpperCAmelCase					:						Optional[Any]=99						,		_UpperCAmelCase					:						Any=32						,		_UpperCAmelCase					:						List[str]=5						,		_UpperCAmelCase					:						Optional[int]=4						,		_UpperCAmelCase					:						int=37						,		_UpperCAmelCase					:						Optional[int]="gelu"						,		_UpperCAmelCase					:						Optional[int]=0.1						,		_UpperCAmelCase					:						Tuple=0.1						,		_UpperCAmelCase					:						Any=512						,		_UpperCAmelCase					:						List[str]=16						,		_UpperCAmelCase					:						Any=2						,		_UpperCAmelCase					:						Optional[Any]=0.02						,		_UpperCAmelCase					:						int=3						,		_UpperCAmelCase					:						Union[str, Any]=4						,		_UpperCAmelCase					:						List[Any]=None						,		)     ->			Any:





								'''simple docstring'''




								_lowerCAmelCase			:			int        =					parent
								_lowerCAmelCase			:			List[Any]        =					batch_size
								_lowerCAmelCase			:			Optional[int]        =					seq_length
								_lowerCAmelCase			:			str        =					is_training
								_lowerCAmelCase			:			int        =					use_input_mask
								_lowerCAmelCase			:			Union[str, Any]        =					use_token_type_ids
								_lowerCAmelCase			:			int        =					use_labels
								_lowerCAmelCase			:			Union[str, Any]        =					vocab_size
								_lowerCAmelCase			:			List[str]        =					hidden_size
								_lowerCAmelCase			:			List[str]        =					num_hidden_layers
								_lowerCAmelCase			:			str        =					num_attention_heads
								_lowerCAmelCase			:			Tuple        =					intermediate_size
								_lowerCAmelCase			:			Dict        =					hidden_act
								_lowerCAmelCase			:			List[str]        =					hidden_dropout_prob
								_lowerCAmelCase			:			Tuple        =					attention_probs_dropout_prob
								_lowerCAmelCase			:			Optional[int]        =					max_position_embeddings
								_lowerCAmelCase			:			Optional[Any]        =					type_vocab_size
								_lowerCAmelCase			:			Tuple        =					type_sequence_label_size
								_lowerCAmelCase			:			str        =					initializer_range
								_lowerCAmelCase			:			Dict        =					num_labels
								_lowerCAmelCase			:			Optional[Any]        =					num_choices
								_lowerCAmelCase			:			Optional[Any]        =					scope



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any					)     ->			int:





								'''simple docstring'''




								_lowerCAmelCase			:			List[Any]        =					ids_tensor([self.batch_size, self.seq_length]						,		self.vocab_size					)

								_lowerCAmelCase			:			Optional[Any]        =					None
								if self.use_input_mask:
												_lowerCAmelCase			:			Tuple        =					random_attention_mask([self.batch_size, self.seq_length]					)

								_lowerCAmelCase			:			Dict        =					None
								if self.use_token_type_ids:
												_lowerCAmelCase			:			int        =					ids_tensor([self.batch_size, self.seq_length]						,		self.type_vocab_size					)

								_lowerCAmelCase			:			int        =					None
								_lowerCAmelCase			:			Tuple        =					None
								_lowerCAmelCase			:			Union[str, Any]        =					None
								if self.use_labels:
												_lowerCAmelCase			:			List[str]        =					ids_tensor([self.batch_size]						,		self.type_sequence_label_size					)
												_lowerCAmelCase			:			List[Any]        =					ids_tensor([self.batch_size, self.seq_length]						,		self.num_labels					)
												_lowerCAmelCase			:			Tuple        =					ids_tensor([self.batch_size]						,		self.num_choices					)

								_lowerCAmelCase			:			int        =					self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



				def 					SCREAMING_SNAKE_CASE				(					self					:						int					)     ->			Optional[int]:





								'''simple docstring'''




								return YosoConfig(
								    vocab_size=self.vocab_size						,		hidden_size=self.hidden_size						,		num_hidden_layers=self.num_hidden_layers						,		num_attention_heads=self.num_attention_heads						,		intermediate_size=self.intermediate_size						,		hidden_act=self.hidden_act						,		hidden_dropout_prob=self.hidden_dropout_prob						,		attention_probs_dropout_prob=self.attention_probs_dropout_prob						,		max_position_embeddings=self.max_position_embeddings						,		type_vocab_size=self.type_vocab_size						,		is_decoder=_UpperCAmelCase						,		initializer_range=self.initializer_range						,		)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple					)     ->			Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					self.get_config()
								_lowerCAmelCase			:			Union[str, Any]        =					300
								return config



				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[Any]					)     ->			List[Any]:





								'''simple docstring'''




								(
								    (
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,								
								)			:			List[str]        =					self.prepare_config_and_inputs()

								_lowerCAmelCase			:			Dict        =					True
								_lowerCAmelCase			:			Any        =					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]					)
								_lowerCAmelCase			:			Tuple        =					ids_tensor([self.batch_size, self.seq_length]						,		vocab_size=2					)

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_mask,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								    encoder_hidden_states,
								    encoder_attention_mask,
								)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						Tuple						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						Any					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					YosoModel(config=_UpperCAmelCase					)
								model.to(_UpperCAmelCase					)
								model.eval()
								_lowerCAmelCase			:			List[str]        =					model(_UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase					)
								_lowerCAmelCase			:			Optional[int]        =					model(_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase					)
								_lowerCAmelCase			:			List[str]        =					model(_UpperCAmelCase					)
								self.parent.assertEqual(result.last_hidden_state.shape						,		(self.batch_size, self.seq_length, self.hidden_size)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[int]						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						Tuple						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						Tuple						,		_UpperCAmelCase					:						Optional[Any]						,		_UpperCAmelCase					:						List[Any]						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						Optional[int]						,		)     ->			int:





								'''simple docstring'''




								_lowerCAmelCase			:			Union[str, Any]        =					True
								_lowerCAmelCase			:			List[Any]        =					YosoModel(_UpperCAmelCase					)
								model.to(_UpperCAmelCase					)
								model.eval()
								_lowerCAmelCase			:			Union[str, Any]        =					model(
								    _UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase						,		encoder_hidden_states=_UpperCAmelCase						,		encoder_attention_mask=_UpperCAmelCase						,		)
								_lowerCAmelCase			:			int        =					model(
								    _UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase						,		encoder_hidden_states=_UpperCAmelCase						,		)
								_lowerCAmelCase			:			Union[str, Any]        =					model(_UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase					)
								self.parent.assertEqual(result.last_hidden_state.shape						,		(self.batch_size, self.seq_length, self.hidden_size)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						Union[str, Any]						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Union[str, Any]						,		_UpperCAmelCase					:						Dict						,		_UpperCAmelCase					:						str					)     ->			str:





								'''simple docstring'''




								_lowerCAmelCase			:			Union[str, Any]        =					YosoForMaskedLM(config=_UpperCAmelCase					)
								model.to(_UpperCAmelCase					)
								model.eval()
								_lowerCAmelCase			:			int        =					model(_UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase						,		labels=_UpperCAmelCase					)
								self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.seq_length, self.vocab_size)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						Tuple						,		_UpperCAmelCase					:						Dict						,		_UpperCAmelCase					:						List[str]						,		_UpperCAmelCase					:						Dict						,		_UpperCAmelCase					:						List[str]						,		_UpperCAmelCase					:						Dict					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					YosoForQuestionAnswering(config=_UpperCAmelCase					)
								model.to(_UpperCAmelCase					)
								model.eval()
								_lowerCAmelCase			:			str        =					model(
								    _UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase						,		start_positions=_UpperCAmelCase						,		end_positions=_UpperCAmelCase						,		)
								self.parent.assertEqual(result.start_logits.shape						,		(self.batch_size, self.seq_length)					)
								self.parent.assertEqual(result.end_logits.shape						,		(self.batch_size, self.seq_length)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						int						,		_UpperCAmelCase					:						Optional[Any]						,		_UpperCAmelCase					:						List[Any]						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						Union[str, Any]					)     ->			Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			Union[str, Any]        =					self.num_labels
								_lowerCAmelCase			:			Any        =					YosoForSequenceClassification(_UpperCAmelCase					)
								model.to(_UpperCAmelCase					)
								model.eval()
								_lowerCAmelCase			:			Optional[Any]        =					model(_UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase						,		labels=_UpperCAmelCase					)
								self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.num_labels)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]						,		_UpperCAmelCase					:						Tuple						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Optional[Any]						,		_UpperCAmelCase					:						Union[str, Any]						,		_UpperCAmelCase					:						List[Any]					)     ->			str:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					self.num_labels
								_lowerCAmelCase			:			List[Any]        =					YosoForTokenClassification(config=_UpperCAmelCase					)
								model.to(_UpperCAmelCase					)
								model.eval()
								_lowerCAmelCase			:			Optional[Any]        =					model(_UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase						,		labels=_UpperCAmelCase					)
								self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.seq_length, self.num_labels)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						int						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						Union[str, Any]						,		_UpperCAmelCase					:						Optional[Any]						,		_UpperCAmelCase					:						Union[str, Any]						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						Union[str, Any]					)     ->			List[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					self.num_choices
								_lowerCAmelCase			:			Tuple        =					YosoForMultipleChoice(config=_UpperCAmelCase					)
								model.to(_UpperCAmelCase					)
								model.eval()
								_lowerCAmelCase			:			Any        =					input_ids.unsqueeze(1					).expand(-1						,		self.num_choices						,		-1					).contiguous()
								_lowerCAmelCase			:			str        =					token_type_ids.unsqueeze(1					).expand(-1						,		self.num_choices						,		-1					).contiguous()
								_lowerCAmelCase			:			Optional[int]        =					input_mask.unsqueeze(1					).expand(-1						,		self.num_choices						,		-1					).contiguous()
								_lowerCAmelCase			:			List[str]        =					model(
								    _UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		token_type_ids=_UpperCAmelCase						,		labels=_UpperCAmelCase						,		)
								self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.num_choices)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any					)     ->			str:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[Any]        =					self.prepare_config_and_inputs()
								(
								    (
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,			(
								    _lowerCAmelCase					
								)    ,								
								)			:			Optional[Any]        =					config_and_inputs
								_lowerCAmelCase			:			Union[str, Any]        =					{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
								return config, inputs_dict






@require_torch
class 				__snake_case     (_SCREAMING_SNAKE_CASE  ,		_SCREAMING_SNAKE_CASE  ,		unittest.TestCase						):
				lowerCAmelCase__			=   (
				    (
				        YosoModel,
				        YosoForMaskedLM,
				        YosoForMultipleChoice,
				        YosoForQuestionAnswering,
				        YosoForSequenceClassification,
				        YosoForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				lowerCAmelCase__			=   False
				lowerCAmelCase__			=   False
				lowerCAmelCase__			=   False

				lowerCAmelCase__			=   ()
				lowerCAmelCase__			=   (
				    {
				        "feature-extraction": YosoModel,
				        "fill-mask": YosoForMaskedLM,
				        "question-answering": YosoForQuestionAnswering,
				        "text-classification": YosoForSequenceClassification,
				        "token-classification": YosoForTokenClassification,
				        "zero-shot": YosoForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			str:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[int]        =					YosoModelTester(self					)
								_lowerCAmelCase			:			Optional[Any]        =					ConfigTester(self						,		config_class=_UpperCAmelCase						,		hidden_size=37					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						int					)     ->			int:





								'''simple docstring'''




								self.config_tester.run_common_tests()



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[str]					)     ->			Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]					)     ->			Dict:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[Any]        =					self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
												_lowerCAmelCase			:			List[str]        =					type
												self.model_tester.create_and_check_model(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any					)     ->			str:





								'''simple docstring'''




								_lowerCAmelCase			:			List[Any]        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str					)     ->			str:





								'''simple docstring'''




								_lowerCAmelCase			:			int        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple					)     ->			List[str]:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[int]        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]					)     ->			Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[Any]        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple					)     ->			List[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase					)



				@slow
				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			Union[str, Any]:





								'''simple docstring'''




								for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_lowerCAmelCase			:			List[Any]        =					YosoModel.from_pretrained(_UpperCAmelCase					)
												self.assertIsNotNone(_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple					)     ->			Union[str, Any]:





								'''simple docstring'''




								return






@require_torch
class 				__snake_case     (unittest.TestCase						):



				@slow
				def 					SCREAMING_SNAKE_CASE				(					self					:						str					)     ->			Dict:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[int]        =					YosoModel.from_pretrained("""uw-madison/yoso-4096"""					)
								_lowerCAmelCase			:			List[Any]        =					torch.tensor([[0, 1, 2, 3, 4, 5]]					)

								with torch.no_grad():
												_lowerCAmelCase			:			Dict        =					model(_UpperCAmelCase					)[0]

								_lowerCAmelCase			:			Optional[int]        =					torch.Size((1, 6, 768)					)
								self.assertEqual(output.shape						,		_UpperCAmelCase					)

								_lowerCAmelCase			:			List[Any]        =					torch.tensor(
								    [[[-0.0611, 0.1242, 0.0840], [0.0280, -0.0048, 0.1125], [0.0106, 0.0226, 0.0751]]]					)

								self.assertTrue(torch.allclose(output[:, :3, :3]						,		_UpperCAmelCase						,		atol=1E-4					)					)



				@slow
				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[Any]        =					YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""					)
								_lowerCAmelCase			:			Optional[Any]        =					torch.tensor([[0, 1, 2, 3, 4, 5]]					)

								with torch.no_grad():
												_lowerCAmelCase			:			int        =					model(_UpperCAmelCase					)[0]

								_lowerCAmelCase			:			Optional[int]        =					5_0265

								_lowerCAmelCase			:			List[str]        =					torch.Size((1, 6, vocab_size)					)
								self.assertEqual(output.shape						,		_UpperCAmelCase					)

								_lowerCAmelCase			:			List[Any]        =					torch.tensor(
								    [[[-2.1313, -3.7285, -2.2407], [-2.7047, -3.3314, -2.6408], [0.0629, -2.5166, -0.3356]]]					)

								self.assertTrue(torch.allclose(output[:, :3, :3]						,		_UpperCAmelCase						,		atol=1E-4					)					)



				@slow
				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[int]					)     ->			Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[Any]        =					YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""					)
								_lowerCAmelCase			:			List[str]        =					torch.arange(4096					).unsqueeze(0					)

								with torch.no_grad():
												_lowerCAmelCase			:			Tuple        =					model(_UpperCAmelCase					)[0]

								_lowerCAmelCase			:			Dict        =					5_0265

								_lowerCAmelCase			:			Union[str, Any]        =					torch.Size((1, 4096, vocab_size)					)
								self.assertEqual(output.shape						,		_UpperCAmelCase					)

								_lowerCAmelCase			:			int        =					torch.tensor(
								    [[[-2.3914, -4.3742, -5.0956], [-4.0988, -4.2384, -7.0406], [-3.1427, -3.7192, -6.6800]]]					)

								self.assertTrue(torch.allclose(output[:, :3, :3]						,		_UpperCAmelCase						,		atol=1E-4					)					)







def     __SCREAMING_SNAKE_CASE      (     lowercase_					)				->		bool:

    """simple docstring"""


    if not isinstance(lowercase_    ,   lowercase_					):
        __UpperCamelCase     						=  F"Input value of [number={number}] must be an integer"
        raise TypeError(lowercase_					)
    if number < 0 or number % 2 == 1:
        return False
    __UpperCamelCase     						=  int(number**0.5					)
    return number == number_sqrt * (number_sqrt + 1)


if __name__ == "__main__":
 import doctest

 doctest.testmod()






