
import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__	=							get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
@require_tokenizers
class        lowerCAmelCase__      (   __lowercase		,	unittest.TestCase	):
						UpperCamelCase_							: int	=		XGLMTokenizer
						UpperCamelCase_							: Tuple	=		XGLMTokenizerFast
						UpperCamelCase_							: List[Any]	=		True
						UpperCamelCase_							: Dict	=		True

						def  A_				(			self				)					->      int:


								'''simple docstring'''

								super().setUp()

								# We have a SentencePiece fixture for testing
								_UpperCamelCase						    =	XGLMTokenizer(a       ,							keep_accents=a				)
								tokenizer.save_pretrained(self.tmpdirname				)

						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	"""<pad>"""
								_UpperCamelCase						    =	1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(a				)       ,							a				)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(a				)       ,							a				)

						def  A_				(			self				)					->      str:


								'''simple docstring'''

								_UpperCamelCase						    =	list(self.get_tokenizer().get_vocab().keys()				)

								self.assertEqual(vocab_keys[0]       ,							"""<s>"""				)
								self.assertEqual(vocab_keys[1]       ,							"""<pad>"""				)
								self.assertEqual(len(a				)       ,							10_08				)

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								self.assertEqual(self.get_tokenizer().vocab_size       ,							10_08				)

						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	XGLMTokenizer(a       ,							keep_accents=a				)

								_UpperCamelCase						    =	tokenizer.tokenize("""This is a test"""				)
								self.assertListEqual(a       ,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(a				)       ,							[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]       ,							)

								_UpperCamelCase						    =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
								self.assertListEqual(
								    a       ,							[
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """é""",
								        """.""",
								    ]       ,							)
								_UpperCamelCase						    =	tokenizer.convert_tokens_to_ids(a				)
								self.assertListEqual(
								    a       ,							[
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]       ,							)

								_UpperCamelCase						    =	tokenizer.convert_ids_to_tokens(a				)
								self.assertListEqual(
								    a       ,							[
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """<unk>""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """<unk>""",
								        """.""",
								    ]       ,							)

						@cached_property
						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								return XGLMTokenizer.from_pretrained("""facebook/xglm-564M"""				)

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								with tempfile.NamedTemporaryFile() as f:
										shutil.copyfile(a       ,							f.name				)
										_UpperCamelCase						    =	XGLMTokenizer(f.name       ,							keep_accents=a				)
										_UpperCamelCase						    =	pickle.dumps(a				)
								pickle.loads(a				)

						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								if not self.test_rust_tokenizer:
										return

								_UpperCamelCase						    =	self.get_tokenizer()
								_UpperCamelCase						    =	self.get_rust_tokenizer()

								_UpperCamelCase						    =	"""I was born in 92000, and this is falsé."""

								_UpperCamelCase						    =	tokenizer.tokenize(a				)
								_UpperCamelCase						    =	rust_tokenizer.tokenize(a				)
								self.assertListEqual(a       ,							a				)

								_UpperCamelCase						    =	tokenizer.encode(a       ,							add_special_tokens=a				)
								_UpperCamelCase						    =	rust_tokenizer.encode(a       ,							add_special_tokens=a				)
								self.assertListEqual(a       ,							a				)

								_UpperCamelCase						    =	self.get_rust_tokenizer()
								_UpperCamelCase						    =	tokenizer.encode(a				)
								_UpperCamelCase						    =	rust_tokenizer.encode(a				)
								self.assertListEqual(a       ,							a				)

						@slow
						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	"""Hello World!"""
								_UpperCamelCase						    =	[2, 3_12_27, 44_47, 35]

								self.assertListEqual(a       ,							self.big_tokenizer.encode(a				)				)

						@slow
						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	(
								    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
								    """ add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth"""
								)
								# fmt: off
								_UpperCamelCase						    =	[2, 10_18, 67, 11, 19_88, 26_17, 56_31, 2_78, 11, 34_07, 48, 7_16_30, 2_80_85, 4, 32_34, 1_57, 13, 6, 5, 6, 4, 35_26, 7_68, 15, 6_59, 57, 2_98, 39_83, 8_64, 1_29, 21, 6, 5, 1_36_75, 3_77, 6_52, 75_80, 1_03_41, 1_55, 28_17, 4_22, 16_66, 7, 16_74, 53, 1_13, 20_22_77, 1_78_92, 33, 60, 87, 4, 32_34, 1_57, 61, 26_67, 5_23_76, 19, 88, 23, 7_35]
								# fmt: on

								self.assertListEqual(a       ,							self.big_tokenizer.encode(a				)				)







						@slow
						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	{
								    """input_ids""": [[2, 10_88_25, 11_63, 15, 8_80_10, 4_73, 1_58_98, 1_57, 1_36_72, 18_57, 3_12, 8, 23_80_21, 11_63, 53, 1_36_72, 18_57, 3_12, 8, 5_32_83, 18_23_96, 8, 1_85_66, 16, 3_67_33, 41_01, 8, 2_30, 24_40_17, 12_25_53, 7, 15, 13_25_97, 4, 2_93, 1_25_11, 76_10, 4, 34_14, 13_25_97, 9, 4, 3_23_61, 3_62, 4, 7_34, 2_85_12, 3_25_69, 18, 4, 3_23_61, 2_60_96, 1_49_82, 73, 1_87_15, 2_14_33, 23_52_61, 15, 4_92, 1_24_27, 16, 53, 1_87_15, 2_14_33, 6_54_54, 15, 2_36_59, 5_63, 16, 2_78, 5_97, 28_43, 5_95, 79_31, 18_23_96, 6_41_86, 22, 8_86, 5_95, 13_29_81, 53, 2_55_40, 34_49, 4_39_82, 3_99_01, 59_51, 8_78, 3_30, 4, 2_76_94, 8_02_69, 3_12, 53, 65_17, 1_17_80, 6_11, 2_04_08, 5], [2, 6, 13_25_97, 67, 4_28_97, 33, 5_92, 8, 16_37_29, 2_55_40, 3_61, 13_69_97, 10_95_14, 17_32_30, 7, 5_01, 60, 10_29_13, 1_96, 56_31, 2_35, 6_32_43, 4_73, 6, 23_17_57, 74, 52_77, 79_05, 53, 30_95, 3_73_17, 22, 4_54, 18_38_74, 5], [2, 2_68, 3_12_98, 4_65_30, 6, 13_29_35, 4_38_31, 7, 5_97, 32, 24, 36_88, 98_65, 5]],
								    """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
								}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=a       ,							model_name="""facebook/xglm-564M"""       ,							padding=a       ,							)





import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    NezhaForMaskedLM,
		    NezhaForMultipleChoice,
		    NezhaForNextSentencePrediction,
		    NezhaForPreTraining,
		    NezhaForQuestionAnswering,
		    NezhaForSequenceClassification,
		    NezhaForTokenClassification,
		    NezhaModel,
		)
		from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST







class        lowerCAmelCase__      :

						def __init__(			self       ,							a       ,							a=13       ,							a=7       ,							a=True       ,							a=True       ,							a=True       ,							a=True       ,							a=99       ,							a=32       ,							a=5       ,							a=4       ,							a=37       ,							a="gelu"       ,							a=0.1       ,							a=0.1       ,							a=1_28       ,							a=32       ,							a=16       ,							a=2       ,							a=0.02       ,							a=3       ,							a=4       ,							a=None       ,							)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	parent
								_UpperCamelCase						    =	batch_size
								_UpperCamelCase						    =	seq_length
								_UpperCamelCase						    =	is_training
								_UpperCamelCase						    =	use_input_mask
								_UpperCamelCase						    =	use_token_type_ids
								_UpperCamelCase						    =	use_labels
								_UpperCamelCase						    =	vocab_size
								_UpperCamelCase						    =	hidden_size
								_UpperCamelCase						    =	num_hidden_layers
								_UpperCamelCase						    =	num_attention_heads
								_UpperCamelCase						    =	intermediate_size
								_UpperCamelCase						    =	hidden_act
								_UpperCamelCase						    =	hidden_dropout_prob
								_UpperCamelCase						    =	attention_probs_dropout_prob
								_UpperCamelCase						    =	max_position_embeddings
								_UpperCamelCase						    =	type_vocab_size
								_UpperCamelCase						    =	type_sequence_label_size
								_UpperCamelCase						    =	initializer_range
								_UpperCamelCase						    =	num_labels
								_UpperCamelCase						    =	num_choices
								_UpperCamelCase						    =	scope

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size				)

								_UpperCamelCase						    =	None
								if self.use_input_mask:
										_UpperCamelCase						    =	random_attention_mask([self.batch_size, self.seq_length]				)

								_UpperCamelCase						    =	None
								if self.use_token_type_ids:
										_UpperCamelCase						    =	ids_tensor([self.batch_size, self.seq_length]       ,							self.type_vocab_size				)

								_UpperCamelCase						    =	None
								_UpperCamelCase						    =	None
								_UpperCamelCase						    =	None
								if self.use_labels:
										_UpperCamelCase						    =	ids_tensor([self.batch_size]       ,							self.type_sequence_label_size				)
										_UpperCamelCase						    =	ids_tensor([self.batch_size, self.seq_length]       ,							self.num_labels				)
										_UpperCamelCase						    =	ids_tensor([self.batch_size]       ,							self.num_choices				)

								_UpperCamelCase						    =	self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								return NezhaConfig(
								    vocab_size=self.vocab_size       ,							hidden_size=self.hidden_size       ,							num_hidden_layers=self.num_hidden_layers       ,							num_attention_heads=self.num_attention_heads       ,							intermediate_size=self.intermediate_size       ,							hidden_act=self.hidden_act       ,							hidden_dropout_prob=self.hidden_dropout_prob       ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,							max_position_embeddings=self.max_position_embeddings       ,							type_vocab_size=self.type_vocab_size       ,							is_decoder=a       ,							initializer_range=self.initializer_range       ,							)

						def  A_				(			self				)					->      int:


								'''simple docstring'''

								(
								    (
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	
								)						    =	self.prepare_config_and_inputs()

								_UpperCamelCase						    =	True
								_UpperCamelCase						    =	floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
								_UpperCamelCase						    =	ids_tensor([self.batch_size, self.seq_length]       ,							vocab_size=2				)

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_mask,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								    encoder_hidden_states,
								    encoder_attention_mask,
								)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaModel(config=a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(a       ,							attention_mask=a       ,							token_type_ids=a				)
								_UpperCamelCase						    =	model(a       ,							token_type_ids=a				)
								_UpperCamelCase						    =	model(a				)
								self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)				)
								self.parent.assertEqual(result.pooler_output.shape       ,							(self.batch_size, self.hidden_size)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	True
								_UpperCamelCase						    =	NezhaModel(a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(
								    a       ,							attention_mask=a       ,							token_type_ids=a       ,							encoder_hidden_states=a       ,							encoder_attention_mask=a       ,							)
								_UpperCamelCase						    =	model(
								    a       ,							attention_mask=a       ,							token_type_ids=a       ,							encoder_hidden_states=a       ,							)
								_UpperCamelCase						    =	model(a       ,							attention_mask=a       ,							token_type_ids=a				)
								self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)				)
								self.parent.assertEqual(result.pooler_output.shape       ,							(self.batch_size, self.hidden_size)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaForMaskedLM(config=a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(a       ,							attention_mask=a       ,							token_type_ids=a       ,							labels=a				)
								self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.vocab_size)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaForNextSentencePrediction(config=a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(
								    a       ,							attention_mask=a       ,							token_type_ids=a       ,							labels=a       ,							)
								self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, 2)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaForPreTraining(config=a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(
								    a       ,							attention_mask=a       ,							token_type_ids=a       ,							labels=a       ,							next_sentence_label=a       ,							)
								self.parent.assertEqual(result.prediction_logits.shape       ,							(self.batch_size, self.seq_length, self.vocab_size)				)
								self.parent.assertEqual(result.seq_relationship_logits.shape       ,							(self.batch_size, 2)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaForQuestionAnswering(config=a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(
								    a       ,							attention_mask=a       ,							token_type_ids=a       ,							start_positions=a       ,							end_positions=a       ,							)
								self.parent.assertEqual(result.start_logits.shape       ,							(self.batch_size, self.seq_length)				)
								self.parent.assertEqual(result.end_logits.shape       ,							(self.batch_size, self.seq_length)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.num_labels
								_UpperCamelCase						    =	NezhaForSequenceClassification(a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(a       ,							attention_mask=a       ,							token_type_ids=a       ,							labels=a				)
								self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_labels)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	self.num_labels
								_UpperCamelCase						    =	NezhaForTokenClassification(config=a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	model(a       ,							attention_mask=a       ,							token_type_ids=a       ,							labels=a				)
								self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.num_labels)				)

						def  A_				(			self       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	self.num_choices
								_UpperCamelCase						    =	NezhaForMultipleChoice(config=a				)
								model.to(a				)
								model.eval()
								_UpperCamelCase						    =	input_ids.unsqueeze(1				).expand(-1       ,							self.num_choices       ,							-1				).contiguous()
								_UpperCamelCase						    =	token_type_ids.unsqueeze(1				).expand(-1       ,							self.num_choices       ,							-1				).contiguous()
								_UpperCamelCase						    =	input_mask.unsqueeze(1				).expand(-1       ,							self.num_choices       ,							-1				).contiguous()
								_UpperCamelCase						    =	model(
								    a       ,							attention_mask=a       ,							token_type_ids=a       ,							labels=a       ,							)
								self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_choices)				)







						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								_UpperCamelCase						    =	self.prepare_config_and_inputs()
								(
								    (
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	
								)						    =	config_and_inputs
								_UpperCamelCase						    =	{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
								return config, inputs_dict







@require_torch
class        lowerCAmelCase__      (   __lowercase		,	__lowercase		,	__lowercase		,	unittest.TestCase	):
						UpperCamelCase_							: int	=		(
						    (
						        NezhaModel,
						        NezhaForMaskedLM,
						        NezhaForMultipleChoice,
						        NezhaForNextSentencePrediction,
						        NezhaForPreTraining,
						        NezhaForQuestionAnswering,
						        NezhaForSequenceClassification,
						        NezhaForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						UpperCamelCase_							: Dict	=		(
						    {
						        "feature-extraction": NezhaModel,
						        "fill-mask": NezhaForMaskedLM,
						        "question-answering": NezhaForQuestionAnswering,
						        "text-classification": NezhaForSequenceClassification,
						        "token-classification": NezhaForTokenClassification,
						        "zero-shot": NezhaForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						UpperCamelCase_							: Dict	=		True

						def  A_				(			self       ,							a       ,							a       ,							a=False				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	super()._prepare_for_class(a       ,							a       ,							return_labels=a				)

								if return_labels:
										if model_class in get_values(a				):
												_UpperCamelCase						    =	torch.zeros(
												    (self.model_tester.batch_size, self.model_tester.seq_length)       ,							dtype=torch.long       ,							device=a				)
												_UpperCamelCase						    =	torch.zeros(
												    self.model_tester.batch_size       ,							dtype=torch.long       ,							device=a				)
								return inputs_dict

						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaModelTester(self				)
								_UpperCamelCase						    =	ConfigTester(self       ,							config_class=a       ,							hidden_size=37				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								self.config_tester.run_common_tests()

						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*a				)

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(*a				)

						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								(
								    (
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	(
								    _UpperCamelCase
								)    ,	
								)						    =	self.model_tester.prepare_config_and_inputs_for_decoder()

								_UpperCamelCase						    =	None

								self.model_tester.create_and_check_model_as_decoder(
								    a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							a       ,							)

						def  A_				(			self				)					->      str:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*a				)

						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*a				)

						def  A_				(			self				)					->      str:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_next_sequence_prediction(*a				)

						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_pretraining(*a				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*a				)

						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*a				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*a				)

						@slow
						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										_UpperCamelCase						    =	NezhaModel.from_pretrained(a				)
										self.assertIsNotNone(a				)







						@slow
						@require_torch_gpu
						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase    ,	_UpperCamelCase						    =	self.model_tester.prepare_config_and_inputs_for_common()
								for model_class in self.all_model_classes:
										# NezhaForMultipleChoice behaves incorrectly in JIT environments.
										if model_class == NezhaForMultipleChoice:
												return

										_UpperCamelCase						    =	True
										_UpperCamelCase						    =	model_class(config=a				)

										_UpperCamelCase						    =	self._prepare_for_class(a       ,							a				)
										_UpperCamelCase						    =	torch.jit.trace(
										    a       ,							(inputs_dict["""input_ids"""].to("""cpu"""				), inputs_dict["""attention_mask"""].to("""cpu"""				))				)

										with tempfile.TemporaryDirectory() as tmp:
												torch.jit.save(a       ,							os.path.join(a       ,							"""bert.pt"""				)				)
												_UpperCamelCase						    =	torch.jit.load(os.path.join(a       ,							"""bert.pt"""				)       ,							map_location=a				)
												loaded(inputs_dict["""input_ids"""].to(a				)       ,							inputs_dict["""attention_mask"""].to(a				)				)







@require_torch
class        lowerCAmelCase__      (   unittest.TestCase	):

						@slow
						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaModel.from_pretrained("""sijunhe/nezha-cn-base"""				)
								_UpperCamelCase						    =	torch.tensor([[0, 1, 2, 3, 4, 5]]				)
								_UpperCamelCase						    =	torch.tensor([[0, 1, 1, 1, 1, 1]]				)
								with torch.no_grad():
										_UpperCamelCase						    =	model(a       ,							attention_mask=a				)[0]
								_UpperCamelCase						    =	torch.Size((1, 6, 7_68)				)
								self.assertEqual(output.shape       ,							a				)
								_UpperCamelCase						    =	torch.tensor([[[0.0685, 0.2441, 0.1102], [0.0600, 0.1906, 0.1349], [0.0221, 0.0819, 0.0586]]]				)

								self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,							a       ,							atol=1e-4				)				)







						@slow
						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	NezhaForMaskedLM.from_pretrained("""sijunhe/nezha-cn-base"""				)
								_UpperCamelCase						    =	torch.tensor([[0, 1, 2, 3, 4, 5]]				)
								_UpperCamelCase						    =	torch.tensor([[1, 1, 1, 1, 1, 1]]				)
								with torch.no_grad():
										_UpperCamelCase						    =	model(a       ,							attention_mask=a				)[0]
								_UpperCamelCase						    =	torch.Size((1, 6, 2_11_28)				)
								self.assertEqual(output.shape       ,							a				)
								_UpperCamelCase						    =	torch.tensor(
								    [[-2.7939, -1.7902, -2.2189], [-2.8585, -1.8908, -2.3723], [-2.6499, -1.7750, -2.2558]]				)

								self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,							a       ,							atol=1e-4				)				)



