




import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    XLMForMultipleChoice,
	    XLMForQuestionAnswering,
	    XLMForQuestionAnsweringSimple,
	    XLMForSequenceClassification,
	    XLMForTokenClassification,
	    XLMModel,
	    XLMWithLMHeadModel,
	)
	from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST



class 							SCREAMING_SNAKE_CASE__						:





					"""simple docstring"""






					def __init__( self					:  str							,		__A					:  List[Any]							,		__A					:  List[str]=1_3							,		__A					:  Dict=7							,		__A					:  Dict=True							,		__A					:  Dict=True							,		__A					:  int=True							,		__A					:  Any=True							,		__A					:  Optional[Any]=True							,		__A					:  int=False							,		__A					:  Dict=False							,		__A					:  Dict=False							,		__A					:  Dict=2							,		__A					:  Dict=9_9							,		__A					:  Any=0							,		__A					:  Optional[Any]=3_2							,		__A					:  List[str]=5							,		__A					:  str=4							,		__A					:  int=0.1							,		__A					:  int=0.1							,		__A					:  str=5_1_2							,		__A					:  Tuple=2							,		__A					:  Tuple=0.0_2							,		__A					:  Union[str, Any]=2							,		__A					:  Optional[int]=4							,		__A					:  List[Any]="last"							,		__A					:  List[Any]=True							,		__A					:  Dict=None							,		__A					:  str=0							,		):
						snake_case__			:				int       =  parent
						snake_case__			:				str       =  batch_size
						snake_case__			:				int       =  seq_length
						snake_case__			:				Tuple       =  is_training
						snake_case__			:				Any       =  use_input_lengths
						snake_case__			:				Optional[Any]       =  use_token_type_ids
						snake_case__			:				Optional[int]       =  use_labels
						snake_case__			:				int       =  gelu_activation
						snake_case__			:				Tuple       =  sinusoidal_embeddings
						snake_case__			:				Dict       =  causal
						snake_case__			:				Optional[Any]       =  asm
						snake_case__			:				Any       =  n_langs
						snake_case__			:				Any       =  vocab_size
						snake_case__			:				Union[str, Any]       =  n_special
						snake_case__			:				Any       =  hidden_size
						snake_case__			:				Optional[Any]       =  num_hidden_layers
						snake_case__			:				List[Any]       =  num_attention_heads
						snake_case__			:				Optional[int]       =  hidden_dropout_prob
						snake_case__			:				Any       =  attention_probs_dropout_prob
						snake_case__			:				List[str]       =  max_position_embeddings
						snake_case__			:				Union[str, Any]       =  type_sequence_label_size
						snake_case__			:				Optional[Any]       =  initializer_range
						snake_case__			:				Union[str, Any]       =  num_labels
						snake_case__			:				List[Any]       =  num_choices
						snake_case__			:				Optional[int]       =  summary_type
						snake_case__			:				Optional[int]       =  use_proj
						snake_case__			:				Optional[int]       =  scope
						snake_case__			:				int       =  bos_token_id



					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				Any       =  ids_tensor([self.batch_size, self.seq_length]							,		self.vocab_size	)
						snake_case__			:				str       =  random_attention_mask([self.batch_size, self.seq_length]	)

						snake_case__			:				Union[str, Any]       =  None
						if self.use_input_lengths:
							snake_case__			:				List[Any]       =  (
							    ids_tensor([self.batch_size]							,		vocab_size=2	) + self.seq_length - 2
							)  # small variation of seq_length

						snake_case__			:				List[Any]       =  None
						if self.use_token_type_ids:
							snake_case__			:				int       =  ids_tensor([self.batch_size, self.seq_length]							,		self.n_langs	)

						snake_case__			:				int       =  None
						snake_case__			:				Dict       =  None
						snake_case__			:				Union[str, Any]       =  None
						if self.use_labels:
							snake_case__			:				str       =  ids_tensor([self.batch_size]							,		self.type_sequence_label_size	)
							snake_case__			:				int       =  ids_tensor([self.batch_size, self.seq_length]							,		self.num_labels	)
							snake_case__			:				Union[str, Any]       =  ids_tensor([self.batch_size]							,		2	).float()
							snake_case__			:				str       =  ids_tensor([self.batch_size]							,		self.num_choices	)

						snake_case__			:				Optional[Any]       =  self.get_config()

						return (
						    config,
						    input_ids,
						    token_type_ids,
						    input_lengths,
						    sequence_labels,
						    token_labels,
						    is_impossible_labels,
						    choice_labels,
						    input_mask,
						)



					def 	_lowercase					( self					:  List[Any]	):
						return XLMConfig(
						    vocab_size=self.vocab_size							,		n_special=self.n_special							,		emb_dim=self.hidden_size							,		n_layers=self.num_hidden_layers							,		n_heads=self.num_attention_heads							,		dropout=self.hidden_dropout_prob							,		attention_dropout=self.attention_probs_dropout_prob							,		gelu_activation=self.gelu_activation							,		sinusoidal_embeddings=self.sinusoidal_embeddings							,		asm=self.asm							,		causal=self.causal							,		n_langs=self.n_langs							,		max_position_embeddings=self.max_position_embeddings							,		initializer_range=self.initializer_range							,		summary_type=self.summary_type							,		use_proj=self.use_proj							,		num_labels=self.num_labels							,		bos_token_id=self.bos_token_id							,		)



					def 	_lowercase					( self					:  str							,		__A					:  str							,		__A					:  Optional[Any]							,		__A					:  Any							,		__A					:  Any							,		__A					:  int							,		__A					:  Tuple							,		__A					:  int							,		__A					:  Optional[Any]							,		__A					:  Tuple							,		):
						snake_case__			:				Optional[int]       =  XLMModel(config=__A	)
						model.to(__A	)
						model.eval()
						snake_case__			:				List[str]       =  model(__A							,		lengths=__A							,		langs=__A	)
						snake_case__			:				Optional[int]       =  model(__A							,		langs=__A	)
						snake_case__			:				int       =  model(__A	)
						self.parent.assertEqual(result.last_hidden_state.shape							,		(self.batch_size, self.seq_length, self.hidden_size)	)



					def 	_lowercase					( self					:  str							,		__A					:  Dict							,		__A					:  Optional[int]							,		__A					:  int							,		__A					:  Optional[int]							,		__A					:  Optional[Any]							,		__A					:  List[Any]							,		__A					:  str							,		__A					:  Any							,		__A					:  Tuple							,		):
						snake_case__			:				List[str]       =  XLMWithLMHeadModel(__A	)
						model.to(__A	)
						model.eval()

						snake_case__			:				List[str]       =  model(__A							,		token_type_ids=__A							,		labels=__A	)
						self.parent.assertEqual(result.loss.shape							,		()	)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.seq_length, self.vocab_size)	)



					def 	_lowercase					( self					:  int							,		__A					:  Optional[Any]							,		__A					:  Any							,		__A					:  List[str]							,		__A					:  Union[str, Any]							,		__A					:  str							,		__A					:  str							,		__A					:  int							,		__A					:  Dict							,		__A					:  Union[str, Any]							,		):
						snake_case__			:				Union[str, Any]       =  XLMForQuestionAnsweringSimple(__A	)
						model.to(__A	)
						model.eval()

						snake_case__			:				Any       =  model(__A	)

						snake_case__			:				int       =  model(__A							,		start_positions=__A							,		end_positions=__A	)
						snake_case__			:				int       =  outputs
						self.parent.assertEqual(result.start_logits.shape							,		(self.batch_size, self.seq_length)	)
						self.parent.assertEqual(result.end_logits.shape							,		(self.batch_size, self.seq_length)	)



					def 	_lowercase					( self					:  Optional[int]							,		__A					:  List[Any]							,		__A					:  int							,		__A					:  str							,		__A					:  str							,		__A					:  List[str]							,		__A					:  Optional[Any]							,		__A					:  Tuple							,		__A					:  List[Any]							,		__A					:  Dict							,		):
						snake_case__			:				Optional[Any]       =  XLMForQuestionAnswering(__A	)
						model.to(__A	)
						model.eval()

						snake_case__			:				Dict       =  model(__A	)

						snake_case__			:				Tuple       =  model(
						    __A							,		start_positions=__A							,		end_positions=__A							,		cls_index=__A							,		is_impossible=__A							,		p_mask=__A							,		)

						snake_case__			:				List[str]       =  model(
						    __A							,		start_positions=__A							,		end_positions=__A							,		cls_index=__A							,		is_impossible=__A							,		)

						(snake_case__						)			:				Union[str, Any]       =  result_with_labels.to_tuple()

						snake_case__			:				Union[str, Any]       =  model(__A							,		start_positions=__A							,		end_positions=__A	)

						(snake_case__						)			:				List[Any]       =  result_with_labels.to_tuple()

						self.parent.assertEqual(result_with_labels.loss.shape							,		()	)
						self.parent.assertEqual(result.start_top_log_probs.shape							,		(self.batch_size, model.config.start_n_top)	)
						self.parent.assertEqual(result.start_top_index.shape							,		(self.batch_size, model.config.start_n_top)	)
						self.parent.assertEqual(
						    result.end_top_log_probs.shape							,		(self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
						self.parent.assertEqual(
						    result.end_top_index.shape							,		(self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
						self.parent.assertEqual(result.cls_logits.shape							,		(self.batch_size,)	)



					def 	_lowercase					( self					:  int							,		__A					:  List[Any]							,		__A					:  Optional[Any]							,		__A					:  List[Any]							,		__A					:  Dict							,		__A					:  Tuple							,		__A					:  List[str]							,		__A					:  Tuple							,		__A					:  Any							,		__A					:  Optional[int]							,		):
						snake_case__			:				Union[str, Any]       =  XLMForSequenceClassification(__A	)
						model.to(__A	)
						model.eval()

						snake_case__			:				List[Any]       =  model(__A	)
						snake_case__			:				Any       =  model(__A							,		labels=__A	)
						self.parent.assertEqual(result.loss.shape							,		()	)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.type_sequence_label_size)	)



					def 	_lowercase					( self					:  Optional[Any]							,		__A					:  Any							,		__A					:  Optional[Any]							,		__A					:  Union[str, Any]							,		__A					:  Dict							,		__A					:  Any							,		__A					:  List[Any]							,		__A					:  Dict							,		__A					:  List[str]							,		__A					:  str							,		):
						snake_case__			:				int       =  self.num_labels
						snake_case__			:				Any       =  XLMForTokenClassification(__A	)
						model.to(__A	)
						model.eval()

						snake_case__			:				str       =  model(__A							,		attention_mask=__A							,		labels=__A	)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.seq_length, self.num_labels)	)



					def 	_lowercase					( self					:  Dict							,		__A					:  Optional[Any]							,		__A					:  Optional[int]							,		__A					:  Dict							,		__A					:  List[str]							,		__A					:  Dict							,		__A					:  Union[str, Any]							,		__A					:  Any							,		__A					:  str							,		__A					:  List[str]							,		):
						snake_case__			:				str       =  self.num_choices
						snake_case__			:				Optional[Any]       =  XLMForMultipleChoice(config=__A	)
						model.to(__A	)
						model.eval()
						snake_case__			:				Any       =  input_ids.unsqueeze(1	).expand(-1							,		self.num_choices							,		-1	).contiguous()
						snake_case__			:				str       =  token_type_ids.unsqueeze(1	).expand(-1							,		self.num_choices							,		-1	).contiguous()
						snake_case__			:				Optional[Any]       =  input_mask.unsqueeze(1	).expand(-1							,		self.num_choices							,		-1	).contiguous()
						snake_case__			:				Tuple       =  model(
						    __A							,		attention_mask=__A							,		token_type_ids=__A							,		labels=__A							,		)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.num_choices)	)






					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				Any       =  self.prepare_config_and_inputs()
						(
						    snake_case__						
						)			:				List[Any]       =  config_and_inputs
						snake_case__			:				Optional[Any]       =  {"input_ids": input_ids, "token_type_ids": token_type_ids, "lengths": input_lengths}
						return config, inputs_dict



@require_torch
class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_			,       UpperCamelCase_			,       UpperCamelCase_			,       unittest.TestCase						):





					"""simple docstring"""






					a_     =		(
					    (
					        XLMModel,
					        XLMWithLMHeadModel,
					        XLMForQuestionAnswering,
					        XLMForSequenceClassification,
					        XLMForQuestionAnsweringSimple,
					        XLMForTokenClassification,
					        XLMForMultipleChoice,
					    )
					    if is_torch_available()
					    else ()
					)
					a_     =		(
					    (XLMWithLMHeadModel,) if is_torch_available() else ()
					)  # TODO (PVP): Check other models whether language generation is also applicable
					a_     =		(
					    {
					        "feature-extraction": XLMModel,
					        "fill-mask": XLMWithLMHeadModel,
					        "question-answering": XLMForQuestionAnsweringSimple,
					        "text-classification": XLMForSequenceClassification,
					        "text-generation": XLMWithLMHeadModel,
					        "token-classification": XLMForTokenClassification,
					        "zero-shot": XLMForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)



					def 	_lowercase					( self					:  List[Any]							,		__A					:  str							,		__A					:  Optional[int]							,		__A					:  Dict							,		__A					:  str							,		__A					:  List[Any]	):
						if (
						    pipeline_test_casse_name == "QAPipelineTests"
						    and tokenizer_name is not None
						    and not tokenizer_name.endswith("Fast"	)
						):
							# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
							# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
							# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
							return True

						return False



					def 	_lowercase					( self					:  Any							,		__A					:  Dict							,		__A					:  Any							,		__A					:  Tuple=False	):
						snake_case__			:				List[Any]       =  super()._prepare_for_class(__A							,		__A							,		return_labels=__A	)

						if return_labels:
							if model_class.__name__ == "XLMForQuestionAnswering":
								snake_case__			:				Optional[Any]       =  torch.zeros(
								    self.model_tester.batch_size							,		dtype=torch.long							,		device=__A	)
								snake_case__			:				Union[str, Any]       =  torch.zeros(
								    self.model_tester.batch_size							,		dtype=torch.long							,		device=__A	)

						return inputs_dict



					def 	_lowercase					( self					:  Any	):
						snake_case__			:				Optional[int]       =  XLMModelTester(self	)
						snake_case__			:				Optional[Any]       =  ConfigTester(self							,		config_class=__A							,		emb_dim=3_7	)



					def 	_lowercase					( self					:  List[str]	):
						self.config_tester.run_common_tests()



					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				str       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_xlm_model(*__A	)



					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				Any       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_xlm_lm_head(*__A	)



					def 	_lowercase					( self					:  Tuple	):
						snake_case__			:				List[Any]       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_xlm_simple_qa(*__A	)



					def 	_lowercase					( self					:  Tuple	):
						snake_case__			:				Dict       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_xlm_qa(*__A	)



					def 	_lowercase					( self					:  int	):
						snake_case__			:				Union[str, Any]       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_xlm_sequence_classif(*__A	)



					def 	_lowercase					( self					:  Dict	):
						snake_case__			:				Optional[int]       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_xlm_token_classif(*__A	)



					def 	_lowercase					( self					:  Any	):
						snake_case__			:				List[Any]       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_xlm_for_multiple_choice(*__A	)



					def 	_lowercase					( self					:  Union[str, Any]							,		__A					:  Any							,		__A					:  List[str]							,		__A					:  str							,		__A					:  str							,		__A					:  str							,		__A					:  Tuple=False							,		__A					:  Optional[int]=1	):
						self.assertIsInstance(__A							,		__A	)
						self.assertListEqual(
						    [isinstance(__A							,		__A	) for iter_attentions in attentions]							,		[True] * len(__A	)	)
						self.assertEqual(len(__A	)							,		(max_length - min_length) * num_beam_groups	)

						for idx, iter_attentions in enumerate(__A	):
							# adds PAD dummy token
							snake_case__			:				List[str]       =  min_length + idx + 1
							snake_case__			:				Optional[int]       =  min_length + idx + 1

							snake_case__			:				Tuple       =  (
							    batch_size * num_beam_groups,
							    config.num_attention_heads,
							    tgt_len,
							    src_len,
							)
							# check attn size
							self.assertListEqual(
							    [layer_attention.shape for layer_attention in iter_attentions]							,		[expected_shape] * len(__A	)	)



					def 	_lowercase					( self					:  Tuple							,		__A					:  Any							,		__A					:  List[str]							,		__A					:  Any							,		__A					:  Tuple							,		__A					:  Optional[Any]							,		__A					:  str=False							,		__A					:  int=1	):
						self.assertIsInstance(__A							,		__A	)
						self.assertListEqual(
						    [isinstance(__A							,		__A	) for iter_hidden_states in hidden_states]							,		[True] * len(__A	)							,		)
						self.assertEqual(len(__A	)							,		(max_length - min_length) * num_beam_groups	)

						for idx, iter_hidden_states in enumerate(__A	):
							# adds PAD dummy token
							snake_case__			:				Union[str, Any]       =  min_length + idx + 1
							snake_case__			:				Tuple       =  (batch_size * num_beam_groups, seq_len, config.hidden_size)
							# check hidden size
							self.assertListEqual(
							    [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]							,		[expected_shape] * len(__A	)							,		)
						pass






					@slow
					def 	_lowercase					( self					:  int	):
						for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							snake_case__			:				List[str]       =  XLMModel.from_pretrained(__A	)
							self.assertIsNotNone(__A	)



@require_torch
class 							SCREAMING_SNAKE_CASE__						(    unittest.TestCase						):





					"""simple docstring"""






					@slow
					def 	_lowercase					( self					:  List[Any]	):
						snake_case__			:				Optional[int]       =  XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"	)
						model.to(__A	)
						snake_case__			:				Union[str, Any]       =  torch.tensor([[1_4, 4_4_7]]							,		dtype=torch.long							,		device=__A	)  # the president
						snake_case__			:				int       =  [
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						    1_4,
						    4_4_7,
						]  # the president the president the president the president the president the president the president the president the president the president
						# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
						snake_case__			:				Tuple       =  model.generate(__A							,		do_sample=__A	)
						self.assertListEqual(output_ids[0].cpu().numpy().tolist()							,		__A	)











from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import (
	    TFConvBertForMaskedLM,
	    TFConvBertForMultipleChoice,
	    TFConvBertForQuestionAnswering,
	    TFConvBertForSequenceClassification,
	    TFConvBertForTokenClassification,
	    TFConvBertModel,
	)



class 							SCREAMING_SNAKE_CASE__						:





					"""simple docstring"""






					def __init__( self					:  Union[str, Any]							,		__A					:  str							,		__A					:  Optional[Any]=1_3							,		__A					:  Dict=7							,		__A					:  List[str]=True							,		__A					:  Any=True							,		__A					:  str=True							,		__A					:  Optional[Any]=True							,		__A					:  List[str]=9_9							,		__A					:  Dict=3_2							,		__A					:  Tuple=2							,		__A					:  Tuple=4							,		__A					:  Dict=3_7							,		__A					:  Tuple="gelu"							,		__A					:  Any=0.1							,		__A					:  str=0.1							,		__A					:  int=5_1_2							,		__A					:  Union[str, Any]=1_6							,		__A					:  Optional[int]=2							,		__A					:  Union[str, Any]=0.0_2							,		__A					:  Tuple=3							,		__A					:  Union[str, Any]=4							,		__A					:  Optional[int]=None							,		):
						snake_case__			:				Optional[int]       =  parent
						snake_case__			:				Optional[Any]       =  1_3
						snake_case__			:				int       =  7
						snake_case__			:				Optional[int]       =  True
						snake_case__			:				Optional[Any]       =  True
						snake_case__			:				List[str]       =  True
						snake_case__			:				int       =  True
						snake_case__			:				Optional[int]       =  9_9
						snake_case__			:				Union[str, Any]       =  3_8_4
						snake_case__			:				Optional[Any]       =  2
						snake_case__			:				Union[str, Any]       =  4
						snake_case__			:				Any       =  3_7
						snake_case__			:				Any       =  "gelu"
						snake_case__			:				str       =  0.1
						snake_case__			:				Optional[Any]       =  0.1
						snake_case__			:				Union[str, Any]       =  5_1_2
						snake_case__			:				Optional[Any]       =  1_6
						snake_case__			:				List[Any]       =  2
						snake_case__			:				Optional[int]       =  0.0_2
						snake_case__			:				Dict       =  3
						snake_case__			:				Any       =  4
						snake_case__			:				int       =  1_2_8
						snake_case__			:				Dict       =  2
						snake_case__			:				Any       =  9
						snake_case__			:				List[str]       =  1
						snake_case__			:				List[Any]       =  None



					def 	_lowercase					( self					:  List[str]	):
						snake_case__			:				Optional[int]       =  ids_tensor([self.batch_size, self.seq_length]							,		self.vocab_size	)

						snake_case__			:				str       =  None
						if self.use_input_mask:
							snake_case__			:				str       =  random_attention_mask([self.batch_size, self.seq_length]	)

						snake_case__			:				Union[str, Any]       =  None
						if self.use_token_type_ids:
							snake_case__			:				Tuple       =  ids_tensor([self.batch_size, self.seq_length]							,		self.type_vocab_size	)

						snake_case__			:				Optional[Any]       =  None
						snake_case__			:				Any       =  None
						snake_case__			:				Tuple       =  None
						if self.use_labels:
							snake_case__			:				List[str]       =  ids_tensor([self.batch_size]							,		self.type_sequence_label_size	)
							snake_case__			:				str       =  ids_tensor([self.batch_size, self.seq_length]							,		self.num_labels	)
							snake_case__			:				int       =  ids_tensor([self.batch_size]							,		self.num_choices	)

						snake_case__			:				int       =  ConvBertConfig(
						    vocab_size=self.vocab_size							,		hidden_size=self.hidden_size							,		num_hidden_layers=self.num_hidden_layers							,		num_attention_heads=self.num_attention_heads							,		intermediate_size=self.intermediate_size							,		hidden_act=self.hidden_act							,		hidden_dropout_prob=self.hidden_dropout_prob							,		attention_probs_dropout_prob=self.attention_probs_dropout_prob							,		max_position_embeddings=self.max_position_embeddings							,		type_vocab_size=self.type_vocab_size							,		initializer_range=self.initializer_range							,		return_dict=__A							,		)

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



					def 	_lowercase					( self					:  Dict							,		__A					:  Dict							,		__A					:  Dict							,		__A					:  Union[str, Any]							,		__A					:  Optional[int]							,		__A					:  Any							,		__A					:  Union[str, Any]							,		__A					:  Tuple	):
						snake_case__			:				Optional[int]       =  TFConvBertModel(config=__A	)
						snake_case__			:				Optional[int]       =  {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

						snake_case__			:				List[str]       =  [input_ids, input_mask]
						snake_case__			:				Union[str, Any]       =  model(__A	)

						snake_case__			:				str       =  model(__A	)

						self.parent.assertEqual(result.last_hidden_state.shape							,		(self.batch_size, self.seq_length, self.hidden_size)	)



					def 	_lowercase					( self					:  Union[str, Any]							,		__A					:  List[Any]							,		__A					:  Any							,		__A					:  Union[str, Any]							,		__A					:  int							,		__A					:  Optional[Any]							,		__A					:  Dict							,		__A					:  Optional[int]	):
						snake_case__			:				List[str]       =  TFConvBertForMaskedLM(config=__A	)
						snake_case__			:				Any       =  {
						    "input_ids": input_ids,
						    "attention_mask": input_mask,
						    "token_type_ids": token_type_ids,
						}
						snake_case__			:				int       =  model(__A	)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.seq_length, self.vocab_size)	)



					def 	_lowercase					( self					:  Tuple							,		__A					:  Union[str, Any]							,		__A					:  List[Any]							,		__A					:  Any							,		__A					:  List[Any]							,		__A					:  List[Any]							,		__A					:  Optional[int]							,		__A					:  List[str]	):
						snake_case__			:				Any       =  self.num_labels
						snake_case__			:				List[Any]       =  TFConvBertForSequenceClassification(config=__A	)
						snake_case__			:				Any       =  {
						    "input_ids": input_ids,
						    "attention_mask": input_mask,
						    "token_type_ids": token_type_ids,
						}

						snake_case__			:				Optional[int]       =  model(__A	)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.num_labels)	)



					def 	_lowercase					( self					:  int							,		__A					:  List[Any]							,		__A					:  Union[str, Any]							,		__A					:  Optional[Any]							,		__A					:  List[Any]							,		__A					:  Union[str, Any]							,		__A					:  Union[str, Any]							,		__A					:  Optional[int]	):
						snake_case__			:				Optional[Any]       =  self.num_choices
						snake_case__			:				Any       =  TFConvBertForMultipleChoice(config=__A	)
						snake_case__			:				Optional[int]       =  tf.tile(tf.expand_dims(__A							,		1	)							,		(1, self.num_choices, 1)	)
						snake_case__			:				Optional[Any]       =  tf.tile(tf.expand_dims(__A							,		1	)							,		(1, self.num_choices, 1)	)
						snake_case__			:				Optional[int]       =  tf.tile(tf.expand_dims(__A							,		1	)							,		(1, self.num_choices, 1)	)
						snake_case__			:				int       =  {
						    "input_ids": multiple_choice_inputs_ids,
						    "attention_mask": multiple_choice_input_mask,
						    "token_type_ids": multiple_choice_token_type_ids,
						}
						snake_case__			:				Optional[Any]       =  model(__A	)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.num_choices)	)



					def 	_lowercase					( self					:  List[str]							,		__A					:  Tuple							,		__A					:  str							,		__A					:  Union[str, Any]							,		__A					:  Union[str, Any]							,		__A					:  Any							,		__A					:  int							,		__A					:  Tuple	):
						snake_case__			:				Dict       =  self.num_labels
						snake_case__			:				str       =  TFConvBertForTokenClassification(config=__A	)
						snake_case__			:				Optional[Any]       =  {
						    "input_ids": input_ids,
						    "attention_mask": input_mask,
						    "token_type_ids": token_type_ids,
						}
						snake_case__			:				List[str]       =  model(__A	)
						self.parent.assertEqual(result.logits.shape							,		(self.batch_size, self.seq_length, self.num_labels)	)



					def 	_lowercase					( self					:  Optional[int]							,		__A					:  Union[str, Any]							,		__A					:  List[Any]							,		__A					:  List[str]							,		__A					:  Any							,		__A					:  Any							,		__A					:  Optional[int]							,		__A					:  Optional[Any]	):
						snake_case__			:				Any       =  TFConvBertForQuestionAnswering(config=__A	)
						snake_case__			:				List[str]       =  {
						    "input_ids": input_ids,
						    "attention_mask": input_mask,
						    "token_type_ids": token_type_ids,
						}

						snake_case__			:				int       =  model(__A	)
						self.parent.assertEqual(result.start_logits.shape							,		(self.batch_size, self.seq_length)	)
						self.parent.assertEqual(result.end_logits.shape							,		(self.batch_size, self.seq_length)	)






					def 	_lowercase					( self					:  Any	):
						snake_case__			:				List[Any]       =  self.prepare_config_and_inputs()
						(
						    (
						    snake_case__
						),						(
						    snake_case__
						),						(
						    snake_case__
						),						(
						    snake_case__
						),						(
						    snake_case__
						),						(
						    snake_case__
						),						(
						    snake_case__
						),						
						)			:				List[str]       =  config_and_inputs
						snake_case__			:				Tuple       =  {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
						return config, inputs_dict



@require_tf
class 							SCREAMING_SNAKE_CASE__						(    UpperCamelCase_			,       UpperCamelCase_			,       unittest.TestCase						):





					"""simple docstring"""






					a_     =		(
					    (
					        TFConvBertModel,
					        TFConvBertForMaskedLM,
					        TFConvBertForQuestionAnswering,
					        TFConvBertForSequenceClassification,
					        TFConvBertForTokenClassification,
					        TFConvBertForMultipleChoice,
					    )
					    if is_tf_available()
					    else ()
					)
					a_     =		(
					    {
					        "feature-extraction": TFConvBertModel,
					        "fill-mask": TFConvBertForMaskedLM,
					        "question-answering": TFConvBertForQuestionAnswering,
					        "text-classification": TFConvBertForSequenceClassification,
					        "token-classification": TFConvBertForTokenClassification,
					        "zero-shot": TFConvBertForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					a_     =		False
					a_     =		False
					a_     =		False



					def 	_lowercase					( self					:  int	):
						snake_case__			:				Optional[Any]       =  TFConvBertModelTester(self	)
						snake_case__			:				List[str]       =  ConfigTester(self							,		config_class=__A							,		hidden_size=3_7	)



					def 	_lowercase					( self					:  List[Any]	):
						self.config_tester.run_common_tests()



					def 	_lowercase					( self					:  Any	):
						snake_case__			:				List[str]       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*__A	)



					def 	_lowercase					( self					:  Union[str, Any]	):
						snake_case__			:				Any       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*__A	)



					def 	_lowercase					( self					:  Dict	):
						snake_case__			:				str       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_multiple_choice(*__A	)



					def 	_lowercase					( self					:  Optional[Any]	):
						snake_case__			:				List[str]       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*__A	)



					def 	_lowercase					( self					:  Optional[int]	):
						snake_case__			:				int       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*__A	)



					def 	_lowercase					( self					:  Dict	):
						snake_case__			:				Any       =  self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*__A	)



					@slow
					def 	_lowercase					( self					:  Dict	):
						snake_case__,						snake_case__			:				Tuple       =  self.model_tester.prepare_config_and_inputs_for_common()
						snake_case__			:				int       =  True
						snake_case__			:				int       =  True

						if hasattr(__A							,		"use_cache"	):
							snake_case__			:				Optional[Any]       =  True

						snake_case__			:				Dict       =  getattr(self.model_tester							,		"encoder_seq_length"							,		self.model_tester.seq_length	)
						snake_case__			:				List[str]       =  getattr(self.model_tester							,		"key_length"							,		__A	)

						for model_class in self.all_model_classes:
							snake_case__			:				Tuple       =  self._prepare_for_class(__A							,		__A	)
							snake_case__			:				List[str]       =  model_class(__A	)
							snake_case__			:				List[Any]       =  len(model(__A	)	)

							with tempfile.TemporaryDirectory() as tmpdirname:
								model.save_pretrained(__A							,		saved_model=__A	)
								snake_case__			:				str       =  os.path.join(__A							,		"saved_model"							,		"1"	)
								snake_case__			:				str       =  tf.keras.models.load_model(__A	)
								snake_case__			:				Optional[Any]       =  model(__A	)

								if self.is_encoder_decoder:
									snake_case__			:				Tuple       =  outputs["encoder_hidden_states"]
									snake_case__			:				str       =  outputs["encoder_attentions"]
								else:
									snake_case__			:				Dict       =  outputs["hidden_states"]
									snake_case__			:				Tuple       =  outputs["attentions"]

								self.assertEqual(len(__A	)							,		__A	)

								snake_case__			:				int       =  getattr(
								    self.model_tester							,		"expected_num_hidden_layers"							,		self.model_tester.num_hidden_layers + 1	)

								self.assertEqual(len(__A	)							,		__A	)
								self.assertListEqual(
								    list(output_hidden_states[0].shape[-2:]	)							,		[self.model_tester.seq_length, self.model_tester.hidden_size]							,		)

								self.assertEqual(len(__A	)							,		self.model_tester.num_hidden_layers	)
								self.assertListEqual(
								    list(output_attentions[0].shape[-3:]	)							,		[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]							,		)



					@slow
					def 	_lowercase					( self					:  Tuple	):
						snake_case__			:				Optional[Any]       =  TFConvBertModel.from_pretrained("YituTech/conv-bert-base"	)
						self.assertIsNotNone(__A	)






					def 	_lowercase					( self					:  List[str]	):
						snake_case__,						snake_case__			:				Optional[Any]       =  self.model_tester.prepare_config_and_inputs_for_common()
						snake_case__			:				Optional[Any]       =  True
						snake_case__			:				List[Any]       =  getattr(self.model_tester							,		"decoder_seq_length"							,		self.model_tester.seq_length	)
						snake_case__			:				int       =  getattr(self.model_tester							,		"encoder_seq_length"							,		self.model_tester.seq_length	)
						snake_case__			:				Any       =  getattr(self.model_tester							,		"key_length"							,		__A	)
						snake_case__			:				List[Any]       =  getattr(self.model_tester							,		"key_length"							,		__A	)

						def check_decoder_attentions_output(__A					:  Optional[int]	):
							snake_case__			:				Optional[Any]       =  len(__A	)
							self.assertEqual(out_len % 2							,		0	)
							snake_case__			:				Optional[int]       =  outputs.decoder_attentions
							self.assertEqual(len(__A	)							,		self.model_tester.num_hidden_layers	)
							self.assertListEqual(
							    list(decoder_attentions[0].shape[-3:]	)							,		[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]							,		)

						def check_encoder_attentions_output(__A					:  Any	):
							snake_case__			:				List[Any]       =  [
							    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
							]
							self.assertEqual(len(__A	)							,		self.model_tester.num_hidden_layers	)
							self.assertListEqual(
							    list(attentions[0].shape[-3:]	)							,		[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]							,		)

						for model_class in self.all_model_classes:
							snake_case__			:				Optional[int]       =  True
							snake_case__			:				Any       =  False
							snake_case__			:				Dict       =  model_class(__A	)
							snake_case__			:				List[Any]       =  model(self._prepare_for_class(__A							,		__A	)	)
							snake_case__			:				Dict       =  len(__A	)
							self.assertEqual(config.output_hidden_states							,		__A	)
							check_encoder_attentions_output(__A	)

							if self.is_encoder_decoder:
								snake_case__			:				str       =  model_class(__A	)
								snake_case__			:				List[Any]       =  model(self._prepare_for_class(__A							,		__A	)	)
								self.assertEqual(config.output_hidden_states							,		__A	)
								check_decoder_attentions_output(__A	)

							# Check that output attentions can also be changed via the config
							del inputs_dict["output_attentions"]
							snake_case__			:				Optional[int]       =  True
							snake_case__			:				Optional[Any]       =  model_class(__A	)
							snake_case__			:				Union[str, Any]       =  model(self._prepare_for_class(__A							,		__A	)	)
							self.assertEqual(config.output_hidden_states							,		__A	)
							check_encoder_attentions_output(__A	)

							# Check attention is always last and order is fine
							snake_case__			:				Optional[int]       =  True
							snake_case__			:				List[Any]       =  True
							snake_case__			:				Any       =  model_class(__A	)
							snake_case__			:				str       =  model(self._prepare_for_class(__A							,		__A	)	)

							self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)							,		len(__A	)	)
							self.assertEqual(model.config.output_hidden_states							,		__A	)
							check_encoder_attentions_output(__A	)



@require_tf
class 							SCREAMING_SNAKE_CASE__						(    unittest.TestCase						):





					"""simple docstring"""






					@slow
					def 	_lowercase					( self					:  int	):
						snake_case__			:				int       =  TFConvBertModel.from_pretrained("YituTech/conv-bert-base"	)
						snake_case__			:				int       =  tf.constant([[0, 1, 2, 3, 4, 5]]	)
						snake_case__			:				str       =  model(__A	)[0]

						snake_case__			:				int       =  [1, 6, 7_6_8]
						self.assertEqual(output.shape							,		__A	)

						snake_case__			:				List[Any]       =  tf.constant(
						    [
						        [
						            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
						            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
						            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
						        ]
						    ]	)
						tf.debugging.assert_near(output[:, :3, :3]							,		__A							,		atol=1e-4	)



