




'''simple docstring'''





import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    XLMForMultipleChoice,
		    XLMForQuestionAnswering,
		    XLMForQuestionAnsweringSimple,
		    XLMForSequenceClassification,
		    XLMForTokenClassification,
		    XLMModel,
		    XLMWithLMHeadModel,
		)
		from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST




class 					_snake_case						:




			def __init__(   self							,				a__							,				a__=13							,				a__=7							,				a__=True							,				a__=True							,				a__=True							,				a__=True							,				a__=True							,				a__=False							,				a__=False							,				a__=False							,				a__=2							,				a__=99							,				a__=0							,				a__=32							,				a__=5							,				a__=4							,				a__=0.1							,				a__=0.1							,				a__=512							,				a__=2							,				a__=0.0_2							,				a__=2							,				a__=4							,				a__="last"							,				a__=True							,				a__=None							,				a__=0							,				)							->							List[Any]:

										'''simple docstring'''




										snake_case_         = parent
										snake_case_         = batch_size
										snake_case_         = seq_length
										snake_case_         = is_training
										snake_case_         = use_input_lengths
										snake_case_         = use_token_type_ids
										snake_case_         = use_labels
										snake_case_         = gelu_activation
										snake_case_         = sinusoidal_embeddings
										snake_case_         = causal
										snake_case_         = asm
										snake_case_         = n_langs
										snake_case_         = vocab_size
										snake_case_         = n_special
										snake_case_         = hidden_size
										snake_case_         = num_hidden_layers
										snake_case_         = num_attention_heads
										snake_case_         = hidden_dropout_prob
										snake_case_         = attention_probs_dropout_prob
										snake_case_         = max_position_embeddings
										snake_case_         = type_sequence_label_size
										snake_case_         = initializer_range
										snake_case_         = num_labels
										snake_case_         = num_choices
										snake_case_         = summary_type
										snake_case_         = use_proj
										snake_case_         = scope
										snake_case_         = bos_token_id




			def   lowerCAmelCase__							(   self				)							->							Optional[Any]:

										'''simple docstring'''




										snake_case_         = ids_tensor([self.batch_size, self.seq_length]							,				self.vocab_size				)
										snake_case_         = random_attention_mask([self.batch_size, self.seq_length]				)

										snake_case_         = None
										if self.use_input_lengths:
																	snake_case_         = (
																	    ids_tensor([self.batch_size]							,				vocab_size=2				) + self.seq_length - 2
																	)  # small variation of seq_length

										snake_case_         = None
										if self.use_token_type_ids:
																	snake_case_         = ids_tensor([self.batch_size, self.seq_length]							,				self.n_langs				)

										snake_case_         = None
										snake_case_         = None
										snake_case_         = None
										if self.use_labels:
																	snake_case_         = ids_tensor([self.batch_size]							,				self.type_sequence_label_size				)
																	snake_case_         = ids_tensor([self.batch_size, self.seq_length]							,				self.num_labels				)
																	snake_case_         = ids_tensor([self.batch_size]							,				2				).float()
																	snake_case_         = ids_tensor([self.batch_size]							,				self.num_choices				)

										snake_case_         = self.get_config()

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    input_lengths,
										    sequence_labels,
										    token_labels,
										    is_impossible_labels,
										    choice_labels,
										    input_mask,
										)




			def   lowerCAmelCase__							(   self				)							->							int:

										'''simple docstring'''




										return XLMConfig(
										    vocab_size=self.vocab_size							,				n_special=self.n_special							,				emb_dim=self.hidden_size							,				n_layers=self.num_hidden_layers							,				n_heads=self.num_attention_heads							,				dropout=self.hidden_dropout_prob							,				attention_dropout=self.attention_probs_dropout_prob							,				gelu_activation=self.gelu_activation							,				sinusoidal_embeddings=self.sinusoidal_embeddings							,				asm=self.asm							,				causal=self.causal							,				n_langs=self.n_langs							,				max_position_embeddings=self.max_position_embeddings							,				initializer_range=self.initializer_range							,				summary_type=self.summary_type							,				use_proj=self.use_proj							,				num_labels=self.num_labels							,				bos_token_id=self.bos_token_id							,				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				)							->							Union[str, Any]:

										'''simple docstring'''




										snake_case_         = XLMModel(config=UpperCAmelCase__				)
										model.to(UpperCAmelCase__				)
										model.eval()
										snake_case_         = model(UpperCAmelCase__							,				lengths=UpperCAmelCase__							,				langs=UpperCAmelCase__				)
										snake_case_         = model(UpperCAmelCase__							,				langs=UpperCAmelCase__				)
										snake_case_         = model(UpperCAmelCase__				)
										self.parent.assertEqual(result.last_hidden_state.shape							,				(self.batch_size, self.seq_length, self.hidden_size)				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				)							->							Any:

										'''simple docstring'''




										snake_case_         = XLMWithLMHeadModel(UpperCAmelCase__				)
										model.to(UpperCAmelCase__				)
										model.eval()

										snake_case_         = model(UpperCAmelCase__							,				token_type_ids=UpperCAmelCase__							,				labels=UpperCAmelCase__				)
										self.parent.assertEqual(result.loss.shape							,				()				)
										self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.seq_length, self.vocab_size)				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				)							->							List[Any]:

										'''simple docstring'''




										snake_case_         = XLMForQuestionAnsweringSimple(UpperCAmelCase__				)
										model.to(UpperCAmelCase__				)
										model.eval()

										snake_case_         = model(UpperCAmelCase__				)

										snake_case_         = model(UpperCAmelCase__							,				start_positions=UpperCAmelCase__							,				end_positions=UpperCAmelCase__				)
										snake_case_         = outputs
										self.parent.assertEqual(result.start_logits.shape							,				(self.batch_size, self.seq_length)				)
										self.parent.assertEqual(result.end_logits.shape							,				(self.batch_size, self.seq_length)				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				)							->							Optional[Any]:

										'''simple docstring'''




										snake_case_         = XLMForQuestionAnswering(UpperCAmelCase__				)
										model.to(UpperCAmelCase__				)
										model.eval()

										snake_case_         = model(UpperCAmelCase__				)

										snake_case_         = model(
										    UpperCAmelCase__							,				start_positions=UpperCAmelCase__							,				end_positions=UpperCAmelCase__							,				cls_index=UpperCAmelCase__							,				is_impossible=UpperCAmelCase__							,				p_mask=UpperCAmelCase__							,				)

										snake_case_         = model(
										    UpperCAmelCase__							,				start_positions=UpperCAmelCase__							,				end_positions=UpperCAmelCase__							,				cls_index=UpperCAmelCase__							,				is_impossible=UpperCAmelCase__							,				)

										((snake_case_				)   , 				)         = result_with_labels.to_tuple()

										snake_case_         = model(UpperCAmelCase__							,				start_positions=UpperCAmelCase__							,				end_positions=UpperCAmelCase__				)

										((snake_case_				)   , 				)         = result_with_labels.to_tuple()

										self.parent.assertEqual(result_with_labels.loss.shape							,				()				)
										self.parent.assertEqual(result.start_top_log_probs.shape							,				(self.batch_size, model.config.start_n_top)				)
										self.parent.assertEqual(result.start_top_index.shape							,				(self.batch_size, model.config.start_n_top)				)
										self.parent.assertEqual(
										    result.end_top_log_probs.shape							,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)				)
										self.parent.assertEqual(
										    result.end_top_index.shape							,				(self.batch_size, model.config.start_n_top * model.config.end_n_top)				)
										self.parent.assertEqual(result.cls_logits.shape							,				(self.batch_size,)				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				)							->							int:

										'''simple docstring'''




										snake_case_         = XLMForSequenceClassification(UpperCAmelCase__				)
										model.to(UpperCAmelCase__				)
										model.eval()

										snake_case_         = model(UpperCAmelCase__				)
										snake_case_         = model(UpperCAmelCase__							,				labels=UpperCAmelCase__				)
										self.parent.assertEqual(result.loss.shape							,				()				)
										self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.type_sequence_label_size)				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				)							->							str:

										'''simple docstring'''




										snake_case_         = self.num_labels
										snake_case_         = XLMForTokenClassification(UpperCAmelCase__				)
										model.to(UpperCAmelCase__				)
										model.eval()

										snake_case_         = model(UpperCAmelCase__							,				attention_mask=UpperCAmelCase__							,				labels=UpperCAmelCase__				)
										self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.seq_length, self.num_labels)				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__							,				)							->							List[str]:

										'''simple docstring'''




										snake_case_         = self.num_choices
										snake_case_         = XLMForMultipleChoice(config=UpperCAmelCase__				)
										model.to(UpperCAmelCase__				)
										model.eval()
										snake_case_         = input_ids.unsqueeze(1				).expand(-1							,				self.num_choices							,				-1				).contiguous()
										snake_case_         = token_type_ids.unsqueeze(1				).expand(-1							,				self.num_choices							,				-1				).contiguous()
										snake_case_         = input_mask.unsqueeze(1				).expand(-1							,				self.num_choices							,				-1				).contiguous()
										snake_case_         = model(
										    UpperCAmelCase__							,				attention_mask=UpperCAmelCase__							,				token_type_ids=UpperCAmelCase__							,				labels=UpperCAmelCase__							,				)
										self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.num_choices)				)




			def   lowerCAmelCase__							(   self				)							->							Dict:

										'''simple docstring'''




										snake_case_         = self.prepare_config_and_inputs()
										(
										    (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , (
										    snake_case_				
										)   , 				
										)         = config_and_inputs
										snake_case_         = {"input_ids": input_ids, "token_type_ids": token_type_ids, "lengths": input_lengths}
										return config, inputs_dict




@require_torch
class 					_snake_case						(      UpperCAmelCase__		,		UpperCAmelCase__		,		UpperCAmelCase__		,		unittest.TestCase				):
			lowerCAmelCase_	: List[str]					=       (
			    (
			        XLMModel,
			        XLMWithLMHeadModel,
			        XLMForQuestionAnswering,
			        XLMForSequenceClassification,
			        XLMForQuestionAnsweringSimple,
			        XLMForTokenClassification,
			        XLMForMultipleChoice,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCAmelCase_	: Optional[Any]					=       (
			    (XLMWithLMHeadModel,) if is_torch_available() else ()
			)  # TODO (PVP): Check other models whether language generation is also applicable
			lowerCAmelCase_	: Any					=       (
			    {
			        "feature-extraction": XLMModel,
			        "fill-mask": XLMWithLMHeadModel,
			        "question-answering": XLMForQuestionAnsweringSimple,
			        "text-classification": XLMForSequenceClassification,
			        "text-generation": XLMWithLMHeadModel,
			        "token-classification": XLMForTokenClassification,
			        "zero-shot": XLMForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__				)							->							str:

										'''simple docstring'''




										if (
										    pipeline_test_casse_name == "QAPipelineTests"
										    and tokenizer_name is not None
										    and not tokenizer_name.endswith("Fast"				)
										):
																	# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
																	# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
																	# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
																	return True

										return False




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__=False				)							->							Tuple:

										'''simple docstring'''




										snake_case_         = super()._prepare_for_class(UpperCAmelCase__							,				UpperCAmelCase__							,				return_labels=UpperCAmelCase__				)

										if return_labels:
																	if model_class.__name__ == "XLMForQuestionAnswering":
																								snake_case_         = torch.zeros(
																								    self.model_tester.batch_size							,				dtype=torch.long							,				device=UpperCAmelCase__				)
																								snake_case_         = torch.zeros(
																								    self.model_tester.batch_size							,				dtype=torch.long							,				device=UpperCAmelCase__				)

										return inputs_dict




			def   lowerCAmelCase__							(   self				)							->							List[Any]:

										'''simple docstring'''




										snake_case_         = XLMModelTester(self				)
										snake_case_         = ConfigTester(self							,				config_class=UpperCAmelCase__							,				emb_dim=37				)




			def   lowerCAmelCase__							(   self				)							->							Union[str, Any]:

										'''simple docstring'''




										self.config_tester.run_common_tests()




			def   lowerCAmelCase__							(   self				)							->							Optional[int]:

										'''simple docstring'''




										snake_case_         = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_xlm_model(*UpperCAmelCase__				)




			def   lowerCAmelCase__							(   self				)							->							int:

										'''simple docstring'''




										snake_case_         = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_xlm_lm_head(*UpperCAmelCase__				)




			def   lowerCAmelCase__							(   self				)							->							List[Any]:

										'''simple docstring'''




										snake_case_         = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_xlm_simple_qa(*UpperCAmelCase__				)




			def   lowerCAmelCase__							(   self				)							->							Tuple:

										'''simple docstring'''




										snake_case_         = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_xlm_qa(*UpperCAmelCase__				)




			def   lowerCAmelCase__							(   self				)							->							int:

										'''simple docstring'''




										snake_case_         = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_xlm_sequence_classif(*UpperCAmelCase__				)




			def   lowerCAmelCase__							(   self				)							->							int:

										'''simple docstring'''




										snake_case_         = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_xlm_token_classif(*UpperCAmelCase__				)




			def   lowerCAmelCase__							(   self				)							->							Union[str, Any]:

										'''simple docstring'''




										snake_case_         = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_xlm_for_multiple_choice(*UpperCAmelCase__				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__=False							,				a__=1				)							->							int:

										'''simple docstring'''




										self.assertIsInstance(UpperCAmelCase__							,				UpperCAmelCase__				)
										self.assertListEqual(
										    [isinstance(UpperCAmelCase__							,				UpperCAmelCase__				) for iter_attentions in attentions]							,				[True] * len(UpperCAmelCase__				)				)
										self.assertEqual(len(UpperCAmelCase__				)							,				(max_length - min_length) * num_beam_groups				)

										for idx, iter_attentions in enumerate(UpperCAmelCase__				):
																	# adds PAD dummy token
																	snake_case_         = min_length + idx + 1
																	snake_case_         = min_length + idx + 1

																	snake_case_         = (
																	    batch_size * num_beam_groups,
																	    config.num_attention_heads,
																	    tgt_len,
																	    src_len,
																	)
																	# check attn size
																	self.assertListEqual(
																	    [layer_attention.shape for layer_attention in iter_attentions]							,				[expected_shape] * len(UpperCAmelCase__				)				)




			def   lowerCAmelCase__							(   self							,				a__							,				a__							,				a__							,				a__							,				a__							,				a__=False							,				a__=1				)							->							Dict:

										'''simple docstring'''




										self.assertIsInstance(UpperCAmelCase__							,				UpperCAmelCase__				)
										self.assertListEqual(
										    [isinstance(UpperCAmelCase__							,				UpperCAmelCase__				) for iter_hidden_states in hidden_states]							,				[True] * len(UpperCAmelCase__				)							,				)
										self.assertEqual(len(UpperCAmelCase__				)							,				(max_length - min_length) * num_beam_groups				)

										for idx, iter_hidden_states in enumerate(UpperCAmelCase__				):
																	# adds PAD dummy token
																	snake_case_         = min_length + idx + 1
																	snake_case_         = (batch_size * num_beam_groups, seq_len, config.hidden_size)
																	# check hidden size
																	self.assertListEqual(
																	    [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]							,				[expected_shape] * len(UpperCAmelCase__				)							,				)
										pass




			@slow
			def   lowerCAmelCase__							(   self				)							->							int:

										'''simple docstring'''




										for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	snake_case_         = XLMModel.from_pretrained(UpperCAmelCase__				)
																	self.assertIsNotNone(UpperCAmelCase__				)




@require_torch
class 					_snake_case						(      unittest.TestCase				):




			@slow
			def   lowerCAmelCase__							(   self				)							->							Dict:

										'''simple docstring'''




										snake_case_         = XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"				)
										model.to(UpperCAmelCase__				)
										snake_case_         = torch.tensor([[14, 447]]							,				dtype=torch.long							,				device=UpperCAmelCase__				)  # the president
										snake_case_         = [
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										    14,
										    447,
										]  # the president the president the president the president the president the president the president the president the president the president
										# TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
										snake_case_         = model.generate(UpperCAmelCase__							,				do_sample=UpperCAmelCase__				)
										self.assertListEqual(output_ids[0].cpu().numpy().tolist()							,				UpperCAmelCase__				)










from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase      :						str							=  logging.get_logger(__name__)

_lowerCamelCase      :						List[str]							=  {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/config.json"""}


class 							UpperCamelCase_       (					UpperCAmelCase__	):

    '''simple docstring'''






    UpperCAmelCase__				    = '''openai-gpt'''
    UpperCAmelCase__				    = {
        '''max_position_embeddings''': '''n_positions''',
        '''hidden_size''': '''n_embd''',
        '''num_attention_heads''': '''n_head''',
        '''num_hidden_layers''': '''n_layer''',
    }


    def __init__(     self :      Union[str, Any]   ,					UpperCAmelCase__ :      Dict=40_478   ,					UpperCAmelCase__ :      str=512   ,					UpperCAmelCase__ :      Union[str, Any]=768   ,					UpperCAmelCase__ :      Optional[Any]=12   ,					UpperCAmelCase__ :      Any=12   ,					UpperCAmelCase__ :      Optional[Any]="gelu"   ,					UpperCAmelCase__ :      Any=0.1   ,					UpperCAmelCase__ :      List[str]=0.1   ,					UpperCAmelCase__ :      Tuple=0.1   ,					UpperCAmelCase__ :      List[str]=1e-5   ,					UpperCAmelCase__ :      int=0.02   ,					UpperCAmelCase__ :      Any="cls_index"   ,					UpperCAmelCase__ :      Optional[Any]=True   ,					UpperCAmelCase__ :      List[Any]=None   ,					UpperCAmelCase__ :      List[Any]=True   ,					UpperCAmelCase__ :      Optional[Any]=0.1   ,					**UpperCAmelCase__ :      Dict   ,					)		->Any:





           '''simple docstring'''

           A__ 					=       vocab_size
           A__ 					=       n_positions
           A__ 					=       n_embd
           A__ 					=       n_layer
           A__ 					=       n_head
           A__ 					=       afn
           A__ 					=       resid_pdrop
           A__ 					=       embd_pdrop
           A__ 					=       attn_pdrop
           A__ 					=       layer_norm_epsilon
           A__ 					=       initializer_range
           A__ 					=       summary_type
           A__ 					=       summary_use_proj
           A__ 					=       summary_activation
           A__ 					=       summary_first_dropout
           A__ 					=       summary_proj_to_labels
           super().__init__(**UpperCAmelCase__)


