



import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       XLMForMultipleChoice,
       XLMForQuestionAnswering,
       XLMForQuestionAnsweringSimple,
       XLMForSequenceClassification,
       XLMForTokenClassification,
       XLMModel,
       XLMWithLMHeadModel,
   )
   from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST



class     _UpperCAmelCase			:




     def __init__(				self ,						a__ ,						a__=1_3 ,						a__=7 ,						a__=True ,						a__=True ,						a__=True ,						a__=True ,						a__=True ,						a__=False ,						a__=False ,						a__=False ,						a__=2 ,						a__=9_9 ,						a__=0 ,						a__=3_2 ,						a__=5 ,						a__=4 ,						a__=0.1 ,						a__=0.1 ,						a__=5_1_2 ,						a__=2 ,						a__=0.0_2 ,						a__=2 ,						a__=4 ,						a__="last" ,						a__=True ,						a__=None ,						a__=0 ,						):
         A__		 =					parent
         A__		 =					batch_size
         A__		 =					seq_length
         A__		 =					is_training
         A__		 =					use_input_lengths
         A__		 =					use_token_type_ids
         A__		 =					use_labels
         A__		 =					gelu_activation
         A__		 =					sinusoidal_embeddings
         A__		 =					causal
         A__		 =					asm
         A__		 =					n_langs
         A__		 =					vocab_size
         A__		 =					n_special
         A__		 =					hidden_size
         A__		 =					num_hidden_layers
         A__		 =					num_attention_heads
         A__		 =					hidden_dropout_prob
         A__		 =					attention_probs_dropout_prob
         A__		 =					max_position_embeddings
         A__		 =					type_sequence_label_size
         A__		 =					initializer_range
         A__		 =					num_labels
         A__		 =					num_choices
         A__		 =					summary_type
         A__		 =					use_proj
         A__		 =					scope
         A__		 =					bos_token_id




     def 		snake_case_		(				self):
         A__		 =					ids_tensor([self.batch_size, self.seq_length] ,						self.vocab_size)
         A__		 =					random_attention_mask([self.batch_size, self.seq_length])

         A__		 =					None
         if self.use_input_lengths:
             A__		 =					(
                 ids_tensor([self.batch_size] ,						vocab_size=2) + self.seq_length - 2
             )  # small variation of seq_length

         A__		 =					None
         if self.use_token_type_ids:
             A__		 =					ids_tensor([self.batch_size, self.seq_length] ,						self.n_langs)

         A__		 =					None
         A__		 =					None
         A__		 =					None
         if self.use_labels:
             A__		 =					ids_tensor([self.batch_size] ,						self.type_sequence_label_size)
             A__		 =					ids_tensor([self.batch_size, self.seq_length] ,						self.num_labels)
             A__		 =					ids_tensor([self.batch_size] ,						2).float()
             A__		 =					ids_tensor([self.batch_size] ,						self.num_choices)

         A__		 =					self.get_config()

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )




     def 		snake_case_		(				self):
         return XLMConfig(
             vocab_size=self.vocab_size ,						n_special=self.n_special ,						emb_dim=self.hidden_size ,						n_layers=self.num_hidden_layers ,						n_heads=self.num_attention_heads ,						dropout=self.hidden_dropout_prob ,						attention_dropout=self.attention_probs_dropout_prob ,						gelu_activation=self.gelu_activation ,						sinusoidal_embeddings=self.sinusoidal_embeddings ,						asm=self.asm ,						causal=self.causal ,						n_langs=self.n_langs ,						max_position_embeddings=self.max_position_embeddings ,						initializer_range=self.initializer_range ,						summary_type=self.summary_type ,						use_proj=self.use_proj ,						num_labels=self.num_labels ,						bos_token_id=self.bos_token_id ,						)




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMModel(config=a__)
         model.to(a__)
         model.eval()
         A__		 =					model(a__ ,						lengths=a__ ,						langs=a__)
         A__		 =					model(a__ ,						langs=a__)
         A__		 =					model(a__)
         self.parent.assertEqual(result.last_hidden_state.shape ,						(self.batch_size, self.seq_length, self.hidden_size))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMWithLMHeadModel(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__ ,						token_type_ids=a__ ,						labels=a__)
         self.parent.assertEqual(result.loss.shape ,						())
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.seq_length, self.vocab_size))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMForQuestionAnsweringSimple(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__)

         A__		 =					model(a__ ,						start_positions=a__ ,						end_positions=a__)
         A__		 =					outputs
         self.parent.assertEqual(result.start_logits.shape ,						(self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape ,						(self.batch_size, self.seq_length))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMForQuestionAnswering(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__)

         A__		 =					model(
             a__ ,						start_positions=a__ ,						end_positions=a__ ,						cls_index=a__ ,						is_impossible=a__ ,						p_mask=a__ ,						)

         A__		 =					model(
             a__ ,						start_positions=a__ ,						end_positions=a__ ,						cls_index=a__ ,						is_impossible=a__ ,						)

         ((A__) ,	)		 =					result_with_labels.to_tuple()

         A__		 =					model(a__ ,						start_positions=a__ ,						end_positions=a__)

         ((A__) ,	)		 =					result_with_labels.to_tuple()

         self.parent.assertEqual(result_with_labels.loss.shape ,						())
         self.parent.assertEqual(result.start_top_log_probs.shape ,						(self.batch_size, model.config.start_n_top))
         self.parent.assertEqual(result.start_top_index.shape ,						(self.batch_size, model.config.start_n_top))
         self.parent.assertEqual(
             result.end_top_log_probs.shape ,						(self.batch_size, model.config.start_n_top * model.config.end_n_top))
         self.parent.assertEqual(
             result.end_top_index.shape ,						(self.batch_size, model.config.start_n_top * model.config.end_n_top))
         self.parent.assertEqual(result.cls_logits.shape ,						(self.batch_size,))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					XLMForSequenceClassification(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__)
         A__		 =					model(a__ ,						labels=a__)
         self.parent.assertEqual(result.loss.shape ,						())
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.type_sequence_label_size))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					self.num_labels
         A__		 =					XLMForTokenClassification(a__)
         model.to(a__)
         model.eval()

         A__		 =					model(a__ ,						attention_mask=a__ ,						labels=a__)
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.seq_length, self.num_labels))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						):
         A__		 =					self.num_choices
         A__		 =					XLMForMultipleChoice(config=a__)
         model.to(a__)
         model.eval()
         A__		 =					input_ids.unsqueeze(1).expand(-1 ,						self.num_choices ,						-1).contiguous()
         A__		 =					token_type_ids.unsqueeze(1).expand(-1 ,						self.num_choices ,						-1).contiguous()
         A__		 =					input_mask.unsqueeze(1).expand(-1 ,						self.num_choices ,						-1).contiguous()
         A__		 =					model(
             a__ ,						attention_mask=a__ ,						token_type_ids=a__ ,						labels=a__ ,						)
         self.parent.assertEqual(result.logits.shape ,						(self.batch_size, self.num_choices))




     def 		snake_case_		(				self):
         A__		 =					self.prepare_config_and_inputs()
         (
             (
             A__
         ) ,	(
             A__
         ) ,	(
             A__
         ) ,	(
             A__
         ) ,	(
             A__
         ) ,	(
             A__
         ) ,	(
             A__
         ) ,	(
             A__
         ) ,	(
             A__
         ) ,	
         )		 =					config_and_inputs
         A__		 =					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''lengths''': input_lengths}
         return config, inputs_dict







@require_torch
class     _UpperCAmelCase			(       A__			,      A__			,      A__			,      unittest.TestCase     ):
     UpperCamelCase__		    =				(
         (
             XLMModel,
             XLMWithLMHeadModel,
             XLMForQuestionAnswering,
             XLMForSequenceClassification,
             XLMForQuestionAnsweringSimple,
             XLMForTokenClassification,
             XLMForMultipleChoice,
         )
         if is_torch_available()
         else ()
     )
     UpperCamelCase__		    =				(
         (XLMWithLMHeadModel,) if is_torch_available() else ()
     )  # TODO (PVP): Check other models whether language generation is also applicable
     UpperCamelCase__		    =				(
         {
             '''feature-extraction''': XLMModel,
             '''fill-mask''': XLMWithLMHeadModel,
             '''question-answering''': XLMForQuestionAnsweringSimple,
             '''text-classification''': XLMForSequenceClassification,
             '''text-generation''': XLMWithLMHeadModel,
             '''token-classification''': XLMForTokenClassification,
             '''zero-shot''': XLMForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__):
         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith('''Fast''')
         ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

         return False




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__=False):
         A__		 =					super()._prepare_for_class(a__ ,						a__ ,						return_labels=a__)

         if return_labels:
             if model_class.__name__ == "XLMForQuestionAnswering":
                 A__		 =					torch.zeros(
                     self.model_tester.batch_size ,						dtype=torch.long ,						device=a__)
                 A__		 =					torch.zeros(
                     self.model_tester.batch_size ,						dtype=torch.long ,						device=a__)

         return inputs_dict




     def 		snake_case_		(				self):
         A__		 =					XLMModelTester(self)
         A__		 =					ConfigTester(self ,						config_class=a__ ,						emb_dim=3_7)




     def 		snake_case_		(				self):
         self.config_tester.run_common_tests()




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_model(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_lm_head(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_simple_qa(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_qa(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_sequence_classif(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_token_classif(*a__)




     def 		snake_case_		(				self):
         A__		 =					self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_multiple_choice(*a__)




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__=False ,						a__=1):
         self.assertIsInstance(a__ ,						a__)
         self.assertListEqual(
             [isinstance(a__ ,						a__) for iter_attentions in attentions] ,						[True] * len(a__))
         self.assertEqual(len(a__) ,						(max_length - min_length) * num_beam_groups)

         for idx, iter_attentions in enumerate(a__):
             # adds PAD dummy token
             A__		 =					min_length + idx + 1
             A__		 =					min_length + idx + 1

             A__		 =					(
                 batch_size * num_beam_groups,
                 config.num_attention_heads,
                 tgt_len,
                 src_len,
             )
             # check attn size
             self.assertListEqual(
                 [layer_attention.shape for layer_attention in iter_attentions] ,						[expected_shape] * len(a__))




     def 		snake_case_		(				self ,						a__ ,						a__ ,						a__ ,						a__ ,						a__ ,						a__=False ,						a__=1):
         self.assertIsInstance(a__ ,						a__)
         self.assertListEqual(
             [isinstance(a__ ,						a__) for iter_hidden_states in hidden_states] ,						[True] * len(a__) ,						)
         self.assertEqual(len(a__) ,						(max_length - min_length) * num_beam_groups)

         for idx, iter_hidden_states in enumerate(a__):
             # adds PAD dummy token
             A__		 =					min_length + idx + 1
             A__		 =					(batch_size * num_beam_groups, seq_len, config.hidden_size)
             # check hidden size
             self.assertListEqual(
                 [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states] ,						[expected_shape] * len(a__) ,						)
         pass




     @slow
     def 		snake_case_		(				self):
         for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             A__		 =					XLMModel.from_pretrained(a__)
             self.assertIsNotNone(a__)







@require_torch
class     _UpperCAmelCase			(       unittest.TestCase     ):




     @slow
     def 		snake_case_		(				self):
         A__		 =					XLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048''')
         model.to(a__)
         A__		 =					torch.tensor([[1_4, 4_4_7]] ,						dtype=torch.long ,						device=a__)  # the president
         A__		 =					[
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
             1_4,
             4_4_7,
         ]  # the president the president the president the president the president the president the president the president the president the president
         # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
         A__		 =					model.generate(a__ ,						do_sample=a__)
         self.assertListEqual(output_ids[0].cpu().numpy().tolist() ,						a__)





import jax.numpy as jnp

from ...utils import logging
from ..ta.modeling_flax_ta import FlaxTaEncoderModel, FlaxTaForConditionalGeneration, FlaxTaModel
from .configuration_mta import MTaConfig


_lowercase						  =		logging.get_logger(__name__)

_lowercase						  =		"T5Config"



def 		lowerCAmelCase__    (					UpperCamelCase_				:       jnp.array ,       UpperCamelCase_				:       int ,       UpperCamelCase_				:       int	)->  jnp.ndarray:
    A__		 =					jnp.zeros_like(UpperCamelCase_	)
    A__		 =					shifted_input_ids.at[:, 1:].set(input_ids[:, :-1]	)
    A__		 =					shifted_input_ids.at[:, 0].set(UpperCamelCase_	)

    A__		 =					jnp.where(shifted_input_ids == -1_0_0 ,       UpperCamelCase_ ,       UpperCamelCase_	)
    return shifted_input_ids



class     _UpperCAmelCase			(       A__     ):
     UpperCamelCase__		    =				'''mt5'''
     UpperCamelCase__		    =				MTaConfig







class     _UpperCAmelCase			(       A__     ):
     UpperCamelCase__		    =				'''mt5'''
     UpperCamelCase__		    =				MTaConfig







class     _UpperCAmelCase			(       A__     ):
     UpperCamelCase__		    =				'''mt5'''
     UpperCamelCase__		    =				MTaConfig
