
'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import (
       TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
       TFXLMForMultipleChoice,
       TFXLMForQuestionAnsweringSimple,
       TFXLMForSequenceClassification,
       TFXLMForTokenClassification,
       TFXLMModel,
       TFXLMWithLMHeadModel,
       XLMConfig,
   )







class 	__lowerCAmelCase    :



     def __init__(self						,	lowerCAmelCase__						,	):
          _UpperCAmelCase					:  Dict         =    parent
          _UpperCAmelCase					:  Dict         =    1_3
          _UpperCAmelCase					:  Optional[Any]         =    7
          _UpperCAmelCase					:  Optional[Any]         =    True
          _UpperCAmelCase					:  str         =    True
          _UpperCAmelCase					:  Optional[Any]         =    True
          _UpperCAmelCase					:  Dict         =    True
          _UpperCAmelCase					:  Tuple         =    True
          _UpperCAmelCase					:  int         =    False
          _UpperCAmelCase					:  Optional[Any]         =    False
          _UpperCAmelCase					:  List[Any]         =    False
          _UpperCAmelCase					:  Any         =    2
          _UpperCAmelCase					:  int         =    9_9
          _UpperCAmelCase					:  List[Any]         =    0
          _UpperCAmelCase					:  Optional[Any]         =    3_2
          _UpperCAmelCase					:  Tuple         =    2
          _UpperCAmelCase					:  int         =    4
          _UpperCAmelCase					:  Dict         =    0.1
          _UpperCAmelCase					:  Any         =    0.1
          _UpperCAmelCase					:  List[Any]         =    5_1_2
          _UpperCAmelCase					:  Optional[int]         =    1_6
          _UpperCAmelCase					:  List[Any]         =    2
          _UpperCAmelCase					:  Optional[int]         =    0.0_2
          _UpperCAmelCase					:  Dict         =    3
          _UpperCAmelCase					:  Union[str, Any]         =    4
          _UpperCAmelCase					:  Union[str, Any]         =    """last"""
          _UpperCAmelCase					:  List[Any]         =    True
          _UpperCAmelCase					:  str         =    None
          _UpperCAmelCase					:  Union[str, Any]         =    0



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Optional[Any]         =    ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size	)
          _UpperCAmelCase					:  Union[str, Any]         =    random_attention_mask([self.batch_size, self.seq_length]						,	dtype=tf.floataa	)

          _UpperCAmelCase					:  Optional[Any]         =    None
          if self.use_input_lengths:
               _UpperCAmelCase					:  str         =    (
                   ids_tensor([self.batch_size]						,	vocab_size=2	) + self.seq_length - 2
               )  # small variation of seq_length

          _UpperCAmelCase					:  List[str]         =    None
          if self.use_token_type_ids:
               _UpperCAmelCase					:  str         =    ids_tensor([self.batch_size, self.seq_length]						,	self.n_langs	)

          _UpperCAmelCase					:  str         =    None
          _UpperCAmelCase					:  Optional[Any]         =    None
          _UpperCAmelCase					:  Tuple         =    None
          if self.use_labels:
               _UpperCAmelCase					:  Optional[int]         =    ids_tensor([self.batch_size]						,	self.type_sequence_label_size	)
               _UpperCAmelCase					:  List[str]         =    ids_tensor([self.batch_size, self.seq_length]						,	self.num_labels	)
               _UpperCAmelCase					:  str         =    ids_tensor([self.batch_size]						,	2						,	dtype=tf.floataa	)
               _UpperCAmelCase					:  str         =    ids_tensor([self.batch_size]						,	self.num_choices	)

          _UpperCAmelCase					:  Any         =    XLMConfig(
              vocab_size=self.vocab_size						,	n_special=self.n_special						,	emb_dim=self.hidden_size						,	n_layers=self.num_hidden_layers						,	n_heads=self.num_attention_heads						,	dropout=self.hidden_dropout_prob						,	attention_dropout=self.attention_probs_dropout_prob						,	gelu_activation=self.gelu_activation						,	sinusoidal_embeddings=self.sinusoidal_embeddings						,	asm=self.asm						,	causal=self.causal						,	n_langs=self.n_langs						,	max_position_embeddings=self.max_position_embeddings						,	initializer_range=self.initializer_range						,	summary_type=self.summary_type						,	use_proj=self.use_proj						,	bos_token_id=self.bos_token_id						,	)

          return (
              config,
              input_ids,
              token_type_ids,
              input_lengths,
              sequence_labels,
              token_labels,
              is_impossible_labels,
              choice_labels,
              input_mask,
          )



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	):
          _UpperCAmelCase					:  Tuple         =    TFXLMModel(config=lowerCAmelCase__	)
          _UpperCAmelCase					:  Union[str, Any]         =    {"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
          _UpperCAmelCase					:  Tuple         =    model(lowerCAmelCase__	)

          _UpperCAmelCase					:  List[Any]         =    [input_ids, input_mask]
          _UpperCAmelCase					:  List[Any]         =    model(lowerCAmelCase__	)
          self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size)	)



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	):
          _UpperCAmelCase					:  Tuple         =    TFXLMWithLMHeadModel(lowerCAmelCase__	)

          _UpperCAmelCase					:  Optional[Any]         =    {"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
          _UpperCAmelCase					:  List[Any]         =    model(lowerCAmelCase__	)

          _UpperCAmelCase					:  Tuple         =    outputs

          self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.vocab_size)	)



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	):
          _UpperCAmelCase					:  Tuple         =    TFXLMForQuestionAnsweringSimple(lowerCAmelCase__	)

          _UpperCAmelCase					:  Any         =    {"""input_ids""": input_ids, """lengths""": input_lengths}

          _UpperCAmelCase					:  Any         =    model(lowerCAmelCase__	)

          self.parent.assertEqual(result.start_logits.shape						,	(self.batch_size, self.seq_length)	)
          self.parent.assertEqual(result.end_logits.shape						,	(self.batch_size, self.seq_length)	)



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	):
          _UpperCAmelCase					:  Union[str, Any]         =    TFXLMForSequenceClassification(lowerCAmelCase__	)

          _UpperCAmelCase					:  int         =    {"""input_ids""": input_ids, """lengths""": input_lengths}

          _UpperCAmelCase					:  List[Any]         =    model(lowerCAmelCase__	)

          self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.type_sequence_label_size)	)



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	):
          _UpperCAmelCase					:  Optional[Any]         =    self.num_labels
          _UpperCAmelCase					:  Tuple         =    TFXLMForTokenClassification(config=lowerCAmelCase__	)
          _UpperCAmelCase					:  int         =    {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
          _UpperCAmelCase					:  int         =    model(lowerCAmelCase__	)
          self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.num_labels)	)



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	):
          _UpperCAmelCase					:  Any         =    self.num_choices
          _UpperCAmelCase					:  Union[str, Any]         =    TFXLMForMultipleChoice(config=lowerCAmelCase__	)
          _UpperCAmelCase					:  List[str]         =    tf.tile(tf.expand_dims(lowerCAmelCase__						,	1	)						,	(1, self.num_choices, 1)	)
          _UpperCAmelCase					:  Optional[Any]         =    tf.tile(tf.expand_dims(lowerCAmelCase__						,	1	)						,	(1, self.num_choices, 1)	)
          _UpperCAmelCase					:  Union[str, Any]         =    tf.tile(tf.expand_dims(lowerCAmelCase__						,	1	)						,	(1, self.num_choices, 1)	)
          _UpperCAmelCase					:  Tuple         =    {
              """input_ids""": multiple_choice_inputs_ids,
              """attention_mask""": multiple_choice_input_mask,
              """token_type_ids""": multiple_choice_token_type_ids,
          }
          _UpperCAmelCase					:  List[Any]         =    model(lowerCAmelCase__	)
          self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.num_choices)	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Dict         =    self.prepare_config_and_inputs()
          (
              (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     (
              _UpperCAmelCase
          )  ,     
          )					:  Optional[int]         =    config_and_inputs
          _UpperCAmelCase					:  List[Any]         =    {
              """input_ids""": input_ids,
              """token_type_ids""": token_type_ids,
              """langs""": token_type_ids,
              """lengths""": input_lengths,
          }
          return config, inputs_dict







@require_tf
class 	__lowerCAmelCase    (  __a       ,					__a       ,					unittest.TestCase				):
     snake_case   :    int							      =      (
         (
             TFXLMModel,
             TFXLMWithLMHeadModel,
             TFXLMForSequenceClassification,
             TFXLMForQuestionAnsweringSimple,
             TFXLMForTokenClassification,
             TFXLMForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     snake_case   :    int							      =      (
         (TFXLMWithLMHeadModel,) if is_tf_available() else ()
     )  # TODO (PVP): Check other models whether language generation is also applicable
     snake_case   :    int							      =      (
         {
             """feature-extraction""": TFXLMModel,
             """fill-mask""": TFXLMWithLMHeadModel,
             """question-answering""": TFXLMForQuestionAnsweringSimple,
             """text-classification""": TFXLMForSequenceClassification,
             """text-generation""": TFXLMWithLMHeadModel,
             """token-classification""": TFXLMForTokenClassification,
             """zero-shot""": TFXLMForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     snake_case   :    Dict							      =      False
     snake_case   :    Tuple							      =      False



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__						,	lowerCAmelCase__	):
          if (
              pipeline_test_casse_name == "QAPipelineTests"
              and tokenizer_name is not None
              and not tokenizer_name.endswith("""Fast"""	)
          ):
               # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
               # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
               # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
               return True

          return False



     def     snake_case_ (self	):
          _UpperCAmelCase					:  List[Any]         =    TFXLMModelTester(self	)
          _UpperCAmelCase					:  Any         =    ConfigTester(self						,	config_class=lowerCAmelCase__						,	emb_dim=3_7	)



     def     snake_case_ (self	):
          self.config_tester.run_common_tests()



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Optional[int]         =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_model(*lowerCAmelCase__	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Dict         =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_lm_head(*lowerCAmelCase__	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  int         =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_qa(*lowerCAmelCase__	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Dict         =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_sequence_classif(*lowerCAmelCase__	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  int         =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_for_token_classification(*lowerCAmelCase__	)



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Optional[int]         =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_for_multiple_choice(*lowerCAmelCase__	)



     @slow
     def     snake_case_ (self	):
          for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               _UpperCAmelCase					:  Optional[Any]         =    TFXLMModel.from_pretrained(lowerCAmelCase__	)
               self.assertIsNotNone(lowerCAmelCase__	)







@require_tf
class 	__lowerCAmelCase    (  unittest.TestCase				):



     @slow
     def     snake_case_ (self	):
          _UpperCAmelCase					:  Union[str, Any]         =    TFXLMWithLMHeadModel.from_pretrained("""xlm-mlm-en-2048"""	)
          _UpperCAmelCase					:  str         =    tf.convert_to_tensor([[1_4, 4_4_7]]						,	dtype=tf.intaa	)  # the president
          _UpperCAmelCase					:  Optional[int]         =    [
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
              1_4,
              4_4_7,
          ]  # the president the president the president the president the president the president the president the president the president the president
          # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
          _UpperCAmelCase					:  int         =    model.generate(lowerCAmelCase__						,	do_sample=lowerCAmelCase__	)
          self.assertListEqual(output_ids[0].numpy().tolist()						,	lowerCAmelCase__	)










import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


lowerCamelCase_						     =  logging.get_logger(__name__)




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					Any:
														'''simple docstring'''

														super().__init__()
														if version.parse(torch.__version__   ) < version.parse('1.12.0'   ):
																					raise ImportError(
																					    f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
																					    'PytorchGELUTanh. Please upgrade torch.'   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return nn.functional.gelu(lowercase_			,			approximate='tanh'   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi   ) * (input + 0.04_47_15 * torch.pow(lowercase_			,			3.0   ))   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self			,			lowercase_ = False   )					->					Optional[Any]:
														'''simple docstring'''

														super().__init__()
														if use_gelu_python:
																					lowerCAmelCase_										=     self._gelu_python
														else:
																					lowerCAmelCase_										=     nn.functional.gelu

							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0   )   ))






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return self.act(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1.0 + torch.tanh(input * 0.79_78_84_56_08 * (1.0 + 0.04_47_15 * input * input)   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * torch.sigmoid(1.7_02 * input   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self			,			lowercase_			,			lowercase_   )					->					Optional[int]:
														'''simple docstring'''

														if min > max:
																					raise ValueError(f'''min should be < max (got min: {min}, max: {max})'''   )

														super().__init__()
														lowerCAmelCase_										=     min
														lowerCAmelCase_										=     max






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return torch.clip(gelu(lowercase_   )			,			self.min			,			self.max   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					Union[str, Any]:
														'''simple docstring'''

														super().__init__()
														lowerCAmelCase_										=     math.sqrt(2 / math.pi   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_47_15 * torch.pow(lowercase_			,			3   ))   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return nn.functional.silu(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def __init__(		self   )					->					List[Any]:
														'''simple docstring'''

														super().__init__()
														if version.parse(torch.__version__   ) < version.parse('1.9.0'   ):
																					lowerCAmelCase_										=     self._mish_python
														else:
																					lowerCAmelCase_										=     nn.functional.mish

							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input * torch.tanh(nn.functional.softplus(lowercase_   )   )






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return self.act(lowercase_   )




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Tensor:
														'''simple docstring'''

														return input




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_			,			lowercase_=0.70_71_07			,			lowercase_=0.28_20_95   )					->					Optional[Any]:
														'''simple docstring'''

														lowerCAmelCase_										=     (input - mu).div(sigma * math.sqrt(2.0   )   )
														return 0.5 * (1.0 + torch.erf(lowercase_   ))




class 							a_ (						nn.Module		):


							'''simple docstring'''






							def      _lowercase       (		self			,			lowercase_   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     nn.functional.relu(lowercase_   )
														lowerCAmelCase_										=     torch.square(lowercase_   )
														return squared




class 							a_ (						a_		):


							'''simple docstring'''






							def __getitem__(		self			,			lowercase_   )					->					Any:
														'''simple docstring'''

														lowerCAmelCase_										=     super().__getitem__(lowercase_   )
														lowerCAmelCase_						,   lowerCAmelCase_										=     content if isinstance(lowercase_			,			lowercase_   ) else (content, {})
														return cls(**lowercase_   )


lowerCamelCase_						     =  {
    """gelu""": GELUActivation,
    """gelu_10""": (ClippedGELUActivation, {"""min""": -1_0, """max""": 1_0}),
    """gelu_fast""": FastGELUActivation,
    """gelu_new""": NewGELUActivation,
    """gelu_python""": (GELUActivation, {"""use_gelu_python""": True}),
    """gelu_pytorch_tanh""": PytorchGELUTanh,
    """gelu_accurate""": AccurateGELUActivation,
    """laplace""": LaplaceActivation,
    """linear""": LinearActivation,
    """mish""": MishActivation,
    """quick_gelu""": QuickGELUActivation,
    """relu""": nn.ReLU,
    """relu2""": ReLUSquaredActivation,
    """relu6""": nn.ReLUa,
    """sigmoid""": nn.Sigmoid,
    """silu""": SiLUActivation,
    """swish""": SiLUActivation,
    """tanh""": nn.Tanh,
}
lowerCamelCase_						     =  ClassInstantier(ACTaCLS)
def      lowerCamelCase					(						a_	)	->			str:
							if activation_string in ACTaFN:
														return ACTaFN[activation_string]
							else:
														raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()	)}'''	)


# For backwards compatibility with: from activations import gelu_python
lowerCamelCase_						     =  get_activation("""gelu_python""")
lowerCamelCase_						     =  get_activation("""gelu_new""")
lowerCamelCase_						     =  get_activation("""gelu""")
lowerCamelCase_						     =  get_activation("""gelu_fast""")
lowerCamelCase_						     =  get_activation("""quick_gelu""")
lowerCamelCase_						     =  get_activation("""silu""")
lowerCamelCase_						     =  get_activation("""mish""")



lowerCamelCase_						     =  get_activation("""linear""")



