


'''simple docstring'''



import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__UpperCAmelCase			:str 				=   logging.get_logger(__name__)



class 	a				(		nn.Module       ):


     """simple docstring"""






     def __init__(				self     :		Dict			)						->							Union[str, Any]:
          super().__init__()
          if version.parse(torch.__version__			) < version.parse('''1.12.0'''			):
               raise ImportError(
                   f'You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '
                   '''PytorchGELUTanh. Please upgrade torch.'''			)






     def  lowerCamelCase__ (				self     :		Dict							, snake_case     :		int			)						->							Tensor:
          return nn.functional.gelu(snake_case							, approximate='''tanh'''			)



class 	a				(		nn.Module       ):


     """simple docstring"""






     def  lowerCamelCase__ (				self     :		Optional[Any]							, snake_case     :		List[Any]			)						->							Tensor:
          return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi			) * (input + 0.044_715 * torch.pow(snake_case							, 3.0			))			))



class 	a				(		nn.Module       ):


     """simple docstring"""






     def __init__(				self     :		List[Any]							, snake_case     :		str = False			)						->							int:
          super().__init__()
          if use_gelu_python:
               __UpperCAmelCase	: Tuple  							=			self._gelu_python
          else:
               __UpperCAmelCase	: Dict  							=			nn.functional.gelu

     def  lowerCamelCase__ (				self     :		int							, snake_case     :		Dict			)						->							Tensor:
          return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0			)			))






     def  lowerCamelCase__ (				self     :		str							, snake_case     :		Dict			)						->							Tensor:
          return self.act(snake_case			)



class 	a				(		nn.Module       ):


     """simple docstring"""






     def  lowerCamelCase__ (				self     :		Dict							, snake_case     :		List[str]			)						->							Tensor:
          return 0.5 * input * (1.0 + torch.tanh(input * 0.7_978_845_608 * (1.0 + 0.044_715 * input * input)			))



class 	a				(		nn.Module       ):


     """simple docstring"""






     def  lowerCamelCase__ (				self     :		int							, snake_case     :		str			)						->							Tensor:
          return input * torch.sigmoid(1.702 * input			)



class 	a				(		nn.Module       ):


     """simple docstring"""






     def __init__(				self     :		Optional[int]							, snake_case     :		Any							, snake_case     :		Any			)						->							Union[str, Any]:



          if min > max:
               raise ValueError(f'min should be < max (got min: {min}, max: {max})'			)

          super().__init__()
          __UpperCAmelCase	: Tuple  							=			min
          __UpperCAmelCase	: List[str]  							=			max






     def  lowerCamelCase__ (				self     :		Union[str, Any]							, snake_case     :		Any			)						->							Tensor:
          return torch.clip(gelu(snake_case			)							, self.min							, self.max			)



class 	a				(		nn.Module       ):


     """simple docstring"""






     def __init__(				self     :		str			)						->							List[str]:
          super().__init__()
          __UpperCAmelCase	: int  							=			math.sqrt(2 / math.pi			)






     def  lowerCamelCase__ (				self     :		List[str]							, snake_case     :		List[str]			)						->							Tensor:
          return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044_715 * torch.pow(snake_case							, 3			))			))



class 	a				(		nn.Module       ):


     """simple docstring"""






     def  lowerCamelCase__ (				self     :		Dict							, snake_case     :		Tuple			)						->							Tensor:
          return nn.functional.silu(snake_case			)



class 	a				(		nn.Module       ):


     """simple docstring"""






     def __init__(				self     :		Union[str, Any]			)						->							Union[str, Any]:
          super().__init__()
          if version.parse(torch.__version__			) < version.parse('''1.9.0'''			):
               __UpperCAmelCase	: str  							=			self._mish_python
          else:
               __UpperCAmelCase	: List[str]  							=			nn.functional.mish

     def  lowerCamelCase__ (				self     :		Optional[Any]							, snake_case     :		int			)						->							Tensor:
          return input * torch.tanh(nn.functional.softplus(snake_case			)			)






     def  lowerCamelCase__ (				self     :		Optional[Any]							, snake_case     :		Optional[Any]			)						->							Tensor:
          return self.act(snake_case			)



class 	a				(		nn.Module       ):


     """simple docstring"""






     def  lowerCamelCase__ (				self     :		Optional[Any]							, snake_case     :		str			)						->							Tensor:
          return input



class 	a				(		nn.Module       ):


     """simple docstring"""






     def  lowerCamelCase__ (				self     :		Dict							, snake_case     :		Optional[Any]							, snake_case     :		Tuple=0.707_107							, snake_case     :		Any=0.282_095			)						->							int:
          __UpperCAmelCase	: int  							=			(input - mu).div(sigma * math.sqrt(2.0			)			)
          return 0.5 * (1.0 + torch.erf(snake_case			))



class 	a				(		nn.Module       ):


     """simple docstring"""






     def  lowerCamelCase__ (				self     :		Any							, snake_case     :		Optional[int]			)						->							Optional[Any]:
          __UpperCAmelCase	: Dict  							=			nn.functional.relu(snake_case			)
          __UpperCAmelCase	: List[str]  							=			torch.square(snake_case			)
          return squared



class 	a				(		UpperCamelCase__       ):


     """simple docstring"""






     def __getitem__(				self     :		int							, snake_case     :		List[Any]			)						->							List[str]:
          __UpperCAmelCase	: str  							=			super().__getitem__(snake_case			)
          __UpperCAmelCase							,   __UpperCAmelCase	: Dict  							=			content if isinstance(snake_case							, snake_case			) else (content, {})
          return cls(**snake_case			)



__UpperCAmelCase			:List[str] 				=   {
    "gelu": GELUActivation,
    "gelu_10": (ClippedGELUActivation, {"min": -1_0, "max": 1_0}),
    "gelu_fast": FastGELUActivation,
    "gelu_new": NewGELUActivation,
    "gelu_python": (GELUActivation, {"use_gelu_python": True}),
    "gelu_pytorch_tanh": PytorchGELUTanh,
    "gelu_accurate": AccurateGELUActivation,
    "laplace": LaplaceActivation,
    "linear": LinearActivation,
    "mish": MishActivation,
    "quick_gelu": QuickGELUActivation,
    "relu": nn.ReLU,
    "relu2": ReLUSquaredActivation,
    "relu6": nn.ReLUa,
    "sigmoid": nn.Sigmoid,
    "silu": SiLUActivation,
    "swish": SiLUActivation,
    "tanh": nn.Tanh,
}
__UpperCAmelCase			:List[Any] 				=   ClassInstantier(ACTaCLS)




def 	_a	(						_lowercase					:	Union[str, Any]  ):





     '''simple docstring'''






     if activation_string in ACTaFN:
          return ACTaFN[activation_string]
     else:
          raise KeyError(F'function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()  )}'  )


# For backwards compatibility with: from activations import gelu_python
__UpperCAmelCase			:int 				=   get_activation("gelu_python")
__UpperCAmelCase			:Optional[Any] 				=   get_activation("gelu_new")
__UpperCAmelCase			:Any 				=   get_activation("gelu")
__UpperCAmelCase			:Any 				=   get_activation("gelu_fast")
__UpperCAmelCase			:Dict 				=   get_activation("quick_gelu")
__UpperCAmelCase			:int 				=   get_activation("silu")
__UpperCAmelCase			:Union[str, Any] 				=   get_activation("mish")

__UpperCAmelCase			:Tuple 				=   get_activation("linear")





from __future__ import annotations

import unittest

from transformers import CTRLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers.models.ctrl.modeling_tf_ctrl import (
       TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
       TFCTRLForSequenceClassification,
       TFCTRLLMHeadModel,
       TFCTRLModel,
   )




class 			a__       (					UpperCamelCase__      ):







     def __init__(	self  ,		A  ,		)  ->    List[Any]:



          '''simple docstring'''



          a      =      parent
          a      =      13
          a      =      7
          a      =      True
          a      =      True
          a      =      True
          a      =      True
          a      =      True
          a      =      99
          a      =      32
          a      =      2
          a      =      4
          a      =      37
          a      =      "gelu"
          a      =      0.1
          a      =      0.1
          a      =      512
          a      =      16
          a      =      2
          a      =      0.0_2
          a      =      3
          a      =      4
          a      =      None
          a      =      self.vocab_size - 1







     def 			lowerCAmelCase_				(	self )  ->    Union[str, Any]:



          '''simple docstring'''



          a      =      ids_tensor([self.batch_size, self.seq_length]  ,		self.vocab_size )

          a      =      None
          if self.use_input_mask:
               a      =      random_attention_mask([self.batch_size, self.seq_length] )

          a      =      None
          if self.use_token_type_ids:
               a      =      ids_tensor([self.batch_size, self.seq_length]  ,		self.type_vocab_size )

          a      =      None
          if self.use_mc_token_ids:
               a      =      ids_tensor([self.batch_size, self.num_choices]  ,		self.seq_length )

          a      =      None
          a      =      None
          a      =      None
          if self.use_labels:
               a      =      ids_tensor([self.batch_size]  ,		self.type_sequence_label_size )
               a      =      ids_tensor([self.batch_size, self.seq_length]  ,		self.num_labels )
               a      =      ids_tensor([self.batch_size]  ,		self.num_choices )

          a      =      CTRLConfig(
              vocab_size=self.vocab_size  ,		n_embd=self.hidden_size  ,		n_layer=self.num_hidden_layers  ,		n_head=self.num_attention_heads  ,		n_positions=self.max_position_embeddings  ,		pad_token_id=self.pad_token_id  ,		)

          a      =      ids_tensor([self.num_hidden_layers, self.num_attention_heads]  ,		2 )

          return (
              config,
              input_ids,
              input_mask,
              head_mask,
              token_type_ids,
              mc_token_ids,
              sequence_labels,
              token_labels,
              choice_labels,
          )







     def 			lowerCAmelCase_				(	self  ,		A  ,		A  ,		A  ,		A  ,		A  ,		*A )  ->    Optional[int]:



          '''simple docstring'''



          a      =      TFCTRLModel(config=A )
          a      =      {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
          a      =      model(A )

          a      =      [input_ids, None, input_mask]  # None is the input for 'past'
          a      =      model(A )

          a      =      model(A )

          self.parent.assertEqual(result.last_hidden_state.shape  ,		(self.batch_size, self.seq_length, self.hidden_size) )







     def 			lowerCAmelCase_				(	self  ,		A  ,		A  ,		A  ,		A  ,		A  ,		*A )  ->    Tuple:



          '''simple docstring'''



          a      =      TFCTRLLMHeadModel(config=A )
          a      =      {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
          a      =      model(A )
          self.parent.assertEqual(result.logits.shape  ,		(self.batch_size, self.seq_length, self.vocab_size) )







     def 			lowerCAmelCase_				(	self  ,		A  ,		A  ,		A  ,		A  ,		A  ,		*A )  ->    List[Any]:



          '''simple docstring'''



          a      =      self.num_labels
          a      =      ids_tensor([self.batch_size]  ,		self.type_sequence_label_size )
          a      =      {
              "input_ids": input_ids,
              "token_type_ids": token_type_ids,
              "labels": sequence_labels,
          }
          a      =      TFCTRLForSequenceClassification(A )
          a      =      model(A )
          self.parent.assertEqual(result.logits.shape  ,		(self.batch_size, self.num_labels) )







     def 			lowerCAmelCase_				(	self )  ->    Union[str, Any]:



          '''simple docstring'''



          a      =      self.prepare_config_and_inputs()

          (
              (
              a
          )  ,      (
              a
          )  ,      (
              a
          )  ,      (
              a
          )  ,      (
              a
          )  ,      (
              a
          )  ,      (
              a
          )  ,      (
              a
          )  ,      (
              a
          )  ,      
          )      =      config_and_inputs

          a      =      {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict







@require_tf
class 			a__       (					UpperCamelCase__   ,					UpperCamelCase__   ,					unittest.TestCase      ):
     a		:			Any							=	(TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
     a		:			Optional[Any]							=	(TFCTRLLMHeadModel,) if is_tf_available() else ()
     a		:			List[Any]							=	(
         {
             """feature-extraction""": TFCTRLModel,
             """text-classification""": TFCTRLForSequenceClassification,
             """text-generation""": TFCTRLLMHeadModel,
             """zero-shot""": TFCTRLForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     a		:			Union[str, Any]							=	False
     a		:			str							=	False







     def 			lowerCAmelCase_				(	self  ,		A  ,		A  ,		A  ,		A  ,		A )  ->    List[str]:



          '''simple docstring'''



          if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
               # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
               # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
               # config could not be created.
               return True

          return False







     def 			lowerCAmelCase_				(	self )  ->    Optional[int]:



          '''simple docstring'''



          a      =      TFCTRLModelTester(self )
          a      =      ConfigTester(self  ,		config_class=A  ,		n_embd=37 )







     def 			lowerCAmelCase_				(	self )  ->    Tuple:



          '''simple docstring'''



          self.config_tester.run_common_tests()







     def 			lowerCAmelCase_				(	self )  ->    Optional[Any]:



          '''simple docstring'''



          a      =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_ctrl_model(*A )







     def 			lowerCAmelCase_				(	self )  ->    List[str]:



          '''simple docstring'''



          a      =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_ctrl_lm_head(*A )







     def 			lowerCAmelCase_				(	self )  ->    Optional[Any]:



          '''simple docstring'''



          a      =      self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_ctrl_for_sequence_classification(*A )







     def 			lowerCAmelCase_				(	self )  ->    Tuple:



          '''simple docstring'''



          a  ,      a      =      self.model_tester.prepare_config_and_inputs_for_common()
          a      =      [TFCTRLLMHeadModel]
          a      =      [TFCTRLForSequenceClassification]

          for model_class in self.all_model_classes:
               a      =      model_class(A )
               model.build()  # may be needed for the get_bias() call below
               assert isinstance(model.get_input_embeddings()  ,		tf.keras.layers.Layer )

               if model_class in list_lm_models:
                    a      =      model.get_output_embeddings()
                    assert isinstance(A  ,		tf.keras.layers.Layer )
                    a      =      model.get_bias()
                    assert isinstance(A  ,		A )
                    for k, v in name.items():
                         assert isinstance(A  ,		tf.Variable )
               elif model_class in list_other_models_with_output_ebd:
                    a      =      model.get_output_embeddings()
                    assert isinstance(A  ,		tf.keras.layers.Layer )
                    a      =      model.get_bias()
                    assert name is None
               else:
                    a      =      model.get_output_embeddings()
                    assert x is None
                    a      =      model.get_bias()
                    assert name is None







     @slow
     def 			lowerCAmelCase_				(	self )  ->    Tuple:



          '''simple docstring'''



          for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               a      =      TFCTRLModel.from_pretrained(A )
               self.assertIsNotNone(A )







@require_tf
class 			a__       (					unittest.TestCase      ):







     @slow
     def 			lowerCAmelCase_				(	self )  ->    Any:



          '''simple docstring'''



          a      =      TFCTRLLMHeadModel.from_pretrained("ctrl" )
          a      =      tf.convert_to_tensor([[11859, 0, 1611, 8]]  ,		dtype=tf.intaa )  # Legal the president is
          a      =      [
              11859,
              0,
              1611,
              8,
              5,
              150,
              26449,
              2,
              19,
              348,
              469,
              3,
              2595,
              48,
              20740,
              246533,
              246533,
              19,
              30,
              5,
          ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

          a      =      model.generate(A  ,		do_sample=A )
          self.assertListEqual(output_ids[0].numpy().tolist()  ,		A )

