






from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:		List[str]		     =			logging.get_logger(__name__)

A_		:		int		     =			{'ctrl': 'https://huggingface.co/ctrl/resolve/main/config.json'}




class     _lowerCAmelCase(	UpperCAmelCase_ ):





       """simple docstring"""







       a    :		int									='''ctrl'''
       a    :		Tuple									=['''past_key_values''']
       a    :		Optional[int]									={
           '''max_position_embeddings''': '''n_positions''',
           '''hidden_size''': '''n_embd''',
           '''num_attention_heads''': '''n_head''',
           '''num_hidden_layers''': '''n_layer''',
       }





       def __init__( self  ,      _lowerCamelCase=2_4_6_5_3_4  ,      _lowerCamelCase=2_5_6  ,      _lowerCamelCase=1_2_8_0  ,      _lowerCamelCase=8_1_9_2  ,      _lowerCamelCase=4_8  ,      _lowerCamelCase=1_6  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=1e-6  ,      _lowerCamelCase=0.0_2  ,      _lowerCamelCase=True  ,      **_lowerCamelCase  ,      ):
         UpperCamelCase_: int   				=    vocab_size
         UpperCamelCase_: Union[str, Any]   				=    n_positions
         UpperCamelCase_: Any   				=    n_embd
         UpperCamelCase_: Optional[Any]   				=    n_layer
         UpperCamelCase_: Dict   				=    n_head
         UpperCamelCase_: Tuple   				=    dff
         UpperCamelCase_: Union[str, Any]   				=    resid_pdrop
         UpperCamelCase_: Any   				=    embd_pdrop
         UpperCamelCase_: Dict   				=    layer_norm_epsilon
         UpperCamelCase_: str   				=    initializer_range

         UpperCamelCase_: Any   				=    use_cache

         super().__init__(**_lowerCamelCase							)







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
   import torch

   from transformers.activations import gelu_new, gelu_python, get_activation




@require_torch
class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       def       _a   ( self							):
         UpperCamelCase_: Any   				=    torch.tensor([-1_0_0, -1, -0.1, 0, 0.1, 1.0, 1_0_0]							)
         UpperCamelCase_: List[str]   				=    get_activation('gelu'							)
         self.assertTrue(torch.allclose(gelu_python(_lowerCamelCase							)  ,      torch_builtin(_lowerCamelCase							)							)							)
         self.assertFalse(torch.allclose(gelu_python(_lowerCamelCase							)  ,      gelu_new(_lowerCamelCase							)							)							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    torch.tensor([-1_0_0, -1, -0.1, 0, 0.1, 1.0, 1_0_0]							)
         UpperCamelCase_: Union[str, Any]   				=    get_activation('gelu'							)
         UpperCamelCase_: int   				=    get_activation('gelu_10'							)

         UpperCamelCase_: Union[str, Any]   				=    torch_builtin(_lowerCamelCase							)
         UpperCamelCase_: List[str]   				=    geluaa(_lowerCamelCase							)

         UpperCamelCase_: Dict   				=    torch.where(y_gelu_aa < 1_0.0  ,      1  ,      0							)

         self.assertTrue(torch.max(_lowerCamelCase							).item() == 1_0.0							)
         self.assertTrue(torch.allclose(y_gelu * clipped_mask  ,      y_gelu_aa * clipped_mask							)							)





       def       _a   ( self							):
         get_activation('gelu'							)
         get_activation('gelu_10'							)
         get_activation('gelu_fast'							)
         get_activation('gelu_new'							)
         get_activation('gelu_python'							)
         get_activation('gelu_pytorch_tanh'							)
         get_activation('linear'							)
         get_activation('mish'							)
         get_activation('quick_gelu'							)
         get_activation('relu'							)
         get_activation('sigmoid'							)
         get_activation('silu'							)
         get_activation('swish'							)
         get_activation('tanh'							)
         with self.assertRaises(_lowerCamelCase							):
           get_activation('bogus'							)
         with self.assertRaises(_lowerCamelCase							):
           get_activation(_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    get_activation('gelu'							)
         UpperCamelCase_: str   				=    1
         UpperCamelCase_: int   				=    get_activation('gelu'							)
         self.assertEqual(acta.a  ,      1							)
         with self.assertRaises(_lowerCamelCase							):
           UpperCamelCase_: Tuple   				=    acta.a