





from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFFunnelBaseModel,
         TFFunnelForMaskedLM,
         TFFunnelForMultipleChoice,
         TFFunnelForPreTraining,
         TFFunnelForQuestionAnswering,
         TFFunnelForSequenceClassification,
         TFFunnelForTokenClassification,
         TFFunnelModel,
     )




class 					_lowerCamelCase			:



      """simple docstring"""






      def __init__(			self					,    UpperCAmelCase					,    UpperCAmelCase=13					,    UpperCAmelCase=7					,    UpperCAmelCase=True					,    UpperCAmelCase=True					,    UpperCAmelCase=True					,    UpperCAmelCase=True					,    UpperCAmelCase=99					,    UpperCAmelCase=[1, 1, 2]					,    UpperCAmelCase=1					,    UpperCAmelCase=32					,    UpperCAmelCase=4					,    UpperCAmelCase=8					,    UpperCAmelCase=37					,    UpperCAmelCase="gelu_new"					,    UpperCAmelCase=0.1					,    UpperCAmelCase=0.1					,    UpperCAmelCase=0.0					,    UpperCAmelCase=512					,    UpperCAmelCase=3					,    UpperCAmelCase=0.02					,    UpperCAmelCase=3					,    UpperCAmelCase=4					,    UpperCAmelCase=None					,    UpperCAmelCase=False					,    )							-> Any:






        '''simple docstring'''






        __snake_case :					int          =							parent
        __snake_case :					int          =							batch_size
        __snake_case :					List[str]          =							seq_length
        __snake_case :					Union[str, Any]          =							is_training
        __snake_case :					Tuple          =							use_input_mask
        __snake_case :					List[str]          =							use_token_type_ids
        __snake_case :					List[str]          =							use_labels
        __snake_case :					Any          =							vocab_size
        __snake_case :					Union[str, Any]          =							block_sizes
        __snake_case :					Optional[int]          =							num_decoder_layers
        __snake_case :					Tuple          =							d_model
        __snake_case :					str          =							n_head
        __snake_case :					int          =							d_head
        __snake_case :					Union[str, Any]          =							d_inner
        __snake_case :					Dict          =							hidden_act
        __snake_case :					Optional[Any]          =							hidden_dropout
        __snake_case :					Union[str, Any]          =							attention_dropout
        __snake_case :					Any          =							activation_dropout
        __snake_case :					str          =							max_position_embeddings
        __snake_case :					Any          =							type_vocab_size
        __snake_case :					Union[str, Any]          =							2
        __snake_case :					List[Any]          =							num_labels
        __snake_case :					str          =							num_choices
        __snake_case :					Optional[int]          =							scope
        __snake_case :					int          =							initializer_std

        # Used in the tests to check the size of the first attention layer
        __snake_case :					Optional[int]          =							n_head
        # Used in the tests to check the size of the first hidden state
        __snake_case :					Union[str, Any]          =							self.d_model
        # Used in the tests to check the number of output hidden states/attentions
        __snake_case :					Union[str, Any]          =							sum(self.block_sizes ) + (0 if base else self.num_decoder_layers)
        # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
        # the last hidden state of the first block (which is the first hidden state of the decoder).
        if not base:
          __snake_case :					Union[str, Any]          =							self.num_hidden_layers + 2







      def     UpperCAmelCase		(			self )							-> Optional[int]:






        '''simple docstring'''






        __snake_case :					List[str]          =							ids_tensor([self.batch_size, self.seq_length]					,    self.vocab_size )

        __snake_case :					Optional[int]          =							None
        if self.use_input_mask:
          __snake_case :					Optional[Any]          =							random_attention_mask([self.batch_size, self.seq_length] )

        __snake_case :					Dict          =							None
        if self.use_token_type_ids:
          __snake_case :					List[Any]          =							ids_tensor([self.batch_size, self.seq_length]					,    self.type_vocab_size )

        __snake_case :					Union[str, Any]          =							None
        __snake_case :					str          =							None
        __snake_case :					Dict          =							None
        if self.use_labels:
          __snake_case :					str          =							ids_tensor([self.batch_size]					,    self.type_sequence_label_size )
          __snake_case :					List[Any]          =							ids_tensor([self.batch_size, self.seq_length]					,    self.num_labels )
          __snake_case :					Any          =							ids_tensor([self.batch_size]					,    self.num_choices )

        __snake_case :					Dict          =							FunnelConfig(
            vocab_size=self.vocab_size					,    block_sizes=self.block_sizes					,    num_decoder_layers=self.num_decoder_layers					,    d_model=self.d_model					,    n_head=self.n_head					,    d_head=self.d_head					,    d_inner=self.d_inner					,    hidden_act=self.hidden_act					,    hidden_dropout=self.hidden_dropout					,    attention_dropout=self.attention_dropout					,    activation_dropout=self.activation_dropout					,    max_position_embeddings=self.max_position_embeddings					,    type_vocab_size=self.type_vocab_size					,    initializer_std=self.initializer_std					,    )

        return (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
            choice_labels,
        )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> int:






        '''simple docstring'''






        __snake_case :					Dict          =							TFFunnelModel(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Any          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __snake_case :					Optional[int]          =							model(__SCREAMING_SNAKE_CASE )

        __snake_case :					Optional[int]          =							[input_ids, input_mask]
        __snake_case :					List[str]          =							model(__SCREAMING_SNAKE_CASE )

        __snake_case :					Optional[int]          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.last_hidden_state.shape					,    (self.batch_size, self.seq_length, self.d_model) )

        __snake_case :					List[str]          =							False
        __snake_case :					List[Any]          =							TFFunnelModel(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					List[str]          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.last_hidden_state.shape					,    (self.batch_size, self.seq_length, self.d_model) )

        __snake_case :					Any          =							False
        __snake_case :					Tuple          =							TFFunnelModel(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Tuple          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.last_hidden_state.shape					,    (self.batch_size, self.seq_length, self.d_model) )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> Optional[Any]:






        '''simple docstring'''






        __snake_case :					str          =							TFFunnelBaseModel(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					List[str]          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __snake_case :					str          =							model(__SCREAMING_SNAKE_CASE )

        __snake_case :					List[Any]          =							[input_ids, input_mask]
        __snake_case :					Dict          =							model(__SCREAMING_SNAKE_CASE )

        __snake_case :					Optional[Any]          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.last_hidden_state.shape					,    (self.batch_size, 2, self.d_model) )

        __snake_case :					Union[str, Any]          =							False
        __snake_case :					Tuple          =							TFFunnelBaseModel(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Dict          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.last_hidden_state.shape					,    (self.batch_size, 3, self.d_model) )

        __snake_case :					str          =							False
        __snake_case :					int          =							TFFunnelBaseModel(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Dict          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.last_hidden_state.shape					,    (self.batch_size, 2, self.d_model) )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> Union[str, Any]:






        '''simple docstring'''






        __snake_case :					int          =							TFFunnelForPreTraining(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Any          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __snake_case :					Dict          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.logits.shape					,    (self.batch_size, self.seq_length) )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> str:






        '''simple docstring'''






        __snake_case :					int          =							TFFunnelForMaskedLM(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Any          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __snake_case :					Tuple          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.logits.shape					,    (self.batch_size, self.seq_length, self.vocab_size) )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> Tuple:






        '''simple docstring'''






        __snake_case :					List[Any]          =							self.num_labels
        __snake_case :					List[str]          =							TFFunnelForSequenceClassification(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Dict          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __snake_case :					Union[str, Any]          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.logits.shape					,    (self.batch_size, self.num_labels) )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> Optional[Any]:






        '''simple docstring'''






        __snake_case :					str          =							self.num_choices
        __snake_case :					Optional[Any]          =							TFFunnelForMultipleChoice(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Tuple          =							tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE					,    1 )					,    (1, self.num_choices, 1) )
        __snake_case :					Dict          =							tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE					,    1 )					,    (1, self.num_choices, 1) )
        __snake_case :					List[Any]          =							tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE					,    1 )					,    (1, self.num_choices, 1) )
        __snake_case :					Dict          =							{
            '''input_ids''': multiple_choice_inputs_ids,
            '''attention_mask''': multiple_choice_input_mask,
            '''token_type_ids''': multiple_choice_token_type_ids,
        }
        __snake_case :					Tuple          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.logits.shape					,    (self.batch_size, self.num_choices) )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> Dict:






        '''simple docstring'''






        __snake_case :					Union[str, Any]          =							self.num_labels
        __snake_case :					Any          =							TFFunnelForTokenClassification(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					List[str]          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __snake_case :					List[str]          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.logits.shape					,    (self.batch_size, self.seq_length, self.num_labels) )







      def     UpperCAmelCase		(			self					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    UpperCAmelCase					,    )							-> str:






        '''simple docstring'''






        __snake_case :					str          =							TFFunnelForQuestionAnswering(config=__SCREAMING_SNAKE_CASE )
        __snake_case :					Dict          =							{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __snake_case :					Optional[Any]          =							model(__SCREAMING_SNAKE_CASE )
        self.parent.assertEqual(result.start_logits.shape					,    (self.batch_size, self.seq_length) )
        self.parent.assertEqual(result.end_logits.shape					,    (self.batch_size, self.seq_length) )







      def     UpperCAmelCase		(			self )							-> List[str]:






        '''simple docstring'''






        __snake_case :					str          =							self.prepare_config_and_inputs()
        (
            __snake_case
        ) :					List[str]          =							config_and_inputs
        __snake_case :					Dict          =							{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
        return config, inputs_dict




@require_tf
class 					_lowerCamelCase			(     UpperCamelCase_			, UpperCamelCase_			, unittest.TestCase     ):



      """simple docstring"""






      UpperCAmelCase_    :   int      =(
          (
              TFFunnelModel,
              TFFunnelForMaskedLM,
              TFFunnelForPreTraining,
              TFFunnelForQuestionAnswering,
              TFFunnelForTokenClassification,
          )
          if is_tf_available()
          else ()
      )
      UpperCAmelCase_    :   Union[str, Any]      =(
          {
              "feature-extraction": (TFFunnelBaseModel, TFFunnelModel),
              "fill-mask": TFFunnelForMaskedLM,
              "question-answering": TFFunnelForQuestionAnswering,
              "text-classification": TFFunnelForSequenceClassification,
              "token-classification": TFFunnelForTokenClassification,
              "zero-shot": TFFunnelForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      UpperCAmelCase_    :   Optional[Any]      =False
      UpperCAmelCase_    :   List[str]      =False







      def     UpperCAmelCase		(			self )							-> Tuple:






        '''simple docstring'''






        __snake_case :					Union[str, Any]          =							TFFunnelModelTester(self )
        __snake_case :					Tuple          =							ConfigTester(self					,    config_class=__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> List[str]:






        '''simple docstring'''






        self.config_tester.run_common_tests()







      def     UpperCAmelCase		(			self )							-> str:






        '''simple docstring'''






        __snake_case :					List[str]          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> Any:






        '''simple docstring'''






        __snake_case :					Tuple          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_pretraining(*__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> Any:






        '''simple docstring'''






        __snake_case :					str          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> List[str]:






        '''simple docstring'''






        __snake_case :					Optional[int]          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> Any:






        '''simple docstring'''






        __snake_case :					Tuple          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE )




@require_tf
class 					_lowerCamelCase			(     UpperCamelCase_			, unittest.TestCase     ):



      """simple docstring"""






      UpperCAmelCase_    :   Union[str, Any]      =(
          (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
      )
      UpperCAmelCase_    :   Tuple      =False
      UpperCAmelCase_    :   Optional[Any]      =False







      def     UpperCAmelCase		(			self )							-> List[str]:






        '''simple docstring'''






        __snake_case :					List[str]          =							TFFunnelModelTester(self					,    base=__SCREAMING_SNAKE_CASE )
        __snake_case :					List[str]          =							ConfigTester(self					,    config_class=__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> List[str]:






        '''simple docstring'''






        self.config_tester.run_common_tests()







      def     UpperCAmelCase		(			self )							-> int:






        '''simple docstring'''






        __snake_case :					Tuple          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_base_model(*__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> int:






        '''simple docstring'''






        __snake_case :					Tuple          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE )







      def     UpperCAmelCase		(			self )							-> Union[str, Any]:






        '''simple docstring'''






        __snake_case :					int          =							self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE )



from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase  =logging.get_logger(__name__)

lowerCamelCase  ={
    "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json",
}




class 					_lowerCamelCase			(   UpperCamelCase_			):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		 =	'''gpt_neox_japanese'''






     def __init__(				self    ,    __SCREAMING_SNAKE_CASE=3_2_0_0_0    ,    __SCREAMING_SNAKE_CASE=2_5_6_0    ,    __SCREAMING_SNAKE_CASE=3_2    ,    __SCREAMING_SNAKE_CASE=3_2    ,    __SCREAMING_SNAKE_CASE=4    ,    __SCREAMING_SNAKE_CASE="gelu"    ,    __SCREAMING_SNAKE_CASE=1.00    ,    __SCREAMING_SNAKE_CASE=1_0_0_0_0    ,    __SCREAMING_SNAKE_CASE=2_0_4_8    ,    __SCREAMING_SNAKE_CASE=0.02    ,    __SCREAMING_SNAKE_CASE=1e-5    ,    __SCREAMING_SNAKE_CASE=True    ,    __SCREAMING_SNAKE_CASE=3_1_9_9_6    ,    __SCREAMING_SNAKE_CASE=3_1_9_9_9    ,    __SCREAMING_SNAKE_CASE=0.1    ,    __SCREAMING_SNAKE_CASE=0.0    ,    **__SCREAMING_SNAKE_CASE    ,    )	-> List[str]:




           """simple docstring"""







           super().__init__(bos_token_id=__SCREAMING_SNAKE_CASE    ,    eos_token_id=__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)
           UpperCamelCase__     : List[str]											=					vocab_size
           UpperCamelCase__     : int											=					max_position_embeddings
           UpperCamelCase__     : Union[str, Any]											=					hidden_size
           UpperCamelCase__     : Any											=					num_hidden_layers
           UpperCamelCase__     : List[Any]											=					num_attention_heads
           UpperCamelCase__     : int											=					intermediate_multiple_size
           UpperCamelCase__     : List[Any]											=					hidden_act
           UpperCamelCase__     : Tuple											=					rotary_pct
           UpperCamelCase__     : Optional[int]											=					rotary_emb_base
           UpperCamelCase__     : Optional[Any]											=					initializer_range
           UpperCamelCase__     : Optional[int]											=					layer_norm_eps
           UpperCamelCase__     : Optional[Any]											=					use_cache
           UpperCamelCase__     : List[str]											=					attention_dropout
           UpperCamelCase__     : Optional[int]											=					hidden_dropout


