






import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      NezhaForMaskedLM,
      NezhaForMultipleChoice,
      NezhaForNextSentencePrediction,
      NezhaForPreTraining,
      NezhaForQuestionAnswering,
      NezhaForSequenceClassification,
      NezhaForTokenClassification,
      NezhaModel,
  )
  from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST
class 		SCREAMING_SNAKE_CASE__	:

  '''simple docstring'''







  def __init__(      self		:			int       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			int=13       ,				lowerCamelCase		:			Union[str, Any]=7       ,				lowerCamelCase		:			Optional[Any]=True       ,				lowerCamelCase		:			Dict=True       ,				lowerCamelCase		:			List[Any]=True       ,				lowerCamelCase		:			Optional[Any]=True       ,				lowerCamelCase		:			List[str]=99       ,				lowerCamelCase		:			Tuple=32       ,				lowerCamelCase		:			Any=5       ,				lowerCamelCase		:			Any=4       ,				lowerCamelCase		:			Optional[int]=37       ,				lowerCamelCase		:			Optional[Any]="gelu"       ,				lowerCamelCase		:			Dict=0.1       ,				lowerCamelCase		:			Any=0.1       ,				lowerCamelCase		:			Tuple=128       ,				lowerCamelCase		:			Dict=32       ,				lowerCamelCase		:			int=16       ,				lowerCamelCase		:			Dict=2       ,				lowerCamelCase		:			Optional[int]=0.02       ,				lowerCamelCase		:			Any=3       ,				lowerCamelCase		:			List[str]=4       ,				lowerCamelCase		:			Dict=None       ,				)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    parent
         _UpperCAmelCase							=    batch_size
         _UpperCAmelCase							=    seq_length
         _UpperCAmelCase							=    is_training
         _UpperCAmelCase							=    use_input_mask
         _UpperCAmelCase							=    use_token_type_ids
         _UpperCAmelCase							=    use_labels
         _UpperCAmelCase							=    vocab_size
         _UpperCAmelCase							=    hidden_size
         _UpperCAmelCase							=    num_hidden_layers
         _UpperCAmelCase							=    num_attention_heads
         _UpperCAmelCase							=    intermediate_size
         _UpperCAmelCase							=    hidden_act
         _UpperCAmelCase							=    hidden_dropout_prob
         _UpperCAmelCase							=    attention_probs_dropout_prob
         _UpperCAmelCase							=    max_position_embeddings
         _UpperCAmelCase							=    type_vocab_size
         _UpperCAmelCase							=    type_sequence_label_size
         _UpperCAmelCase							=    initializer_range
         _UpperCAmelCase							=    num_labels
         _UpperCAmelCase							=    num_choices
         _UpperCAmelCase							=    scope


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	Optional[int]:


         """simple docstring"""
         _UpperCAmelCase							=    ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size							)

         _UpperCAmelCase							=    None
         if self.use_input_mask:
                _UpperCAmelCase							=    random_attention_mask([self.batch_size, self.seq_length]							)

         _UpperCAmelCase							=    None
         if self.use_token_type_ids:
                _UpperCAmelCase							=    ids_tensor([self.batch_size, self.seq_length]       ,				self.type_vocab_size							)

         _UpperCAmelCase							=    None
         _UpperCAmelCase							=    None
         _UpperCAmelCase							=    None
         if self.use_labels:
                _UpperCAmelCase							=    ids_tensor([self.batch_size]       ,				self.type_sequence_label_size							)
                _UpperCAmelCase							=    ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels							)
                _UpperCAmelCase							=    ids_tensor([self.batch_size]       ,				self.num_choices							)

         _UpperCAmelCase							=    self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


  def  lowerCamelCase       (      self		:			Any							)						->	Any:


         """simple docstring"""
         return NezhaConfig(
             vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				type_vocab_size=self.type_vocab_size       ,				is_decoder=lowerCamelCase       ,				initializer_range=self.initializer_range       ,				)


  def  lowerCamelCase       (      self		:			int							)						->	List[str]:


         """simple docstring"""
         (
             (
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			
         )							=    self.prepare_config_and_inputs()

         _UpperCAmelCase							=    True
         _UpperCAmelCase							=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]							)
         _UpperCAmelCase							=    ids_tensor([self.batch_size, self.seq_length]       ,				vocab_size=2							)

         return (
             config,
             input_ids,
             token_type_ids,
             input_mask,
             sequence_labels,
             token_labels,
             choice_labels,
             encoder_hidden_states,
             encoder_attention_mask,
         )


  def  lowerCamelCase       (      self		:			Optional[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Tuple							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaModel(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase							)
         _UpperCAmelCase							=    model(lowerCamelCase       ,				token_type_ids=lowerCamelCase							)
         _UpperCAmelCase							=    model(lowerCamelCase							)
         self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)							)
         self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)							)


  def  lowerCamelCase       (      self		:			Union[str, Any]       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			List[Any]       ,				)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    True
         _UpperCAmelCase							=    NezhaModel(lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				encoder_hidden_states=lowerCamelCase       ,				encoder_attention_mask=lowerCamelCase       ,				)
         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				encoder_hidden_states=lowerCamelCase       ,				)
         _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase							)
         self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)							)
         self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)							)


  def  lowerCamelCase       (      self		:			Any       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Tuple							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaForMaskedLM(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)							)


  def  lowerCamelCase       (      self		:			Any       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Optional[Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaForNextSentencePrediction(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase       ,				)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, 2)							)


  def  lowerCamelCase       (      self		:			Dict       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Any							)						->	Dict:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaForPreTraining(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase       ,				next_sentence_label=lowerCamelCase       ,				)
         self.parent.assertEqual(result.prediction_logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)							)
         self.parent.assertEqual(result.seq_relationship_logits.shape       ,				(self.batch_size, 2)							)


  def  lowerCamelCase       (      self		:			List[Any]       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Optional[int]							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaForQuestionAnswering(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				start_positions=lowerCamelCase       ,				end_positions=lowerCamelCase       ,				)
         self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)							)
         self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)							)


  def  lowerCamelCase       (      self		:			int       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Any       ,				lowerCamelCase		:			str							)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    self.num_labels
         _UpperCAmelCase							=    NezhaForSequenceClassification(lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_labels)							)


  def  lowerCamelCase       (      self		:			Optional[int]       ,				lowerCamelCase		:			List[str]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			List[str]							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    self.num_labels
         _UpperCAmelCase							=    NezhaForTokenClassification(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)							)


  def  lowerCamelCase       (      self		:			List[Any]       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Union[str, Any]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			List[Any]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			List[Any]							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    self.num_choices
         _UpperCAmelCase							=    NezhaForMultipleChoice(config=lowerCamelCase							)
         model.to(lowerCamelCase							)
         model.eval()
         _UpperCAmelCase							=    input_ids.unsqueeze(1							).expand(-1       ,				self.num_choices       ,				-1							).contiguous()
         _UpperCAmelCase							=    token_type_ids.unsqueeze(1							).expand(-1       ,				self.num_choices       ,				-1							).contiguous()
         _UpperCAmelCase							=    input_mask.unsqueeze(1							).expand(-1       ,				self.num_choices       ,				-1							).contiguous()
         _UpperCAmelCase							=    model(
             lowerCamelCase       ,				attention_mask=lowerCamelCase       ,				token_type_ids=lowerCamelCase       ,				labels=lowerCamelCase       ,				)
         self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_choices)							)





  def  lowerCamelCase       (      self		:			Dict							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    self.prepare_config_and_inputs()
         (
             (
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			
         )							=    config_and_inputs
         _UpperCAmelCase							=    {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
         return config, inputs_dict



@require_torch
class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase					,      UpperCAmelCase					,      UpperCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  (
      (
          NezhaModel,
          NezhaForMaskedLM,
          NezhaForMultipleChoice,
          NezhaForNextSentencePrediction,
          NezhaForPreTraining,
          NezhaForQuestionAnswering,
          NezhaForSequenceClassification,
          NezhaForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  _lowerCamelCase    =  (
      {
          '''feature-extraction''': NezhaModel,
          '''fill-mask''': NezhaForMaskedLM,
          '''question-answering''': NezhaForQuestionAnswering,
          '''text-classification''': NezhaForSequenceClassification,
          '''token-classification''': NezhaForTokenClassification,
          '''zero-shot''': NezhaForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  _lowerCamelCase    =  True


  def  lowerCamelCase       (      self		:			Optional[Any]       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Union[str, Any]=False							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    super()._prepare_for_class(lowerCamelCase       ,				lowerCamelCase       ,				return_labels=lowerCamelCase							)

         if return_labels:
                if model_class in get_values(lowerCamelCase							):
                       _UpperCAmelCase							=    torch.zeros(
                           (self.model_tester.batch_size, self.model_tester.seq_length)       ,				dtype=torch.long       ,				device=lowerCamelCase							)
                       _UpperCAmelCase							=    torch.zeros(
                           self.model_tester.batch_size       ,				dtype=torch.long       ,				device=lowerCamelCase							)
         return inputs_dict


  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaModelTester(self							)
         _UpperCAmelCase							=    ConfigTester(self       ,				config_class=lowerCamelCase       ,				hidden_size=37							)


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	int:


         """simple docstring"""
         self.config_tester.run_common_tests()


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	Optional[int]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_model_as_decoder(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Tuple							)						->	str:


         """simple docstring"""
         # This regression test was failing with PyTorch < 1.3
         (
             (
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			(
             _UpperCAmelCase
         )	,			
         )							=    self.model_tester.prepare_config_and_inputs_for_decoder()

         _UpperCAmelCase							=    None

         self.model_tester.create_and_check_model_as_decoder(
             lowerCamelCase       ,				lowerCamelCase       ,				lowerCamelCase       ,				lowerCamelCase       ,				lowerCamelCase       ,				lowerCamelCase       ,				lowerCamelCase       ,				lowerCamelCase       ,				lowerCamelCase       ,				)


  def  lowerCamelCase       (      self		:			List[str]							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Any							)						->	Union[str, Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			List[str]							)						->	Optional[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_next_sequence_prediction(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Dict							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_pretraining(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			List[str]							)						->	str:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Union[str, Any]							)						->	Any:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase							)


  def  lowerCamelCase       (      self		:			Optional[int]							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*lowerCamelCase							)


  @slow
  def  lowerCamelCase       (      self		:			Optional[int]							)						->	Optional[Any]:


         """simple docstring"""
         for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                _UpperCAmelCase							=    NezhaModel.from_pretrained(lowerCamelCase							)
                self.assertIsNotNone(lowerCamelCase							)





  @slow
  @require_torch_gpu
  def  lowerCamelCase       (      self		:			Tuple							)						->	Any:


         """simple docstring"""
         _UpperCAmelCase	,			_UpperCAmelCase							=    self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
                # NezhaForMultipleChoice behaves incorrectly in JIT environments.
                if model_class == NezhaForMultipleChoice:
                       return

                _UpperCAmelCase							=    True
                _UpperCAmelCase							=    model_class(config=lowerCamelCase							)

                _UpperCAmelCase							=    self._prepare_for_class(lowerCamelCase       ,				lowerCamelCase							)
                _UpperCAmelCase							=    torch.jit.trace(
                    lowerCamelCase       ,				(inputs_dict["""input_ids"""].to("""cpu"""							), inputs_dict["""attention_mask"""].to("""cpu"""							))							)

                with tempfile.TemporaryDirectory() as tmp:
                       torch.jit.save(lowerCamelCase       ,				os.path.join(lowerCamelCase       ,				"""bert.pt"""							)							)
                       _UpperCAmelCase							=    torch.jit.load(os.path.join(lowerCamelCase       ,				"""bert.pt"""							)       ,				map_location=lowerCamelCase							)
                       loaded(inputs_dict["""input_ids"""].to(lowerCamelCase							)       ,				inputs_dict["""attention_mask"""].to(lowerCamelCase							)							)



@require_torch
class 		SCREAMING_SNAKE_CASE__	(  unittest.TestCase     ):

  '''simple docstring'''







  @slow
  def  lowerCamelCase       (      self		:			Dict							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaModel.from_pretrained("""sijunhe/nezha-cn-base"""							)
         _UpperCAmelCase							=    torch.tensor([[0, 1, 2, 3, 4, 5]]							)
         _UpperCAmelCase							=    torch.tensor([[0, 1, 1, 1, 1, 1]]							)
         with torch.no_grad():
                _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase							)[0]
         _UpperCAmelCase							=    torch.Size((1, 6, 768)							)
         self.assertEqual(output.shape       ,				lowerCamelCase							)
         _UpperCAmelCase							=    torch.tensor([[[0.0685, 0.2441, 0.1102], [0.0600, 0.1906, 0.1349], [0.0221, 0.0819, 0.0586]]]							)

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,				lowerCamelCase       ,				atol=1E-4							)							)





  @slow
  def  lowerCamelCase       (      self		:			str							)						->	Dict:


         """simple docstring"""
         _UpperCAmelCase							=    NezhaForMaskedLM.from_pretrained("""sijunhe/nezha-cn-base"""							)
         _UpperCAmelCase							=    torch.tensor([[0, 1, 2, 3, 4, 5]]							)
         _UpperCAmelCase							=    torch.tensor([[1, 1, 1, 1, 1, 1]]							)
         with torch.no_grad():
                _UpperCAmelCase							=    model(lowerCamelCase       ,				attention_mask=lowerCamelCase							)[0]
         _UpperCAmelCase							=    torch.Size((1, 6, 2_1128)							)
         self.assertEqual(output.shape       ,				lowerCamelCase							)
         _UpperCAmelCase							=    torch.tensor(
             [[-2.7939, -1.7902, -2.2189], [-2.8585, -1.8908, -2.3723], [-2.6499, -1.7750, -2.2558]]							)

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,				lowerCamelCase       ,				atol=1E-4							)							)

'''simple docstring'''





from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFSequenceClassifierOutput,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import logging
from .configuration_regnet import RegNetConfig


SCREAMING_SNAKE_CASE     =       logging.get_logger(__name__)

# General docstring
SCREAMING_SNAKE_CASE     =       """RegNetConfig"""

# Base docstring
SCREAMING_SNAKE_CASE     =       """facebook/regnet-y-040"""
SCREAMING_SNAKE_CASE     =       [1, 1_0_8_8, 7, 7]

# Image classification docstring
SCREAMING_SNAKE_CASE     =       """facebook/regnet-y-040"""
SCREAMING_SNAKE_CASE     =       """tabby, tabby cat"""

SCREAMING_SNAKE_CASE     =       [
    """facebook/regnet-y-040""",
    # See all regnet models at https://huggingface.co/models?filter=regnet
]






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  str			,				snake_case__		:  int			,				snake_case__		:  int = 3			,				snake_case__		:  int = 1			,				snake_case__		:  int = 1			,				snake_case__		:  Optional[str] = "relu"			,				**snake_case__		:  Dict			,				):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   # The padding and conv has been verified in
   # https://colab.research.google.com/gist/sayakpaul/854bc10eeaf21c9ee2119e0b9f3841a7/scratchpad.ipynb
   UpperCAmelCase__      :					Any				       =							tf.keras.layers.ZeroPaddingaD(padding=kernel_size // 2 )
   UpperCAmelCase__      :					Union[str, Any]				       =							tf.keras.layers.ConvaD(
       filters=snake_case__			,				kernel_size=snake_case__			,				strides=snake_case__			,				padding="VALID"			,				groups=snake_case__			,				use_bias=snake_case__			,				name="convolution"			,				)
   UpperCAmelCase__      :					Any				       =							tf.keras.layers.BatchNormalization(epsilon=1e-5			,				momentum=0.9			,				name="normalization" )
   UpperCAmelCase__      :					Tuple				       =							ACTaFN[activation] if activation is not None else tf.identity







 def 							UpperCamelCase  (					self		:  str			,				snake_case__		:  str ):



   '''simple docstring'''




   UpperCAmelCase__      :					str				       =							self.convolution(self.padding(snake_case__ ) )
   UpperCAmelCase__      :					Dict				       =							self.normalization(snake_case__ )
   UpperCAmelCase__      :					Any				       =							self.activation(snake_case__ )
   return hidden_state






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  List[Any]			,				snake_case__		:  RegNetConfig			,				**snake_case__		:  Optional[int] ):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   UpperCAmelCase__      :					Any				       =							config.num_channels
   UpperCAmelCase__      :					Optional[Any]				       =							TFRegNetConvLayer(
       out_channels=config.embedding_size			,				kernel_size=3			,				stride=2			,				activation=config.hidden_act			,				name="embedder"			,				)







 def 							UpperCamelCase  (					self		:  int			,				snake_case__		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							shape_list(snake_case__ )[1]
   if tf.executing_eagerly() and num_channels != self.num_channels:
     raise ValueError(
         "Make sure that the channel dimension of the pixel values match with the one set in the configuration." )

   # When running on CPU, `tf.keras.layers.Conv2D` doesn't support `NCHW` format.
   # So change the input format from `NCHW` to `NHWC`.
   # shape = (batch_size, in_height, in_width, in_channels=num_channels)
   UpperCAmelCase__      :					List[str]				       =							tf.transpose(snake_case__			,				perm=(0, 2, 3, 1) )
   UpperCAmelCase__      :					Dict				       =							self.embedder(snake_case__ )
   return hidden_state






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  Optional[int]			,				snake_case__		:  int			,				snake_case__		:  int = 2			,				**snake_case__		:  Optional[int] ):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   UpperCAmelCase__      :					int				       =							tf.keras.layers.ConvaD(
       filters=snake_case__			,				kernel_size=1			,				strides=snake_case__			,				use_bias=snake_case__			,				name="convolution" )
   UpperCAmelCase__      :					str				       =							tf.keras.layers.BatchNormalization(epsilon=1e-5			,				momentum=0.9			,				name="normalization" )







 def 							UpperCamelCase  (					self		:  str			,				snake_case__		:  tf.Tensor			,				snake_case__		:  bool = False ):



   '''simple docstring'''




   return self.normalization(self.convolution(snake_case__ )			,				training=snake_case__ )






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  List[str]			,				snake_case__		:  int			,				snake_case__		:  int			,				**snake_case__		:  Union[str, Any] ):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   UpperCAmelCase__      :					Any				       =							tf.keras.layers.GlobalAveragePoolingaD(keepdims=snake_case__			,				name="pooler" )
   UpperCAmelCase__      :					Optional[Any]				       =							[
       tf.keras.layers.ConvaD(filters=snake_case__			,				kernel_size=1			,				activation="relu"			,				name="attention.0" ),
       tf.keras.layers.ConvaD(filters=snake_case__			,				kernel_size=1			,				activation="sigmoid"			,				name="attention.2" ),
   ]







 def 							UpperCamelCase  (					self		:  Optional[int]			,				snake_case__		:  Any ):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							self.pooler(snake_case__ )
   for layer_module in self.attention:
     UpperCAmelCase__      :					Dict				       =							layer_module(snake_case__ )
   UpperCAmelCase__      :					int				       =							hidden_state * pooled
   return hidden_state






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  List[str]			,				snake_case__		:  RegNetConfig			,				snake_case__		:  int			,				snake_case__		:  int			,				snake_case__		:  int = 1			,				**snake_case__		:  str ):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   UpperCAmelCase__      :					Optional[Any]				       =							in_channels != out_channels or stride != 1
   UpperCAmelCase__      :					Optional[int]				       =							max(1			,				out_channels // config.groups_width )
   UpperCAmelCase__      :					Tuple				       =							(
       TFRegNetShortCut(snake_case__			,				stride=snake_case__			,				name="shortcut" )
       if should_apply_shortcut
       else tf.keras.layers.Activation("linear"			,				name="shortcut" )
   )
   # `self.layers` instead of `self.layer` because that is a reserved argument.
   UpperCAmelCase__      :					int				       =							[
       TFRegNetConvLayer(snake_case__			,				kernel_size=1			,				activation=config.hidden_act			,				name="layer.0" ),
       TFRegNetConvLayer(
           snake_case__			,				stride=snake_case__			,				groups=snake_case__			,				activation=config.hidden_act			,				name="layer.1" ),
       TFRegNetConvLayer(snake_case__			,				kernel_size=1			,				activation=snake_case__			,				name="layer.2" ),
   ]
   UpperCAmelCase__      :					List[Any]				       =							ACTaFN[config.hidden_act]







 def 							UpperCamelCase  (					self		:  Tuple			,				snake_case__		:  Optional[int] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							hidden_state
   for layer_module in self.layers:
     UpperCAmelCase__      :					List[str]				       =							layer_module(snake_case__ )
   UpperCAmelCase__      :					Optional[Any]				       =							self.shortcut(snake_case__ )
   hidden_state += residual
   UpperCAmelCase__      :					Union[str, Any]				       =							self.activation(snake_case__ )
   return hidden_state






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  List[Any]			,				snake_case__		:  RegNetConfig			,				snake_case__		:  int			,				snake_case__		:  int			,				snake_case__		:  int = 1			,				**snake_case__		:  List[Any] ):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   UpperCAmelCase__      :					int				       =							in_channels != out_channels or stride != 1
   UpperCAmelCase__      :					Union[str, Any]				       =							max(1			,				out_channels // config.groups_width )
   UpperCAmelCase__      :					Dict				       =							(
       TFRegNetShortCut(snake_case__			,				stride=snake_case__			,				name="shortcut" )
       if should_apply_shortcut
       else tf.keras.layers.Activation("linear"			,				name="shortcut" )
   )
   UpperCAmelCase__      :					Optional[Any]				       =							[
       TFRegNetConvLayer(snake_case__			,				kernel_size=1			,				activation=config.hidden_act			,				name="layer.0" ),
       TFRegNetConvLayer(
           snake_case__			,				stride=snake_case__			,				groups=snake_case__			,				activation=config.hidden_act			,				name="layer.1" ),
       TFRegNetSELayer(snake_case__			,				reduced_channels=int(round(in_channels / 4 ) )			,				name="layer.2" ),
       TFRegNetConvLayer(snake_case__			,				kernel_size=1			,				activation=snake_case__			,				name="layer.3" ),
   ]
   UpperCAmelCase__      :					Dict				       =							ACTaFN[config.hidden_act]







 def 							UpperCamelCase  (					self		:  Optional[int]			,				snake_case__		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					List[str]				       =							hidden_state
   for layer_module in self.layers:
     UpperCAmelCase__      :					Tuple				       =							layer_module(snake_case__ )
   UpperCAmelCase__      :					Dict				       =							self.shortcut(snake_case__ )
   hidden_state += residual
   UpperCAmelCase__      :					List[Any]				       =							self.activation(snake_case__ )
   return hidden_state






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  List[Any]			,				snake_case__		:  RegNetConfig			,				snake_case__		:  int			,				snake_case__		:  int			,				snake_case__		:  int = 2			,				snake_case__		:  int = 2			,				**snake_case__		:  Any ):



   '''simple docstring'''




   super().__init__(**snake_case__ )

   UpperCAmelCase__      :					Any				       =							TFRegNetXLayer if config.layer_type == "x" else TFRegNetYLayer
   UpperCAmelCase__      :					int				       =							[
       # downsampling is done in the first layer with stride of 2
       layer(snake_case__			,				snake_case__			,				snake_case__			,				stride=snake_case__			,				name="layers.0" ),
       *[layer(snake_case__			,				snake_case__			,				snake_case__			,				name=F"""layers.{i+1}""" ) for i in range(depth - 1 )],
   ]







 def 							UpperCamelCase  (					self		:  Dict			,				snake_case__		:  Tuple ):



   '''simple docstring'''




   for layer_module in self.layers:
     UpperCAmelCase__      :					Tuple				       =							layer_module(snake_case__ )
   return hidden_state






class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''







 def __init__(					self		:  Dict			,				snake_case__		:  RegNetConfig			,				**snake_case__		:  Any ):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   UpperCAmelCase__      :					Union[str, Any]				       =							[]
   # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
   self.stages.append(
       TFRegNetStage(
           snake_case__			,				config.embedding_size			,				config.hidden_sizes[0]			,				stride=2 if config.downsample_in_first_stage else 1			,				depth=config.depths[0]			,				name="stages.0"			,				) )
   UpperCAmelCase__      :					List[Any]				       =							zip(config.hidden_sizes			,				config.hidden_sizes[1:] )
   for i, ((in_channels, out_channels), depth) in enumerate(zip(snake_case__			,				config.depths[1:] ) ):
     self.stages.append(TFRegNetStage(snake_case__			,				snake_case__			,				snake_case__			,				depth=snake_case__			,				name=F"""stages.{i+1}""" ) )







 def 							UpperCamelCase  (					self		:  Dict			,				snake_case__		:  tf.Tensor			,				snake_case__		:  bool = False			,				snake_case__		:  bool = True ):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							() if output_hidden_states else None

   for stage_module in self.stages:
     if output_hidden_states:
       UpperCAmelCase__      :					Optional[Any]				       =							hidden_states + (hidden_state,)

     UpperCAmelCase__      :					Dict				       =							stage_module(snake_case__ )

   if output_hidden_states:
     UpperCAmelCase__      :					Any				       =							hidden_states + (hidden_state,)

   if not return_dict:
     return tuple(v for v in [hidden_state, hidden_states] if v is not None )

   return TFBaseModelOutputWithNoAttention(last_hidden_state=snake_case__			,				hidden_states=snake_case__ )






@keras_serializable
class 							UpperCAmelCase_							(    tf.keras.layers.Layer				):

 '''simple docstring'''
 lowercase_					:    Dict				=							RegNetConfig







 def __init__(					self		:  int			,				snake_case__		:  str			,				**snake_case__		:  Union[str, Any] ):



   '''simple docstring'''




   super().__init__(**snake_case__ )
   UpperCAmelCase__      :					List[Any]				       =							config
   UpperCAmelCase__      :					Dict				       =							TFRegNetEmbeddings(snake_case__			,				name="embedder" )
   UpperCAmelCase__      :					int				       =							TFRegNetEncoder(snake_case__			,				name="encoder" )
   UpperCAmelCase__      :					int				       =							tf.keras.layers.GlobalAveragePoolingaD(keepdims=snake_case__			,				name="pooler" )







 @unpack_inputs
 def 							UpperCamelCase  (					self		:  Dict			,				snake_case__		:  tf.Tensor			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  bool = False			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[Any]				       =							(
       output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
   )
   UpperCAmelCase__      :					Tuple				       =							return_dict if return_dict is not None else self.config.use_return_dict

   UpperCAmelCase__      :					str				       =							self.embedder(snake_case__			,				training=snake_case__ )

   UpperCAmelCase__      :					Optional[Any]				       =							self.encoder(
       snake_case__			,				output_hidden_states=snake_case__			,				return_dict=snake_case__			,				training=snake_case__ )

   UpperCAmelCase__      :					Optional[int]				       =							encoder_outputs[0]
   UpperCAmelCase__      :					Optional[Any]				       =							self.pooler(snake_case__ )

   # Change to NCHW output format have uniformity in the modules
   UpperCAmelCase__      :					List[Any]				       =							tf.transpose(snake_case__			,				perm=(0, 3, 1, 2) )
   UpperCAmelCase__      :					Any				       =							tf.transpose(snake_case__			,				perm=(0, 3, 1, 2) )

   # Change the other hidden state outputs to NCHW as well
   if output_hidden_states:
     UpperCAmelCase__      :					Optional[int]				       =							tuple([tf.transpose(snake_case__			,				perm=(0, 3, 1, 2) ) for h in encoder_outputs[1]] )

   if not return_dict:
     return (last_hidden_state, pooled_output) + encoder_outputs[1:]

   return TFBaseModelOutputWithPoolingAndNoAttention(
       last_hidden_state=snake_case__			,				pooler_output=snake_case__			,				hidden_states=hidden_states if output_hidden_states else encoder_outputs.hidden_states			,				)






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    List[Any]				=							RegNetConfig
 lowercase_					:    Optional[int]				=							"regnet"
 lowercase_					:    Optional[Any]				=							"pixel_values"







 @property
 def 							UpperCamelCase  (					self		:  Any ):



   '''simple docstring'''




   return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 2_24, 2_24)			,				dtype=tf.floataa )}



SCREAMING_SNAKE_CASE     =       R"""
    Parameters:
    This model is a Tensorflow
    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a
    regular Tensorflow Module and refer to the Tensorflow documentation for all matter related to general usage and
    behavior.
        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""

SCREAMING_SNAKE_CASE     =       R"""
    Args:
        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConveNextImageProcessor.__call__`] for details.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
"""






@add_start_docstrings(
    "The bare RegNet model outputting raw features without any specific head on top."					,				A					,				)
class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''







 def __init__(					self		:  Union[str, Any]			,				snake_case__		:  RegNetConfig			,				*snake_case__		:  Any			,				**snake_case__		:  List[str] ):



   '''simple docstring'''




   super().__init__(snake_case__			,				*snake_case__			,				**snake_case__ )
   UpperCAmelCase__      :					Any				       =							TFRegNetMainLayer(snake_case__			,				name="regnet" )







 @unpack_inputs
 @add_start_docstrings_to_model_forward(snake_case__ )
 @add_code_sample_docstrings(
     checkpoint=_CHECKPOINT_FOR_DOC			,				output_type=snake_case__			,				config_class=_CONFIG_FOR_DOC			,				modality="vision"			,				expected_output=_EXPECTED_OUTPUT_SHAPE			,				)
 def 							UpperCamelCase  (					self		:  Optional[int]			,				snake_case__		:  tf.Tensor			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  Optional[bool] = None			,				snake_case__		:  int=False			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							(
       output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
   )
   UpperCAmelCase__      :					str				       =							return_dict if return_dict is not None else self.config.use_return_dict

   UpperCAmelCase__      :					List[str]				       =							self.regnet(
       pixel_values=snake_case__			,				output_hidden_states=snake_case__			,				return_dict=snake_case__			,				training=snake_case__			,				)
   if not return_dict:
     return (outputs[0],) + outputs[1:]

   return TFBaseModelOutputWithPoolingAndNoAttention(
       last_hidden_state=outputs.last_hidden_state			,				pooler_output=outputs.pooler_output			,				hidden_states=outputs.hidden_states			,				)






@add_start_docstrings(
    "\n    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "					,				A					,				)
class 							UpperCAmelCase_							(    A					,				A				):

 '''simple docstring'''







 def __init__(					self		:  List[str]			,				snake_case__		:  RegNetConfig			,				*snake_case__		:  Optional[Any]			,				**snake_case__		:  int ):



   '''simple docstring'''




   super().__init__(snake_case__			,				*snake_case__			,				**snake_case__ )
   UpperCAmelCase__      :					Optional[Any]				       =							config.num_labels
   UpperCAmelCase__      :					str				       =							TFRegNetMainLayer(snake_case__			,				name="regnet" )
   # classification head
   UpperCAmelCase__      :					Optional[int]				       =							[
       tf.keras.layers.Flatten(),
       tf.keras.layers.Dense(config.num_labels			,				name="classifier.1" ) if config.num_labels > 0 else tf.identity,
   ]







 @unpack_inputs
 @add_start_docstrings_to_model_forward(snake_case__ )
 @add_code_sample_docstrings(
     checkpoint=_IMAGE_CLASS_CHECKPOINT			,				output_type=snake_case__			,				config_class=_CONFIG_FOR_DOC			,				expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT			,				)
 def 							UpperCamelCase  (					self		:  List[str]			,				snake_case__		:  tf.Tensor = None			,				snake_case__		:  tf.Tensor = None			,				snake_case__		:  bool = None			,				snake_case__		:  bool = None			,				snake_case__		:  List[Any]=False			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Optional[int]				       =							(
       output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
   )
   UpperCAmelCase__      :					List[Any]				       =							return_dict if return_dict is not None else self.config.use_return_dict

   UpperCAmelCase__      :					Dict				       =							self.regnet(
       snake_case__			,				output_hidden_states=snake_case__			,				return_dict=snake_case__			,				training=snake_case__ )

   UpperCAmelCase__      :					Dict				       =							outputs.pooler_output if return_dict else outputs[1]

   UpperCAmelCase__      :					Union[str, Any]				       =							self.classifier[0](snake_case__ )
   UpperCAmelCase__      :					str				       =							self.classifier[1](snake_case__ )

   UpperCAmelCase__      :					Any				       =							None if labels is None else self.hf_compute_loss(labels=snake_case__			,				logits=snake_case__ )

   if not return_dict:
     UpperCAmelCase__      :					Tuple				       =							(logits,) + outputs[2:]
     return ((loss,) + output) if loss is not None else output

   return TFSequenceClassifierOutput(loss=snake_case__			,				logits=snake_case__			,				hidden_states=outputs.hidden_states )



