





'''simple docstring'''




import torch
from torch import nn


class 					SCREAMING_SNAKE_CASE       (nn.Module				):


     def __init__(		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase=1		,      _UpperCAmelCase=False):

          '''simple docstring'''






          super().__init__()

          __A		:					Optional[Any]						=			n_token
          __A		:					Tuple						=			d_embed
          __A		:					Dict						=			d_proj

          __A		:					List[Any]						=			cutoffs + [n_token]
          __A		:					Optional[int]						=			[0] + self.cutoffs
          __A		:					Optional[int]						=			div_val

          __A		:					Any						=			self.cutoffs[0]
          __A		:					Optional[Any]						=			len(self.cutoffs) - 1
          __A		:					Union[str, Any]						=			self.shortlist_size + self.n_clusters

          if self.n_clusters > 0:
               __A		:					str						=			nn.Parameter(torch.zeros(self.n_clusters		,      self.d_embed))
               __A		:					str						=			nn.Parameter(torch.zeros(self.n_clusters))

          __A		:					Any						=			nn.ModuleList()
          __A		:					str						=			nn.ParameterList()

          if div_val == 1:
               for i in range(len(self.cutoffs)):
                    if d_proj != d_embed:
                         self.out_projs.append(nn.Parameter(torch.FloatTensor(_UpperCAmelCase		,      _UpperCAmelCase)))
                    else:
                         self.out_projs.append(_UpperCAmelCase)

               self.out_layers.append(nn.Linear(_UpperCAmelCase		,      _UpperCAmelCase))
          else:
               for i in range(len(self.cutoffs)):
                    __A					,__A		:					List[Any]						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    __A		:					List[str]						=			d_embed // (div_val**i)

                    self.out_projs.append(nn.Parameter(torch.FloatTensor(_UpperCAmelCase		,      _UpperCAmelCase)))

                    self.out_layers.append(nn.Linear(_UpperCAmelCase		,      r_idx - l_idx))

          __A		:					Tuple						=			keep_order


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          if proj is None:
               __A		:					Any						=			nn.functional.linear(_UpperCAmelCase		,      _UpperCAmelCase		,      bias=_UpperCAmelCase)
          else:
               # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
               __A		:					Optional[int]						=			nn.functional.linear(_UpperCAmelCase		,      proj.t().contiguous())
               __A		:					Optional[Any]						=			nn.functional.linear(_UpperCAmelCase		,      _UpperCAmelCase		,      bias=_UpperCAmelCase)
               # else:
               #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
               #     if bias is not None:
               #         logit = logit + bias

          return logit


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase=None		,      _UpperCAmelCase=False):

          '''simple docstring'''






          if labels is not None:
               # Shift so that tokens < n predict n
               __A		:					Any						=			hidden[..., :-1, :].contiguous()
               __A		:					int						=			labels[..., 1:].contiguous()
               __A		:					int						=			hidden.view(-1		,      hidden.size(-1))
               __A		:					Union[str, Any]						=			labels.view(-1)
               if hidden.size(0) != labels.size(0):
                    raise RuntimeError('Input and labels should have the same size in the batch dimension.')
          else:
               __A		:					Optional[int]						=			hidden.view(-1		,      hidden.size(-1))

          if self.n_clusters == 0:
               __A		:					int						=			self._compute_logit(_UpperCAmelCase		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0])
               if labels is not None:
                    __A		:					str						=			labels != -100
                    __A		:					Dict						=			torch.zeros_like(_UpperCAmelCase		,      dtype=hidden.dtype		,      device=hidden.device)
                    __A		:					Dict						=			(
                        -nn.functional.log_softmax(_UpperCAmelCase		,      dim=-1)[mask].gather(1		,      labels[mask].unsqueeze(1)).squeeze(1)
                    )
               else:
                    __A		:					List[Any]						=			nn.functional.log_softmax(_UpperCAmelCase		,      dim=-1)
          else:
               # construct weights and biases
               __A					,__A		:					Union[str, Any]						=			[], []
               for i in range(len(self.cutoffs)):
                    if self.div_val == 1:
                         __A					,__A		:					Union[str, Any]						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                         __A		:					List[Any]						=			self.out_layers[0].weight[l_idx:r_idx]
                         __A		:					List[Any]						=			self.out_layers[0].bias[l_idx:r_idx]
                    else:
                         __A		:					Dict						=			self.out_layers[i].weight
                         __A		:					Dict						=			self.out_layers[i].bias

                    if i == 0:
                         __A		:					str						=			torch.cat([weight_i, self.cluster_weight]		,      dim=0)
                         __A		:					List[str]						=			torch.cat([bias_i, self.cluster_bias]		,      dim=0)

                    weights.append(_UpperCAmelCase)
                    biases.append(_UpperCAmelCase)

               __A					,__A					,__A		:					Any						=			weights[0], biases[0], self.out_projs[0]

               __A		:					str						=			self._compute_logit(_UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase)
               __A		:					Tuple						=			nn.functional.log_softmax(_UpperCAmelCase		,      dim=1)

               if labels is None:
                    __A		:					Dict						=			hidden.new_empty((head_logit.size(0), self.n_token))
               else:
                    __A		:					Optional[int]						=			torch.zeros_like(_UpperCAmelCase		,      dtype=hidden.dtype		,      device=hidden.device)

               __A		:					Any						=			0
               __A		:					int						=			[0] + self.cutoffs
               for i in range(len(_UpperCAmelCase) - 1):
                    __A					,__A		:					List[Any]						=			cutoff_values[i], cutoff_values[i + 1]

                    if labels is not None:
                         __A		:					Tuple						=			(labels >= l_idx) & (labels < r_idx)
                         __A		:					Tuple						=			mask_i.nonzero().squeeze()

                         if indices_i.numel() == 0:
                              continue

                         __A		:					Optional[int]						=			labels.index_select(0		,      _UpperCAmelCase) - l_idx
                         __A		:					Optional[int]						=			head_logprob.index_select(0		,      _UpperCAmelCase)
                         __A		:					List[Any]						=			hidden.index_select(0		,      _UpperCAmelCase)
                    else:
                         __A		:					List[str]						=			hidden

                    if i == 0:
                         if labels is not None:
                              __A		:					str						=			head_logprob_i.gather(1		,      target_i[:, None]).squeeze(1)
                         else:
                              __A		:					List[str]						=			head_logprob[:, : self.cutoffs[0]]
                    else:
                         __A					,__A					,__A		:					Tuple						=			weights[i], biases[i], self.out_projs[i]

                         __A		:					Dict						=			self._compute_logit(_UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase)
                         __A		:					int						=			nn.functional.log_softmax(_UpperCAmelCase		,      dim=1)
                         __A		:					List[Any]						=			self.cutoffs[0] + i - 1  # No probability for the head cluster
                         if labels is not None:
                              __A		:					Optional[int]						=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                  1		,      target_i[:, None]).squeeze(1)
                         else:
                              __A		:					List[Any]						=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                              __A		:					Dict						=			logprob_i

                    if labels is not None:
                         if (hasattr(self		,      'keep_order') and self.keep_order) or keep_order:
                              out.index_copy_(0		,      _UpperCAmelCase		,      -logprob_i)
                         else:
                              out[offset : offset + logprob_i.size(0)].copy_(-logprob_i)
                         offset += logprob_i.size(0)

          return out


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase):

          '''simple docstring'''






          if self.n_clusters == 0:
               __A		:					Tuple						=			self._compute_logit(_UpperCAmelCase		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0])
               return nn.functional.log_softmax(_UpperCAmelCase		,      dim=-1)
          else:
               # construct weights and biases
               __A					,__A		:					Tuple						=			[], []
               for i in range(len(self.cutoffs)):
                    if self.div_val == 1:
                         __A					,__A		:					int						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                         __A		:					str						=			self.out_layers[0].weight[l_idx:r_idx]
                         __A		:					Optional[int]						=			self.out_layers[0].bias[l_idx:r_idx]
                    else:
                         __A		:					Optional[int]						=			self.out_layers[i].weight
                         __A		:					Optional[int]						=			self.out_layers[i].bias

                    if i == 0:
                         __A		:					List[Any]						=			torch.cat([weight_i, self.cluster_weight]		,      dim=0)
                         __A		:					Union[str, Any]						=			torch.cat([bias_i, self.cluster_bias]		,      dim=0)

                    weights.append(_UpperCAmelCase)
                    biases.append(_UpperCAmelCase)

               __A					,__A					,__A		:					Optional[int]						=			weights[0], biases[0], self.out_projs[0]
               __A		:					Any						=			self._compute_logit(_UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase)

               __A		:					Optional[int]						=			hidden.new_empty((head_logit.size(0), self.n_token))
               __A		:					str						=			nn.functional.log_softmax(_UpperCAmelCase		,      dim=1)

               __A		:					Dict						=			[0] + self.cutoffs
               for i in range(len(_UpperCAmelCase) - 1):
                    __A					,__A		:					Optional[Any]						=			cutoff_values[i], cutoff_values[i + 1]

                    if i == 0:
                         __A		:					List[Any]						=			head_logprob[:, : self.cutoffs[0]]
                    else:
                         __A					,__A					,__A		:					Any						=			weights[i], biases[i], self.out_projs[i]

                         __A		:					str						=			self._compute_logit(_UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase)
                         __A		:					Optional[Any]						=			nn.functional.log_softmax(_UpperCAmelCase		,      dim=1)

                         __A		:					int						=			head_logprob[:, -i] + tail_logprob_i
                         __A		:					Tuple						=			logprob_i

               return out






'''simple docstring'''




from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFConvBertForMaskedLM,
      TFConvBertForMultipleChoice,
      TFConvBertForQuestionAnswering,
      TFConvBertForSequenceClassification,
      TFConvBertForTokenClassification,
      TFConvBertModel,
  )


class 					SCREAMING_SNAKE_CASE       :


     def __init__(		self		,      _UpperCAmelCase		,      _UpperCAmelCase=13		,      _UpperCAmelCase=7		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=99		,      _UpperCAmelCase=32		,      _UpperCAmelCase=2		,      _UpperCAmelCase=4		,      _UpperCAmelCase=37		,      _UpperCAmelCase="gelu"		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=512		,      _UpperCAmelCase=16		,      _UpperCAmelCase=2		,      _UpperCAmelCase=0.02		,      _UpperCAmelCase=3		,      _UpperCAmelCase=4		,      _UpperCAmelCase=None		,      ):

          '''simple docstring'''






          __A		:					Optional[int]						=			parent
          __A		:					str						=			13
          __A		:					List[Any]						=			7
          __A		:					List[str]						=			True
          __A		:					str						=			True
          __A		:					Optional[Any]						=			True
          __A		:					int						=			True
          __A		:					Dict						=			99
          __A		:					Dict						=			384
          __A		:					Any						=			2
          __A		:					int						=			4
          __A		:					Optional[Any]						=			37
          __A		:					Optional[int]						=			'gelu'
          __A		:					Dict						=			0.1
          __A		:					Optional[int]						=			0.1
          __A		:					Any						=			512
          __A		:					int						=			16
          __A		:					List[str]						=			2
          __A		:					str						=			0.02
          __A		:					Any						=			3
          __A		:					str						=			4
          __A		:					Union[str, Any]						=			128
          __A		:					int						=			2
          __A		:					List[Any]						=			9
          __A		:					List[Any]						=			1
          __A		:					List[Any]						=			None


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Union[str, Any]						=			ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size)

          __A		:					str						=			None
          if self.use_input_mask:
               __A		:					List[Any]						=			random_attention_mask([self.batch_size, self.seq_length])

          __A		:					Optional[Any]						=			None
          if self.use_token_type_ids:
               __A		:					Optional[Any]						=			ids_tensor([self.batch_size, self.seq_length]		,      self.type_vocab_size)

          __A		:					Optional[int]						=			None
          __A		:					List[str]						=			None
          __A		:					Dict						=			None
          if self.use_labels:
               __A		:					Dict						=			ids_tensor([self.batch_size]		,      self.type_sequence_label_size)
               __A		:					List[Any]						=			ids_tensor([self.batch_size, self.seq_length]		,      self.num_labels)
               __A		:					str						=			ids_tensor([self.batch_size]		,      self.num_choices)

          __A		:					List[Any]						=			ConvBertConfig(
              vocab_size=self.vocab_size		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      intermediate_size=self.intermediate_size		,      hidden_act=self.hidden_act		,      hidden_dropout_prob=self.hidden_dropout_prob		,      attention_probs_dropout_prob=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      type_vocab_size=self.type_vocab_size		,      initializer_range=self.initializer_range		,      return_dict=_UpperCAmelCase		,      )

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					int						=			TFConvBertModel(config=_UpperCAmelCase)
          __A		:					Optional[Any]						=			{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

          __A		:					Tuple						=			[input_ids, input_mask]
          __A		:					Any						=			model(_UpperCAmelCase)

          __A		:					Dict						=			model(_UpperCAmelCase)

          self.parent.assertEqual(result.last_hidden_state.shape		,      (self.batch_size, self.seq_length, self.hidden_size))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					str						=			TFConvBertForMaskedLM(config=_UpperCAmelCase)
          __A		:					str						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          __A		:					str						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Optional[int]						=			self.num_labels
          __A		:					Any						=			TFConvBertForSequenceClassification(config=_UpperCAmelCase)
          __A		:					Optional[Any]						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          __A		:					Dict						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Tuple						=			self.num_choices
          __A		:					List[str]						=			TFConvBertForMultipleChoice(config=_UpperCAmelCase)
          __A		:					int						=			tf.tile(tf.expand_dims(_UpperCAmelCase		,      1)		,      (1, self.num_choices, 1))
          __A		:					Optional[Any]						=			tf.tile(tf.expand_dims(_UpperCAmelCase		,      1)		,      (1, self.num_choices, 1))
          __A		:					List[Any]						=			tf.tile(tf.expand_dims(_UpperCAmelCase		,      1)		,      (1, self.num_choices, 1))
          __A		:					int						=			{
              'input_ids': multiple_choice_inputs_ids,
              'attention_mask': multiple_choice_input_mask,
              'token_type_ids': multiple_choice_token_type_ids,
          }
          __A		:					Optional[Any]						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_choices))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					List[Any]						=			self.num_labels
          __A		:					List[Any]						=			TFConvBertForTokenClassification(config=_UpperCAmelCase)
          __A		:					str						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          __A		:					int						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.num_labels))


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Optional[Any]						=			TFConvBertForQuestionAnswering(config=_UpperCAmelCase)
          __A		:					Any						=			{
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          __A		:					Union[str, Any]						=			model(_UpperCAmelCase)
          self.parent.assertEqual(result.start_logits.shape		,      (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape		,      (self.batch_size, self.seq_length))


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			self.prepare_config_and_inputs()
          (
              (
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,(
              __A
          )					,
          )		:					Union[str, Any]						=			config_and_inputs
          __A		:					List[str]						=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
          return config, inputs_dict




@require_tf
class 					SCREAMING_SNAKE_CASE       (a__   ,						a__   ,						unittest.TestCase				):
     lowerCAmelCase						     =			(
         (
             TFConvBertModel,
             TFConvBertForMaskedLM,
             TFConvBertForQuestionAnswering,
             TFConvBertForSequenceClassification,
             TFConvBertForTokenClassification,
             TFConvBertForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     lowerCAmelCase						     =			(
         {
             '''feature-extraction''': TFConvBertModel,
             '''fill-mask''': TFConvBertForMaskedLM,
             '''question-answering''': TFConvBertForQuestionAnswering,
             '''text-classification''': TFConvBertForSequenceClassification,
             '''token-classification''': TFConvBertForTokenClassification,
             '''zero-shot''': TFConvBertForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     lowerCAmelCase						     =			False
     lowerCAmelCase						     =			False
     lowerCAmelCase						     =			False


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					int						=			TFConvBertModelTester(self)
          __A		:					str						=			ConfigTester(self		,      config_class=_UpperCAmelCase		,      hidden_size=37)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          self.config_tester.run_common_tests()


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Tuple						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					str						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[Any]						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[Any]						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					int						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					int						=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase)


     @slow
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A		:					Union[str, Any]						=			self.model_tester.prepare_config_and_inputs_for_common()
          __A		:					List[str]						=			True
          __A		:					List[str]						=			True

          if hasattr(_UpperCAmelCase		,      'use_cache'):
               __A		:					List[Any]						=			True

          __A		:					str						=			getattr(self.model_tester		,      'encoder_seq_length'		,      self.model_tester.seq_length)
          __A		:					Union[str, Any]						=			getattr(self.model_tester		,      'key_length'		,      _UpperCAmelCase)

          for model_class in self.all_model_classes:
               __A		:					List[str]						=			self._prepare_for_class(_UpperCAmelCase		,      _UpperCAmelCase)
               __A		:					Optional[int]						=			model_class(_UpperCAmelCase)
               __A		:					Optional[Any]						=			len(model(_UpperCAmelCase))

               with tempfile.TemporaryDirectory() as tmpdirname:
                    model.save_pretrained(_UpperCAmelCase		,      saved_model=_UpperCAmelCase)
                    __A		:					Union[str, Any]						=			os.path.join(_UpperCAmelCase		,      'saved_model'		,      '1')
                    __A		:					Tuple						=			tf.keras.models.load_model(_UpperCAmelCase)
                    __A		:					str						=			model(_UpperCAmelCase)

                    if self.is_encoder_decoder:
                         __A		:					Optional[int]						=			outputs['encoder_hidden_states']
                         __A		:					str						=			outputs['encoder_attentions']
                    else:
                         __A		:					List[Any]						=			outputs['hidden_states']
                         __A		:					Optional[Any]						=			outputs['attentions']

                    self.assertEqual(len(_UpperCAmelCase)		,      _UpperCAmelCase)

                    __A		:					str						=			getattr(
                        self.model_tester		,      'expected_num_hidden_layers'		,      self.model_tester.num_hidden_layers + 1)

                    self.assertEqual(len(_UpperCAmelCase)		,      _UpperCAmelCase)
                    self.assertListEqual(
                        list(output_hidden_states[0].shape[-2:])		,      [self.model_tester.seq_length, self.model_tester.hidden_size]		,      )

                    self.assertEqual(len(_UpperCAmelCase)		,      self.model_tester.num_hidden_layers)
                    self.assertListEqual(
                        list(output_attentions[0].shape[-3:])		,      [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]		,      )


     @slow
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Dict						=			TFConvBertModel.from_pretrained('YituTech/conv-bert-base')
          self.assertIsNotNone(_UpperCAmelCase)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A					,__A		:					Dict						=			self.model_tester.prepare_config_and_inputs_for_common()
          __A		:					Any						=			True
          __A		:					str						=			getattr(self.model_tester		,      'decoder_seq_length'		,      self.model_tester.seq_length)
          __A		:					Any						=			getattr(self.model_tester		,      'encoder_seq_length'		,      self.model_tester.seq_length)
          __A		:					int						=			getattr(self.model_tester		,      'key_length'		,      _UpperCAmelCase)
          __A		:					Tuple						=			getattr(self.model_tester		,      'key_length'		,      _UpperCAmelCase)

          def check_decoder_attentions_output(_UpperCAmelCase):
               __A		:					List[str]						=			len(_UpperCAmelCase)
               self.assertEqual(out_len % 2		,      0)
               __A		:					Any						=			outputs.decoder_attentions
               self.assertEqual(len(_UpperCAmelCase)		,      self.model_tester.num_hidden_layers)
               self.assertListEqual(
                   list(decoder_attentions[0].shape[-3:])		,      [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]		,      )

          def check_encoder_attentions_output(_UpperCAmelCase):
               __A		:					str						=			[
                   t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
               ]
               self.assertEqual(len(_UpperCAmelCase)		,      self.model_tester.num_hidden_layers)
               self.assertListEqual(
                   list(attentions[0].shape[-3:])		,      [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]		,      )

          for model_class in self.all_model_classes:
               __A		:					Dict						=			True
               __A		:					Any						=			False
               __A		:					str						=			model_class(_UpperCAmelCase)
               __A		:					List[str]						=			model(self._prepare_for_class(_UpperCAmelCase		,      _UpperCAmelCase))
               __A		:					List[str]						=			len(_UpperCAmelCase)
               self.assertEqual(config.output_hidden_states		,      _UpperCAmelCase)
               check_encoder_attentions_output(_UpperCAmelCase)

               if self.is_encoder_decoder:
                    __A		:					Union[str, Any]						=			model_class(_UpperCAmelCase)
                    __A		:					int						=			model(self._prepare_for_class(_UpperCAmelCase		,      _UpperCAmelCase))
                    self.assertEqual(config.output_hidden_states		,      _UpperCAmelCase)
                    check_decoder_attentions_output(_UpperCAmelCase)

               # Check that output attentions can also be changed via the config
               del inputs_dict["output_attentions"]
               __A		:					int						=			True
               __A		:					Tuple						=			model_class(_UpperCAmelCase)
               __A		:					Dict						=			model(self._prepare_for_class(_UpperCAmelCase		,      _UpperCAmelCase))
               self.assertEqual(config.output_hidden_states		,      _UpperCAmelCase)
               check_encoder_attentions_output(_UpperCAmelCase)

               # Check attention is always last and order is fine
               __A		:					Any						=			True
               __A		:					str						=			True
               __A		:					Union[str, Any]						=			model_class(_UpperCAmelCase)
               __A		:					Union[str, Any]						=			model(self._prepare_for_class(_UpperCAmelCase		,      _UpperCAmelCase))

               self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)		,      len(_UpperCAmelCase))
               self.assertEqual(model.config.output_hidden_states		,      _UpperCAmelCase)
               check_encoder_attentions_output(_UpperCAmelCase)




@require_tf
class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):


     @slow
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Tuple						=			TFConvBertModel.from_pretrained('YituTech/conv-bert-base')
          __A		:					str						=			tf.constant([[0, 1, 2, 3, 4, 5]])
          __A		:					Optional[int]						=			model(_UpperCAmelCase)[0]

          __A		:					List[Any]						=			[1, 6, 768]
          self.assertEqual(output.shape		,      _UpperCAmelCase)

          __A		:					Tuple						=			tf.constant(
              [
                  [
                      [-0.03475493, -0.4686034, -0.30638832],
                      [0.22637248, -0.26988646, -0.7423424],
                      [0.10324868, -0.45013508, -0.58280784],
                  ]
              ])
          tf.debugging.assert_near(output[:, :3, :3]		,      _UpperCAmelCase		,      atol=1e-4)