




'''simple docstring'''

import tensorflow as tf

from ...tf_utils import shape_list



class 		SCREAMING_SNAKE_CASE__  (			tf.keras.layers.Layer	):





			def __init__(      self						:    str      ,		a_						:    str      ,		a_						:    Optional[int]      ,		a_						:    Optional[Any]      ,		a_						:    List[str]      ,		a_						:    int=1      ,		a_						:    str=False      ,		**a_						:    Optional[int]							):



						"""simple docstring"""




						super().__init__(**a_							)

						__snake_case			=		vocab_size
						__snake_case			=		d_embed
						__snake_case			=		d_proj

						__snake_case			=		cutoffs + [vocab_size]
						__snake_case			=		[0] + self.cutoffs
						__snake_case			=		div_val

						__snake_case			=		self.cutoffs[0]
						__snake_case			=		len(self.cutoffs							) - 1
						__snake_case			=		self.shortlist_size + self.n_clusters
						__snake_case			=		keep_order

						__snake_case			=		[]
						__snake_case			=		[]





			def 							A (      self						:    int      ,		a_						:    List[str]							):



						"""simple docstring"""




						if self.n_clusters > 0:
									__snake_case			=		self.add_weight(
									    shape=(self.n_clusters, self.d_embed)      ,		initializer="zeros"      ,		trainable=a_      ,		name="cluster_weight"							)
									__snake_case			=		self.add_weight(
									    shape=(self.n_clusters,)      ,		initializer="zeros"      ,		trainable=a_      ,		name="cluster_bias"							)

						if self.div_val == 1:
									for i in range(len(self.cutoffs							)							):
												if self.d_proj != self.d_embed:
															__snake_case			=		self.add_weight(
															    shape=(self.d_embed, self.d_proj)      ,		initializer="zeros"      ,		trainable=a_      ,		name=f'''out_projs_._{i}'''      ,		)
															self.out_projs.append(a_							)
												else:
															self.out_projs.append(a_							)
												__snake_case			=		self.add_weight(
												    shape=(self.vocab_size, self.d_embed)      ,		initializer="zeros"      ,		trainable=a_      ,		name=f'''out_layers_._{i}_._weight'''      ,		)
												__snake_case			=		self.add_weight(
												    shape=(self.vocab_size,)      ,		initializer="zeros"      ,		trainable=a_      ,		name=f'''out_layers_._{i}_._bias'''      ,		)
												self.out_layers.append((weight, bias)							)
						else:
									for i in range(len(self.cutoffs							)							):
												__snake_case		,     __snake_case			=		self.cutoff_ends[i], self.cutoff_ends[i + 1]
												__snake_case			=		self.d_embed // (self.div_val**i)

												__snake_case			=		self.add_weight(
												    shape=(d_emb_i, self.d_proj)      ,		initializer="zeros"      ,		trainable=a_      ,		name=f'''out_projs_._{i}'''							)
												self.out_projs.append(a_							)
												__snake_case			=		self.add_weight(
												    shape=(r_idx - l_idx, d_emb_i)      ,		initializer="zeros"      ,		trainable=a_      ,		name=f'''out_layers_._{i}_._weight'''      ,		)
												__snake_case			=		self.add_weight(
												    shape=(r_idx - l_idx,)      ,		initializer="zeros"      ,		trainable=a_      ,		name=f'''out_layers_._{i}_._bias'''      ,		)
												self.out_layers.append((weight, bias)							)
						super().build(a_							)





			@staticmethod
			def 							A (      a_						:    str      ,		a_						:    Tuple      ,		a_						:    Tuple      ,		a_						:    List[str]=None							):



						"""simple docstring"""




						__snake_case			=		x
						if proj is not None:
									__snake_case			=		tf.einsum("ibd,ed->ibe"      ,		a_      ,		a_							)
						return tf.einsum("ibd,nd->ibn"      ,		a_      ,		a_							) + b





			@staticmethod
			def 							A (      a_						:    int      ,		a_						:    Optional[Any]							):



						"""simple docstring"""




						__snake_case			=		shape_list(a_							)
						__snake_case			=		tf.range(lp_size[0]      ,		dtype=target.dtype							)
						__snake_case			=		tf.stack([r, target]      ,		1							)
						return tf.gather_nd(a_      ,		a_							)






			def 							A (      self						:    Optional[int]      ,		a_						:    Dict      ,		a_						:    Dict      ,		a_						:    Tuple=True      ,		a_						:    Optional[Any]=False							):



						"""simple docstring"""




						__snake_case			=		0
						if self.n_clusters == 0:
									__snake_case			=		self._logit(a_      ,		self.out_layers[0][0]      ,		self.out_layers[0][1]      ,		self.out_projs[0]							)
									if target is not None:
												__snake_case			=		tf.nn.sparse_softmax_cross_entropy_with_logits(labels=a_      ,		logits=a_							)
									__snake_case			=		tf.nn.log_softmax(a_      ,		axis=-1							)
						else:
									__snake_case			=		shape_list(a_							)
									__snake_case			=		[]
									__snake_case			=		tf.zeros(hidden_sizes[:2]							)
									for i in range(len(self.cutoffs							)							):
												__snake_case		,     __snake_case			=		self.cutoff_ends[i], self.cutoff_ends[i + 1]
												if target is not None:
															__snake_case			=		(target >= l_idx) & (target < r_idx)
															__snake_case			=		tf.where(a_							)
															__snake_case			=		tf.boolean_mask(a_      ,		a_							) - l_idx

												if self.div_val == 1:
															__snake_case			=		self.out_layers[0][0][l_idx:r_idx]
															__snake_case			=		self.out_layers[0][1][l_idx:r_idx]
												else:
															__snake_case			=		self.out_layers[i][0]
															__snake_case			=		self.out_layers[i][1]

												if i == 0:
															__snake_case			=		tf.concat([cur_W, self.cluster_weight]      ,		0							)
															__snake_case			=		tf.concat([cur_b, self.cluster_bias]      ,		0							)

															__snake_case			=		self._logit(a_      ,		a_      ,		a_      ,		self.out_projs[0]							)
															__snake_case			=		tf.nn.log_softmax(a_							)
															out.append(head_logprob[..., : self.cutoffs[0]]							)
															if target is not None:
																		__snake_case			=		tf.boolean_mask(a_      ,		a_							)
																		__snake_case			=		self._gather_logprob(a_      ,		a_							)
												else:
															__snake_case			=		self._logit(a_      ,		a_      ,		a_      ,		self.out_projs[i]							)
															__snake_case			=		tf.nn.log_softmax(a_							)
															__snake_case			=		self.cutoffs[0] + i - 1  # No probability for the head cluster
															__snake_case			=		head_logprob[..., cluster_prob_idx, None] + tail_logprob
															out.append(a_							)
															if target is not None:
																		__snake_case			=		tf.boolean_mask(a_      ,		a_							)
																		__snake_case			=		tf.boolean_mask(a_      ,		a_							)
																		__snake_case			=		self._gather_logprob(a_      ,		a_							)
																		cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
												if target is not None:
															loss += tf.scatter_nd(a_      ,		-cur_logprob      ,		shape_list(a_							)							)
									__snake_case			=		tf.concat(a_      ,		axis=-1							)

						if target is not None:
									if return_mean:
												__snake_case			=		tf.reduce_mean(a_							)
									# Add the training-time loss value to the layer using `self.add_loss()`.
									self.add_loss(a_							)

									# Log the loss as a metric (we could log arbitrary metrics,
									# including different metrics for training and inference.
									self.add_metric(a_      ,		name=self.name      ,		aggregation="mean" if return_mean else ""							)

						return out











'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TFFunnelBaseModel,
							    TFFunnelForMaskedLM,
							    TFFunnelForMultipleChoice,
							    TFFunnelForPreTraining,
							    TFFunnelForQuestionAnswering,
							    TFFunnelForSequenceClassification,
							    TFFunnelForTokenClassification,
							    TFFunnelModel,
							)
class   __UpperCAmelCase				:



					'''simple docstring'''





					def __init__(	self     :						str      ,       _lowercase     :						List[str]      ,       _lowercase     :						Dict=13      ,       _lowercase     :						List[str]=7      ,       _lowercase     :						Union[str, Any]=True      ,       _lowercase     :						int=True      ,       _lowercase     :						int=True      ,       _lowercase     :						Optional[Any]=True      ,       _lowercase     :						List[Any]=99      ,       _lowercase     :						Optional[Any]=[1, 1, 2]      ,       _lowercase     :						Optional[Any]=1      ,       _lowercase     :						List[str]=32      ,       _lowercase     :						Dict=4      ,       _lowercase     :						List[str]=8      ,       _lowercase     :						List[str]=37      ,       _lowercase     :						int="gelu_new"      ,       _lowercase     :						Optional[int]=0.1      ,       _lowercase     :						List[str]=0.1      ,       _lowercase     :						Union[str, Any]=0.0      ,       _lowercase     :						str=512      ,       _lowercase     :						Optional[Any]=3      ,       _lowercase     :						str=0.02      ,       _lowercase     :						Union[str, Any]=3      ,       _lowercase     :						Optional[Any]=4      ,       _lowercase     :						Tuple=None      ,       _lowercase     :						List[Any]=False      ,       )    ->							List[str]:
								A_									= parent
								A_									= batch_size
								A_									= seq_length
								A_									= is_training
								A_									= use_input_mask
								A_									= use_token_type_ids
								A_									= use_labels
								A_									= vocab_size
								A_									= block_sizes
								A_									= num_decoder_layers
								A_									= d_model
								A_									= n_head
								A_									= d_head
								A_									= d_inner
								A_									= hidden_act
								A_									= hidden_dropout
								A_									= attention_dropout
								A_									= activation_dropout
								A_									= max_position_embeddings
								A_									= type_vocab_size
								A_									= 2
								A_									= num_labels
								A_									= num_choices
								A_									= scope
								A_									= initializer_std

								# Used in the tests to check the size of the first attention layer
								A_									= n_head
								# Used in the tests to check the size of the first hidden state
								A_									= self.d_model
								# Used in the tests to check the number of output hidden states/attentions
								A_									= sum(self.block_sizes) + (0 if base else self.num_decoder_layers)
								# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
								# the last hidden state of the first block (which is the first hidden state of the decoder).
								if not base:
											A_									= self.num_hidden_layers + 2



					def     __snake_case						(	self     :						str)    ->							List[str]:
								A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.vocab_size)

								A_									= None
								if self.use_input_mask:
											A_									= random_attention_mask([self.batch_size, self.seq_length])

								A_									= None
								if self.use_token_type_ids:
											A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.type_vocab_size)

								A_									= None
								A_									= None
								A_									= None
								if self.use_labels:
											A_									= ids_tensor([self.batch_size]      ,       self.type_sequence_label_size)
											A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.num_labels)
											A_									= ids_tensor([self.batch_size]      ,       self.num_choices)

								A_									= FunnelConfig(
								    vocab_size=self.vocab_size      ,       block_sizes=self.block_sizes      ,       num_decoder_layers=self.num_decoder_layers      ,       d_model=self.d_model      ,       n_head=self.n_head      ,       d_head=self.d_head      ,       d_inner=self.d_inner      ,       hidden_act=self.hidden_act      ,       hidden_dropout=self.hidden_dropout      ,       attention_dropout=self.attention_dropout      ,       activation_dropout=self.activation_dropout      ,       max_position_embeddings=self.max_position_embeddings      ,       type_vocab_size=self.type_vocab_size      ,       initializer_std=self.initializer_std      ,       )

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_mask,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								)



					def     __snake_case						(	self     :						Tuple      ,       _lowercase     :						Tuple      ,       _lowercase     :						str      ,       _lowercase     :						int      ,       _lowercase     :						Dict      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						str      ,       _lowercase     :						str      ,       )    ->							Dict:
								A_									= TFFunnelModel(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)

								A_									= [input_ids, input_mask]
								A_									= model(_lowercase)

								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))

								A_									= False
								A_									= TFFunnelModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))

								A_									= False
								A_									= TFFunnelModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))



					def     __snake_case						(	self     :						Union[str, Any]      ,       _lowercase     :						int      ,       _lowercase     :						Tuple      ,       _lowercase     :						int      ,       _lowercase     :						int      ,       _lowercase     :						Tuple      ,       _lowercase     :						List[str]      ,       _lowercase     :						Optional[int]      ,       )    ->							Dict:
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)

								A_									= [input_ids, input_mask]
								A_									= model(_lowercase)

								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 2, self.d_model))

								A_									= False
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 3, self.d_model))

								A_									= False
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 2, self.d_model))



					def     __snake_case						(	self     :						int      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						List[str]      ,       _lowercase     :						str      ,       _lowercase     :						Any      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Optional[Any]      ,       )    ->							List[Any]:
								A_									= TFFunnelForPreTraining(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length))



					def     __snake_case						(	self     :						List[str]      ,       _lowercase     :						str      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Tuple      ,       )    ->							Union[str, Any]:
								A_									= TFFunnelForMaskedLM(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.vocab_size))



					def     __snake_case						(	self     :						Union[str, Any]      ,       _lowercase     :						Dict      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Any      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						Optional[int]      ,       _lowercase     :						str      ,       )    ->							Dict:
								A_									= self.num_labels
								A_									= TFFunnelForSequenceClassification(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_labels))



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						int      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Tuple      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						str      ,       )    ->							List[str]:
								A_									= self.num_choices
								A_									= TFFunnelForMultipleChoice(config=_lowercase)
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= {
								    'input_ids': multiple_choice_inputs_ids,
								    'attention_mask': multiple_choice_input_mask,
								    'token_type_ids': multiple_choice_token_type_ids,
								}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_choices))



					def     __snake_case						(	self     :						Any      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Dict      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						int      ,       _lowercase     :						Dict      ,       )    ->							List[str]:
								A_									= self.num_labels
								A_									= TFFunnelForTokenClassification(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.num_labels))



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						str      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Union[str, Any]      ,       )    ->							Optional[int]:
								A_									= TFFunnelForQuestionAnswering(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.start_logits.shape      ,       (self.batch_size, self.seq_length))
								self.parent.assertEqual(result.end_logits.shape      ,       (self.batch_size, self.seq_length))



					def     __snake_case						(	self     :						str)    ->							Union[str, Any]:
								A_									= self.prepare_config_and_inputs()
								(
								    (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, 
								)									= config_and_inputs
								A_									= {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
								return config, inputs_dict





@require_tf
class   __UpperCAmelCase				(     lowerCAmelCase			,lowerCAmelCase			,unittest.TestCase				):



					'''simple docstring'''





					_UpperCamelCase							=		(
					    (
					        TFFunnelModel,
					        TFFunnelForMaskedLM,
					        TFFunnelForPreTraining,
					        TFFunnelForQuestionAnswering,
					        TFFunnelForTokenClassification,
					    )
					    if is_tf_available()
					    else ()
					)
					_UpperCamelCase							=		(
					    {
					        """feature-extraction""": (TFFunnelBaseModel, TFFunnelModel),
					        """fill-mask""": TFFunnelForMaskedLM,
					        """question-answering""": TFFunnelForQuestionAnswering,
					        """text-classification""": TFFunnelForSequenceClassification,
					        """token-classification""": TFFunnelForTokenClassification,
					        """zero-shot""": TFFunnelForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					_UpperCamelCase							=		False
					_UpperCamelCase							=		False



					def     __snake_case						(	self     :						Optional[int])    ->							Optional[Any]:
								A_									= TFFunnelModelTester(self)
								A_									= ConfigTester(self      ,       config_class=_lowercase)



					def     __snake_case						(	self     :						Any)    ->							Optional[Any]:
								self.config_tester.run_common_tests()



					def     __snake_case						(	self     :						Union[str, Any])    ->							Union[str, Any]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_lowercase)



					def     __snake_case						(	self     :						List[Any])    ->							str:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_pretraining(*_lowercase)



					def     __snake_case						(	self     :						Any)    ->							Optional[int]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*_lowercase)



					def     __snake_case						(	self     :						List[str])    ->							Tuple:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*_lowercase)



					def     __snake_case						(	self     :						int)    ->							Dict:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*_lowercase)





@require_tf
class   __UpperCAmelCase				(     lowerCAmelCase			,unittest.TestCase				):



					'''simple docstring'''





					_UpperCamelCase							=		(
					    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
					)
					_UpperCamelCase							=		False
					_UpperCamelCase							=		False



					def     __snake_case						(	self     :						str)    ->							Dict:
								A_									= TFFunnelModelTester(self      ,       base=_lowercase)
								A_									= ConfigTester(self      ,       config_class=_lowercase)



					def     __snake_case						(	self     :						Optional[Any])    ->							Tuple:
								self.config_tester.run_common_tests()



					def     __snake_case						(	self     :						List[str])    ->							List[str]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_base_model(*_lowercase)



					def     __snake_case						(	self     :						Dict)    ->							Dict:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*_lowercase)



					def     __snake_case						(	self     :						Union[str, Any])    ->							int:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*_lowercase)




