



import argparse

import fairseq
import torch

from transformers import UniSpeechSatConfig, UniSpeechSatForCTC, UniSpeechSatForPreTraining, logging


logging.set_verbosity_info()
_lowerCamelCase    :		List[Any]		      =    logging.get_logger(__name__)

_lowerCamelCase    :		Optional[int]		      =    {
    """post_extract_proj""": """feature_projection.projection""",
    """encoder.pos_conv.0""": """encoder.pos_conv_embed.conv""",
    """self_attn.k_proj""": """encoder.layers.*.attention.k_proj""",
    """self_attn.v_proj""": """encoder.layers.*.attention.v_proj""",
    """self_attn.q_proj""": """encoder.layers.*.attention.q_proj""",
    """self_attn.out_proj""": """encoder.layers.*.attention.out_proj""",
    """self_attn_layer_norm""": """encoder.layers.*.layer_norm""",
    """fc1""": """encoder.layers.*.feed_forward.intermediate_dense""",
    """fc2""": """encoder.layers.*.feed_forward.output_dense""",
    """final_layer_norm""": """encoder.layers.*.final_layer_norm""",
    """encoder.layer_norm""": """encoder.layer_norm""",
    """encoder.layer_norm_for_extract""": """layer_norm_for_extract""",
    """w2v_model.layer_norm""": """feature_projection.layer_norm""",
    """quantizer.weight_proj""": """quantizer.weight_proj""",
    """quantizer.vars""": """quantizer.codevectors""",
    """project_q""": """project_q""",
    """final_proj""": """project_hid""",
    """w2v_encoder.proj""": """lm_head""",
    """label_embs_concat""": """label_embeddings_concat""",
    """mask_emb""": """masked_spec_embed""",
    """spk_proj""": """speaker_proj""",
}
_lowerCamelCase    :		Optional[Any]		      =    [
    """lm_head""",
    """quantizer.weight_proj""",
    """quantizer.codevectors""",
    """project_q""",
    """project_hid""",
    """label_embeddings_concat""",
    """speaker_proj""",
    """layer_norm_for_extract""",
]

def        __a			(   __lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase   )     ->     int:
					for attribute in key.split('.'   ):
										SCREAMING_SNAKE_CASE							:     Dict						=				getattr(__lowerCAmelCase      ,	__lowerCAmelCase   )

					if weight_type is not None:
										SCREAMING_SNAKE_CASE							:     str						=				getattr(__lowerCAmelCase      ,	__lowerCAmelCase   ).shape
					else:
										SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				hf_pointer.shape

					if hf_shape != value.shape:
										raise ValueError(
										    F'''Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be'''
										    F''' {value.shape} for {full_name}'''   )

					if weight_type == "weight":
										SCREAMING_SNAKE_CASE							:     Optional[Any]						=				value
					elif weight_type == "weight_g":
										SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				value
					elif weight_type == "weight_v":
										SCREAMING_SNAKE_CASE							:     List[str]						=				value
					elif weight_type == "bias":
										SCREAMING_SNAKE_CASE							:     Any						=				value
					else:
										SCREAMING_SNAKE_CASE							:     Tuple						=				value

					logger.info(F'''{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.'''   )

def        __a			(   __lowerCAmelCase      ,	__lowerCAmelCase   )     ->     str:
					SCREAMING_SNAKE_CASE							:     int						=				[]
					SCREAMING_SNAKE_CASE							:     List[Any]						=				fairseq_model.state_dict()

					SCREAMING_SNAKE_CASE							:     str						=				hf_model.unispeech_sat.feature_extractor

					for name, value in fairseq_dict.items():
										SCREAMING_SNAKE_CASE							:     List[Any]						=				False
										if "conv_layers" in name:
															load_conv_layer(
															    __lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	hf_model.config.feat_extract_norm == 'group'      ,	)
															SCREAMING_SNAKE_CASE							:     Optional[Any]						=				True
										else:
															for key, mapped_key in MAPPING.items():
																				SCREAMING_SNAKE_CASE							:     List[str]						=				'unispeech_sat.' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
																				if key in name or key.split('w2v_model.'   )[-1] == name.split('.'   )[0]:
																									if "layer_norm_for_extract" in name and (".".join(name.split('.'   )[:-1]   ) != key):
																														# special case since naming is very similar
																														continue
																									SCREAMING_SNAKE_CASE							:     Any						=				True
																									if "*" in mapped_key:
																														SCREAMING_SNAKE_CASE							:     Dict						=				name.split(__lowerCAmelCase   )[0].split('.'   )[-2]
																														SCREAMING_SNAKE_CASE							:     int						=				mapped_key.replace('*'      ,	__lowerCAmelCase   )
																									if "weight_g" in name:
																														SCREAMING_SNAKE_CASE							:     Any						=				'weight_g'
																									elif "weight_v" in name:
																														SCREAMING_SNAKE_CASE							:     List[str]						=				'weight_v'
																									elif "bias" in name:
																														SCREAMING_SNAKE_CASE							:     Any						=				'bias'
																									elif "weight" in name:
																														# TODO: don't match quantizer.weight_proj
																														SCREAMING_SNAKE_CASE							:     int						=				'weight'
																									else:
																														SCREAMING_SNAKE_CASE							:     Any						=				None
																									set_recursively(__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase   )
																				continue
										if not is_used:
															unused_weights.append(__lowerCAmelCase   )

					logger.warning(F'''Unused weights: {unused_weights}'''   )

def        __a			(   __lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase   )     ->     Union[str, Any]:
					SCREAMING_SNAKE_CASE							:     Tuple						=				full_name.split('conv_layers.'   )[-1]
					SCREAMING_SNAKE_CASE							:     str						=				name.split('.'   )
					SCREAMING_SNAKE_CASE							:     List[str]						=				int(items[0]   )
					SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				int(items[1]   )

					if type_id == 0:
										if "bias" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''   )
															SCREAMING_SNAKE_CASE							:     int						=				value
															logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''   )
										elif "weight" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''   )
															SCREAMING_SNAKE_CASE							:     List[Any]						=				value
															logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''   )
					elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
										if "bias" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor[layer_id].layer_norm.bias.data.shape} was found.'''   )
															SCREAMING_SNAKE_CASE							:     Optional[int]						=				value
															logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''   )
										elif "weight" in name:
															if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
																				raise ValueError(
																				    F'''{full_name} has size {value.shape}, but'''
																				    F''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''   )
															SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				value
															logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''   )
					else:
										unused_weights.append(__lowerCAmelCase   )

@torch.no_grad()
def        __a			(   __lowerCAmelCase      ,	__lowerCAmelCase      ,	__lowerCAmelCase=None      ,	__lowerCAmelCase=None      ,	__lowerCAmelCase=True   )     ->     Optional[int]:
					if config_path is not None:
										SCREAMING_SNAKE_CASE							:     List[str]						=				UniSpeechSatConfig.from_pretrained(__lowerCAmelCase   )
					else:
										SCREAMING_SNAKE_CASE							:     List[Any]						=				UniSpeechSatConfig()

					SCREAMING_SNAKE_CASE							:     Dict						=				''

					if is_finetuned:
										SCREAMING_SNAKE_CASE							:     Tuple						=				UniSpeechSatForCTC(__lowerCAmelCase   )
					else:
										SCREAMING_SNAKE_CASE							:     List[Any]						=				UniSpeechSatForPreTraining(__lowerCAmelCase   )

					SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Dict						=				fairseq.checkpoint_utils.load_model_ensemble_and_task(
					    [checkpoint_path]      ,	arg_overrides={'data': '/'.join(dict_path.split('/'   )[:-1]   )}   )
					SCREAMING_SNAKE_CASE							:     Dict						=				model[0].eval()

					recursively_load_weights(__lowerCAmelCase      ,	__lowerCAmelCase   )

					hf_wavavec.save_pretrained(__lowerCAmelCase   )


if __name__ == "__main__":
			_lowerCamelCase    :		List[str]		      =    argparse.ArgumentParser()
			parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
			parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
			parser.add_argument("""--dict_path""", default=None, type=str, help="""Path to dict of fine-tuned model""")
			parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
			parser.add_argument(
			    """--not_finetuned""", action="""store_true""", help="""Whether the model to convert is a fine-tuned model or not"""
			)
			_lowerCamelCase    :		Dict		      =    parser.parse_args()
			convert_unispeech_sat_checkpoint(
			    args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
			)




from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFFunnelBaseModel,
			    TFFunnelForMaskedLM,
			    TFFunnelForMultipleChoice,
			    TFFunnelForPreTraining,
			    TFFunnelForQuestionAnswering,
			    TFFunnelForSequenceClassification,
			    TFFunnelForTokenClassification,
			    TFFunnelModel,
			)


class 						lowercase    :
						'''simple docstring'''


						def __init__(  self    :				List[Any]       ,	snake_case    :				Optional[Any]       ,	snake_case    :				int=13       ,	snake_case    :				int=7       ,	snake_case    :				Dict=True       ,	snake_case    :				Union[str, Any]=True       ,	snake_case    :				Union[str, Any]=True       ,	snake_case    :				List[Any]=True       ,	snake_case    :				Tuple=99       ,	snake_case    :				Any=[1, 1, 2]       ,	snake_case    :				Dict=1       ,	snake_case    :				Optional[int]=32       ,	snake_case    :				Union[str, Any]=4       ,	snake_case    :				Optional[Any]=8       ,	snake_case    :				Dict=37       ,	snake_case    :				int="gelu_new"       ,	snake_case    :				Optional[Any]=0.1       ,	snake_case    :				List[str]=0.1       ,	snake_case    :				Any=0.0       ,	snake_case    :				Dict=512       ,	snake_case    :				List[str]=3       ,	snake_case    :				Any=0.02       ,	snake_case    :				List[str]=3       ,	snake_case    :				Optional[Any]=4       ,	snake_case    :				Dict=None       ,	snake_case    :				Any=False       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				parent
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				batch_size
											SCREAMING_SNAKE_CASE							:     List[str]						=				seq_length
											SCREAMING_SNAKE_CASE							:     int						=				is_training
											SCREAMING_SNAKE_CASE							:     Any						=				use_input_mask
											SCREAMING_SNAKE_CASE							:     Tuple						=				use_token_type_ids
											SCREAMING_SNAKE_CASE							:     Dict						=				use_labels
											SCREAMING_SNAKE_CASE							:     Dict						=				vocab_size
											SCREAMING_SNAKE_CASE							:     Dict						=				block_sizes
											SCREAMING_SNAKE_CASE							:     Dict						=				num_decoder_layers
											SCREAMING_SNAKE_CASE							:     int						=				d_model
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				n_head
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				d_head
											SCREAMING_SNAKE_CASE							:     int						=				d_inner
											SCREAMING_SNAKE_CASE							:     List[Any]						=				hidden_act
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				hidden_dropout
											SCREAMING_SNAKE_CASE							:     Any						=				attention_dropout
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				activation_dropout
											SCREAMING_SNAKE_CASE							:     str						=				max_position_embeddings
											SCREAMING_SNAKE_CASE							:     int						=				type_vocab_size
											SCREAMING_SNAKE_CASE							:     str						=				2
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				num_labels
											SCREAMING_SNAKE_CASE							:     Tuple						=				num_choices
											SCREAMING_SNAKE_CASE							:     str						=				scope
											SCREAMING_SNAKE_CASE							:     int						=				initializer_std

											# Used in the tests to check the size of the first attention layer
											SCREAMING_SNAKE_CASE							:     int						=				n_head
											# Used in the tests to check the size of the first hidden state
											SCREAMING_SNAKE_CASE							:     List[str]						=				self.d_model
											# Used in the tests to check the number of output hidden states/attentions
											SCREAMING_SNAKE_CASE							:     List[str]						=				sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
											# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
											# the last hidden state of the first block (which is the first hidden state of the decoder).
											if not base:
																SCREAMING_SNAKE_CASE							:     Dict						=				self.num_hidden_layers + 2







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Any						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size	)

											SCREAMING_SNAKE_CASE							:     Dict						=				None
											if self.use_input_mask:
																SCREAMING_SNAKE_CASE							:     Optional[int]						=				random_attention_mask([self.batch_size, self.seq_length]	)

											SCREAMING_SNAKE_CASE							:     Optional[int]						=				None
											if self.use_token_type_ids:
																SCREAMING_SNAKE_CASE							:     Optional[int]						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size	)

											SCREAMING_SNAKE_CASE							:     Any						=				None
											SCREAMING_SNAKE_CASE							:     Dict						=				None
											SCREAMING_SNAKE_CASE							:     int						=				None
											if self.use_labels:
																SCREAMING_SNAKE_CASE							:     List[Any]						=				ids_tensor([self.batch_size]       ,	self.type_sequence_label_size	)
																SCREAMING_SNAKE_CASE							:     Any						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels	)
																SCREAMING_SNAKE_CASE							:     Dict						=				ids_tensor([self.batch_size]       ,	self.num_choices	)

											SCREAMING_SNAKE_CASE							:     Dict						=				FunnelConfig(
											    vocab_size=self.vocab_size       ,	block_sizes=self.block_sizes       ,	num_decoder_layers=self.num_decoder_layers       ,	d_model=self.d_model       ,	n_head=self.n_head       ,	d_head=self.d_head       ,	d_inner=self.d_inner       ,	hidden_act=self.hidden_act       ,	hidden_dropout=self.hidden_dropout       ,	attention_dropout=self.attention_dropout       ,	activation_dropout=self.activation_dropout       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	initializer_std=self.initializer_std       ,	)

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											)







						def       lowerCamelCase_				(  self    :				Dict       ,	snake_case    :				List[str]       ,	snake_case    :				Tuple       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				Any       ,	snake_case    :				Dict       ,	snake_case    :				Any       ,	snake_case    :				Union[str, Any]       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[str]						=				TFFunnelModel(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Dict						=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				model(snake_case	)

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				[input_ids, input_mask]
											SCREAMING_SNAKE_CASE							:     List[Any]						=				model(snake_case	)

											SCREAMING_SNAKE_CASE							:     int						=				model(snake_case	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.d_model)	)

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				False
											SCREAMING_SNAKE_CASE							:     Any						=				TFFunnelModel(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Dict						=				model(snake_case	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.d_model)	)

											SCREAMING_SNAKE_CASE							:     Any						=				False
											SCREAMING_SNAKE_CASE							:     int						=				TFFunnelModel(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     List[Any]						=				model(snake_case	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.d_model)	)







						def       lowerCamelCase_				(  self    :				str       ,	snake_case    :				Any       ,	snake_case    :				Dict       ,	snake_case    :				Any       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Tuple       ,	snake_case    :				Dict       ,	snake_case    :				Optional[Any]       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				TFFunnelBaseModel(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				model(snake_case	)

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				[input_ids, input_mask]
											SCREAMING_SNAKE_CASE							:     Tuple						=				model(snake_case	)

											SCREAMING_SNAKE_CASE							:     Any						=				model(snake_case	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, 2, self.d_model)	)

											SCREAMING_SNAKE_CASE							:     Optional[int]						=				False
											SCREAMING_SNAKE_CASE							:     int						=				TFFunnelBaseModel(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Any						=				model(snake_case	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, 3, self.d_model)	)

											SCREAMING_SNAKE_CASE							:     int						=				False
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				TFFunnelBaseModel(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				model(snake_case	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, 2, self.d_model)	)







						def       lowerCamelCase_				(  self    :				Optional[int]       ,	snake_case    :				Any       ,	snake_case    :				str       ,	snake_case    :				Optional[int]       ,	snake_case    :				List[str]       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				List[Any]       ,	snake_case    :				Dict       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Any						=				TFFunnelForPreTraining(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     str						=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											SCREAMING_SNAKE_CASE							:     List[str]						=				model(snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length)	)







						def       lowerCamelCase_				(  self    :				List[str]       ,	snake_case    :				Any       ,	snake_case    :				int       ,	snake_case    :				List[str]       ,	snake_case    :				Tuple       ,	snake_case    :				int       ,	snake_case    :				str       ,	snake_case    :				List[Any]       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				TFFunnelForMaskedLM(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											SCREAMING_SNAKE_CASE							:     Tuple						=				model(snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)	)







						def       lowerCamelCase_				(  self    :				Any       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Optional[int]       ,	snake_case    :				Dict       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Dict       ,	snake_case    :				Tuple       ,	snake_case    :				str       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				self.num_labels
											SCREAMING_SNAKE_CASE							:     Any						=				TFFunnelForSequenceClassification(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											SCREAMING_SNAKE_CASE							:     Tuple						=				model(snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_labels)	)







						def       lowerCamelCase_				(  self    :				Optional[Any]       ,	snake_case    :				List[Any]       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Any       ,	snake_case    :				Dict       ,	snake_case    :				Dict       ,	snake_case    :				int       ,	snake_case    :				Tuple       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				self.num_choices
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				TFFunnelForMultipleChoice(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				tf.tile(tf.expand_dims(snake_case       ,	1	)       ,	(1, self.num_choices, 1)	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				tf.tile(tf.expand_dims(snake_case       ,	1	)       ,	(1, self.num_choices, 1)	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				tf.tile(tf.expand_dims(snake_case       ,	1	)       ,	(1, self.num_choices, 1)	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				{
											    'input_ids': multiple_choice_inputs_ids,
											    'attention_mask': multiple_choice_input_mask,
											    'token_type_ids': multiple_choice_token_type_ids,
											}
											SCREAMING_SNAKE_CASE							:     List[str]						=				model(snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_choices)	)







						def       lowerCamelCase_				(  self    :				Any       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Dict       ,	snake_case    :				Any       ,	snake_case    :				int       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				Any       ,	snake_case    :				Optional[Any]       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				self.num_labels
											SCREAMING_SNAKE_CASE							:     List[Any]						=				TFFunnelForTokenClassification(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											SCREAMING_SNAKE_CASE							:     Tuple						=				model(snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.num_labels)	)







						def       lowerCamelCase_				(  self    :				str       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Any       ,	snake_case    :				Dict       ,	snake_case    :				Any       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				List[Any]       ,	snake_case    :				Dict       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				TFFunnelForQuestionAnswering(config=snake_case	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											SCREAMING_SNAKE_CASE							:     Dict						=				model(snake_case	)
											self.parent.assertEqual(result.start_logits.shape       ,	(self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape       ,	(self.batch_size, self.seq_length)	)







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				self.prepare_config_and_inputs()
											(
											    (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  
											)							:     Any						=				config_and_inputs
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict


@require_tf
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		int		=		(
						    (
						        TFFunnelModel,
						        TFFunnelForMaskedLM,
						        TFFunnelForPreTraining,
						        TFFunnelForQuestionAnswering,
						        TFFunnelForTokenClassification,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase			:		Optional[Any]		=		(
						    {
						        'feature-extraction': (TFFunnelBaseModel, TFFunnelModel),
						        'fill-mask': TFFunnelForMaskedLM,
						        'question-answering': TFFunnelForQuestionAnswering,
						        'text-classification': TFFunnelForSequenceClassification,
						        'token-classification': TFFunnelForTokenClassification,
						        'zero-shot': TFFunnelForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						UpperCAmelCase			:		Any		=		False
						UpperCAmelCase			:		Tuple		=		False







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				TFFunnelModelTester(self	)
											SCREAMING_SNAKE_CASE							:     Any						=				ConfigTester(self       ,	config_class=snake_case	)







						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											self.config_tester.run_common_tests()







						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*snake_case	)







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_pretraining(*snake_case	)







						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*snake_case	)







						def       lowerCamelCase_				(  self    :				Union[str, Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*snake_case	)







						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*snake_case	)


@require_tf
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		Any		=		(
						    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
						)
						UpperCAmelCase			:		List[str]		=		False
						UpperCAmelCase			:		Tuple		=		False







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				TFFunnelModelTester(self       ,	base=snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				ConfigTester(self       ,	config_class=snake_case	)







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											self.config_tester.run_common_tests()







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_base_model(*snake_case	)







						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*snake_case	)







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*snake_case	)