






import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


_UpperCAmelCase			:		str   				=    re.compile(r"\s+")





def 	lowerCAmelCase_		(lowercase__						:       Optional[int]       )		->    List[str]:


				'''simple docstring'''




				return {"hash": hashlib.mda(re.sub(lowercase__     ,					''''''     ,					example['''content''']       ).encode('''utf-8'''       )       ).hexdigest()}





def 	lowerCAmelCase_		(lowercase__						:       Dict       )		->    Tuple:


				'''simple docstring'''




				lowerCAmelCase__      			=						[len(lowercase__       ) for line in example['''content'''].splitlines()]
				return {"line_mean": np.mean(lowercase__       ), "line_max": max(lowercase__       )}





def 	lowerCAmelCase_		(lowercase__						:       Tuple       )		->    Optional[int]:


				'''simple docstring'''




				lowerCAmelCase__      			=						np.mean([c.isalnum() for c in example['''content''']]       )
				return {"alpha_frac": alpha_frac}





def 	lowerCAmelCase_		(lowercase__						:       Optional[Any]     ,					lowercase__						:       Dict       )		->    List[Any]:


				'''simple docstring'''




				if example["hash"] in uniques:
								uniques.remove(example['''hash''']       )
								return True
				else:
								return False





def 	lowerCAmelCase_		(lowercase__						:       Optional[int]     ,					lowercase__						:       Any=5       )		->    Optional[Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						['''auto-generated''', '''autogenerated''', '''automatically generated''']
				lowerCAmelCase__      			=						example['''content'''].splitlines()
				for _, line in zip(range(lowercase__       )     ,					lowercase__       ):
								for keyword in keywords:
												if keyword in line.lower():
																return {"autogenerated": True}
				else:
								return {"autogenerated": False}





def 	lowerCAmelCase_		(lowercase__						:       Dict     ,					lowercase__						:       Any=5     ,					lowercase__						:       Union[str, Any]=0.05       )		->    Union[str, Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						['''unit tests''', '''test file''', '''configuration file''']
				lowerCAmelCase__      			=						example['''content'''].splitlines()
				lowerCAmelCase__      			=						0
				lowerCAmelCase__      			=						0
				# first test
				for _, line in zip(range(lowercase__       )     ,					lowercase__       ):
								for keyword in keywords:
												if keyword in line.lower():
																return {"config_or_test": True}
    # second test
				lowerCAmelCase__      			=						example['''content'''].count('''\n'''       )
				lowerCAmelCase__      			=						int(coeff * nlines       )
				for line in lines:
								count_config += line.lower().count('''config'''       )
								count_test += line.lower().count('''test'''       )
								if count_config > threshold or count_test > threshold:
												return {"config_or_test": True}
				return {"config_or_test": False}





def 	lowerCAmelCase_		(lowercase__						:       str       )		->    List[str]:


				'''simple docstring'''




				lowerCAmelCase__      			=						['''def ''', '''class ''', '''for ''', '''while ''']
				lowerCAmelCase__      			=						example['''content'''].splitlines()
				for line in lines:
								for keyword in keywords:
												if keyword in line.lower():
																return {"has_no_keywords": False}
				return {"has_no_keywords": True}





def 	lowerCAmelCase_		(lowercase__						:       List[str]     ,					lowercase__						:       List[Any]=4       )		->    str:


				'''simple docstring'''




				lowerCAmelCase__      			=						example['''content'''].splitlines()
				lowerCAmelCase__      			=						0
				for line in lines:
								counter += line.lower().count('''='''       )
								if counter > minimum:
												return {"has_few_assignments": False}
				return {"has_few_assignments": True}





def 	lowerCAmelCase_		(lowercase__						:       Optional[int]       )		->    Dict:


				'''simple docstring'''




				lowerCAmelCase__      			=						tokenizer(example['''content''']     ,					truncation=lowercase__       )['''input_ids''']
				lowerCAmelCase__      			=						len(example['''content''']       ) / len(lowercase__       )
				return {"ratio": ratio}





def 	lowerCAmelCase_		(lowercase__						:       List[Any]       )		->    Optional[Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						{}
				results.update(get_hash(lowercase__       )       )
				results.update(line_stats(lowercase__       )       )
				results.update(alpha_stats(lowercase__       )       )
				results.update(char_token_ratio(lowercase__       )       )
				results.update(is_autogenerated(lowercase__       )       )
				results.update(is_config_or_test(lowercase__       )       )
				results.update(has_no_keywords(lowercase__       )       )
				results.update(has_few_assignments(lowercase__       )       )
				return results





def 	lowerCAmelCase_		(lowercase__						:       Optional[Any]     ,					lowercase__						:       Tuple     ,					lowercase__						:       Dict       )		->    int:


				'''simple docstring'''




				if not check_uniques(lowercase__     ,					lowercase__       ):
								return False
				elif example["autogenerated"]:
								return False
				elif example["line_max"] > args.line_max:
								return False
				elif example["line_mean"] > args.line_mean:
								return False
				elif example["alpha_frac"] < args.alpha_frac:
								return False
				elif example["ratio"] < args.min_token_ratio:
								return False
				elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
								return False
				elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
								return False
				elif example["has_few_assignments"]:
								return False
				else:
								return True





def 	lowerCAmelCase_		(lowercase__						:       Dict       )		->    Any:


				'''simple docstring'''




				with open(lowercase__     ,					'''rb'''       ) as f_in:
								with gzip.open(str(lowercase__       ) + '''.gz'''     ,					'''wb'''     ,					compresslevel=6       ) as f_out:
												shutil.copyfileobj(lowercase__     ,					lowercase__       )
				os.unlink(lowercase__       )


# Settings
_UpperCAmelCase			:		Optional[Any]   				=    HfArgumentParser(PreprocessingArguments)
_UpperCAmelCase			:		str   				=    parser.parse_args()
if args.num_workers is None:
	_UpperCAmelCase			:		Dict   				=    multiprocessing.cpu_count()
_UpperCAmelCase			:		Optional[int]   				=    AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
_UpperCAmelCase			:		Tuple   				=    time.time()
_UpperCAmelCase			:		Dict   				=    load_dataset(args.dataset_name, split="train")
print(F'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
_UpperCAmelCase			:		Union[str, Any]   				=    time.time()
_UpperCAmelCase			:		int   				=    ds.map(preprocess, num_proc=args.num_workers)
print(F'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
_UpperCAmelCase			:		Optional[int]   				=    set(ds.unique("hash"))
_UpperCAmelCase			:		Optional[Any]   				=    len(uniques) / len(ds)
print(F'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
_UpperCAmelCase			:		Optional[int]   				=    time.time()
_UpperCAmelCase			:		Optional[Any]   				=    ds.filter(filter, fn_kwargs={"uniques": uniques, "args": args})
print(F'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(F'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
	_UpperCAmelCase			:		Union[str, Any]   				=    time.time()
	_UpperCAmelCase			:		str   				=    deduplicate_dataset(ds_filter, args.jaccard_threshold)
	print(F'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
	print(F'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
_UpperCAmelCase			:		List[str]   				=    Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
	with open(output_dir / "duplicate_clusters.json", "w") as f:
		json.dump(duplicate_clusters, f)

_UpperCAmelCase			:		Any   				=    output_dir / "data"
data_dir.mkdir(exist_ok=True)

_UpperCAmelCase			:		Union[str, Any]   				=    time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
	_UpperCAmelCase			:		Union[str, Any]   				=    str(data_dir / F'''file-{file_number+1:012}.json''')
	_UpperCAmelCase			:		Union[str, Any]   				=    min(len(ds_filter), index + args.samples_per_file)
	ds_filter.select(list(range(index, end_index))).to_json(file_path)
	compress_file(file_path)




print(F'''Time to save dataset: {time.time()-t_start:.2f}''')













from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFFunnelBaseModel,
			    TFFunnelForMaskedLM,
			    TFFunnelForMultipleChoice,
			    TFFunnelForPreTraining,
			    TFFunnelForQuestionAnswering,
			    TFFunnelForSequenceClassification,
			    TFFunnelForTokenClassification,
			    TFFunnelModel,
			)
class  lowerCAmelCase_       :




							def __init__(   self     :   Dict ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=13 ,							SCREAMING_SNAKE_CASE_     :   str=7 ,							SCREAMING_SNAKE_CASE_     :   Tuple=True ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=True ,							SCREAMING_SNAKE_CASE_     :   List[Any]=True ,							SCREAMING_SNAKE_CASE_     :   Optional[int]=True ,							SCREAMING_SNAKE_CASE_     :   Optional[int]=99 ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=[1, 1, 2] ,							SCREAMING_SNAKE_CASE_     :   List[str]=1 ,							SCREAMING_SNAKE_CASE_     :   str=32 ,							SCREAMING_SNAKE_CASE_     :   Dict=4 ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=8 ,							SCREAMING_SNAKE_CASE_     :   Dict=37 ,							SCREAMING_SNAKE_CASE_     :   Tuple="gelu_new" ,							SCREAMING_SNAKE_CASE_     :   Dict=0.1 ,							SCREAMING_SNAKE_CASE_     :   Any=0.1 ,							SCREAMING_SNAKE_CASE_     :   Any=0.0 ,							SCREAMING_SNAKE_CASE_     :   Tuple=512 ,							SCREAMING_SNAKE_CASE_     :   str=3 ,							SCREAMING_SNAKE_CASE_     :   int=0.02 ,							SCREAMING_SNAKE_CASE_     :   Optional[int]=3 ,							SCREAMING_SNAKE_CASE_     :   Any=4 ,							SCREAMING_SNAKE_CASE_     :   Optional[int]=None ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=False ,							):
											lowerCAmelCase__      			=						parent
											lowerCAmelCase__      			=						batch_size
											lowerCAmelCase__      			=						seq_length
											lowerCAmelCase__      			=						is_training
											lowerCAmelCase__      			=						use_input_mask
											lowerCAmelCase__      			=						use_token_type_ids
											lowerCAmelCase__      			=						use_labels
											lowerCAmelCase__      			=						vocab_size
											lowerCAmelCase__      			=						block_sizes
											lowerCAmelCase__      			=						num_decoder_layers
											lowerCAmelCase__      			=						d_model
											lowerCAmelCase__      			=						n_head
											lowerCAmelCase__      			=						d_head
											lowerCAmelCase__      			=						d_inner
											lowerCAmelCase__      			=						hidden_act
											lowerCAmelCase__      			=						hidden_dropout
											lowerCAmelCase__      			=						attention_dropout
											lowerCAmelCase__      			=						activation_dropout
											lowerCAmelCase__      			=						max_position_embeddings
											lowerCAmelCase__      			=						type_vocab_size
											lowerCAmelCase__      			=						2
											lowerCAmelCase__      			=						num_labels
											lowerCAmelCase__      			=						num_choices
											lowerCAmelCase__      			=						scope
											lowerCAmelCase__      			=						initializer_std

											# Used in the tests to check the size of the first attention layer
											lowerCAmelCase__      			=						n_head
											# Used in the tests to check the size of the first hidden state
											lowerCAmelCase__      			=						self.d_model
											# Used in the tests to check the number of output hidden states/attentions
											lowerCAmelCase__      			=						sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
											# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
											# the last hidden state of the first block (which is the first hidden state of the decoder).
											if not base:
															lowerCAmelCase__      			=						self.num_hidden_layers + 2




							def 						__snake_case					(   self     :   Optional[int]	):
											lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size	)

											lowerCAmelCase__      			=						None
											if self.use_input_mask:
															lowerCAmelCase__      			=						random_attention_mask([self.batch_size, self.seq_length]	)

											lowerCAmelCase__      			=						None
											if self.use_token_type_ids:
															lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.type_vocab_size	)

											lowerCAmelCase__      			=						None
											lowerCAmelCase__      			=						None
											lowerCAmelCase__      			=						None
											if self.use_labels:
															lowerCAmelCase__      			=						ids_tensor([self.batch_size] ,							self.type_sequence_label_size	)
															lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.num_labels	)
															lowerCAmelCase__      			=						ids_tensor([self.batch_size] ,							self.num_choices	)

											lowerCAmelCase__      			=						FunnelConfig(
											    vocab_size=self.vocab_size ,							block_sizes=self.block_sizes ,							num_decoder_layers=self.num_decoder_layers ,							d_model=self.d_model ,							n_head=self.n_head ,							d_head=self.d_head ,							d_inner=self.d_inner ,							hidden_act=self.hidden_act ,							hidden_dropout=self.hidden_dropout ,							attention_dropout=self.attention_dropout ,							activation_dropout=self.activation_dropout ,							max_position_embeddings=self.max_position_embeddings ,							type_vocab_size=self.type_vocab_size ,							initializer_std=self.initializer_std ,							)

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											)




							def 						__snake_case					(   self     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Dict ,							):
											lowerCAmelCase__      			=						TFFunnelModel(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)

											lowerCAmelCase__      			=						[input_ids, input_mask]
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)

											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.d_model)	)

											lowerCAmelCase__      			=						False
											lowerCAmelCase__      			=						TFFunnelModel(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.d_model)	)

											lowerCAmelCase__      			=						False
											lowerCAmelCase__      			=						TFFunnelModel(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.d_model)	)




							def 						__snake_case					(   self     :   Any ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							):
											lowerCAmelCase__      			=						TFFunnelBaseModel(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)

											lowerCAmelCase__      			=						[input_ids, input_mask]
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)

											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, 2, self.d_model)	)

											lowerCAmelCase__      			=						False
											lowerCAmelCase__      			=						TFFunnelBaseModel(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, 3, self.d_model)	)

											lowerCAmelCase__      			=						False
											lowerCAmelCase__      			=						TFFunnelBaseModel(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, 2, self.d_model)	)




							def 						__snake_case					(   self     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Any ,							):
											lowerCAmelCase__      			=						TFFunnelForPreTraining(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length)	)




							def 						__snake_case					(   self     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   int ,							):
											lowerCAmelCase__      			=						TFFunnelForMaskedLM(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length, self.vocab_size)	)




							def 						__snake_case					(   self     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Any ,							):
											lowerCAmelCase__      			=						self.num_labels
											lowerCAmelCase__      			=						TFFunnelForSequenceClassification(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.num_labels)	)




							def 						__snake_case					(   self     :   int ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							):
											lowerCAmelCase__      			=						self.num_choices
											lowerCAmelCase__      			=						TFFunnelForMultipleChoice(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE_ ,							1	) ,							(1, self.num_choices, 1)	)
											lowerCAmelCase__      			=						tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE_ ,							1	) ,							(1, self.num_choices, 1)	)
											lowerCAmelCase__      			=						tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE_ ,							1	) ,							(1, self.num_choices, 1)	)
											lowerCAmelCase__      			=						{
											    '''input_ids''': multiple_choice_inputs_ids,
											    '''attention_mask''': multiple_choice_input_mask,
											    '''token_type_ids''': multiple_choice_token_type_ids,
											}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.num_choices)	)




							def 						__snake_case					(   self     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							):
											lowerCAmelCase__      			=						self.num_labels
											lowerCAmelCase__      			=						TFFunnelForTokenClassification(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length, self.num_labels)	)




							def 						__snake_case					(   self     :   str ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   Any ,							):
											lowerCAmelCase__      			=						TFFunnelForQuestionAnswering(config=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.start_logits.shape ,							(self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape ,							(self.batch_size, self.seq_length)	)




							def 						__snake_case					(   self     :   List[str]	):
											lowerCAmelCase__      			=						self.prepare_config_and_inputs()
											(
											    (
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		
											)      			=						config_and_inputs
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
											return config, inputs_dict



@require_tf
class  lowerCAmelCase_       (				snake_case__     ,    snake_case__     ,    unittest.TestCase					):
							UpperCamelCase_		:Dict		      =    (
							    (
							        TFFunnelModel,
							        TFFunnelForMaskedLM,
							        TFFunnelForPreTraining,
							        TFFunnelForQuestionAnswering,
							        TFFunnelForTokenClassification,
							    )
							    if is_tf_available()
							    else ()
							)
							UpperCamelCase_		:Dict		      =    (
							    {
							        'feature-extraction': (TFFunnelBaseModel, TFFunnelModel),
							        'fill-mask': TFFunnelForMaskedLM,
							        'question-answering': TFFunnelForQuestionAnswering,
							        'text-classification': TFFunnelForSequenceClassification,
							        'token-classification': TFFunnelForTokenClassification,
							        'zero-shot': TFFunnelForSequenceClassification,
							    }
							    if is_tf_available()
							    else {}
							)
							UpperCamelCase_		:Tuple		      =    False
							UpperCamelCase_		:Tuple		      =    False




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						TFFunnelModelTester(self	)
											lowerCAmelCase__      			=						ConfigTester(self ,							config_class=SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Dict	):
											self.config_tester.run_common_tests()




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_pretraining(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Dict	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_	)



@require_tf
class  lowerCAmelCase_       (				snake_case__     ,    unittest.TestCase					):
							UpperCamelCase_		:Optional[Any]		      =    (
							    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
							)
							UpperCamelCase_		:int		      =    False
							UpperCamelCase_		:Any		      =    False




							def 						__snake_case					(   self     :   List[str]	):
											lowerCAmelCase__      			=						TFFunnelModelTester(self ,							base=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						ConfigTester(self ,							config_class=SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Tuple	):
											self.config_tester.run_common_tests()




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_base_model(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   List[str]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_	)





