
"""simple docstring"""




import argparse
import os
import sys
import urllib.request
import zipfile


snake_case    =		['CoLA', 'SST', 'MRPC', 'QQP', 'STS', 'MNLI', 'SNLI', 'QNLI', 'RTE', 'WNLI', 'diagnostic']
snake_case    =		{
    'CoLA': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4',
    'SST': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',
    'MRPC': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc',
    'QQP': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5',
    'STS': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5',
    'MNLI': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce',
    'SNLI': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df',
    'QNLI': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLIv2.zip?alt=media&token=6fdcf570-0fc5-4631-8456-9505272d1601',
    'RTE': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb',
    'WNLI': 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf',
    'diagnostic': 'https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D',
}

snake_case    =		'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt'
snake_case    =		'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt'

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
					print(f'''Downloading and extracting {task}...'''			)
					SCREAMING_SNAKE_CASE				=	f'''{task}.zip'''
					urllib.request.urlretrieve(TASKaPATH[task],     SCREAMING_SNAKE_CASE_			)
					with zipfile.ZipFile(SCREAMING_SNAKE_CASE_			) as zip_ref:
										zip_ref.extractall(SCREAMING_SNAKE_CASE_			)
					os.remove(SCREAMING_SNAKE_CASE_			)
					print('\tCompleted!'			)

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
					print('Processing MRPC...'			)
					SCREAMING_SNAKE_CASE				=	os.path.join(SCREAMING_SNAKE_CASE_,     'MRPC'			)
					if not os.path.isdir(SCREAMING_SNAKE_CASE_			):
										os.mkdir(SCREAMING_SNAKE_CASE_			)
					if path_to_data:
										SCREAMING_SNAKE_CASE				=	os.path.join(SCREAMING_SNAKE_CASE_,     'msr_paraphrase_train.txt'			)
										SCREAMING_SNAKE_CASE				=	os.path.join(SCREAMING_SNAKE_CASE_,     'msr_paraphrase_test.txt'			)
					else:
										print('Local MRPC data not specified, downloading data from %s' % MRPC_TRAIN			)
										SCREAMING_SNAKE_CASE				=	os.path.join(SCREAMING_SNAKE_CASE_,     'msr_paraphrase_train.txt'			)
										SCREAMING_SNAKE_CASE				=	os.path.join(SCREAMING_SNAKE_CASE_,     'msr_paraphrase_test.txt'			)
										urllib.request.urlretrieve(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)
										urllib.request.urlretrieve(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)
					if not os.path.isfile(SCREAMING_SNAKE_CASE_			):
										raise ValueError(f'''Train data not found at {mrpc_train_file}'''			)
					if not os.path.isfile(SCREAMING_SNAKE_CASE_			):
										raise ValueError(f'''Test data not found at {mrpc_test_file}'''			)
					urllib.request.urlretrieve(TASKaPATH['MRPC'],     os.path.join(SCREAMING_SNAKE_CASE_,     'dev_ids.tsv'			)			)

					SCREAMING_SNAKE_CASE				=	[]
					with open(os.path.join(SCREAMING_SNAKE_CASE_,     'dev_ids.tsv'			),     encoding='utf8'			) as ids_fh:
										for row in ids_fh:
															dev_ids.append(row.strip().split('\t'			)			)

					with open(SCREAMING_SNAKE_CASE_,     encoding='utf8'			) as data_fh, open(
					    os.path.join(SCREAMING_SNAKE_CASE_,     'train.tsv'			),     'w',     encoding='utf8'			) as train_fh, open(os.path.join(SCREAMING_SNAKE_CASE_,     'dev.tsv'			),     'w',     encoding='utf8'			) as dev_fh:
										SCREAMING_SNAKE_CASE				=	data_fh.readline()
										train_fh.write(SCREAMING_SNAKE_CASE_			)
										dev_fh.write(SCREAMING_SNAKE_CASE_			)
										for row in data_fh:
															SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	row.strip().split('\t'			)
															if [ida, ida] in dev_ids:
																				dev_fh.write('%s\t%s\t%s\t%s\t%s\n' % (label, ida, ida, sa, sa)			)
															else:
																				train_fh.write('%s\t%s\t%s\t%s\t%s\n' % (label, ida, ida, sa, sa)			)

					with open(SCREAMING_SNAKE_CASE_,     encoding='utf8'			) as data_fh, open(
					    os.path.join(SCREAMING_SNAKE_CASE_,     'test.tsv'			),     'w',     encoding='utf8'			) as test_fh:
										SCREAMING_SNAKE_CASE				=	data_fh.readline()
										test_fh.write('index\t#1 ID\t#2 ID\t#1 String\t#2 String\n'			)
										for idx, row in enumerate(SCREAMING_SNAKE_CASE_			):
															SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	row.strip().split('\t'			)
															test_fh.write('%d\t%s\t%s\t%s\t%s\n' % (idx, ida, ida, sa, sa)			)
					print('\tCompleted!'			)

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_			):
					print('Downloading and extracting diagnostic...'			)
					if not os.path.isdir(os.path.join(SCREAMING_SNAKE_CASE_,     'diagnostic'			)			):
										os.mkdir(os.path.join(SCREAMING_SNAKE_CASE_,     'diagnostic'			)			)
					SCREAMING_SNAKE_CASE				=	os.path.join(SCREAMING_SNAKE_CASE_,     'diagnostic',     'diagnostic.tsv'			)
					urllib.request.urlretrieve(TASKaPATH['diagnostic'],     SCREAMING_SNAKE_CASE_			)
					print('\tCompleted!'			)
					return

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_			):
					SCREAMING_SNAKE_CASE				=	task_names.split(','			)
					if "all" in task_names:
										SCREAMING_SNAKE_CASE				=	TASKS
					else:
										SCREAMING_SNAKE_CASE				=	[]
										for task_name in task_names:
															if task_name not in TASKS:
																				raise ValueError(f'''Task {task_name} not found!'''			)
															tasks.append(SCREAMING_SNAKE_CASE_			)
					return tasks

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_			):
					SCREAMING_SNAKE_CASE				=	argparse.ArgumentParser()
					parser.add_argument('--data_dir',     help='directory to save data to',     type=SCREAMING_SNAKE_CASE_,     default='glue_data'			)
					parser.add_argument(
					    '--tasks',     help='tasks to download data for as a comma separated string',     type=SCREAMING_SNAKE_CASE_,     default='all'			)
					parser.add_argument(
					    '--path_to_mrpc',     help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt',     type=SCREAMING_SNAKE_CASE_,     default='',     )
					SCREAMING_SNAKE_CASE				=	parser.parse_args(SCREAMING_SNAKE_CASE_			)

					if not os.path.isdir(args.data_dir			):
										os.mkdir(args.data_dir			)
					SCREAMING_SNAKE_CASE				=	get_tasks(args.tasks			)

					for task in tasks:
										if task == "MRPC":
															format_mrpc(args.data_dir,     args.path_to_mrpc			)
										elif task == "diagnostic":
															download_diagnostic(args.data_dir			)
										else:
															download_and_extract(SCREAMING_SNAKE_CASE_,     args.data_dir			)


if __name__ == "__main__":
	sys.exit(main(sys.argv[1:]))


"""simple docstring"""




import unittest

from transformers.testing_utils import is_torch_available, require_torch, tooslow

from ...generation.test_utils import torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    CpmAntConfig,
	    CpmAntForCausalLM,
	    CpmAntModel,
	    CpmAntTokenizer,
	)




@require_torch
class      UpperCamelCase	:

				"""simple docstring"""



				def __init__(				self		,	lowercase__		,	lowercase__=2		,	lowercase__=8		,	lowercase__=True		,	lowercase__=False		,	lowercase__=False		,	lowercase__=False		,	lowercase__=False		,	lowercase__=99		,	lowercase__=32		,	lowercase__=3		,	lowercase__=4		,	lowercase__=37		,	lowercase__=32		,	lowercase__=128		,	lowercase__=8		,	lowercase__=8		,	lowercase__=8		,	lowercase__=1.0		,	lowercase__=True		,	)    ->    str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	parent
									SCREAMING_SNAKE_CASE				=	batch_size
									SCREAMING_SNAKE_CASE				=	seq_length
									SCREAMING_SNAKE_CASE				=	is_training
									SCREAMING_SNAKE_CASE				=	use_token_type_ids
									SCREAMING_SNAKE_CASE				=	use_input_mask
									SCREAMING_SNAKE_CASE				=	use_labels
									SCREAMING_SNAKE_CASE				=	use_mc_token_ids
									SCREAMING_SNAKE_CASE				=	vocab_size
									SCREAMING_SNAKE_CASE				=	hidden_size
									SCREAMING_SNAKE_CASE				=	num_hidden_layers
									SCREAMING_SNAKE_CASE				=	num_attention_heads
									SCREAMING_SNAKE_CASE				=	intermediate_size
									SCREAMING_SNAKE_CASE				=	num_buckets
									SCREAMING_SNAKE_CASE				=	max_distance
									SCREAMING_SNAKE_CASE				=	prompt_length
									SCREAMING_SNAKE_CASE				=	prompt_types
									SCREAMING_SNAKE_CASE				=	segment_types
									SCREAMING_SNAKE_CASE				=	init_std
									SCREAMING_SNAKE_CASE				=	return_dict


				def 							A					(				self			)    ->    Tuple:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	{}
									SCREAMING_SNAKE_CASE				=	ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size			).type(torch.intaa			)
									SCREAMING_SNAKE_CASE				=	False

									SCREAMING_SNAKE_CASE				=	self.get_config()

									return (config, input_ids)


				def 							A					(				self			)    ->    List[Any]:
									"""simple docstring"""







									return CpmAntConfig(
									    vocab_size=self.vocab_size		,	hidden_size=self.hidden_size		,	num_hidden_layers=self.num_hidden_layers		,	num_attention_heads=self.num_attention_heads		,	dim_ff=self.intermediate_size		,	position_bias_num_buckets=self.num_buckets		,	position_bias_max_distance=self.max_distance		,	prompt_types=self.prompt_types		,	prompt_length=self.prompt_length		,	segment_types=self.segment_types		,	use_cache=lowercase__		,	init_std=self.init_std		,	return_dict=self.return_dict		,	)


				def 							A					(				self		,	lowercase__		,	lowercase__		,	*lowercase__			)    ->    Union[str, Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	CpmAntModel(config=lowercase__			)
									model.to(lowercase__			)
									model.eval()

									SCREAMING_SNAKE_CASE				=	model(**lowercase__			).last_hidden_state

									self.parent.assertEqual(hidden_states.shape		,	(self.batch_size, self.seq_length, config.hidden_size)			)


				def 							A					(				self		,	lowercase__		,	lowercase__		,	*lowercase__			)    ->    int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	CpmAntForCausalLM(lowercase__			)
									model.to(lowercase__			)
									SCREAMING_SNAKE_CASE				=	input_ids['input_ids'].to(lowercase__			)
									model.eval()

									SCREAMING_SNAKE_CASE				=	model(**lowercase__			)
									self.parent.assertEqual(
									    model_output.logits.shape		,	(self.batch_size, self.seq_length, config.vocab_size + config.prompt_types * config.prompt_length)		,	)




				def 							A					(				self			)    ->    Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.prepare_config_and_inputs()
									return config, inputs_dict






@require_torch
class      UpperCamelCase	(       __magic_name__  ,      __magic_name__  ,      unittest.TestCase      ):

				"""simple docstring"""



				UpperCAmelCase_				:      List[str]						  =				(CpmAntModel, CpmAntForCausalLM) if is_torch_available() else ()
				UpperCAmelCase_				:      List[Any]						  =				(
				    {"feature-extraction": CpmAntModel, "text-generation": CpmAntForCausalLM} if is_torch_available() else {}
				)

				UpperCAmelCase_				:      List[Any]						  =				False
				UpperCAmelCase_				:      Optional[int]						  =				False
				UpperCAmelCase_				:      Optional[Any]						  =				False
				UpperCAmelCase_				:      List[str]						  =				False
				UpperCAmelCase_				:      Dict						  =				False


				def 							A					(				self			)    ->    Union[str, Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	CpmAntModelTester(self			)
									SCREAMING_SNAKE_CASE				=	ConfigTester(self		,	config_class=lowercase__			)


				def 							A					(				self			)    ->    Tuple:
									"""simple docstring"""







									self.config_tester.create_and_test_config_common_properties()
									self.config_tester.create_and_test_config_to_json_string()
									self.config_tester.create_and_test_config_to_json_file()
									self.config_tester.create_and_test_config_from_and_save_pretrained()
									self.config_tester.check_config_can_be_init_without_params()
									self.config_tester.check_config_arguments_init()


				def 							A					(				self			)    ->    List[str]:
									"""simple docstring"""







									unittest.skip('CPMAnt doesn\'t support input_embeds.'			)(self.test_inputs_embeds			)


				def 							A					(				self			)    ->    Optional[int]:
									"""simple docstring"""







									unittest.skip(
									    'CPMAnt doesn\'t support retain grad in hidden_states or attentions, because prompt management will peel off the output.hidden_states from graph.\
                 So is attentions. We strongly recommand you use loss to tune model.'			)(self.test_retain_grad_hidden_states_attentions			)


				def 							A					(				self			)    ->    List[str]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_cpmant_model(lowercase__		,	lowercase__			)




				def 							A					(				self			)    ->    str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_lm_head_model(lowercase__		,	lowercase__			)






@require_torch
class      UpperCamelCase	(       unittest.TestCase      ):

				"""simple docstring"""



				@tooslow
				def 							A					(				self			)    ->    Any:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	'今天天气真好！'
									SCREAMING_SNAKE_CASE				=	'openbmb/cpm-ant-10b'
									SCREAMING_SNAKE_CASE				=	CpmAntModel.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	CpmAntTokenizer.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	tokenizer(lowercase__		,	return_tensors='pt'			)
									SCREAMING_SNAKE_CASE				=	model(**lowercase__			).last_hidden_state

									SCREAMING_SNAKE_CASE				=	torch.tensor(
									    [[[6.1_708, 5.9_244, 1.0_835], [6.5_207, 6.2_893, -11.3_324], [-1.0_107, -0.0_576, -5.9_577]]]		,	)
									self.assertTrue(torch.allclose(hidden_states[:, :3, :3]		,	lowercase__		,	atol=1E-2			)			)






@require_torch
class      UpperCamelCase	(       unittest.TestCase      ):

				"""simple docstring"""



				@tooslow
				def 							A					(				self			)    ->    str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	'今天天气真好！'
									SCREAMING_SNAKE_CASE				=	'openbmb/cpm-ant-10b'
									SCREAMING_SNAKE_CASE				=	CpmAntForCausalLM.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	CpmAntTokenizer.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	tokenizer(lowercase__		,	return_tensors='pt'			)
									SCREAMING_SNAKE_CASE				=	model(**lowercase__			).logits

									SCREAMING_SNAKE_CASE				=	torch.tensor(
									    [[[-6.4_267, -6.4_083, -6.3_958], [-5.8_802, -5.9_447, -5.7_811], [-5.3_896, -5.4_820, -5.4_295]]]		,	)
									self.assertTrue(torch.allclose(hidden_states[:, :3, :3]		,	lowercase__		,	atol=1E-2			)			)


				@tooslow
				def 							A					(				self			)    ->    Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	'openbmb/cpm-ant-10b'
									SCREAMING_SNAKE_CASE				=	CpmAntForCausalLM.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	CpmAntTokenizer.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	'今天天气不错，'
									SCREAMING_SNAKE_CASE				=	'今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的'
									SCREAMING_SNAKE_CASE				=	tokenizer(lowercase__		,	return_tensors='pt'			)
									SCREAMING_SNAKE_CASE				=	model.generate(**lowercase__			)
									SCREAMING_SNAKE_CASE				=	tokenizer.batch_decode(lowercase__			)
									self.assertEqual(lowercase__		,	lowercase__			)




				@tooslow
				def 							A					(				self			)    ->    str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	'openbmb/cpm-ant-10b'
									SCREAMING_SNAKE_CASE				=	CpmAntForCausalLM.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	CpmAntTokenizer.from_pretrained(lowercase__			)
									SCREAMING_SNAKE_CASE				=	['今天天气不错，', '新年快乐，万事如意！']
									SCREAMING_SNAKE_CASE				=	[
									    '今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的',
									    '新年快乐，万事如意！在这辞旧迎新的美好时刻，我谨代表《农村新技术》杂志社全体同仁，向一直以来关心、支持《农村新技术》杂志发展的各级领导、各界朋友和广大读者致以最诚挚的',
									]
									SCREAMING_SNAKE_CASE				=	tokenizer(lowercase__		,	return_tensors='pt'		,	padding=lowercase__			)
									SCREAMING_SNAKE_CASE				=	model.generate(**lowercase__			)
									SCREAMING_SNAKE_CASE				=	tokenizer.batch_decode(lowercase__			)
									self.assertEqual(lowercase__		,	lowercase__			)
