





'''simple docstring'''




import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


a__					:			Dict          =     ['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']






class 				__snake_case     (					__magic_name__							):





	def __init__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_=None      ,       UpperCamelCase_=1		)      ->		Optional[int]:
		snake_case__	       =   tokenizer
		snake_case__	       =   dataset
		snake_case__	       =   len(UpperCamelCase_		) if n_tasks is None else n_tasks
		snake_case__	       =   n_copies





	def __iter__(		self		)      ->		List[Any]:
		snake_case__	       =   []
		for task in range(self.n_tasks		):
			# without strip, the model generate commented codes ...
			prompts.append(self.tokenizer.eos_token + self.dataset[task]['prompt'].strip()		)
		snake_case__	       =   self.tokenizer(UpperCamelCase_      ,       padding=UpperCamelCase_      ,       return_tensors='pt'		)
		for task in range(self.n_tasks		):
			for _ in range(self.n_copies		):
				yield {
				    "ids": outputs.input_ids[task],
				    "task_id": task,
				    "input_len": outputs.attention_mask[task].sum(),
				}






class 				__snake_case     (					__magic_name__							):





	def __init__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       UpperCamelCase_		)      ->		Union[str, Any]:
		snake_case__	       =   start_length
		snake_case__	       =   eof_strings
		snake_case__	       =   tokenizer





	def __call__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_      ,       **UpperCamelCase_		)      ->		Tuple:
		snake_case__	       =   self.tokenizer.batch_decode(input_ids[:, self.start_length :]		)
		snake_case__	       =   []
		for decoded_generation in decoded_generations:
			done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings		)		)
		return all(UpperCamelCase_		)





def 						__lowerCamelCase			(	UpperCAmelCase_							)   ->Dict:
	snake_case__	       =   re.split('(%s)' % '|'.join(UpperCAmelCase_							)					,				UpperCAmelCase_							)
	# last string should be ""
	return "".join(string_list[:-2]							)




def 						__lowerCamelCase			(	UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_=20					,				**UpperCAmelCase_							)   ->Optional[Any]:
	snake_case__	       =   defaultdict(UpperCAmelCase_							)  # dict of list of generated tokens
	for step, batch in tqdm(enumerate(UpperCAmelCase_							)							):
		with torch.no_grad():
			snake_case__	       =   batch['ids'].shape[-1]
			snake_case__	       =   accelerator.unwrap_model(UpperCAmelCase_							).generate(
			    input_ids=batch['ids'][:, : batch['input_len']]					,				num_return_sequences=UpperCAmelCase_					,				**UpperCAmelCase_							)
			# each task is generated batch_size times
			snake_case__	       =   batch['task_id'].repeat(UpperCAmelCase_							)
			snake_case__	       =   accelerator.pad_across_processes(
			    UpperCAmelCase_					,				dim=1					,				pad_index=tokenizer.pad_token_id							)

			snake_case__       ,  snake_case__	       =   accelerator.gather((generated_tokens, generated_tasks)							)
			snake_case__	       =   generated_tokens.cpu().numpy()
			snake_case__	       =   generated_tasks.cpu().numpy()

			for task, generated_tokens in zip(UpperCAmelCase_					,				UpperCAmelCase_							):
				gen_token_dict[task].append(UpperCAmelCase_							)

	snake_case__	       =   [[] for _ in range(UpperCAmelCase_							)]
	for task, generated_tokens in gen_token_dict.items():
		for s in generated_tokens:
			snake_case__	       =   tokenizer.decode(UpperCAmelCase_					,				skip_special_tokens=UpperCAmelCase_					,				clean_up_tokenization_spaces=UpperCAmelCase_							)
			code_gens[task].append(remove_last_block(UpperCAmelCase_							)							)
	return code_gens




def 						__lowerCamelCase			(	)   ->int:
	# Setup configuration
	snake_case__	       =   HfArgumentParser(UpperCAmelCase_							)
	snake_case__	       =   parser.parse_args()

	transformers.logging.set_verbosity_error()
	# enables code execution in code_eval metric
	snake_case__	       =   args.HF_ALLOW_CODE_EVAL
	# make sure tokenizer plays nice with multiprocessing
	snake_case__	       =   'false'

	if args.num_workers is None:
		snake_case__	       =   multiprocessing.cpu_count()

	# Use dataset load to feed to accelerate
	snake_case__	       =   Accelerator()
	set_seed(args.seed					,				device_specific=UpperCAmelCase_							)

	# Load model and tokenizer
	snake_case__	       =   AutoTokenizer.from_pretrained(args.model_ckpt							)
	snake_case__	       =   tokenizer.eos_token
	snake_case__	       =   AutoModelForCausalLM.from_pretrained(args.model_ckpt							)

	# Generation settings
	snake_case__	       =   {
	    'do_sample': args.do_sample,
	    'temperature': args.temperature,
	    'max_new_tokens': args.max_new_tokens,
	    'top_p': args.top_p,
	    'top_k': args.top_k,
	    'stopping_criteria': StoppingCriteriaList([EndOfFunctionCriteria(0					,				UpperCAmelCase_					,				UpperCAmelCase_							)]							),
	}

	# Load evaluation dataset and metric
	snake_case__	       =   load_dataset('openai_humaneval'							)
	snake_case__	       =   load_metric('code_eval'							)

	snake_case__	       =   args.num_tasks if args.num_tasks is not None else len(human_eval['test']							)
	snake_case__	       =   args.n_samples // args.batch_size

	snake_case__	       =   TokenizedDataset(UpperCAmelCase_					,				human_eval['test']					,				n_copies=UpperCAmelCase_					,				n_tasks=UpperCAmelCase_							)
	# do not confuse args.batch_size, which is actually the num_return_sequences
	snake_case__	       =   DataLoader(UpperCAmelCase_					,				batch_size=1							)

	# Run a quick test to see if code evaluation is enabled
	try:
		snake_case__	       =   code_eval_metric.compute(references=['']					,				predictions=[['']]							)
	except ValueError as exception:
		print(
		    'Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'
		    ' flag to enable code evaluation.'							)
		raise exception

	snake_case__       ,  snake_case__	       =   accelerator.prepare(UpperCAmelCase_					,				UpperCAmelCase_							)

	snake_case__	       =   complete_code(
	    UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_					,				n_tasks=UpperCAmelCase_					,				batch_size=args.batch_size					,				**UpperCAmelCase_					,				)

	if accelerator.is_main_process:
		snake_case__	       =   []

		for task in tqdm(range(UpperCAmelCase_							)							):
			snake_case__	       =   human_eval['test'][task]['test']
			snake_case__	       =   f'''check({human_eval['test'][task]['entry_point']})'''
			references.append('\n' + test_func + '\n' + entry_point							)

		# Evaluate completions with "code_eval" metric
		snake_case__       ,  snake_case__	       =   code_eval_metric.compute(
		    references=UpperCAmelCase_					,				predictions=UpperCAmelCase_					,				num_workers=args.num_workers							)
		print(f'''Results: {pass_at_k}'''							)

		# Save results to json file
		with open(args.output_file					,				'w'							) as fp:
			json.dump(UpperCAmelCase_					,				UpperCAmelCase_							)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
			main()










'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a__					:			str          =     logging.get_logger(__name__)

a__					:			Any          =     {
    '''xlm-mlm-en-2048''': '''https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json''',
    '''xlm-mlm-ende-1024''': '''https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json''',
    '''xlm-mlm-enfr-1024''': '''https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json''',
    '''xlm-mlm-enro-1024''': '''https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json''',
    '''xlm-mlm-tlm-xnli15-1024''': '''https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json''',
    '''xlm-mlm-xnli15-1024''': '''https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json''',
    '''xlm-clm-enfr-1024''': '''https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json''',
    '''xlm-clm-ende-1024''': '''https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json''',
    '''xlm-mlm-17-1280''': '''https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json''',
    '''xlm-mlm-100-1280''': '''https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json''',
}






class 				__snake_case     (					__magic_name__							):
	__lowerCAmelCase       						=  '''xlm'''
	__lowerCAmelCase       						=  {
	    '''hidden_size''': '''emb_dim''',
	    '''num_attention_heads''': '''n_heads''',
	    '''num_hidden_layers''': '''n_layers''',
	    '''n_words''': '''vocab_size''',  # For backward compatibility
	}





	def __init__(		self      ,       UpperCamelCase_=3_0145      ,       UpperCamelCase_=2048      ,       UpperCamelCase_=12      ,       UpperCamelCase_=16      ,       UpperCamelCase_=0.1      ,       UpperCamelCase_=0.1      ,       UpperCamelCase_=True      ,       UpperCamelCase_=False      ,       UpperCamelCase_=False      ,       UpperCamelCase_=False      ,       UpperCamelCase_=1      ,       UpperCamelCase_=True      ,       UpperCamelCase_=512      ,       UpperCamelCase_=2048**-0.5      ,       UpperCamelCase_=1E-1_2      ,       UpperCamelCase_=0.0_2      ,       UpperCamelCase_=0      ,       UpperCamelCase_=1      ,       UpperCamelCase_=2      ,       UpperCamelCase_=3      ,       UpperCamelCase_=5      ,       UpperCamelCase_=True      ,       UpperCamelCase_="first"      ,       UpperCamelCase_=True      ,       UpperCamelCase_=None      ,       UpperCamelCase_=True      ,       UpperCamelCase_=0.1      ,       UpperCamelCase_=5      ,       UpperCamelCase_=5      ,       UpperCamelCase_=0      ,       UpperCamelCase_=0      ,       UpperCamelCase_=2      ,       UpperCamelCase_=0      ,       **UpperCamelCase_      ,       )      ->		List[str]:
		snake_case__	       =   vocab_size
		snake_case__	       =   emb_dim
		snake_case__	       =   n_layers
		snake_case__	       =   n_heads
		snake_case__	       =   dropout
		snake_case__	       =   attention_dropout
		snake_case__	       =   gelu_activation
		snake_case__	       =   sinusoidal_embeddings
		snake_case__	       =   causal
		snake_case__	       =   asm
		snake_case__	       =   n_langs
		snake_case__	       =   use_lang_emb
		snake_case__	       =   layer_norm_eps
		snake_case__	       =   bos_index
		snake_case__	       =   eos_index
		snake_case__	       =   pad_index
		snake_case__	       =   unk_index
		snake_case__	       =   mask_index
		snake_case__	       =   is_encoder
		snake_case__	       =   max_position_embeddings
		snake_case__	       =   embed_init_std
		snake_case__	       =   init_std
		snake_case__	       =   summary_type
		snake_case__	       =   summary_use_proj
		snake_case__	       =   summary_activation
		snake_case__	       =   summary_proj_to_labels
		snake_case__	       =   summary_first_dropout
		snake_case__	       =   start_n_top
		snake_case__	       =   end_n_top
		snake_case__	       =   mask_token_id
		snake_case__	       =   lang_id

		if "n_words" in kwargs:
			snake_case__	       =   kwargs['n_words']

		super().__init__(pad_token_id=UpperCamelCase_      ,       bos_token_id=UpperCamelCase_      ,       **UpperCamelCase_		)






class 				__snake_case     (					__magic_name__							):





	@property
	def   _snake_case   (		self		)      ->		Mapping[str, Mapping[int, str]]:
		if self.task == "multiple-choice":
			snake_case__	       =   {0: 'batch', 1: 'choice', 2: 'sequence'}
		else:
			snake_case__	       =   {0: 'batch', 1: 'sequence'}
		return OrderedDict(
		    [
		        ('input_ids', dynamic_axis),
		        ('attention_mask', dynamic_axis),
		        ('token_type_ids', dynamic_axis),
		    ]		)



