



'''simple docstring'''

import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


snake_case_			=     ["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]


class 	a__     (				_lowercase	):


	def __init__(self : List[Any],					__UpperCAmelCase : List[str],					__UpperCAmelCase : Optional[int],					__UpperCAmelCase : Union[str, Any]=None,					__UpperCAmelCase : int=1      )     ->			int:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Optional[int]      					= tokenizer
					SCREAMING_SNAKE_CASE  :						Tuple      					= dataset
					SCREAMING_SNAKE_CASE  :						str      					= len(__UpperCAmelCase      ) if n_tasks is None else n_tasks
					SCREAMING_SNAKE_CASE  :						Dict      					= n_copies






	def __iter__(self : List[str]      )     ->			str:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Any      					= []
					for task in range(self.n_tasks      ):
									# without strip, the model generate commented codes ...
									prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()      )
					SCREAMING_SNAKE_CASE  :						str      					= self.tokenizer(__UpperCAmelCase,					padding=__UpperCAmelCase,					return_tensors='''pt'''      )
					for task in range(self.n_tasks      ):
									for _ in range(self.n_copies      ):
													yield {
													    "ids": outputs.input_ids[task],
													    "task_id": task,
													    "input_len": outputs.attention_mask[task].sum(),
													}



class 	a__     (				_lowercase	):


	def __init__(self : Any,					__UpperCAmelCase : Union[str, Any],					__UpperCAmelCase : Any,					__UpperCAmelCase : Union[str, Any]      )     ->			Optional[Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= start_length
					SCREAMING_SNAKE_CASE  :						List[str]      					= eof_strings
					SCREAMING_SNAKE_CASE  :						str      					= tokenizer






	def __call__(self : List[Any],					__UpperCAmelCase : Any,					__UpperCAmelCase : int,					**__UpperCAmelCase : Union[str, Any]      )     ->			List[str]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Any      					= self.tokenizer.batch_decode(input_ids[:, self.start_length :]      )
					SCREAMING_SNAKE_CASE  :						List[Any]      					= []
					for decoded_generation in decoded_generations:
									done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings      )      )
					return all(__UpperCAmelCase      )






def       __lowercase  (_SCREAMING_SNAKE_CASE      :List[str]       ):
				SCREAMING_SNAKE_CASE  :						Dict      					= re.split('''(%s)''' % '''|'''.join(_SCREAMING_SNAKE_CASE       )    ,	_SCREAMING_SNAKE_CASE       )
				# last string should be ""
				return "".join(string_list[:-2]       )






def       __lowercase  (_SCREAMING_SNAKE_CASE      :str    ,	_SCREAMING_SNAKE_CASE      :Union[str, Any]    ,	_SCREAMING_SNAKE_CASE      :int    ,	_SCREAMING_SNAKE_CASE      :Tuple    ,	_SCREAMING_SNAKE_CASE      :Optional[int]    ,	_SCREAMING_SNAKE_CASE      :Tuple=20    ,	**_SCREAMING_SNAKE_CASE      :Optional[Any]       ):
				SCREAMING_SNAKE_CASE  :						Any      					= defaultdict(_SCREAMING_SNAKE_CASE       )  # dict of list of generated tokens
				for step, batch in tqdm(enumerate(_SCREAMING_SNAKE_CASE       )       ):
								with torch.no_grad():
												SCREAMING_SNAKE_CASE  :						Optional[Any]      					= batch['''ids'''].shape[-1]
												SCREAMING_SNAKE_CASE  :						Optional[int]      					= accelerator.unwrap_model(_SCREAMING_SNAKE_CASE       ).generate(
												    input_ids=batch['''ids'''][:, : batch['''input_len''']]    ,	num_return_sequences=_SCREAMING_SNAKE_CASE    ,	**_SCREAMING_SNAKE_CASE       )
												# each task is generated batch_size times
												SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= batch['''task_id'''].repeat(_SCREAMING_SNAKE_CASE       )
												SCREAMING_SNAKE_CASE  :						Tuple      					= accelerator.pad_across_processes(
												    _SCREAMING_SNAKE_CASE    ,	dim=1    ,	pad_index=tokenizer.pad_token_id       )

												SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE  :						List[Any]      					= accelerator.gather((generated_tokens, generated_tasks)       )
												SCREAMING_SNAKE_CASE  :						List[str]      					= generated_tokens.cpu().numpy()
												SCREAMING_SNAKE_CASE  :						List[Any]      					= generated_tasks.cpu().numpy()

												for task, generated_tokens in zip(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       ):
																gen_token_dict[task].append(_SCREAMING_SNAKE_CASE       )

				SCREAMING_SNAKE_CASE  :						Tuple      					= [[] for _ in range(_SCREAMING_SNAKE_CASE       )]
				for task, generated_tokens in gen_token_dict.items():
								for s in generated_tokens:
												SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= tokenizer.decode(_SCREAMING_SNAKE_CASE    ,	skip_special_tokens=_SCREAMING_SNAKE_CASE    ,	clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE       )
												code_gens[task].append(remove_last_block(_SCREAMING_SNAKE_CASE       )       )
				return code_gens






def       __lowercase  ():
				# Setup configuration
				SCREAMING_SNAKE_CASE  :						Optional[Any]      					= HfArgumentParser(_SCREAMING_SNAKE_CASE       )
				SCREAMING_SNAKE_CASE  :						Any      					= parser.parse_args()

				transformers.logging.set_verbosity_error()
				# enables code execution in code_eval metric
				SCREAMING_SNAKE_CASE  :						Dict      					= args.HF_ALLOW_CODE_EVAL
				# make sure tokenizer plays nice with multiprocessing
				SCREAMING_SNAKE_CASE  :						Optional[int]      					= '''false'''

				if args.num_workers is None:
								SCREAMING_SNAKE_CASE  :						str      					= multiprocessing.cpu_count()

				# Use dataset load to feed to accelerate
				SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= Accelerator()
				set_seed(args.seed    ,	device_specific=_SCREAMING_SNAKE_CASE       )

				# Load model and tokenizer
				SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= AutoTokenizer.from_pretrained(args.model_ckpt       )
				SCREAMING_SNAKE_CASE  :						Any      					= tokenizer.eos_token
				SCREAMING_SNAKE_CASE  :						Any      					= AutoModelForCausalLM.from_pretrained(args.model_ckpt       )

				# Generation settings
				SCREAMING_SNAKE_CASE  :						int      					= {
				    '''do_sample''': args.do_sample,
				    '''temperature''': args.temperature,
				    '''max_new_tokens''': args.max_new_tokens,
				    '''top_p''': args.top_p,
				    '''top_k''': args.top_k,
				    '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )]       ),
				}

				# Load evaluation dataset and metric
				SCREAMING_SNAKE_CASE  :						Dict      					= load_dataset('''openai_humaneval'''       )
				SCREAMING_SNAKE_CASE  :						List[Any]      					= load_metric('''code_eval'''       )

				SCREAMING_SNAKE_CASE  :						Optional[int]      					= args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']       )
				SCREAMING_SNAKE_CASE  :						int      					= args.n_samples // args.batch_size

				SCREAMING_SNAKE_CASE  :						Any      					= TokenizedDataset(_SCREAMING_SNAKE_CASE    ,	human_eval['''test''']    ,	n_copies=_SCREAMING_SNAKE_CASE    ,	n_tasks=_SCREAMING_SNAKE_CASE       )
				# do not confuse args.batch_size, which is actually the num_return_sequences
				SCREAMING_SNAKE_CASE  :						Tuple      					= DataLoader(_SCREAMING_SNAKE_CASE    ,	batch_size=1       )

				# Run a quick test to see if code evaluation is enabled
				try:
								SCREAMING_SNAKE_CASE  :						Any      					= code_eval_metric.compute(references=['''''']    ,	predictions=[['''''']]       )
				except ValueError as exception:
								print(
								    '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
								    ''' flag to enable code evaluation.'''       )
								raise exception

				SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE  :						Dict      					= accelerator.prepare(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )

				SCREAMING_SNAKE_CASE  :						List[str]      					= complete_code(
				    _SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	n_tasks=_SCREAMING_SNAKE_CASE    ,	batch_size=args.batch_size    ,	**_SCREAMING_SNAKE_CASE    ,	)

				if accelerator.is_main_process:
								SCREAMING_SNAKE_CASE  :						str      					= []

								for task in tqdm(range(_SCREAMING_SNAKE_CASE       )       ):
												SCREAMING_SNAKE_CASE  :						str      					= human_eval['''test'''][task]['''test''']
												SCREAMING_SNAKE_CASE  :						List[Any]      					= F'''check({human_eval["test"][task]["entry_point"]})'''
												references.append('''\n''' + test_func + '''\n''' + entry_point       )

								# Evaluate completions with "code_eval" metric
								SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE  :						Dict      					= code_eval_metric.compute(
								    references=_SCREAMING_SNAKE_CASE    ,	predictions=_SCREAMING_SNAKE_CASE    ,	num_workers=args.num_workers       )
								print(F'''Results: {pass_at_k}'''       )

								# Save results to json file
								with open(args.output_file    ,	'''w'''       ) as fp:
												json.dump(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
	main()









'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_			=     logging.get_logger(__name__)

snake_case_			=     {
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}


class 	a__     (				_lowercase	):
	__magic_name__		:    Optional[Any]	    =     "nllb-moe"
	__magic_name__		:    Optional[Any]	    =     ["past_key_values"]
	__magic_name__		:    Optional[int]	    =     {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}


	def __init__(self : int,					__UpperCAmelCase : Tuple=128112,					__UpperCAmelCase : Any=1024,					__UpperCAmelCase : Optional[Any]=12,					__UpperCAmelCase : Optional[int]=4096,					__UpperCAmelCase : Any=16,					__UpperCAmelCase : Any=12,					__UpperCAmelCase : Optional[Any]=4096,					__UpperCAmelCase : Optional[int]=16,					__UpperCAmelCase : List[Any]=0.05,					__UpperCAmelCase : Dict=0.05,					__UpperCAmelCase : Dict=True,					__UpperCAmelCase : List[Any]=True,					__UpperCAmelCase : Any="relu",					__UpperCAmelCase : Union[str, Any]=1024,					__UpperCAmelCase : Optional[int]=0.1,					__UpperCAmelCase : Tuple=0.1,					__UpperCAmelCase : List[Any]=0.0,					__UpperCAmelCase : Optional[int]=0.02,					__UpperCAmelCase : Tuple=2,					__UpperCAmelCase : int=True,					__UpperCAmelCase : int=False,					__UpperCAmelCase : int="float32",					__UpperCAmelCase : Optional[Any]=False,					__UpperCAmelCase : List[str]=128,					__UpperCAmelCase : Dict=64,					__UpperCAmelCase : Dict=4,					__UpperCAmelCase : Optional[Any]=4,					__UpperCAmelCase : Optional[Any]=0.001,					__UpperCAmelCase : Optional[Any]=0.001,					__UpperCAmelCase : Optional[Any]="all",					__UpperCAmelCase : List[str]=False,					__UpperCAmelCase : Dict=False,					__UpperCAmelCase : Any=1.0,					__UpperCAmelCase : Dict=0.2,					__UpperCAmelCase : int=1,					__UpperCAmelCase : Union[str, Any]=0,					__UpperCAmelCase : Any=2,					__UpperCAmelCase : Union[str, Any]=False,					**__UpperCAmelCase : Dict,					)     ->			List[Any]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Any      					= vocab_size
					SCREAMING_SNAKE_CASE  :						int      					= max_position_embeddings
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= d_model
					SCREAMING_SNAKE_CASE  :						Dict      					= encoder_ffn_dim
					SCREAMING_SNAKE_CASE  :						int      					= encoder_layers
					SCREAMING_SNAKE_CASE  :						str      					= encoder_attention_heads
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= decoder_ffn_dim
					SCREAMING_SNAKE_CASE  :						str      					= decoder_layers
					SCREAMING_SNAKE_CASE  :						Dict      					= decoder_attention_heads
					SCREAMING_SNAKE_CASE  :						Dict      					= dropout
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= attention_dropout
					SCREAMING_SNAKE_CASE  :						Dict      					= activation_dropout
					SCREAMING_SNAKE_CASE  :						List[str]      					= activation_function
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= init_std
					SCREAMING_SNAKE_CASE  :						Any      					= encoder_layerdrop
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= decoder_layerdrop
					SCREAMING_SNAKE_CASE  :						Dict      					= use_cache
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= encoder_layers
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= scale_embedding  # scale factor will be sqrt(d_model) if True
					SCREAMING_SNAKE_CASE  :						List[str]      					= router_z_loss_coef
					SCREAMING_SNAKE_CASE  :						List[str]      					= router_aux_loss_coef
					SCREAMING_SNAKE_CASE  :						int      					= decoder_sparse_step
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= encoder_sparse_step
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= num_experts
					SCREAMING_SNAKE_CASE  :						List[Any]      					= expert_capacity
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= router_bias
					if router_dtype not in ["float32", "float16", "bfloat16"]:
									raise ValueError(F'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''      )
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= router_dtype

					SCREAMING_SNAKE_CASE  :						int      					= router_ignore_padding_tokens
					SCREAMING_SNAKE_CASE  :						Dict      					= batch_prioritized_routing
					SCREAMING_SNAKE_CASE  :						Dict      					= second_expert_policy
					SCREAMING_SNAKE_CASE  :						Tuple      					= normalize_router_prob_before_dropping
					SCREAMING_SNAKE_CASE  :						List[str]      					= moe_eval_capacity_token_fraction
					SCREAMING_SNAKE_CASE  :						List[Any]      					= moe_token_dropout
					SCREAMING_SNAKE_CASE  :						List[Any]      					= output_router_logits
					super().__init__(
					    pad_token_id=__UpperCAmelCase,					bos_token_id=__UpperCAmelCase,					eos_token_id=__UpperCAmelCase,					is_encoder_decoder=__UpperCAmelCase,					decoder_start_token_id=__UpperCAmelCase,					**__UpperCAmelCase,					)




