
import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__a         =   ['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']
class 					lowercase__(							UpperCAmelCase				):



	"""simple docstring"""




	def __init__(			self    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     Dict			,				SCREAMING_SNAKE_CASE_    :     Any			,				SCREAMING_SNAKE_CASE_    :     str=None			,				SCREAMING_SNAKE_CASE_    :     Tuple=1			)			->	Any:
					lowercase_	    =   tokenizer
					lowercase_	    =   dataset
					lowercase_	    =   len(SCREAMING_SNAKE_CASE_			) if n_tasks is None else n_tasks
					lowercase_	    =   n_copies




	def __iter__(			self    :     Optional[Any]			)			->	Optional[int]:
					lowercase_	    =   []
					for task in range(self.n_tasks			):
									# without strip, the model generate commented codes ...
									prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()			)
					lowercase_	    =   self.tokenizer(SCREAMING_SNAKE_CASE_			,				padding=SCREAMING_SNAKE_CASE_			,				return_tensors='''pt'''			)
					for task in range(self.n_tasks			):
									for _ in range(self.n_copies			):
													yield {
													    "ids": outputs.input_ids[task],
													    "task_id": task,
													    "input_len": outputs.attention_mask[task].sum(),
													}

class 					lowercase__(							UpperCAmelCase				):



	"""simple docstring"""




	def __init__(			self    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     int			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]			)			->	List[Any]:
					lowercase_	    =   start_length
					lowercase_	    =   eof_strings
					lowercase_	    =   tokenizer




	def __call__(			self    :     str			,				SCREAMING_SNAKE_CASE_    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     Any			,				**SCREAMING_SNAKE_CASE_    :     Union[str, Any]			)			->	List[str]:
					lowercase_	    =   self.tokenizer.batch_decode(input_ids[:, self.start_length :]			)
					lowercase_	    =   []
					for decoded_generation in decoded_generations:
									done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings			)			)
					return all(SCREAMING_SNAKE_CASE_			)




def 							a		(				snake_case__:   Dict   ):


				'''simple docstring'''
				lowercase_	    =   re.split('''(%s)''' % '''|'''.join(snake_case__   )		,  snake_case__   )
				# last string should be ""
				return "".join(string_list[:-2]   )




def 							a		(				snake_case__:   Union[str, Any]		,  snake_case__:   List[str]		,  snake_case__:   Optional[int]		,  snake_case__:   Optional[Any]		,  snake_case__:   Optional[Any]		,  snake_case__:   List[Any]=20		,  **snake_case__:   str   ):


				'''simple docstring'''
				lowercase_	    =   defaultdict(snake_case__   )  # dict of list of generated tokens
				for step, batch in tqdm(enumerate(snake_case__   )   ):
								with torch.no_grad():
												lowercase_	    =   batch['''ids'''].shape[-1]
												lowercase_	    =   accelerator.unwrap_model(snake_case__   ).generate(
												    input_ids=batch['''ids'''][:, : batch['''input_len''']]		,  num_return_sequences=snake_case__		,  **snake_case__   )
												# each task is generated batch_size times
												lowercase_	    =   batch['''task_id'''].repeat(snake_case__   )
												lowercase_	    =   accelerator.pad_across_processes(
												    snake_case__		,  dim=1		,  pad_index=tokenizer.pad_token_id   )

												lowercase_    ,      lowercase_	    =   accelerator.gather((generated_tokens, generated_tasks)   )
												lowercase_	    =   generated_tokens.cpu().numpy()
												lowercase_	    =   generated_tasks.cpu().numpy()

												for task, generated_tokens in zip(snake_case__		,  snake_case__   ):
																gen_token_dict[task].append(snake_case__   )

				lowercase_	    =   [[] for _ in range(snake_case__   )]
				for task, generated_tokens in gen_token_dict.items():
								for s in generated_tokens:
												lowercase_	    =   tokenizer.decode(snake_case__		,  skip_special_tokens=snake_case__		,  clean_up_tokenization_spaces=snake_case__   )
												code_gens[task].append(remove_last_block(snake_case__   )   )
				return code_gens




def 							a		(				):


				'''simple docstring'''
				# Setup configuration
				lowercase_	    =   HfArgumentParser(snake_case__   )
				lowercase_	    =   parser.parse_args()

				transformers.logging.set_verbosity_error()
				# enables code execution in code_eval metric
				lowercase_	    =   args.HF_ALLOW_CODE_EVAL
				# make sure tokenizer plays nice with multiprocessing
				lowercase_	    =   '''false'''

				if args.num_workers is None:
								lowercase_	    =   multiprocessing.cpu_count()

				# Use dataset load to feed to accelerate
				lowercase_	    =   Accelerator()
				set_seed(args.seed		,  device_specific=snake_case__   )

				# Load model and tokenizer
				lowercase_	    =   AutoTokenizer.from_pretrained(args.model_ckpt   )
				lowercase_	    =   tokenizer.eos_token
				lowercase_	    =   AutoModelForCausalLM.from_pretrained(args.model_ckpt   )

				# Generation settings
				lowercase_	    =   {
				    '''do_sample''': args.do_sample,
				    '''temperature''': args.temperature,
				    '''max_new_tokens''': args.max_new_tokens,
				    '''top_p''': args.top_p,
				    '''top_k''': args.top_k,
				    '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0		,  snake_case__		,  snake_case__   )]   ),
				}

				# Load evaluation dataset and metric
				lowercase_	    =   load_dataset('''openai_humaneval'''   )
				lowercase_	    =   load_metric('''code_eval'''   )

				lowercase_	    =   args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']   )
				lowercase_	    =   args.n_samples // args.batch_size

				lowercase_	    =   TokenizedDataset(snake_case__		,  human_eval['''test''']		,  n_copies=snake_case__		,  n_tasks=snake_case__   )
				# do not confuse args.batch_size, which is actually the num_return_sequences
				lowercase_	    =   DataLoader(snake_case__		,  batch_size=1   )

				# Run a quick test to see if code evaluation is enabled
				try:
								lowercase_	    =   code_eval_metric.compute(references=['''''']		,  predictions=[['''''']]   )
				except ValueError as exception:
								print(
								    '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
								    ''' flag to enable code evaluation.'''   )
								raise exception

				lowercase_    ,      lowercase_	    =   accelerator.prepare(snake_case__		,  snake_case__   )

				lowercase_	    =   complete_code(
				    snake_case__		,  snake_case__		,  snake_case__		,  snake_case__		,  n_tasks=snake_case__		,  batch_size=args.batch_size		,  **snake_case__		,  )

				if accelerator.is_main_process:
								lowercase_	    =   []

								for task in tqdm(range(snake_case__   )   ):
												lowercase_	    =   human_eval['''test'''][task]['''test''']
												lowercase_	    =   F'''check({human_eval['test'][task]['entry_point']})'''
												references.append('''\n''' + test_func + '''\n''' + entry_point   )

								# Evaluate completions with "code_eval" metric
								lowercase_    ,      lowercase_	    =   code_eval_metric.compute(
								    references=snake_case__		,  predictions=snake_case__		,  num_workers=args.num_workers   )
								print(F'''Results: {pass_at_k}'''   )

								# Save results to json file
								with open(args.output_file		,  '''w'''   ) as fp:
												json.dump(snake_case__		,  snake_case__   )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
						main()






import os
from collections import namedtuple

import pytest

from datasets import ClassLabel, Features, Sequence, Value
from datasets.commands.test import TestCommand
from datasets.info import DatasetInfo, DatasetInfosDict


__a         =   namedtuple(
    '_TestCommandArgs',
    [
        'dataset',
        'name',
        'cache_dir',
        'data_dir',
        'all_configs',
        'save_infos',
        'ignore_verifications',
        'force_redownload',
        'clear_cache',
    ],
    defaults=[None, None, None, False, False, False, False, False],
)




def 							a		(				snake_case__:   Dict		,  snake_case__:   List[str]   ):


				'''simple docstring'''
				return (abs(source - target   ) / target) < 0.0_1




@pytest.mark.integration
def 							a		(				snake_case__:   int   ):


				'''simple docstring'''
				lowercase_	    =   _TestCommandArgs(dataset=snake_case__		,  all_configs=snake_case__		,  save_infos=snake_case__   )
				lowercase_	    =   TestCommand(*snake_case__   )
				test_command.run()
				lowercase_	    =   os.path.join(snake_case__		,  '''README.md'''   )
				assert os.path.exists(snake_case__   )
				lowercase_	    =   DatasetInfosDict.from_directory(snake_case__   )
				lowercase_	    =   DatasetInfosDict(
				    {
				        '''default''': DatasetInfo(
				            features=Features(
				                {
				                    '''tokens''': Sequence(Value('''string'''   )   ),
				                    '''ner_tags''': Sequence(
				                        ClassLabel(names=['''O''', '''B-PER''', '''I-PER''', '''B-ORG''', '''I-ORG''', '''B-LOC''', '''I-LOC''']   )   ),
				                    '''langs''': Sequence(Value('''string'''   )   ),
				                    '''spans''': Sequence(Value('''string'''   )   ),
				                }   )		,  splits=[
				                {
				                    '''name''': '''train''',
				                    '''num_bytes''': 2_351_563,
				                    '''num_examples''': 10_000,
				                },
				                {
				                    '''name''': '''validation''',
				                    '''num_bytes''': 238_418,
				                    '''num_examples''': 1_000,
				                },
				            ]		,  download_size=3_940_680		,  dataset_size=2_589_981		,  )
				    }   )
				assert dataset_infos.keys() == expected_dataset_infos.keys()
				for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
								lowercase_    ,      lowercase_	    =   getattr(dataset_infos['''default''']		,  snake_case__   ), getattr(expected_dataset_infos['''default''']		,  snake_case__   )
								if key == "num_bytes":
												assert is_apercent_close(snake_case__		,  snake_case__   )
								elif key == "splits":
												assert list(snake_case__   ) == list(snake_case__   )
												for split in result:
																assert result[split].name == expected[split].name
																assert result[split].num_examples == expected[split].num_examples
																assert is_apercent_close(result[split].num_bytes		,  expected[split].num_bytes   )
								else:
												result == expected




