






import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__a:     Any   					=					["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]
class 		SCREAMING_SNAKE_CASE__	(  __lowercase     ):

  '''simple docstring'''







  def __init__(      self		:			List[Any]       ,				lowerCamelCase		:			Optional[int]       ,				lowerCamelCase		:			Optional[Any]       ,				lowerCamelCase		:			List[Any]=None       ,				lowerCamelCase		:			List[Any]=1							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    tokenizer
         _UpperCAmelCase							=    dataset
         _UpperCAmelCase							=    len(_lowerCAmelCase							) if n_tasks is None else n_tasks
         _UpperCAmelCase							=    n_copies





  def __iter__(      self		:			Union[str, Any]							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    []
         for task in range(self.n_tasks							):
                # without strip, the model generate commented codes ...
                prompts.append(self.tokenizer.eos_token + self.dataset[task]["""prompt"""].strip()							)
         _UpperCAmelCase							=    self.tokenizer(_lowerCAmelCase       ,				padding=_lowerCAmelCase       ,				return_tensors="""pt"""							)
         for task in range(self.n_tasks							):
                for _ in range(self.n_copies							):
                       yield {
                           "ids": outputs.input_ids[task],
                           "task_id": task,
                           "input_len": outputs.attention_mask[task].sum(),
                       }



class 		SCREAMING_SNAKE_CASE__	(  __lowercase     ):

  '''simple docstring'''







  def __init__(      self		:			int       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Dict       ,				lowerCamelCase		:			Union[str, Any]							)						->	Dict:


         """simple docstring"""
         _UpperCAmelCase							=    start_length
         _UpperCAmelCase							=    eof_strings
         _UpperCAmelCase							=    tokenizer





  def __call__(      self		:			List[Any]       ,				lowerCamelCase		:			str       ,				lowerCamelCase		:			Dict       ,				**lowerCamelCase		:			str							)						->	List[Any]:


         """simple docstring"""
         _UpperCAmelCase							=    self.tokenizer.batch_decode(input_ids[:, self.start_length :]							)
         _UpperCAmelCase							=    []
         for decoded_generation in decoded_generations:
                done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings							)							)
         return all(_lowerCAmelCase							)




def       _SCREAMING_SNAKE_CASE (							__snake_case )				->						List[str]:
       _UpperCAmelCase							=    re.split("""(%s)""" % """|""".join(UpperCamelCase__ )   ,      UpperCamelCase__ )
       # last string should be ""
       return "".join(string_list[:-2] )




def       _SCREAMING_SNAKE_CASE (							__snake_case   ,      __snake_case   ,      __snake_case   ,      __snake_case   ,      __snake_case   ,      __snake_case=2_0   ,      **__snake_case )				->						str:
       _UpperCAmelCase							=    defaultdict(UpperCamelCase__ )  # dict of list of generated tokens
       for step, batch in tqdm(enumerate(UpperCamelCase__ ) ):
              with torch.no_grad():
                     _UpperCAmelCase							=    batch['''ids'''].shape[-1]
                     _UpperCAmelCase							=    accelerator.unwrap_model(UpperCamelCase__ ).generate(
                         input_ids=batch["""ids"""][:, : batch["""input_len"""]]   ,      num_return_sequences=UpperCamelCase__   ,      **UpperCamelCase__ )
                     # each task is generated batch_size times
                     _UpperCAmelCase							=    batch['''task_id'''].repeat(UpperCamelCase__ )
                     _UpperCAmelCase							=    accelerator.pad_across_processes(
                         UpperCamelCase__   ,      dim=1   ,      pad_index=tokenizer.pad_token_id )

                     _UpperCAmelCase							=    accelerator.gather((generated_tokens, generated_tasks) )
                     _UpperCAmelCase							=    generated_tokens.cpu().numpy()
                     _UpperCAmelCase							=    generated_tasks.cpu().numpy()

                     for task, generated_tokens in zip(UpperCamelCase__   ,      UpperCamelCase__ ):
                            gen_token_dict[task].append(UpperCamelCase__ )

       _UpperCAmelCase							=    [[] for _ in range(UpperCamelCase__ )]
       for task, generated_tokens in gen_token_dict.items():
              for s in generated_tokens:
                     _UpperCAmelCase							=    tokenizer.decode(UpperCamelCase__   ,      skip_special_tokens=UpperCamelCase__   ,      clean_up_tokenization_spaces=UpperCamelCase__ )
                     code_gens[task].append(remove_last_block(UpperCamelCase__ ) )
       return code_gens




def       _SCREAMING_SNAKE_CASE (							)				->						str:
       _UpperCAmelCase							=    HfArgumentParser(UpperCamelCase__ )
       _UpperCAmelCase							=    parser.parse_args()

       transformers.logging.set_verbosity_error()
       # enables code execution in code_eval metric
       _UpperCAmelCase							=    args.HF_ALLOW_CODE_EVAL
       # make sure tokenizer plays nice with multiprocessing
       _UpperCAmelCase							=    '''false'''

       if args.num_workers is None:
              _UpperCAmelCase							=    multiprocessing.cpu_count()

       # Use dataset load to feed to accelerate
       _UpperCAmelCase							=    Accelerator()
       set_seed(args.seed   ,      device_specific=UpperCamelCase__ )

       # Load model and tokenizer
       _UpperCAmelCase							=    AutoTokenizer.from_pretrained(args.model_ckpt )
       _UpperCAmelCase							=    tokenizer.eos_token
       _UpperCAmelCase							=    AutoModelForCausalLM.from_pretrained(args.model_ckpt )

       # Generation settings
       _UpperCAmelCase							=    {
           '''do_sample''': args.do_sample,
           '''temperature''': args.temperature,
           '''max_new_tokens''': args.max_new_tokens,
           '''top_p''': args.top_p,
           '''top_k''': args.top_k,
           '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0   ,      UpperCamelCase__   ,      UpperCamelCase__ )] ),
       }

       # Load evaluation dataset and metric
       _UpperCAmelCase							=    load_dataset("""openai_humaneval""" )
       _UpperCAmelCase							=    load_metric("""code_eval""" )

       _UpperCAmelCase							=    args.num_tasks if args.num_tasks is not None else len(human_eval["""test"""] )
       _UpperCAmelCase							=    args.n_samples // args.batch_size

       _UpperCAmelCase							=    TokenizedDataset(UpperCamelCase__   ,      human_eval["""test"""]   ,      n_copies=UpperCamelCase__   ,      n_tasks=UpperCamelCase__ )
       # do not confuse args.batch_size, which is actually the num_return_sequences
       _UpperCAmelCase							=    DataLoader(UpperCamelCase__   ,      batch_size=1 )

       # Run a quick test to see if code evaluation is enabled
       try:
              _UpperCAmelCase							=    code_eval_metric.compute(references=[""""""]   ,      predictions=[[""""""]] )
       except ValueError as exception:
              print(
                  """Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"""
                  """ flag to enable code evaluation.""" )
              raise exception

       _UpperCAmelCase							=    accelerator.prepare(UpperCamelCase__   ,      UpperCamelCase__ )

       _UpperCAmelCase							=    complete_code(
           UpperCamelCase__   ,      UpperCamelCase__   ,      UpperCamelCase__   ,      UpperCamelCase__   ,      n_tasks=UpperCamelCase__   ,      batch_size=args.batch_size   ,      **UpperCamelCase__   ,      )

       if accelerator.is_main_process:
              _UpperCAmelCase							=    []

              for task in tqdm(range(UpperCamelCase__ ) ):
                     _UpperCAmelCase							=    human_eval['''test'''][task]['''test''']
                     _UpperCAmelCase							=    f"""check({human_eval["test"][task]["entry_point"]})"""
                     references.append("""\n""" + test_func + """\n""" + entry_point )

              # Evaluate completions with "code_eval" metric
              _UpperCAmelCase							=    code_eval_metric.compute(
                  references=UpperCamelCase__   ,      predictions=UpperCamelCase__   ,      num_workers=args.num_workers )
              print(f"""Results: {pass_at_k}""" )

              # Save results to json file
              with open(args.output_file   ,      """w""" ) as fp:
                     json.dump(UpperCamelCase__   ,      UpperCamelCase__ )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
  main()






import unittest
from dataclasses import dataclass

import pytest

from accelerate.commands.config.config_args import SageMakerConfig
from accelerate.utils import ComputeEnvironment
from accelerate.utils.launch import _convert_nargs_to_dict





@dataclass
class 				a			(      __lowercase			):
				SCREAMING_SNAKE_CASE__     :   Any		      =     ComputeEnvironment.AMAZON_SAGEMAKER
				SCREAMING_SNAKE_CASE__     :   Optional[int]		      =     True
				SCREAMING_SNAKE_CASE__     :   int		      =     '''ml.p3.2xlarge'''
				SCREAMING_SNAKE_CASE__     :   Union[str, Any]		      =     '''accelerate_sagemaker_execution_role'''
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     '''hf-sm'''
				SCREAMING_SNAKE_CASE__     :   Any		      =     '''us-east-1'''
				SCREAMING_SNAKE_CASE__     :   Any		      =     1
				SCREAMING_SNAKE_CASE__     :   Dict		      =     '''accelerate-sagemaker-1'''
				SCREAMING_SNAKE_CASE__     :   Dict		      =     '''1.6'''
				SCREAMING_SNAKE_CASE__     :   int		      =     '''4.4'''
				SCREAMING_SNAKE_CASE__     :   int		      =     '''train.py'''
				SCREAMING_SNAKE_CASE__     :   Tuple		      =     [
				    '''--model_name_or_path''',
				    '''bert''',
				    '''--do_train''',
				    '''False''',
				    '''--epochs''',
				    '''3''',
				    '''--learning_rate''',
				    '''5e-5''',
				    '''--max_steps''',
				    '''50.5''',
				]
				SCREAMING_SNAKE_CASE__     :   List[str]		      =     [
				    '''--model_name_or_path''',
				    '''bert''',
				    '''--do_train''',
				    '''--do_test''',
				    '''False''',
				    '''--do_predict''',
				    '''--epochs''',
				    '''3''',
				    '''--learning_rate''',
				    '''5e-5''',
				    '''--max_steps''',
				    '''50.5''',
				]





class 				a			(      unittest.TestCase			):






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   _convert_nargs_to_dict(MockLaunchConfig.success_training_script_args					)
										assert isinstance(converted_args['''model_name_or_path''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''do_train''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''epochs''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''learning_rate''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''max_steps''']			, _lowerCAmelCase					)

										with pytest.raises(_lowerCAmelCase					):
																_convert_nargs_to_dict(MockLaunchConfig.fail_training_script_args					)



