


import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


A   =			["\nclass", "\ndef", "\n#", "\n@", "\nprint", "\nif"]


class      lowercase__	(       __lowercase		):






  def __init__(       self   :       Tuple		,    _lowercase   :       Dict		,    _lowercase   :       Tuple		,    _lowercase   :       Tuple=None		,    _lowercase   :       Optional[Any]=1			):




         """simple docstring"""







         UpperCAmelCase__					    =  tokenizer
         UpperCAmelCase__					    =  dataset
         UpperCAmelCase__					    =  len(_lowercase			) if n_tasks is None else n_tasks
         UpperCAmelCase__					    =  n_copies






  def __iter__(       self   :       int			):




         """simple docstring"""







         UpperCAmelCase__					    =  []
         for task in range(self.n_tasks			):
                # without strip, the model generate commented codes ...
                prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()			)
         UpperCAmelCase__					    =  self.tokenizer(_lowercase		,    padding=_lowercase		,    return_tensors="pt"			)
         for task in range(self.n_tasks			):
                for _ in range(self.n_copies			):
                       yield {
                           "ids": outputs.input_ids[task],
                           "task_id": task,
                           "input_len": outputs.attention_mask[task].sum(),
                       }




class      lowercase__	(       __lowercase		):






  def __init__(       self   :       Any		,    _lowercase   :       Optional[Any]		,    _lowercase   :       Optional[int]		,    _lowercase   :       str			):




         """simple docstring"""







         UpperCAmelCase__					    =  start_length
         UpperCAmelCase__					    =  eof_strings
         UpperCAmelCase__					    =  tokenizer






  def __call__(       self   :       str		,    _lowercase   :       str		,    _lowercase   :       Union[str, Any]		,    **_lowercase   :       List[Any]			):




         """simple docstring"""







         UpperCAmelCase__					    =  self.tokenizer.batch_decode(input_ids[:, self.start_length :]			)
         UpperCAmelCase__					    =  []
         for decoded_generation in decoded_generations:
                done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings			)			)
         return all(_lowercase			)





def __UpperCAmelCase							(		__A      )    ->       Any:



       '''simple docstring'''





       UpperCAmelCase__					    =  re.split("(%s)" % "|".join(_lowerCamelCase      )			,    _lowerCamelCase      )
       # last string should be ""
       return "".join(string_list[:-2]      )





def __UpperCAmelCase							(		__A			,    __A			,    __A			,    __A			,    __A			,    __A=2_0			,    **__A      )    ->       int:



       '''simple docstring'''





       UpperCAmelCase__					    =  defaultdict(_lowerCamelCase      )  # dict of list of generated tokens
       for step, batch in tqdm(enumerate(_lowerCamelCase      )      ):
              with torch.no_grad():
                     UpperCAmelCase__					    =  batch["""ids"""].shape[-1]
                     UpperCAmelCase__					    =  accelerator.unwrap_model(_lowerCamelCase      ).generate(
                         input_ids=batch["ids"][:, : batch["input_len"]]			,    num_return_sequences=_lowerCamelCase			,    **_lowerCamelCase      )
                     # each task is generated batch_size times
                     UpperCAmelCase__					    =  batch["""task_id"""].repeat(_lowerCamelCase      )
                     UpperCAmelCase__					    =  accelerator.pad_across_processes(
                         _lowerCamelCase			,    dim=1			,    pad_index=tokenizer.pad_token_id      )

                     UpperCAmelCase__					    =  accelerator.gather((generated_tokens, generated_tasks)      )
                     UpperCAmelCase__					    =  generated_tokens.cpu().numpy()
                     UpperCAmelCase__					    =  generated_tasks.cpu().numpy()

                     for task, generated_tokens in zip(_lowerCamelCase			,    _lowerCamelCase      ):
                            gen_token_dict[task].append(_lowerCamelCase      )

       UpperCAmelCase__					    =  [[] for _ in range(_lowerCamelCase      )]
       for task, generated_tokens in gen_token_dict.items():
              for s in generated_tokens:
                     UpperCAmelCase__					    =  tokenizer.decode(_lowerCamelCase			,    skip_special_tokens=_lowerCamelCase			,    clean_up_tokenization_spaces=_lowerCamelCase      )
                     code_gens[task].append(remove_last_block(_lowerCamelCase      )      )
       return code_gens





def __UpperCAmelCase							(		)    ->       List[Any]:



       '''simple docstring'''





       UpperCAmelCase__					    =  HfArgumentParser(_lowerCamelCase      )
       UpperCAmelCase__					    =  parser.parse_args()

       transformers.logging.set_verbosity_error()
       # enables code execution in code_eval metric
       UpperCAmelCase__					    =  args.HF_ALLOW_CODE_EVAL
       # make sure tokenizer plays nice with multiprocessing
       UpperCAmelCase__					    =  """false"""

       if args.num_workers is None:
              UpperCAmelCase__					    =  multiprocessing.cpu_count()

       # Use dataset load to feed to accelerate
       UpperCAmelCase__					    =  Accelerator()
       set_seed(args.seed			,    device_specific=_lowerCamelCase      )

       # Load model and tokenizer
       UpperCAmelCase__					    =  AutoTokenizer.from_pretrained(args.model_ckpt      )
       UpperCAmelCase__					    =  tokenizer.eos_token
       UpperCAmelCase__					    =  AutoModelForCausalLM.from_pretrained(args.model_ckpt      )

       # Generation settings
       UpperCAmelCase__					    =  {
           """do_sample""": args.do_sample,
           """temperature""": args.temperature,
           """max_new_tokens""": args.max_new_tokens,
           """top_p""": args.top_p,
           """top_k""": args.top_k,
           """stopping_criteria""": StoppingCriteriaList([EndOfFunctionCriteria(0			,    _lowerCamelCase			,    _lowerCamelCase      )]      ),
       }

       # Load evaluation dataset and metric
       UpperCAmelCase__					    =  load_dataset("openai_humaneval"      )
       UpperCAmelCase__					    =  load_metric("code_eval"      )

       UpperCAmelCase__					    =  args.num_tasks if args.num_tasks is not None else len(human_eval["test"]      )
       UpperCAmelCase__					    =  args.n_samples // args.batch_size

       UpperCAmelCase__					    =  TokenizedDataset(_lowerCamelCase			,    human_eval["test"]			,    n_copies=_lowerCamelCase			,    n_tasks=_lowerCamelCase      )
       # do not confuse args.batch_size, which is actually the num_return_sequences
       UpperCAmelCase__					    =  DataLoader(_lowerCamelCase			,    batch_size=1      )

       # Run a quick test to see if code evaluation is enabled
       try:
              UpperCAmelCase__					    =  code_eval_metric.compute(references=[""]			,    predictions=[[""]]      )
       except ValueError as exception:
              print(
                  "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
                  " flag to enable code evaluation."      )
              raise exception

       UpperCAmelCase__					    =  accelerator.prepare(_lowerCamelCase			,    _lowerCamelCase      )

       UpperCAmelCase__					    =  complete_code(
           _lowerCamelCase			,    _lowerCamelCase			,    _lowerCamelCase			,    _lowerCamelCase			,    n_tasks=_lowerCamelCase			,    batch_size=args.batch_size			,    **_lowerCamelCase			,    )

       if accelerator.is_main_process:
              UpperCAmelCase__					    =  []

              for task in tqdm(range(_lowerCamelCase      )      ):
                     UpperCAmelCase__					    =  human_eval["""test"""][task]["""test"""]
                     UpperCAmelCase__					    =  F"""check({human_eval["test"][task]["entry_point"]})"""
                     references.append("\n" + test_func + "\n" + entry_point      )

              # Evaluate completions with "code_eval" metric
              UpperCAmelCase__					    =  code_eval_metric.compute(
                  references=_lowerCamelCase			,    predictions=_lowerCamelCase			,    num_workers=args.num_workers      )
              print(F"""Results: {pass_at_k}"""      )

              # Save results to json file
              with open(args.output_file			,    "w"      ) as fp:
                     json.dump(_lowerCamelCase			,    _lowerCamelCase      )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
    main()












'''simple docstring'''






import os

from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home


__UpperCamelCase					    =	HUGGINGFACE_HUB_CACHE


__UpperCamelCase					    =	"config.json"
__UpperCamelCase					    =	"diffusion_pytorch_model.bin"
__UpperCamelCase					    =	"diffusion_flax_model.msgpack"
__UpperCamelCase					    =	"model.onnx"
__UpperCamelCase					    =	"diffusion_pytorch_model.safetensors"
__UpperCamelCase					    =	"weights.pb"
__UpperCamelCase					    =	"https://huggingface.co"
__UpperCamelCase					    =	default_cache_path
__UpperCamelCase					    =	"diffusers_modules"
__UpperCamelCase					    =	os.getenv("HF_MODULES_CACHE", os.path.join(hf_cache_home, "modules"))
__UpperCamelCase					    =	["fp16", "non-ema"]
__UpperCamelCase					    =	".self_attn"


