






"""simple docstring"""







import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


lowerCAmelCase_	:    str            =      ['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']





class   UpperCamelCase_ (       a__  ):

  '''simple docstring'''





  def __init__(      self ,		snake_case__ ,		snake_case__ ,		snake_case__=None ,		snake_case__=1				)			->			Union[str, Any]:



         """simple docstring"""





         UpperCAmelCase	    =			tokenizer
         UpperCAmelCase	    =			dataset
         UpperCAmelCase	    =			len(_A				) if n_tasks is None else n_tasks
         UpperCAmelCase	    =			n_copies



  def __iter__(      self				)			->			str:



         """simple docstring"""





         UpperCAmelCase	    =			[]
         for task in range(self.n_tasks				):
                # without strip, the model generate commented codes ...
                prompts.append(self.tokenizer.eos_token + self.dataset[task]["""prompt"""].strip()				)
         UpperCAmelCase	    =			self.tokenizer(_A ,		padding=_A ,		return_tensors="""pt"""				)
         for task in range(self.n_tasks				):
                for _ in range(self.n_copies				):
                       yield {
                           "ids": outputs.input_ids[task],
                           "task_id": task,
                           "input_len": outputs.attention_mask[task].sum(),
                       }





class   UpperCamelCase_ (       a__  ):

  '''simple docstring'''





  def __init__(      self ,		snake_case__ ,		snake_case__ ,		snake_case__				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			start_length
         UpperCAmelCase	    =			eof_strings
         UpperCAmelCase	    =			tokenizer



  def __call__(      self ,		snake_case__ ,		snake_case__ ,		**snake_case__				)			->			int:



         """simple docstring"""





         UpperCAmelCase	    =			self.tokenizer.batch_decode(input_ids[:, self.start_length :]				)
         UpperCAmelCase	    =			[]
         for decoded_generation in decoded_generations:
                done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings				)				)
         return all(_A				)






def       _lowerCAmelCase       (      lowerCAmelCase		):







       '''simple docstring'''
       UpperCAmelCase	    =			re.split("""(%s)""" % """|""".join(_lowerCamelCase		)	,      _lowerCamelCase		)
       # last string should be ""
       return "".join(string_list[:-2]		)






def       _lowerCAmelCase       (      lowerCAmelCase	,      lowerCAmelCase	,      lowerCAmelCase	,      lowerCAmelCase	,      lowerCAmelCase	,      lowerCAmelCase=20	,      **lowerCAmelCase		):







       '''simple docstring'''
       UpperCAmelCase	    =			defaultdict(_lowerCamelCase		)  # dict of list of generated tokens
       for step, batch in tqdm(enumerate(_lowerCamelCase		)		):
              with torch.no_grad():
                     UpperCAmelCase	    =			batch['ids'].shape[-1]
                     UpperCAmelCase	    =			accelerator.unwrap_model(_lowerCamelCase		).generate(
                         input_ids=batch["""ids"""][:, : batch["""input_len"""]]	,      num_return_sequences=_lowerCamelCase	,      **_lowerCamelCase		)
                     # each task is generated batch_size times
                     UpperCAmelCase	    =			batch['task_id'].repeat(_lowerCamelCase		)
                     UpperCAmelCase	    =			accelerator.pad_across_processes(
                         _lowerCamelCase	,      dim=1	,      pad_index=tokenizer.pad_token_id		)

                     UpperCAmelCase	    =			accelerator.gather((generated_tokens, generated_tasks)		)
                     UpperCAmelCase	    =			generated_tokens.cpu().numpy()
                     UpperCAmelCase	    =			generated_tasks.cpu().numpy()

                     for task, generated_tokens in zip(_lowerCamelCase	,      _lowerCamelCase		):
                            gen_token_dict[task].append(_lowerCamelCase		)

       UpperCAmelCase	    =			[[] for _ in range(_lowerCamelCase		)]
       for task, generated_tokens in gen_token_dict.items():
              for s in generated_tokens:
                     UpperCAmelCase	    =			tokenizer.decode(_lowerCamelCase	,      skip_special_tokens=_lowerCamelCase	,      clean_up_tokenization_spaces=_lowerCamelCase		)
                     code_gens[task].append(remove_last_block(_lowerCamelCase		)		)
       return code_gens






def       _lowerCAmelCase       (      ):







       '''simple docstring'''
       UpperCAmelCase	    =			HfArgumentParser(_lowerCamelCase		)
       UpperCAmelCase	    =			parser.parse_args()

       transformers.logging.set_verbosity_error()
       # enables code execution in code_eval metric
       UpperCAmelCase	    =			args.HF_ALLOW_CODE_EVAL
       # make sure tokenizer plays nice with multiprocessing
       UpperCAmelCase	    =			'false'

       if args.num_workers is None:
              UpperCAmelCase	    =			multiprocessing.cpu_count()

       # Use dataset load to feed to accelerate
       UpperCAmelCase	    =			Accelerator()
       set_seed(args.seed	,      device_specific=_lowerCamelCase		)

       # Load model and tokenizer
       UpperCAmelCase	    =			AutoTokenizer.from_pretrained(args.model_ckpt		)
       UpperCAmelCase	    =			tokenizer.eos_token
       UpperCAmelCase	    =			AutoModelForCausalLM.from_pretrained(args.model_ckpt		)

       # Generation settings
       UpperCAmelCase	    =			{
           'do_sample': args.do_sample,
           'temperature': args.temperature,
           'max_new_tokens': args.max_new_tokens,
           'top_p': args.top_p,
           'top_k': args.top_k,
           'stopping_criteria': StoppingCriteriaList([EndOfFunctionCriteria(0	,      _lowerCamelCase	,      _lowerCamelCase		)]		),
       }

       # Load evaluation dataset and metric
       UpperCAmelCase	    =			load_dataset("""openai_humaneval"""		)
       UpperCAmelCase	    =			load_metric("""code_eval"""		)

       UpperCAmelCase	    =			args.num_tasks if args.num_tasks is not None else len(human_eval["""test"""]		)
       UpperCAmelCase	    =			args.n_samples // args.batch_size

       UpperCAmelCase	    =			TokenizedDataset(_lowerCamelCase	,      human_eval["""test"""]	,      n_copies=_lowerCamelCase	,      n_tasks=_lowerCamelCase		)
       # do not confuse args.batch_size, which is actually the num_return_sequences
       UpperCAmelCase	    =			DataLoader(_lowerCamelCase	,      batch_size=1		)

       # Run a quick test to see if code evaluation is enabled
       try:
              UpperCAmelCase	    =			code_eval_metric.compute(references=[""""""]	,      predictions=[[""""""]]		)
       except ValueError as exception:
              print(
                  """Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"""
                  """ flag to enable code evaluation."""		)
              raise exception

       UpperCAmelCase	    =			accelerator.prepare(_lowerCamelCase	,      _lowerCamelCase		)

       UpperCAmelCase	    =			complete_code(
           _lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase	,      _lowerCamelCase	,      n_tasks=_lowerCamelCase	,      batch_size=args.batch_size	,      **_lowerCamelCase	,      )

       if accelerator.is_main_process:
              UpperCAmelCase	    =			[]

              for task in tqdm(range(_lowerCamelCase		)		):
                     UpperCAmelCase	    =			human_eval['test'][task]['test']
                     UpperCAmelCase	    =			F'''check({human_eval['test'][task]['entry_point']})'''
                     references.append("""\n""" + test_func + """\n""" + entry_point		)

              # Evaluate completions with "code_eval" metric
              UpperCAmelCase	    =			code_eval_metric.compute(
                  references=_lowerCamelCase	,      predictions=_lowerCamelCase	,      num_workers=args.num_workers		)
              print(F'''Results: {pass_at_k}'''		)

              # Save results to json file
              with open(args.output_file	,      """w"""		) as fp:
                     json.dump(_lowerCamelCase	,      _lowerCamelCase		)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
 main()








"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_	:    Any            =      logging.get_logger(__name__)

lowerCAmelCase_	:    Any            =      {
    '''google/canine-s''': '''https://huggingface.co/google/canine-s/resolve/main/config.json''',
    # See all CANINE models at https://huggingface.co/models?filter=canine
}





class   UpperCamelCase_ (       a_  ):
  _A       :					Tuple				=	'canine'



  def __init__(      self ,		snake_case__=7_68 ,		snake_case__=12 ,		snake_case__=12 ,		snake_case__=30_72 ,		snake_case__="gelu" ,		snake_case__=0.1 ,		snake_case__=0.1 ,		snake_case__=1_63_84 ,		snake_case__=16 ,		snake_case__=0.02 ,		snake_case__=1e-12 ,		snake_case__=0 ,		snake_case__=0xE_0_0_0 ,		snake_case__=0xE_0_0_1 ,		snake_case__=4 ,		snake_case__=4 ,		snake_case__=8 ,		snake_case__=1_63_84 ,		snake_case__=1_28 ,		**snake_case__ ,		)			->			int:



         """simple docstring"""





         super().__init__(pad_token_id=snake_case__ ,		bos_token_id=snake_case__ ,		eos_token_id=snake_case__ ,		**snake_case__				)

         UpperCAmelCase	    =			max_position_embeddings
         UpperCAmelCase	    =			hidden_size
         UpperCAmelCase	    =			num_hidden_layers
         UpperCAmelCase	    =			num_attention_heads
         UpperCAmelCase	    =			intermediate_size
         UpperCAmelCase	    =			hidden_act
         UpperCAmelCase	    =			hidden_dropout_prob
         UpperCAmelCase	    =			attention_probs_dropout_prob
         UpperCAmelCase	    =			initializer_range
         UpperCAmelCase	    =			type_vocab_size
         UpperCAmelCase	    =			layer_norm_eps

         # Character config:
         UpperCAmelCase	    =			downsampling_rate
         UpperCAmelCase	    =			upsampling_kernel_size
         UpperCAmelCase	    =			num_hash_functions
         UpperCAmelCase	    =			num_hash_buckets
         UpperCAmelCase	    =			local_transformer_stride
