


import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__lowerCAmelCase     							=			['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']


class  __SCREAMING_SNAKE_CASE       (						lowercase):


     def __init__(			self      :		Any       ,  __UpperCamelCase      :		Optional[int]       ,  __UpperCamelCase      :		Dict       ,  __UpperCamelCase      :		Optional[int]=None       ,  __UpperCamelCase      :		Union[str, Any]=1 ):
       _UpperCAmelCase												=     tokenizer
       _UpperCAmelCase												=     dataset
       _UpperCAmelCase												=     len(UpperCamelCase_ ) if n_tasks is None else n_tasks
       _UpperCAmelCase												=     n_copies



     def __iter__(			self      :		int ):
       _UpperCAmelCase												=     []
       for task in range(self.n_tasks ):
         # without strip, the model generate commented codes ...
         prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip() )
       _UpperCAmelCase												=     self.tokenizer(UpperCamelCase_       ,  padding=UpperCamelCase_       ,  return_tensors="pt" )
       for task in range(self.n_tasks ):
         for _ in range(self.n_copies ):
           yield {
               "ids": outputs.input_ids[task],
               "task_id": task,
               "input_len": outputs.attention_mask[task].sum(),
           }





class  __SCREAMING_SNAKE_CASE       (						lowercase):


     def __init__(			self      :		List[str]       ,  __UpperCamelCase      :		str       ,  __UpperCamelCase      :		int       ,  __UpperCamelCase      :		int ):
       _UpperCAmelCase												=     start_length
       _UpperCAmelCase												=     eof_strings
       _UpperCAmelCase												=     tokenizer



     def __call__(			self      :		Tuple       ,  __UpperCamelCase      :		Optional[Any]       ,  __UpperCamelCase      :		Dict       ,  **__UpperCamelCase      :		List[str] ):
       _UpperCAmelCase												=     self.tokenizer.batch_decode(input_ids[:, self.start_length :] )
       _UpperCAmelCase												=     []
       for decoded_generation in decoded_generations:
         done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings ) )
       return all(UpperCamelCase_ )





def 			__lowerCamelCase			( _lowerCAmelCase	)					->	Dict:
  _UpperCAmelCase												=     re.split("(%s)" % "|".join(UpperCAmelCase_	)						,  UpperCAmelCase_	)
  # last string should be ""
  return "".join(string_list[:-2]	)
def 			__lowerCamelCase			( _lowerCAmelCase						,  _lowerCAmelCase						,  _lowerCAmelCase						,  _lowerCAmelCase						,  _lowerCAmelCase						,  _lowerCAmelCase=20						,  **_lowerCAmelCase	)					->	Optional[Any]:
  _UpperCAmelCase												=     defaultdict(UpperCAmelCase_	)  # dict of list of generated tokens
  for step, batch in tqdm(enumerate(UpperCAmelCase_	)	):
    with torch.no_grad():
      _UpperCAmelCase												=     batch["ids"].shape[-1]
      _UpperCAmelCase												=     accelerator.unwrap_model(UpperCAmelCase_	).generate(
          input_ids=batch["ids"][:, : batch["input_len"]]						,  num_return_sequences=UpperCAmelCase_						,  **UpperCAmelCase_	)
      # each task is generated batch_size times
      _UpperCAmelCase												=     batch["task_id"].repeat(UpperCAmelCase_	)
      _UpperCAmelCase												=     accelerator.pad_across_processes(
          UpperCAmelCase_						,  dim=1						,  pad_index=tokenizer.pad_token_id	)

      _UpperCAmelCase						,		_UpperCAmelCase												=     accelerator.gather((generated_tokens, generated_tasks)	)
      _UpperCAmelCase												=     generated_tokens.cpu().numpy()
      _UpperCAmelCase												=     generated_tasks.cpu().numpy()

      for task, generated_tokens in zip(UpperCAmelCase_						,  UpperCAmelCase_	):
        gen_token_dict[task].append(UpperCAmelCase_	)

  _UpperCAmelCase												=     [[] for _ in range(UpperCAmelCase_	)]
  for task, generated_tokens in gen_token_dict.items():
    for s in generated_tokens:
      _UpperCAmelCase												=     tokenizer.decode(UpperCAmelCase_						,  skip_special_tokens=UpperCAmelCase_						,  clean_up_tokenization_spaces=UpperCAmelCase_	)
      code_gens[task].append(remove_last_block(UpperCAmelCase_	)	)
  return code_gens
def 			__lowerCamelCase			( )					->	int:
  # Setup configuration
  _UpperCAmelCase												=     HfArgumentParser(UpperCAmelCase_	)
  _UpperCAmelCase												=     parser.parse_args()

  transformers.logging.set_verbosity_error()
  # enables code execution in code_eval metric
  _UpperCAmelCase												=     args.HF_ALLOW_CODE_EVAL
  # make sure tokenizer plays nice with multiprocessing
  _UpperCAmelCase												=     "false"

  if args.num_workers is None:
    _UpperCAmelCase												=     multiprocessing.cpu_count()

  # Use dataset load to feed to accelerate
  _UpperCAmelCase												=     Accelerator()
  set_seed(args.seed						,  device_specific=UpperCAmelCase_	)

  # Load model and tokenizer
  _UpperCAmelCase												=     AutoTokenizer.from_pretrained(args.model_ckpt	)
  _UpperCAmelCase												=     tokenizer.eos_token
  _UpperCAmelCase												=     AutoModelForCausalLM.from_pretrained(args.model_ckpt	)

  # Generation settings
  _UpperCAmelCase												=     {
      "do_sample": args.do_sample,
      "temperature": args.temperature,
      "max_new_tokens": args.max_new_tokens,
      "top_p": args.top_p,
      "top_k": args.top_k,
      "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0						,  UpperCAmelCase_						,  UpperCAmelCase_	)]	),
  }

  # Load evaluation dataset and metric
  _UpperCAmelCase												=     load_dataset("openai_humaneval"	)
  _UpperCAmelCase												=     load_metric("code_eval"	)

  _UpperCAmelCase												=     args.num_tasks if args.num_tasks is not None else len(human_eval["test"]	)
  _UpperCAmelCase												=     args.n_samples // args.batch_size

  _UpperCAmelCase												=     TokenizedDataset(UpperCAmelCase_						,  human_eval["test"]						,  n_copies=UpperCAmelCase_						,  n_tasks=UpperCAmelCase_	)
  # do not confuse args.batch_size, which is actually the num_return_sequences
  _UpperCAmelCase												=     DataLoader(UpperCAmelCase_						,  batch_size=1	)

  # Run a quick test to see if code evaluation is enabled
  try:
    _UpperCAmelCase												=     code_eval_metric.compute(references=[""]						,  predictions=[[""]]	)
  except ValueError as exception:
    print(
        "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
        " flag to enable code evaluation."	)
    raise exception

  _UpperCAmelCase						,		_UpperCAmelCase												=     accelerator.prepare(UpperCAmelCase_						,  UpperCAmelCase_	)

  _UpperCAmelCase												=     complete_code(
      UpperCAmelCase_						,  UpperCAmelCase_						,  UpperCAmelCase_						,  UpperCAmelCase_						,  n_tasks=UpperCAmelCase_						,  batch_size=args.batch_size						,  **UpperCAmelCase_						,  )

  if accelerator.is_main_process:
    _UpperCAmelCase												=     []

    for task in tqdm(range(UpperCAmelCase_	)	):
      _UpperCAmelCase												=     human_eval["test"][task]["test"]
      _UpperCAmelCase												=     F'''check({human_eval["test"][task]["entry_point"]})'''
      references.append("\n" + test_func + "\n" + entry_point	)

    # Evaluate completions with "code_eval" metric
    _UpperCAmelCase						,		_UpperCAmelCase												=     code_eval_metric.compute(
        references=UpperCAmelCase_						,  predictions=UpperCAmelCase_						,  num_workers=args.num_workers	)
    print(F'''Results: {pass_at_k}'''	)

    # Save results to json file
    with open(args.output_file						,  "w"	) as fp:
      json.dump(UpperCAmelCase_						,  UpperCAmelCase_	)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
    main()










'''simple docstring'''




import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()






class 				__snake_case     (					unittest.TestCase							):





	def   _snake_case   (		self		)      ->		Dict:
		# clean up the VRAM after each test
		super().tearDown()
		gc.collect()
		torch.cuda.empty_cache()





	@property
	def   _snake_case   (		self		)      ->		str:
		snake_case__	       =   1
		snake_case__	       =   3
		snake_case__	       =   (32, 32)

		snake_case__	       =   floats_tensor((batch_size, num_channels) + sizes      ,       rng=random.Random(0		)		).to(UpperCamelCase_		)
		return image





	@property
	def   _snake_case   (		self		)      ->		Union[str, Any]:
		torch.manual_seed(0		)
		snake_case__	       =   UNetaDConditionModel(
		    block_out_channels=(32, 32, 64)      ,       layers_per_block=2      ,       sample_size=32      ,       in_channels=7      ,       out_channels=4      ,       down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D', 'CrossAttnDownBlock2D')      ,       up_block_types=('CrossAttnUpBlock2D', 'CrossAttnUpBlock2D', 'UpBlock2D')      ,       cross_attention_dim=32      ,       attention_head_dim=8      ,       use_linear_projection=UpperCamelCase_      ,       only_cross_attention=(True, True, False)      ,       num_class_embeds=100      ,       )
		return model





	@property
	def   _snake_case   (		self		)      ->		Optional[Any]:
		torch.manual_seed(0		)
		snake_case__	       =   AutoencoderKL(
		    block_out_channels=[32, 32, 64]      ,       in_channels=3      ,       out_channels=3      ,       down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D', 'DownEncoderBlock2D']      ,       up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D', 'UpDecoderBlock2D']      ,       latent_channels=4      ,       )
		return model





	@property
	def   _snake_case   (		self		)      ->		List[str]:
		torch.manual_seed(0		)
		snake_case__	       =   CLIPTextConfig(
		    bos_token_id=0      ,       eos_token_id=2      ,       hidden_size=32      ,       intermediate_size=37      ,       layer_norm_eps=1E-0_5      ,       num_attention_heads=4      ,       num_hidden_layers=5      ,       pad_token_id=1      ,       vocab_size=1000      ,       hidden_act='gelu'      ,       projection_dim=512      ,       )
		return CLIPTextModel(UpperCamelCase_		)





	def   _snake_case   (		self		)      ->		Union[str, Any]:
		snake_case__	       =   'cpu'  # ensure determinism for the device-dependent torch.Generator
		snake_case__	       =   self.dummy_cond_unet_upscale
		snake_case__	       =   DDPMScheduler()
		snake_case__	       =   DDIMScheduler(prediction_type='v_prediction'		)
		snake_case__	       =   self.dummy_vae
		snake_case__	       =   self.dummy_text_encoder
		snake_case__	       =   CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'		)

		snake_case__	       =   self.dummy_image.cpu().permute(0      ,       2      ,       3      ,       1		)[0]
		snake_case__	       =   Image.fromarray(np.uinta(UpperCamelCase_		)		).convert('RGB'		).resize((64, 64)		)

		# make sure here that pndm scheduler skips prk
		snake_case__	       =   StableDiffusionUpscalePipeline(
		    unet=UpperCamelCase_      ,       low_res_scheduler=UpperCamelCase_      ,       scheduler=UpperCamelCase_      ,       vae=UpperCamelCase_      ,       text_encoder=UpperCamelCase_      ,       tokenizer=UpperCamelCase_      ,       max_noise_level=350      ,       )
		snake_case__	       =   sd_pipe.to(UpperCamelCase_		)
		sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

		snake_case__	       =   'A painting of a squirrel eating a burger'
		snake_case__	       =   torch.Generator(device=UpperCamelCase_		).manual_seed(0		)
		snake_case__	       =   sd_pipe(
		    [prompt]      ,       image=UpperCamelCase_      ,       generator=UpperCamelCase_      ,       guidance_scale=6.0      ,       noise_level=20      ,       num_inference_steps=2      ,       output_type='np'      ,       )

		snake_case__	       =   output.images

		snake_case__	       =   torch.Generator(device=UpperCamelCase_		).manual_seed(0		)
		snake_case__	       =   sd_pipe(
		    [prompt]      ,       image=UpperCamelCase_      ,       generator=UpperCamelCase_      ,       guidance_scale=6.0      ,       noise_level=20      ,       num_inference_steps=2      ,       output_type='np'      ,       return_dict=UpperCamelCase_      ,       )[0]

		snake_case__	       =   image[0, -3:, -3:, -1]
		snake_case__	       =   image_from_tuple[0, -3:, -3:, -1]

		snake_case__	       =   low_res_image.size[0] * 4
		assert image.shape == (1, expected_height_width, expected_height_width, 3)
		snake_case__	       =   np.array([0.3_1_1_3, 0.3_9_1_0, 0.4_2_7_2, 0.4_8_5_9, 0.5_0_6_1, 0.4_6_5_2, 0.5_3_6_2, 0.5_7_1_5, 0.5_6_6_1]		)

		assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2
		assert np.abs(image_from_tuple_slice.flatten() - expected_slice		).max() < 1E-2





	def   _snake_case   (		self		)      ->		List[str]:
		snake_case__	       =   'cpu'  # ensure determinism for the device-dependent torch.Generator
		snake_case__	       =   self.dummy_cond_unet_upscale
		snake_case__	       =   DDPMScheduler()
		snake_case__	       =   DDIMScheduler(prediction_type='v_prediction'		)
		snake_case__	       =   self.dummy_vae
		snake_case__	       =   self.dummy_text_encoder
		snake_case__	       =   CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'		)

		snake_case__	       =   self.dummy_image.cpu().permute(0      ,       2      ,       3      ,       1		)[0]
		snake_case__	       =   Image.fromarray(np.uinta(UpperCamelCase_		)		).convert('RGB'		).resize((64, 64)		)

		# make sure here that pndm scheduler skips prk
		snake_case__	       =   StableDiffusionUpscalePipeline(
		    unet=UpperCamelCase_      ,       low_res_scheduler=UpperCamelCase_      ,       scheduler=UpperCamelCase_      ,       vae=UpperCamelCase_      ,       text_encoder=UpperCamelCase_      ,       tokenizer=UpperCamelCase_      ,       max_noise_level=350      ,       )
		snake_case__	       =   sd_pipe.to(UpperCamelCase_		)
		sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

		snake_case__	       =   'A painting of a squirrel eating a burger'
		snake_case__	       =   sd_pipe(
		    2 * [prompt]      ,       image=2 * [low_res_image]      ,       guidance_scale=6.0      ,       noise_level=20      ,       num_inference_steps=2      ,       output_type='np'      ,       )
		snake_case__	       =   output.images
		assert image.shape[0] == 2

		snake_case__	       =   torch.Generator(device=UpperCamelCase_		).manual_seed(0		)
		snake_case__	       =   sd_pipe(
		    [prompt]      ,       image=UpperCamelCase_      ,       generator=UpperCamelCase_      ,       num_images_per_prompt=2      ,       guidance_scale=6.0      ,       noise_level=20      ,       num_inference_steps=2      ,       output_type='np'      ,       )
		snake_case__	       =   output.images
		assert image.shape[0] == 2





	@unittest.skipIf(torch_device != 'cuda'      ,       'This test requires a GPU'		)
	def   _snake_case   (		self		)      ->		str:
		snake_case__	       =   self.dummy_cond_unet_upscale
		snake_case__	       =   DDPMScheduler()
		snake_case__	       =   DDIMScheduler(prediction_type='v_prediction'		)
		snake_case__	       =   self.dummy_vae
		snake_case__	       =   self.dummy_text_encoder
		snake_case__	       =   CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'		)

		snake_case__	       =   self.dummy_image.cpu().permute(0      ,       2      ,       3      ,       1		)[0]
		snake_case__	       =   Image.fromarray(np.uinta(UpperCamelCase_		)		).convert('RGB'		).resize((64, 64)		)

		# put models in fp16, except vae as it overflows in fp16
		snake_case__	       =   unet.half()
		snake_case__	       =   text_encoder.half()

		# make sure here that pndm scheduler skips prk
		snake_case__	       =   StableDiffusionUpscalePipeline(
		    unet=UpperCamelCase_      ,       low_res_scheduler=UpperCamelCase_      ,       scheduler=UpperCamelCase_      ,       vae=UpperCamelCase_      ,       text_encoder=UpperCamelCase_      ,       tokenizer=UpperCamelCase_      ,       max_noise_level=350      ,       )
		snake_case__	       =   sd_pipe.to(UpperCamelCase_		)
		sd_pipe.set_progress_bar_config(disable=UpperCamelCase_		)

		snake_case__	       =   'A painting of a squirrel eating a burger'
		snake_case__	       =   torch.manual_seed(0		)
		snake_case__	       =   sd_pipe(
		    [prompt]      ,       image=UpperCamelCase_      ,       generator=UpperCamelCase_      ,       num_inference_steps=2      ,       output_type='np'      ,       ).images

		snake_case__	       =   low_res_image.size[0] * 4
		assert image.shape == (1, expected_height_width, expected_height_width, 3)






@slow
@require_torch_gpu
class 				__snake_case     (					unittest.TestCase							):





	def   _snake_case   (		self		)      ->		List[str]:
		# clean up the VRAM after each test
		super().tearDown()
		gc.collect()
		torch.cuda.empty_cache()





	def   _snake_case   (		self		)      ->		Optional[int]:
		snake_case__	       =   load_image(
		    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
		    '/sd2-upscale/low_res_cat.png'		)
		snake_case__	       =   load_numpy(
		    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
		    '/upsampled_cat.npy'		)

		snake_case__	       =   'stabilityai/stable-diffusion-x4-upscaler'
		snake_case__	       =   StableDiffusionUpscalePipeline.from_pretrained(UpperCamelCase_		)
		pipe.to(UpperCamelCase_		)
		pipe.set_progress_bar_config(disable=UpperCamelCase_		)
		pipe.enable_attention_slicing()

		snake_case__	       =   'a cat sitting on a park bench'

		snake_case__	       =   torch.manual_seed(0		)
		snake_case__	       =   pipe(
		    prompt=UpperCamelCase_      ,       image=UpperCamelCase_      ,       generator=UpperCamelCase_      ,       output_type='np'      ,       )
		snake_case__	       =   output.images[0]

		assert image.shape == (512, 512, 3)
		assert np.abs(expected_image - image		).max() < 1E-3





	def   _snake_case   (		self		)      ->		List[Any]:
		snake_case__	       =   load_image(
		    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
		    '/sd2-upscale/low_res_cat.png'		)
		snake_case__	       =   load_numpy(
		    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
		    '/upsampled_cat_fp16.npy'		)

		snake_case__	       =   'stabilityai/stable-diffusion-x4-upscaler'
		snake_case__	       =   StableDiffusionUpscalePipeline.from_pretrained(
		    UpperCamelCase_      ,       torch_dtype=torch.floataa      ,       )
		pipe.to(UpperCamelCase_		)
		pipe.set_progress_bar_config(disable=UpperCamelCase_		)
		pipe.enable_attention_slicing()

		snake_case__	       =   'a cat sitting on a park bench'

		snake_case__	       =   torch.manual_seed(0		)
		snake_case__	       =   pipe(
		    prompt=UpperCamelCase_      ,       image=UpperCamelCase_      ,       generator=UpperCamelCase_      ,       output_type='np'      ,       )
		snake_case__	       =   output.images[0]

		assert image.shape == (512, 512, 3)
		assert np.abs(expected_image - image		).max() < 5E-1





	def   _snake_case   (		self		)      ->		int:
		torch.cuda.empty_cache()
		torch.cuda.reset_max_memory_allocated()
		torch.cuda.reset_peak_memory_stats()

		snake_case__	       =   load_image(
		    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
		    '/sd2-upscale/low_res_cat.png'		)

		snake_case__	       =   'stabilityai/stable-diffusion-x4-upscaler'
		snake_case__	       =   StableDiffusionUpscalePipeline.from_pretrained(
		    UpperCamelCase_      ,       torch_dtype=torch.floataa      ,       )
		pipe.to(UpperCamelCase_		)
		pipe.set_progress_bar_config(disable=UpperCamelCase_		)
		pipe.enable_attention_slicing(1		)
		pipe.enable_sequential_cpu_offload()

		snake_case__	       =   'a cat sitting on a park bench'

		snake_case__	       =   torch.manual_seed(0		)
		snake_case__	       =   pipe(
		    prompt=UpperCamelCase_      ,       image=UpperCamelCase_      ,       generator=UpperCamelCase_      ,       num_inference_steps=5      ,       output_type='np'      ,       )

		snake_case__	       =   torch.cuda.max_memory_allocated()
		# make sure that less than 2.9 GB is allocated
		assert mem_bytes < 2.9 * 10**9



